D-Net

package eu.dnetlib.msro.workflows.nodes.repohi;

import java.util.Map;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.env.Env;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.enabling.locators.comparators.HandledDatastructuresComparator;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

public class CreateMDStoreJobNode extends BlackboardJobNode {

	private static final Log log = LogFactory.getLog(CreateMDStoreJobNode.class);

	private String format;

	private String layout;

	private String interpretation;

	private String outputPrefix = "mdstore";

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(MDStoreService.class, new HandledDatastructuresComparator());

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) {

		log.info("preparing blackboard job: " + getFormat());

		job.setAction("CREATE");

		job.getParameters().put("format", getFormat());

		job.getParameters().put("interpretation", getInterpretation());

		job.getParameters().put("layout", getLayout());

	}

	public String getFormat() {

		return format;

	}

	public void setFormat(final String format) {

		this.format = format;

	}

	public String getInterpretation() {

		return interpretation;

	}

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

	public String getLayout() {

		return layout;

	}

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	public String getOutputPrefix() {

		return outputPrefix;

	}

	public void setOutputPrefix(final String outputPrefix) {

		this.outputPrefix = outputPrefix;

	}

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new BlackboardWorkflowJobListener(engine, token) {

			@Override

			protected void populateEnv(final Env env, final Map<String, String> responseParams) {

				env.setAttribute(getOutputPrefix() + "format", format);

				env.setAttribute(getOutputPrefix() + "layout", layout);

				env.setAttribute(getOutputPrefix() + "interpretation", interpretation);

				env.setAttribute(getOutputPrefix() + "id", responseParams.get("id"));

			}

		};

	}

}

package eu.dnetlib.msro.workflows.nodes.repohi;

import java.util.List;

import javax.annotation.Resource;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants.WorkflowStatus;

public class UpdateMetaWfStatusJobNode extends SimpleJobNode {

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String metaWfId = token.getFullEnv().getAttribute("META_WORKFLOW_ID");

		final String dsId = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ID);

		final String ifaceId = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE);

		updateDatasource(dsId, ifaceId);

		if (isReady(metaWfId)) {

			serviceLocator.getService(ISRegistryService.class).updateProfileNode(metaWfId, "//CONFIGURATION/@status", "'" + WorkflowStatus.EXECUTABLE + "'");

		}

		return Arc.DEFAULT_ARC;

	}

	protected void updateDatasource(final String dsId, final String ifaceId) throws Exception {

		serviceLocator.getService(ISRegistryService.class).updateProfileNode(dsId, "//INTERFACE[@id = '" + ifaceId + "']/@active", "'true'");

	}

	private boolean isReady(final String metaWfId) throws Exception {

		final String query = "for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')"

				+ "//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value eq '" + metaWfId + "']//WORKFLOW/@id "

				+ "for $y in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType') " + "where $y//RESOURCE_IDENTIFIER/@value = $x "

				+ "return $y//PARAM[@required='true' and string-length(text()) = 0]/@managedBy/string()";

		final List<String> list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query);

		if (list.contains("system")) { throw new MSROException("A system param is missing in profile: " + metaWfId); }

		return list.isEmpty();

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-msro-service</artifactId>

	<packaging>jar</packaging>

	<version>3.1.12</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-msro-service/tags/dnet-msro-service-3.1.12</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>opensymphony</groupId>

			<artifactId>quartz</artifactId>

			<version>1.6.6</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-msro-service-api</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-data-utility-cleaner-rmi</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-client</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service-rmi</artifactId>

			<version>[1.1.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-datasource-manager-rmi</artifactId>

			<version>[4.0.0,5.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[2.1.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-mongo-logging</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.googlecode</groupId>

			<artifactId>sarasvati</artifactId>

			<version>1.0.3</version>

		</dependency>

		<dependency>

			<groupId>com.googlecode</groupId>

			<artifactId>sarasvati-visual</artifactId>

			<version>1.0.3</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service-rmi</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-enabling-database-api</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-objectstore-rmi</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-data-transformation-service-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-data-provision-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-runtime</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>javax.mail</groupId>

			<artifactId>mail</artifactId>

			<version>1.4</version>

		</dependency>

		<dependency>

			<groupId>org.codehaus.groovy</groupId>

			<artifactId>groovy-all</artifactId>

			<version>2.1.6</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.3</version>

		</dependency>

	</dependencies>

	<properties>

		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

	</properties>

</project>

package eu.dnetlib.msro.workflows.nodes.mdstore;

import java.util.Map;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.env.Env;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.enabling.resultset.rmi.ResultSetException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

public class StoreMDStoreRecordsJobNode extends BlackboardJobNode implements ProgressJobNode {

	private static final Log log = LogFactory.getLog(StoreMDStoreRecordsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String eprParam;

	private String mdId;

	private String storingType;

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	private ResultsetProgressProvider progressProvider;

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(MDStoreService.class, mdId);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws ResultSetException {

		job.setAction("FEED");

		final String eprS = token.getEnv().getAttribute(getEprParam());

		this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), eprS);

		job.getParameters().put("epr", progressProvider.getEpr().toString());

		job.getParameters().put("storingType", getStoringType());

		job.getParameters().put("mdId", getMdId());

	}

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new BlackboardWorkflowJobListener(engine, token) {

			@Override

			protected void populateEnv(final Env env, final Map<String, String> responseParams) {

				log.info("Total records: " + responseParams.get("total"));

				log.info("Number of stored records: " + responseParams.get("stored"));

				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", responseParams.get("total"));

				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "stored", responseParams.get("stored"));

			}

		};

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

	public String getMdId() {

		return mdId;

	}

	public void setMdId(final String mdId) {

		this.mdId = mdId;

	}

	public String getStoringType() {

		return storingType;

	}

	public void setStoringType(final String storingType) {

		this.storingType = storingType;

	}

	public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {

		return processCountingResultSetFactory;

	}

	@Required

	public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {

		this.processCountingResultSetFactory = processCountingResultSetFactory;

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

}

package eu.dnetlib.msro.workflows.nodes.mdstore;

import java.util.Iterator;

import java.util.List;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.data.mdstore.MDStoreServiceException;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class MultipleMdStoreIterator.

 */

public class MultipleMdStoreIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(MultipleMdStoreIterator.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** The service locator. */

	@Resource

	private UniqueServiceLocator serviceLocator;

	/** The current iterator. */

	private Iterator<String> currentIterator;

	/** The result set client factory. */

	@Autowired

	private ResultSetClientFactory resultSetClientFactory;

	/**

	 * Instantiates a new multiple md store iterator.

	 *

	 * @param mdstoreLocator

	 *            the mdstore locator

	 * @param mdIds

	 *            the md ids

	 */

	public MultipleMdStoreIterator(final UniqueServiceLocator serviceLocator, final List<String> mdIds, final ResultSetClientFactory resultSetClientFactory) {

		this.serviceLocator = serviceLocator;

		this.resultSetClientFactory = resultSetClientFactory;

		log.info(String.format("iterating over mdIds: '%s'", mdIds));

		this.currentIterator = Iterables.concat(Iterables.transform(mdIds, new Function<String, Iterable<String>>() {

			@Override

			public Iterable<String> apply(final String mdId) {

				log.debug(String.format("current mdId '%s'", mdId));

				try {

					W3CEndpointReference epr = serviceLocator.getService(MDStoreService.class, mdId).deliverMDRecords(mdId, "", "", "");

					return resultSetClientFactory.getClient(epr);

				} catch (MDStoreServiceException e) {

					log.error(e);

					throw new RuntimeException(String.format("unable to iterate over %s", mdId), e);

				}

			}

		})).iterator();

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see java.util.Iterator#hasNext()

	 */

	@Override

	public boolean hasNext() {

		if (currentIterator == null) return false;

		return currentIterator.hasNext();

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see java.util.Iterator#next()

	 */

	@Override

	public String next() {

		return currentIterator.next();

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see java.util.Iterator#remove()

	 */

	@Override

	public void remove() {

		throw new UnsupportedOperationException("Cannot remove");

	}

}

package eu.dnetlib.msro.workflows.nodes.mdstore;

import java.util.List;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import com.google.gson.Gson;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.IterableResultSetFactory;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class FetchMultipleMDStores extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(FetchMultipleMDStores.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Resource

	private UniqueServiceLocator serviceLocator;

	private List<String> mdId;

	private String eprParam;

	/** The result set factory. */

	@Resource(name = "iterableResultSetFactory")

	private IterableResultSetFactory resultSetFactory;

	/** The result set client factory. */

	@Autowired

	private ResultSetClientFactory resultSetClientFactory;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		if (getMdId() == null) {

			final String mdIdsJson = token.getEnv().getAttribute("mdId");

			@SuppressWarnings("unchecked")

			final List<String> mdIds = new Gson().fromJson(mdIdsJson, List.class);

			log.info(String.format("fetching records for %s mdstores", mdIds.size()));

			if (log.isDebugEnabled()) {

				log.debug(String.format("fetching records for mdstores: %s", mdIdsJson));

			}

			setMdId(mdIds);

		}

		final MultipleMdStoreIterable iter = new MultipleMdStoreIterable(serviceLocator, getMdId(), resultSetClientFactory);

		final W3CEndpointReference epr = resultSetFactory.createIterableResultSet(iter);

		token.getEnv().setAttribute(getEprParam(), epr.toString());

		return Arc.DEFAULT_ARC;

	}

	public List<String> getMdId() {

		return mdId;

	}

	public void setMdId(final List<String> mdId) {

		this.mdId = mdId;

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

}

package eu.dnetlib.msro.workflows.nodes.mdstore;

import java.util.Iterator;

import java.util.List;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class MultipleMdStoreIterable.

 */

public class MultipleMdStoreIterable implements Iterable<String> {

	/** The service locator. */

	private UniqueServiceLocator serviceLocator;

	/** The md ids. */

	private List<String> mdIds;

	/** The result set client factory. */

	private ResultSetClientFactory resultSetClientFactory;

	/**

	 * Instantiates a new multiple md store iterable.

	 *

	 * @param serviceLocator

	 *            the service locator

	 * @param mdIds

	 *            the md ids

	 * @param resultSetClientFactory

	 *            the result set client factory

	 */

	public MultipleMdStoreIterable(final UniqueServiceLocator serviceLocator, final List<String> mdIds, final ResultSetClientFactory resultSetClientFactory) {

		super();

		this.serviceLocator = serviceLocator;

		this.mdIds = mdIds;

		this.resultSetClientFactory = resultSetClientFactory;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<String> iterator() {

		return new MultipleMdStoreIterator(serviceLocator, mdIds, resultSetClientFactory);

	}

}

package eu.dnetlib.msro.workflows.nodes.info;

import java.io.StringReader;

import java.util.Map;

import javax.annotation.Resource;

import org.dom4j.Document;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import com.google.common.collect.Maps;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerService;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class ObjectStoreToApiExtraFieldJobNode extends SimpleJobNode {

	private String objId;

	private String datasourceId;

	private String datasourceInterface;

	private String extraFieldForTotal;

	private String extraFieldForDate;

	private String extraFieldForObjId;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String xq = "for $x in collection('/db/DRIVER/ObjectStoreDSResources/ObjectStoreDSResourceType') " + "where $x//RESOURCE_IDENTIFIER/@value='"

				+ objId + "' " + "return concat($x//COUNT_STORE, ' @=@ ', $x//LAST_STORAGE_DATE)";

		final String s = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xq);

		final String[] arr = s.split(" @=@ ");

		final Map<String, String> map = getCurrentExtraFields(datasourceId, datasourceInterface);

		map.put(extraFieldForTotal, arr[0].trim());

		map.put(extraFieldForDate, arr[1].trim());

		map.put(extraFieldForObjId, objId);

		serviceLocator.getService(DatasourceManagerService.class).bulkUpdateApiExtraFields(datasourceId, datasourceInterface, map);

		return Arc.DEFAULT_ARC;

	}

	private Map<String, String> getCurrentExtraFields(final String repoId, final String ifaceId) throws Exception {

		final Map<String, String> res = Maps.newHashMap();

		final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfile(repoId);

		final SAXReader reader = new SAXReader();

		final Document doc = reader.read(new StringReader(profile));

		final Node ifcNode = doc.selectSingleNode("//INTERFACE[@id='" + ifaceId + "']");

		if (ifcNode != null) {

			for (Object o : ifcNode.selectNodes("./INTERFACE_EXTRA_FIELD")) {

				res.put(((Node) o).valueOf("@name"), ((Node) o).getText());

			}

		}

		return res;

	}

	public String getDatasourceId() {

		return datasourceId;

	}

	public void setDatasourceId(final String datasourceId) {

		this.datasourceId = datasourceId;

	}

	public String getDatasourceInterface() {

		return datasourceInterface;

	}

	public void setDatasourceInterface(final String datasourceInterface) {

		this.datasourceInterface = datasourceInterface;

	}

	public String getExtraFieldForTotal() {

		return extraFieldForTotal;

	}

	public void setExtraFieldForTotal(final String extraFieldForTotal) {

		this.extraFieldForTotal = extraFieldForTotal;

	}

	public String getExtraFieldForDate() {

		return extraFieldForDate;

	}

	public void setExtraFieldForDate(final String extraFieldForDate) {

		this.extraFieldForDate = extraFieldForDate;

	}

	/**

	 * @return the objId

	 */

	public String getObjId() {

		return objId;

	}

	/**

	 * @param objId

	 *            the objId to set

	 */

	public void setObjId(final String objId) {

		this.objId = objId;

	}

	/**

	 * @return the extraFieldForObjId

	 */

	public String getExtraFieldForObjId() {

		return extraFieldForObjId;

	}

	/**

	 * @param extraFieldForObjId

	 *            the extraFieldForObjId to set

	 */

	public void setExtraFieldForObjId(final String extraFieldForObjId) {

		this.extraFieldForObjId = extraFieldForObjId;

	}

}

package eu.dnetlib.msro.workflows.nodes.db;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.database.rmi.DatabaseService;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

public class UpdateDbJobNode extends BlackboardJobNode implements ProgressJobNode {

	private String db;

	private String dbParam;

	private String eprParam;

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	private ResultsetProgressProvider progressProvider;

	private static final Log log = LogFactory.getLog(UpdateDbJobNode.class);

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(DatabaseService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		if (db == null || db.isEmpty()) {

			db = token.getEnv().getAttribute(getDbParam());

		}

		log.info("preparing blackboard job to update DB: " + db);

		final String epr = token.getEnv().getAttribute(getEprParam());

		this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);

		job.setAction("IMPORT");

		job.getParameters().put("db", db);

		job.getParameters().put("epr", this.progressProvider.getEpr().toString());

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

	public String getDb() {

		return db;

	}

	public void setDb(final String db) {

		this.db = db;

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

	public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {

		return processCountingResultSetFactory;

	}

	@Required

	public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {

		this.processCountingResultSetFactory = processCountingResultSetFactory;

	}

	public String getDbParam() {

		return dbParam;

	}

	public void setDbParam(final String dbParam) {

		this.dbParam = dbParam;

	}

}

package eu.dnetlib.msro.workflows.nodes.mdstore;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class FetchMDStoreRecordsJobNode extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(FetchMDStoreRecordsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM

	@Resource

	private UniqueServiceLocator serviceLocator;

	private String mdId;

	private String mdFormat;

	private String eprParam;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		if (getMdId() == null) {

			setMdId(token.getEnv().getAttribute("mdId"));

		}

		if (getMdFormat() == null) {

			setMdFormat(token.getEnv().getAttribute("mdFormat"));

		}

		log.info("getting MDRecords from: " + getMdId());

		final W3CEndpointReference epr = serviceLocator.getService(MDStoreService.class, getMdId()).deliverMDRecords(getMdId(), "", "", "");

		if (epr == null) { throw new MSROException("unable to read MDRecords from: " + getMdId()); }

		token.getEnv().setAttribute(getEprParam(), epr.toString());

		return Arc.DEFAULT_ARC;

	}

	public String getMdId() {

		return mdId;

	}

	public void setMdId(final String mdId) {

		this.mdId = mdId;

	}

	public String getMdFormat() {

		return mdFormat;

	}

	public void setMdFormat(final String mdFormat) {

		this.mdFormat = mdFormat;

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

}

package eu.dnetlib.msro.workflows.nodes.download;

import java.util.Map;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.env.Env;

import eu.dnetlib.data.download.rmi.DownloadService;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

// TODO: Auto-generated Javadoc

/**

 * The Class DownloadFromMetadata is a job node that send a blackboard message to the Download service to start to download file from url

 * retrieved by Metadata .

 */

public class DownloadFromMetadataJobNode extends BlackboardJobNode implements ProgressJobNode {

	private static final Log log = LogFactory.getLog(DownloadFromMetadataJobNode.class);

	protected String regularExpression;

	/** The inputepr param. */

	private String inputeprParam;

	/** The obejct store id. */

	private String objectStoreID;

	/** The plugin. */

	private String plugin;

	/** The protocol. */

	private String protocol;

	/** The mime type. */

	private String mimeType;

	private int numberOfThreads = -1;

	private int connectTimeoutMs = -1;

	private int readTimeoutMs = -1;

	private int sleepTimeMs = 0;

	/** The process counting result set factory. */

	@Autowired

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	/** The progress provider. */

	private ResultsetProgressProvider progressProvider;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.msro.workflows.nodes.BlackboardJobNode#obtainServiceId(com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(DownloadService.class);

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.msro.workflows.nodes.BlackboardJobNode#prepareJob(eu.dnetlib.enabling.tools.blackboard.BlackboardJob,

	 * com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		job.setAction("DOWNLOAD");

		final String eprS = token.getEnv().getAttribute(getInputeprParam());

		this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), eprS);

		job.getParameters().put("epr", progressProvider.getEpr().toString());

		job.getParameters().put("protocol", getProtocol());

		job.getParameters().put("plugin", getPlugin());

		job.getParameters().put("mimeType", getMimeType());

		job.getParameters().put("objectStoreID", getObjectStoreID());

		if (getNumberOfThreads() > 0) {

			job.getParameters().put("numberOfThreads", "" + getNumberOfThreads());

		}

		if (getConnectTimeoutMs() > 0) {

			job.getParameters().put("connectTimeoutMs", "" + getConnectTimeoutMs());

		}

		if (getReadTimeoutMs() > 0) {

			job.getParameters().put("readTimeoutMs", "" + getReadTimeoutMs());

		}

		if (getSleepTimeMs() > 0) {

			job.getParameters().put("sleepTimeMs", "" + getSleepTimeMs());

		}

		if (!StringUtils.isEmpty(getRegularExpression())){

			job.getParameters().put("regularExpressions", getRegularExpression());

		}

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.msro.workflows.nodes.BlackboardJobNode#generateBlackboardListener(com.googlecode.sarasvati.Engine,

	 * com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new BlackboardWorkflowJobListener(engine, token) {

			@Override

			protected void populateEnv(final Env env, final Map<String, String> responseParams) {

				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", responseParams.get("total"));

				final String base64 = responseParams.get("report");

				if (StringUtils.isNotBlank(base64) && Base64.isBase64(base64.getBytes())) {

					final String report = new String(Base64.decodeBase64(base64.getBytes()));

					log.info("found download report");

					log.debug(report);

					env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "report", report);

				} else {

					log.warn("unable to find or decode download report");

				}

			}

		};

	}

	/**

	 * Gets the inputepr param.

	 *

	 * @return the inputeprParam

	 */

	public String getInputeprParam() {

		return inputeprParam;

	}

	/**

	 * Sets the inputepr param.

	 *

	 * @param inputeprParam

	 *            the inputeprParam to set

	 */

	public void setInputeprParam(final String inputeprParam) {

		this.inputeprParam = inputeprParam;

	}

	/**

	 * Gets the object store id.

	 *

	 * @return the objectStoreID

	 */

	public String getObjectStoreID() {

		return objectStoreID;

	}

	/**

	 * Sets the object store id.

	 *

	 * @param objectStoreID

	 *            the objectStoreID to set

	 */

	public void setObjectStoreID(final String objectStoreID) {

		this.objectStoreID = objectStoreID;

	}

	/**

	 * Gets the plugin.

	 *

	 * @return the plugin

	 */

	public String getPlugin() {

		return plugin;

	}

	/**

	 * Sets the plugin.

	 *

	 * @param plugin

	 *            the plugin to set

	 */

	public void setPlugin(final String plugin) {

		this.plugin = plugin;

	}

	/**

	 * Gets the protocol.

	 *

	 * @return the protol

	 */

	public String getProtocol() {

		return protocol;

	}

	/**

	 * Sets the protocol.

	 *

	 * @param protol

	 *            the protol to set

	 */

	public void setProtocol(final String protol) {

		this.protocol = protol;

	}

	/**

	 * Gets the mime type.

	 *

	 * @return the mimeType

	 */

	public String getMimeType() {

		return mimeType;

	}

	/**

	 * Sets the mime type.

	 *

	 * @param mimeType

	 *            the mimeType to set

	 */

	public void setMimeType(final String mimeType) {

		this.mimeType = mimeType;

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.msro.workflows.nodes.ProgressJobNode#getProgressProvider()

	 */

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

	public String getRegularExpression() {

		return regularExpression;

	}

	public void setRegularExpression(final String regularExpression) {

		this.regularExpression = regularExpression;

	}

	public int getNumberOfThreads() {

		return numberOfThreads;

	}

	public void setNumberOfThreads(final int numberOfThreads) {

		this.numberOfThreads = numberOfThreads;

	}

	public int getConnectTimeoutMs() {

		return connectTimeoutMs;

	}

	public void setConnectTimeoutMs(final int connectTimeoutMs) {

		this.connectTimeoutMs = connectTimeoutMs;

	}

	public int getReadTimeoutMs() {

		return readTimeoutMs;

	}

	public void setReadTimeoutMs(final int readTimeoutMs) {

		this.readTimeoutMs = readTimeoutMs;

	}

	public int getSleepTimeMs() {

		return sleepTimeMs;

	}

	public void setSleepTimeMs(final int sleepTimeMs) {

		this.sleepTimeMs = sleepTimeMs;

	}

}

package eu.dnetlib.msro.workflows.nodes.db;

import java.io.IOException;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.StringUtils;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.database.rmi.DatabaseService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;

public class QueryDbJobNode extends AsyncJobNode {

	private String db;

	private String dbParam;

	private String dbProperty;

	private String sql;

	private String sqlForSize;

	private String xslt;

	private String outputEprParam;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String sqlText = fetchSqlAsText(sql);

		W3CEndpointReference epr = null;

		final DatabaseService dbService = serviceLocator.getService(DatabaseService.class);

		if (StringUtils.isNotBlank(xslt)) {

			final String xsltText = IOUtils.toString(getClass().getResourceAsStream(xslt));

			if (StringUtils.isBlank(sqlForSize)) {

				epr = dbService.xsltSearchSQL(findDb(token), sqlText, xsltText);

			} else {

				epr = dbService.alternativeXsltSearchSQL(findDb(token), sqlText, fetchSqlAsText(sqlForSize), xsltText);

			}

		} else {

			if (StringUtils.isBlank(sqlForSize)) {

				epr = dbService.searchSQL(findDb(token), sqlText);

			} else {

				epr = dbService.alternativeSearchSQL(findDb(token), sqlText, fetchSqlAsText(sqlForSize));

			}

		}

		token.getEnv().setAttribute(outputEprParam, epr.toString());

		return Arc.DEFAULT_ARC;

	}

	private String fetchSqlAsText(final String path) throws IOException {

		return IOUtils.toString(getClass().getResourceAsStream(path));

	}

	private String findDb(final NodeToken token) {

		if (dbParam != null && !dbParam.isEmpty()) {

			return token.getEnv().getAttribute(dbParam);

		} else if (dbProperty != null && !dbProperty.isEmpty()) {

			return getPropertyFetcher().getProperty(dbProperty);

		} else {

			return db;

		}

	}

	public String getDb() {

		return db;

	}

	public void setDb(final String db) {

		this.db = db;

	}

	public String getDbParam() {

		return dbParam;

	}

	public void setDbParam(final String dbParam) {

		this.dbParam = dbParam;

	}

	public String getSql() {

		return sql;

	}

	public void setSql(final String sql) {

		this.sql = sql;

	}

	public String getXslt() {

		return xslt;

	}

	public void setXslt(final String xslt) {

		this.xslt = xslt;

	}

	public String getOutputEprParam() {

		return outputEprParam;

	}

	public void setOutputEprParam(final String outputEprParam) {

		this.outputEprParam = outputEprParam;

	}

	public String getDbProperty() {

		return dbProperty;

	}

	public void setDbProperty(final String dbProperty) {

		this.dbProperty = dbProperty;

	}

	public String getSqlForSize() {

		return sqlForSize;

	}

	public void setSqlForSize(final String sqlForSize) {

		this.sqlForSize = sqlForSize;

	}

}

package eu.dnetlib.msro.workflows.nodes.repohi;

import java.io.IOException;

import java.io.StringWriter;

import javax.annotation.Resource;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants.WorkflowStatus;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.StringEscapeUtils;

public class RegisterMetaWfJobNode extends SimpleJobNode {

	@Resource

	private UniqueServiceLocator serviceLocator;

	private String adminEmail;

	private String wfName;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String dsId = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ID);

		final String dsName = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_NAME);

		final String ifaceId = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE);

		final String metaWfId = registerDatasourceWorkflow(dsId, dsName, ifaceId);

		token.getFullEnv().setAttribute("META_WORKFLOW_ID", metaWfId);

		token.getFullEnv().setAttribute("META_WORKFLOW_STATUS", WorkflowStatus.ASSIGNED.toString());

		return Arc.DEFAULT_ARC;

	}

	public String registerDatasourceWorkflow(final String dsId, final String dsName, final String ifaceId) throws ISRegistryException, IOException {

		final StringWriter sw = new StringWriter();

		IOUtils.copy(getClass().getResourceAsStream("/eu/dnetlib/msro/workflows/templates/meta-workflow.xml.st"), sw);

		final StringTemplate st = new StringTemplate(sw.toString());

		st.setAttribute("dsId", dsId);

		st.setAttribute("ifaceId", ifaceId);

		st.setAttribute("dsName", StringEscapeUtils.escapeXml(dsName));

		st.setAttribute("section", "dataproviders");

		st.setAttribute("wfName", StringEscapeUtils.escapeXml(getWfName()));

		st.setAttribute("wfFamily", StringEscapeUtils.escapeXml(getWfName()));

		st.setAttribute("adminEmail", adminEmail);

		st.setAttribute("status", WorkflowStatus.ASSIGNED.toString());

		return serviceLocator.getService(ISRegistryService.class).registerProfile(st.toString());

	}

	public String getWfName() {

		return wfName;

	}

	public void setWfName(final String wfName) {

		this.wfName = wfName;

	}

	public String getAdminEmail() {

		return adminEmail;

	}

	public void setAdminEmail(final String adminEmail) {

		this.adminEmail = adminEmail;

	}

}

package eu.dnetlib.msro.workflows.nodes.repohi;

import java.util.Map;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.env.Env;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

public class CreateObjectStoreJobNode extends BlackboardJobNode {

	private static final Log log = LogFactory.getLog(CreateObjectStoreJobNode.class);

	private String interpretation;

	private String outputPrefix = "objectStore_";

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(ObjectStoreService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) {

		log.info("preparing blackboard job for the creation of the objectStore ");

		String basePath  = token.getEnv().getAttribute("objectStoreBasePath");

		job.setAction("CREATE");

		job.getParameters().put("interpretation", interpretation);

		if (!StringUtils.isEmpty(basePath)) {

			job.getParameters().put("basePath", basePath);

		}

	}

	public String getInterpretation() {

		return interpretation;

	}

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

	public String getOutputPrefix() {

		return outputPrefix;

	}

	public void setOutputPrefix(final String outputPrefix) {

		this.outputPrefix = outputPrefix;

	}

	@Override

	protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {

		return new BlackboardWorkflowJobListener(engine, token) {

			@Override

			protected void populateEnv(final Env env, final Map<String, String> responseParams) {

				env.setAttribute(getOutputPrefix() + "interpretation", interpretation);

				env.setAttribute(getOutputPrefix() + "id", responseParams.get("objectStoreId"));

			}

		};