D-Net

package eu.dnetlib.msro.workflows.nodes.collect;

import java.io.StringReader;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.dom4j.Document;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.collector.rmi.CollectorService;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class DateRangeCollectRecordsJobNode extends SimpleJobNode {

	@Resource

	private UniqueServiceLocator serviceLocator;

	private String datasourceId;

	private String datasourceInterface;

	private String eprParam;

	private String fromDateParam;

	private String untilDateParam;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfile(datasourceId);

		final Document doc = new SAXReader().read(new StringReader(profile));

		final Node ifcNode = doc.selectSingleNode("//INTERFACE[@id='" + datasourceInterface + "']");

		final String fromDate = token.getEnv().getAttribute(getFromDateParam());

		final String untilDate = token.getEnv().getAttribute(getUntilDateParam());

		final InterfaceDescriptor interfaceDescriptor = InterfaceDescriptor.newInstance(ifcNode);

		final W3CEndpointReference epr = serviceLocator.getService(CollectorService.class).dateRangeCollect(interfaceDescriptor, fromDate, untilDate);

		token.getEnv().setAttribute(getEprParam(), epr.toString());

		return Arc.DEFAULT_ARC;

	}

	public String getFromDateParam() {

		return fromDateParam;

	}

	public void setFromDateParam(final String fromDateParam) {

		this.fromDateParam = fromDateParam;

	}

	public String getUntilDateParam() {

		return untilDateParam;

	}

	public void setUntilDateParam(final String untilDateParam) {

		this.untilDateParam = untilDateParam;

	}

	public String getDatasourceId() {

		return datasourceId;

	}

	public void setDatasourceId(final String datasourceId) {

		this.datasourceId = datasourceId;

	}

	public String getDatasourceInterface() {

		return datasourceInterface;

	}

	public void setDatasourceInterface(final String datasourceInterface) {

		this.datasourceInterface = datasourceInterface;

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-msro-service</artifactId>

	<packaging>jar</packaging>

	<version>2.2.2</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-msro-service/tags/dnet-msro-service-2.2.2</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>opensymphony</groupId>

			<artifactId>quartz</artifactId>

			<version>1.6.6</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-msro-service-api</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-data-utility-cleaner-rmi</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-client</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service-rmi</artifactId>

			<version>[1.1.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[2.1.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-mongo-logging</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-datasource-common</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.googlecode</groupId>

			<artifactId>sarasvati</artifactId>

			<version>1.0.3</version>

		</dependency>

		<dependency>

			<groupId>com.googlecode</groupId>

			<artifactId>sarasvati-visual</artifactId>

			<version>1.0.3</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service-rmi</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-enabling-database-api</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-objectstore-rmi</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-data-transformation-service-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-data-provision-rmi</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-runtime</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>javax.mail</groupId>

			<artifactId>mail</artifactId>

			<version>1.4</version>

		</dependency>

		<dependency>

			<groupId>org.codehaus.groovy</groupId>

			<artifactId>groovy-all</artifactId>

			<version>2.1.6</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.3</version>

		</dependency>

	</dependencies>

	<properties>

		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

	</properties>

</project>

package eu.dnetlib.msro.workflows.nodes.is;

import java.io.StringReader;

import java.util.List;

import javax.annotation.Resource;

import javax.xml.XMLConstants;

import javax.xml.transform.stream.StreamSource;

import javax.xml.validation.Schema;

import javax.xml.validation.SchemaFactory;

import javax.xml.validation.Validator;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public class ValidateProfilesJobNode extends SimpleJobNode implements ProgressJobNode {

	@Resource

	private UniqueServiceLocator serviceLocator;

	private int total = 0;

	private int current = 0;

	private static final Log log = LogFactory.getLog(ValidateProfilesJobNode.class);

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final ISLookUpService lookup = serviceLocator.getService(ISLookUpService.class);

		final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);

		int validTotals = 0;

		int invalidTotals = 0;

		final List<String> list = lookup.listResourceTypes();

		this.total = list.size();

		this.current = 0;

		for (String resourceType : list) {

			int valid = 0;

			int invalid = 0;

			final String schemaSource = lookup.getResourceTypeSchema(resourceType);

			final Schema schema = schemaFactory.newSchema(new StreamSource(new StringReader(schemaSource)));

			final Validator validator = schema.newValidator();

			for (String profile : lookup.quickSearchProfile("/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value='" + resourceType + "']")) {

				try {

					validator.validate(new StreamSource(new StringReader(profile)));

					valid++;

				} catch (Exception e) {

					invalid++;

				}

			}

			validTotals += valid;

			invalidTotals += invalid;

			final String message = String.format("Valid: %s, Invalid: %s, Total: %s", valid, invalid, valid + invalid);

			token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + resourceType, message);

			log.info("Validation of " + resourceType + " profiles: " + message);

		}

		if (invalidTotals > 0) { throw new MSROException("Validation wf has found " + invalidTotals + " invalid profiles"); }

		return Arc.DEFAULT_ARC;

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return new ProgressProvider() {

			@Override

			public boolean isInaccurate() {

				return false;

			}

			@Override

			public int getTotalValue() {

				return total;

			}

			@Override

			public int getCurrentValue() {

				return current;

			}

		};

	}

}

package eu.dnetlib.msro.workflows.nodes;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

public class SetEnvParameterJobNode extends SimpleJobNode {

	private String parameterName;

	private String parameterValue;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		token.getEnv().setAttribute(parameterName, parameterValue);

		return Arc.DEFAULT_ARC;

	}

	public String getParameterName() {

		return parameterName;

	}

	public void setParameterName(final String parameterName) {

		this.parameterName = parameterName;

	}

	public String getParameterValue() {

		return parameterValue;

	}

	public void setParameterValue(final String parameterValue) {

		this.parameterValue = parameterValue;

	}

}

package eu.dnetlib.msro.workflows.nodes.info;

import java.io.StringReader;

import java.util.Map;

import javax.annotation.Resource;

import org.dom4j.Document;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.collect.Maps;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.datasource.common.utils.DatasourceUpdater;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

public class MDStoreToApiExtraFieldJobNode extends SimpleJobNode {

	private String mdId;

	private String datasourceId;

	private String datasourceInterface;

	private String extraFieldForTotal;

	private String extraFieldForDate;

	private String extraFieldForMdId;

	@Resource

	private UniqueServiceLocator serviceLocator;

	private DatasourceUpdater datasourceUpdater;

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String xq = "for $x in collection('/db/DRIVER/MDStoreDSResources/MDStoreDSResourceType') " +

				"where $x//RESOURCE_IDENTIFIER/@value='" + mdId + "' " +

				"return concat($x//NUMBER_OF_RECORDS, ' @=@ ', $x//LAST_STORAGE_DATE)";

		final String s = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xq);

		final String[] arr = s.split(" @=@ ");

		final Map<String, String> map = getCurrentExtraFields(datasourceId, datasourceInterface);

		map.put(extraFieldForTotal, arr[0].trim());

		map.put(extraFieldForDate, arr[1].trim());

		map.put(extraFieldForMdId, mdId);

		datasourceUpdater.updateApiExtraFields(datasourceId, datasourceInterface, map);

		return Arc.DEFAULT_ARC;

	}

	private Map<String, String> getCurrentExtraFields(final String repoId, final String ifaceId) throws Exception {

		final Map<String, String> res = Maps.newHashMap();

		final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfile(repoId);

		final SAXReader reader = new SAXReader();

		final Document doc = reader.read(new StringReader(profile));

		final Node ifcNode = doc.selectSingleNode("//INTERFACE[@id='" + ifaceId + "']");

		if (ifcNode != null) {

			for (Object o : ifcNode.selectNodes("./INTERFACE_EXTRA_FIELD")) {

				res.put(((Node) o).valueOf("@name"), ((Node) o).getText());

			}

		}

		return res;

	}

	public String getMdId() {

		return mdId;

	}

	public void setMdId(final String mdId) {

		this.mdId = mdId;

	}

	public String getDatasourceId() {

		return datasourceId;

	}

	public void setDatasourceId(final String datasourceId) {

		this.datasourceId = datasourceId;

	}

	public String getDatasourceInterface() {

		return datasourceInterface;

	}

	public void setDatasourceInterface(final String datasourceInterface) {

		this.datasourceInterface = datasourceInterface;

	}

	public String getExtraFieldForTotal() {

		return extraFieldForTotal;

	}

	public void setExtraFieldForTotal(final String extraFieldForTotal) {

		this.extraFieldForTotal = extraFieldForTotal;

	}

	public String getExtraFieldForDate() {

		return extraFieldForDate;

	}

	public void setExtraFieldForDate(final String extraFieldForDate) {

		this.extraFieldForDate = extraFieldForDate;

	}

	public String getExtraFieldForMdId() {

		return extraFieldForMdId;

	}

	public void setExtraFieldForMdId(final String extraFieldForMdId) {

		this.extraFieldForMdId = extraFieldForMdId;

	}

	public DatasourceUpdater getDatasourceUpdater() {

		return datasourceUpdater;

	}

	@Required

	public void setDatasourceUpdater(final DatasourceUpdater datasourceUpdater) {

		this.datasourceUpdater = datasourceUpdater;

	}

}

package eu.dnetlib.msro.workflows.nodes.repobye;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.mdstore.MDStoreService;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

public class DeleteMDStoreJobNode extends BlackboardJobNode {

	private static final Log log = LogFactory.getLog(DeleteMDStoreJobNode.class);

	private String mdstoreId;

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(MDStoreService.class, mdstoreId);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) {

		log.info("preparing blackboard job: delete of mdstore " + getMdstoreId());

		job.setAction("DELETE");

		job.getParameters().put("id", getMdstoreId());

	}

	public String getMdstoreId() {

		return mdstoreId;

	}

	public void setMdstoreId(final String mdstoreId) {

		this.mdstoreId = mdstoreId;

	}

}

package eu.dnetlib.msro.workflows.nodes.repobye;

import java.io.StringReader;

import java.io.StringWriter;

import javax.annotation.Resource;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public class DeleteMetaWfJobNode extends SimpleJobNode {

	private String metaWfId;

	@Resource

	private UniqueServiceLocator serviceLocator;

	private static final Log log = LogFactory.getLog(DeleteMetaWfJobNode.class);

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfile(metaWfId);

		final Document doc = new SAXReader().read(new StringReader(profile));

		final String dsId = doc.valueOf("//DATAPROVIDER/@id");

		final String dsName = doc.valueOf("//DATAPROVIDER/text()");

		final String ifaceId = doc.valueOf("//DATAPROVIDER/@interface");

		final String destroyWfId = doc.valueOf("//CONFIGURATION/@destroyWorkflow");

		log.info("Removing a MetaWf of dataprovider: " + dsId);

		token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ID, dsId);

		token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_NAME, dsName);

		token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE, ifaceId);

		final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class);

		for (Object o : doc.selectNodes("//WORKFLOW")) {

			final String wfId = ((Node) o).valueOf("@id");

			try {

				registry.deleteProfile(wfId);

				log.info(" - Deleted Workflow: " + wfId);

			} catch (Exception e) {

				log.error(" - (ERR) Error deleting profile " + wfId);

			}

		}

		registry.deleteProfile(metaWfId);

		log.info(" - Deleted MetaWorkflow: " + metaWfId);

		registry.deleteProfile(destroyWfId);

		log.info(" - Deleted destroy workflow: " + destroyWfId);

		verifyDatasource(dsId, ifaceId);

		return Arc.DEFAULT_ARC;

	}

	private void verifyDatasource(final String dsId, final String ifaceId) throws Exception {

		final StringWriter sw = new StringWriter();

		sw.append("for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType') where");

		sw.append("  $x//DATAPROVIDER/@id = '" + dsId + "' and ");

		sw.append("  $x//DATAPROVIDER/@interface = '" + ifaceId + "' and ");

		sw.append("  $x//RESOURCE_IDENTIFIER/@value != '" + metaWfId + "' ");

		sw.append("return $x//RESOURCE_IDENTIFIER/@value/string()");

		final boolean active = !serviceLocator.getService(ISLookUpService.class).quickSearchProfile(sw.toString()).isEmpty();

		log.info(" - Updating iface, active status: " + active);

		updateIfaceActivationStatus(dsId, ifaceId, active);

	}

	protected void updateIfaceActivationStatus(final String dsId, final String ifaceId, final boolean active) throws Exception {

		serviceLocator.getService(ISRegistryService.class).updateProfileNode(dsId, "//INTERFACE[@id = '" + ifaceId + "']/@active", "'" + active + "'");

	}

	public String getMetaWfId() {

		return metaWfId;

	}

	public void setMetaWfId(final String metaWfId) {

		this.metaWfId = metaWfId;

	}

}

package eu.dnetlib.msro.workflows.nodes.index;

import java.io.IOException;

import java.io.StringReader;

import java.io.StringWriter;

import javax.annotation.Resource;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.dom.DOMResult;

import javax.xml.transform.dom.DOMSource;

import javax.xml.transform.stream.StreamResult;

import javax.xml.transform.stream.StreamSource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.provision.index.rmi.IndexService;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.XSLTMappedResultSetFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.enabling.resultset.rmi.ResultSetException;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

public class UpdateIndexJobNode extends BlackboardJobNode implements ProgressJobNode {

	private static final Log log = LogFactory.getLog(UpdateIndexJobNode.class);

	private String eprParam;

	private String indexId;

	private String format;

	private String layout;

	private String feedingType;

	private String defaultIndexId;

	/**

	 * xslt mapped resultset factory.

	 */

	private XSLTMappedResultSetFactory xsltRSFactory;

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	private ResultsetProgressProvider progressProvider;

	/**

	 * Stylesheet which transforms a layout to another stylesheet which converts a input record to a index record.

	 */

	private org.springframework.core.io.Resource layoutToRecordStylesheet;

	/**

	 * service locator.

	 */

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(IndexService.class);

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws ResultSetException, ISLookUpException, IOException, TransformerException {

		log.info("preparing blackboard job update index: " + getIndexId());

		final W3CEndpointReference epr = new EPRUtils().getEpr(token.getEnv().getAttribute(getEprParam()));

		final W3CEndpointReference mappedEpr = prepareForIndexing(epr, getFormat(), getLayout());

		progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), mappedEpr);

		job.setAction("FEED");

		job.getParameters().put("resultset_epr", encode(progressProvider.getEpr().toString()));

		job.getParameters().put("id", getIndexId());

		job.getParameters().put("feeding_type", getFeedingType());

		job.getParameters().put("backend_Id", defaultIndexId);

	}

	// helpers

	/**

	 * Transforms each mdstore record into a index record.

	 * 

	 * @param mdStoreRsetEpr

	 *            mdstore resulsetset

	 * @param layout

	 *            layout

	 * @param format

	 *            format

	 * @return resultset with transformed records

	 * @throws ISLookUpException

	 *             could happen

	 * @throws IOException

	 *             could happen

	 * @throws TransformerException

	 *             could happen

	 */

	protected W3CEndpointReference prepareForIndexing(final W3CEndpointReference mdStoreRsetEpr, final String format, final String layout)

			throws ISLookUpException, IOException, TransformerException {

		final TransformerFactory factory = TransformerFactory.newInstance();

		final Transformer layoutTransformer = factory.newTransformer(new StreamSource(getLayoutToRecordStylesheet().getInputStream()));

		final DOMResult layoutToXsltXslt = new DOMResult();

		layoutTransformer.setParameter("format", format);

		layoutTransformer.transform(new StreamSource(new StringReader(getLayoutSource(format, layout))), layoutToXsltXslt);

		dumpXslt(factory, layoutToXsltXslt);

		return getXsltRSFactory().createMappedResultSet(mdStoreRsetEpr, new DOMSource(layoutToXsltXslt.getNode()),

				"dynamic layout xslt for " + format + ", " + layout);

	}

	private String getLayoutSource(final String format, final String layout) throws ISLookUpDocumentNotFoundException, ISLookUpException {

		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(

				"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + format + "']//LAYOUT[@name='" + layout

						+ "']");

	}

	private void dumpXslt(final TransformerFactory factory, final DOMResult layoutToXsltXslt) throws TransformerConfigurationException, TransformerException {

		if (log.isDebugEnabled()) {

			final StringWriter buffer = new StringWriter();

			factory.newTransformer().transform(new DOMSource(layoutToXsltXslt.getNode()), new StreamResult(buffer));

			log.debug(buffer.toString());

		}

	}

	private String encode(final String epr) {

		return new String(Base64.encodeBase64(epr.getBytes()));

	}

	// setters and getters

	public String getIndexId() {

		return indexId;

	}

	public void setIndexId(final String indexId) {

		this.indexId = indexId;

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

	public String getFeedingType() {

		return feedingType;

	}

	public void setFeedingType(final String feedingType) {

		this.feedingType = feedingType;

	}

	public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {

		return processCountingResultSetFactory;

	}

	@Required

	public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {

		this.processCountingResultSetFactory = processCountingResultSetFactory;

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return progressProvider;

	}

	public org.springframework.core.io.Resource getLayoutToRecordStylesheet() {

		return layoutToRecordStylesheet;

	}

	@Required

	public void setLayoutToRecordStylesheet(final org.springframework.core.io.Resource layoutToRecordStylesheet) {

		this.layoutToRecordStylesheet = layoutToRecordStylesheet;

	}

	public String getFormat() {

		return format;

	}

	public void setFormat(final String format) {

		this.format = format;

	}

	public String getLayout() {

		return layout;

	}

	public void setLayout(final String layout) {

		this.layout = layout;

	}

	public XSLTMappedResultSetFactory getXsltRSFactory() {

		return xsltRSFactory;

	}

	@Required

	public void setXsltRSFactory(final XSLTMappedResultSetFactory xsltRSFactory) {

		this.xsltRSFactory = xsltRSFactory;

	}

	/**

	 * @return the defaultIndexId

	 */

	public String getDefaultIndexId() {

		return defaultIndexId;

	}

	/**

	 * @param defaultIndexId

	 *            the defaultIndexId to set

	 */

	@Required

	public void setDefaultIndexId(final String defaultIndexId) {

		this.defaultIndexId = defaultIndexId;

	}

}

package eu.dnetlib.msro.workflows.nodes.blackboard;

import javax.annotation.Resource;

import javax.xml.ws.Endpoint;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.enabling.is.sn.rmi.ISSNException;

import eu.dnetlib.enabling.is.sn.rmi.ISSNService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.soap.EndpointReferenceBuilder;

public class BlackboardSubscriber {

	/**

	 * Logger.

	 */

	private static final Log log = LogFactory.getLog(BlackboardSubscriber.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * service locator.

	 */

	@Resource

	private UniqueServiceLocator serviceLocator;

	/**

	 * injected EPR builder.

	 */

	@Resource(name = "jaxwsEndpointReferenceBuilder")

	private EndpointReferenceBuilder<Endpoint> eprBuilder;

	/**

	 * notification endpoint (normally the msro service).

	 */

	private Endpoint endpoint;

	/**

	 * performs the subscription.

	 * 

	 * @throws ISSNException

	 *             could happen

	 */

	public void subscribeAll() throws ISSNException {

		log.info("Subscribing msro service");

		final W3CEndpointReference endpointReference = eprBuilder.getEndpointReference(getEndpoint());

		serviceLocator.getService(ISSNService.class).subscribe(endpointReference, "UPDATE/*/*/RESOURCE_PROFILE/BODY/BLACKBOARD/LAST_RESPONSE", 0);

	}

	public Endpoint getEndpoint() {

		return endpoint;

	}

	@Required

	public void setEndpoint(final Endpoint endpoint) {

		this.endpoint = endpoint;

	}

}

package eu.dnetlib.msro.workflows.nodes.objectStore;

import java.util.List;

import java.util.Set;

import javax.annotation.Resource;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

/**

 * The Class RetrieveMdStoreId is a job node used to retrieve the correct MDStore from which extract the url of the file to download.

 * metadata format and interpretation are injected as properties

 */

public class RetrieveMdStoreId extends SimpleJobNode {

	/** The metadata format. */

	private String metadataFormat;

	/** The interpretation. */

	private String interpretation;

	/** The provider id. */

	private String providerId;

	/** The service locator. */

	@Resource

	private UniqueServiceLocator serviceLocator;

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected String execute(final NodeToken token) throws Exception {

		String workflowQuery =

				"for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType') where($x//DATAPROVIDER/@id='%s') return  distinct-values($x//WORKFLOW/@id/string())";

		List<String> result = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(String.format(workflowQuery, providerId));

		if (result.size() == 0) { throw new RuntimeException("there is no mdStore Associated to the provider " + token.getEnv().getAttribute(getProviderId())); }

		Set<String> workflowIds = Sets.newHashSet(result);

		Set<String> metadataIds = getMdStores(workflowIds);

		Gson g = new GsonBuilder().disableHtmlEscaping().create();

		token.getEnv().setAttribute("mdId", g.toJson(metadataIds));

		token.getEnv().setAttribute("mdFormat", getMetadataFormat());

		return Arc.DEFAULT_ARC;

	}

	private Set<String> getMdStores(final Set<String> workflowsId) {

		try {

			String query = "//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s']//PARAM[./@category/string()='MDSTORE_ID']/text()";

			Set<String> mdStores = Sets.newHashSet();

			if (workflowsId == null) { return null; }

			for (String workflowId : workflowsId) {

				List<String> result = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(String.format(query, workflowId));

				Set<String> metadataIds = Sets.newHashSet(result);

				mdStores.addAll(getRightMetadataId(Lists.newArrayList(metadataIds)));

			}

			return mdStores;

		} catch (ISLookUpException e) {

			return null;

		}

	}

	/**

	 * Gets the right metadata id whith the format metadataFormat and interpretation interpretation

	 *

	 * @return the right metadata id

	 * @throws ISLookUpException

	 */

	private Set<String> getRightMetadataId(final Iterable<String> ids) throws ISLookUpException {

		String query =

				"let $x:=//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='%s'] return concat($x//METADATA_FORMAT/text(), '::<<>>::', $x//METADATA_FORMAT_INTERPRETATION/text())";

		Set<String> result = Sets.newHashSet();

		for (String id : ids) {

			List<String> results = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(String.format(query, id));

			if (results.size() > 0) {

				String[] values = results.get(0).split("::<<>>::");

				if (metadataFormat.equals(values[0]) && interpretation.equals(values[1])) {

					result.add(id);

				}

			}

		}

		return result;

	}

	/**

	 * Gets the interpretation.

	 *

	 * @return the interpretation

	 */

	public String getInterpretation() {

		return interpretation;

	}

	/**

	 * Sets the interpretation.

	 *

	 * @param interpretation

	 *            the interpretation to set

	 */

	@Required

	public void setInterpretation(final String interpretation) {

		this.interpretation = interpretation;

	}

	/**

	 * Gets the metadata format.

	 *

	 * @return the metadataFormat

	 */

	public String getMetadataFormat() {

		return metadataFormat;

	}

	/**

	 * Sets the metadata format.

	 *

	 * @param metadataFormat

	 *            the metadataFormat to set

	 */

	@Required

	public void setMetadataFormat(final String metadataFormat) {

		this.metadataFormat = metadataFormat;

	}

	/**

	 * Gets the provider id.

	 *

	 * @return the providerId

	 */

	public String getProviderId() {

		return providerId;

	}

	/**

	 * Sets the provider id.

	 *

	 * @param providerId

	 *            the providerId to set

	 */

	public void setProviderId(final String providerId) {

		this.providerId = providerId;

	}

}

package eu.dnetlib.msro.workflows.nodes;

import java.lang.reflect.Field;

import java.util.HashSet;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Set;

import javax.annotation.Resource;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.BeanNameAware;

import com.google.common.collect.Maps;

import com.google.gson.Gson;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.mem.MemNode;

import eu.dnetlib.conf.PropertyFetcher;

public abstract class SarasvatiJobNode extends MemNode implements BeanNameAware {

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(SarasvatiJobNode.class);

	private String beanName;

	/**

	 * parameter values injected as json map into subclasses instance beans.

	 */

	private String params;

	/**

	 * parameter values retrieved from the sarasvati Env.

	 */

	private String envParams;

	/**

	 * parameter values retrieved from the system properties.

	 */

	private String sysParams;

	@Resource(name = "propertyFetcher")

	private PropertyFetcher propertyFetcher;

	@Override

	public void execute(final Engine engine, final NodeToken token) {

		final String prefix = "system:node:" + getName() + ":";

		for (Field field : getClass().getDeclaredFields()) {

			try {

				field.setAccessible(true);

				token.getEnv().setAttribute(prefix + field.getName(), "" + field.get(this));

			} catch (Exception e) {

				log.warn("Error accessing value of field: " + field.getName());

			}

		}

	}

	/**

	 * builds a parameter map overriding values having the same name in the following order: sysparams < envparams < params

	 * 

	 * @param token

	 * @return

	 */

	protected Map<String, String> parseJsonParameters(final NodeToken token) {

		final Map<String, String> res = Maps.newHashMap();

		for (Entry<?, ?> e : asMapEntries(getSysParams())) {

			String propertyName = e.getValue().toString();

			if (!getPropertyFetcher().getProps().containsKey(propertyName)) {

				log.warn("unable to find system property: '" + propertyName + "'");

			}

			String propertyValue = getPropertyFetcher().getProperty(propertyName);

			res.put(e.getKey().toString(), propertyValue);

		}

		for (Entry<?, ?> e : asMapEntries(getEnvParams())) {

			final String param = e.getValue().toString();

			if (token.getEnv().hasAttribute(param)) {

				res.put(e.getKey().toString(), token.getEnv().getAttribute(param));

			} else {

				res.put(e.getKey().toString(), token.getFullEnv().getAttribute(param));

			}

		}

		for (Entry<?, ?> e : asMapEntries(getParams())) {

			res.put(e.getKey().toString(), e.getValue().toString());

		}

		return res;

	}

	@SuppressWarnings("unchecked")

	private Set<Map.Entry<?, ?>> asMapEntries(final String s) {

		return isSet(s) ? new Gson().fromJson(s, Map.class).entrySet() : new HashSet<Map.Entry<?, ?>>();

	}

	private boolean isSet(final String s) {

		return (s != null) && !s.isEmpty();

	}

	// ///////////////

	public String getBeanName() {

		return beanName;

	}

	@Override

	public void setBeanName(final String beanName) {

		this.beanName = beanName;

	}

	public PropertyFetcher getPropertyFetcher() {

		return propertyFetcher;

	}

	public String getParams() {

		return params;

	}

	public void setParams(final String params) {

		this.params = params;

	}

	public String getEnvParams() {

		return envParams;

	}

	public void setEnvParams(final String envParams) {

		this.envParams = envParams;

	}

	public String getSysParams() {

		return sysParams;

	}

	public void setSysParams(final String sysParams) {

		this.sysParams = sysParams;

	}

}

package eu.dnetlib.msro.workflows.nodes.objectStore;

import javax.annotation.Resource;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

// TODO: Auto-generated Javadoc

/**

 * The Class UpdateObjectStoreSizeJobNode.

 */

public class UpdateObjectStoreSizeJobNode extends SimpleJobNode {

	/** The object store id. */

	private String objectStoreIdParam;

	/** The service locator. */

	@Resource

	private UniqueServiceLocator serviceLocator;

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)

	 */

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class);

		int size = serviceLocator.getService(ObjectStoreService.class, objectStoreIdParam).getSize(objectStoreIdParam);

		String now = DateUtils.now_ISO8601();

		String mdstoreXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + objectStoreIdParam + "']"

				+ "return update value $x//LAST_STORAGE_DATE with '" + now + "'";

		registry.executeXUpdate(mdstoreXUpdate);

		String mdstoreNumberXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + objectStoreIdParam + "']"

				+ "return update value $x//COUNT_STORE with '" + size + "'";

		registry.executeXUpdate(mdstoreNumberXUpdate);

		mdstoreNumberXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + objectStoreIdParam + "']"

				+ "return update value $x//STORE_SIZE with '" + size + "'";

		registry.executeXUpdate(mdstoreNumberXUpdate);

		return Arc.DEFAULT_ARC;

	}

	/**

	 * Gets the object store id param.

	 *

	 * @return the objectStoreIdParam

	 */

	public String getObjectStoreIdParam() {

		return objectStoreIdParam;

	}

	/**

	 * Sets the object store id param.

	 *

	 * @param objectStoreIdParam

	 *            the new object store id param

	 */

	public void setObjectStoreIdParam(final String objectStoreIdParam) {

		this.objectStoreIdParam = objectStoreIdParam;

	}

}

package eu.dnetlib.msro.workflows.nodes.objectStore;

import java.io.StringReader;

import java.util.Iterator;

import java.util.Map;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.xml.sax.InputSource;

import com.googlecode.sarasvati.Engine;

import com.googlecode.sarasvati.NodeToken;

import com.googlecode.sarasvati.env.Env;

import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;

import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;

import eu.dnetlib.enabling.resultset.IterableResultSetFactory;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;

import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;

import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public class DownloadIntoObjectStoreJobNode extends BlackboardJobNode implements ProgressJobNode {

	private static final Log log = LogFactory.getLog(DownloadIntoObjectStoreJobNode.class);

	class MetadataObjectIterator implements Iterator<String> {

		private Iterator<String> inputIterator;

		private String mime;

		public MetadataObjectIterator(final Iterator<String> inputIterator, final String xpath, final String mime) {

			this.inputIterator = inputIterator;

		}

		@Override

		public boolean hasNext() {

			return inputIterator.hasNext();

		}

		@Override

		public String next() {

			try {

				String record = inputIterator.next();

				XPath xpath = XPathFactory.newInstance().newXPath();

				InputSource doc = new InputSource(new StringReader(record));

				String identifier = xpath.evaluate(getIdXpath(), doc);

				MetadataObjectRecord objectrecord = new MetadataObjectRecord(identifier, record, mime);

				return objectrecord.toJSON();

			} catch (Exception e) {

				return null;

			}

		}

		@Override

		public void remove() {

		}

	}

	private String eprParam;

	private String objectStoreId;

	private String idXpath; // "//*[local-name()='objIdentifier']

	private String contentDescription;

	private String objectIsInsideEpr;

	private IterableResultSetFactory iterableResultSetFactory;

	private ResultSetClientFactory resultSetClientFactory;

	private ResultsetProgressProvider progressProvider;

	private ProcessCountingResultSetFactory processCountingResultSetFactory;

	@Override

	protected String obtainServiceId(final NodeToken token) {

		return getServiceLocator().getServiceId(ObjectStoreService.class);

	}

	public String getEprParam() {

		return eprParam;

	}

	public void setEprParam(final String eprParam) {

		this.eprParam = eprParam;

	}

	public String getObjectStoreId() {

		return objectStoreId;

	}

	public void setObjectStoreId(final String objectStoreId) {

		this.objectStoreId = objectStoreId;

	}

	public ProgressProvider getProgressProvider(final NodeToken token) {

		return progressProvider;

	}

	@Override

	protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {

		job.setAction("FEEDOBJECT");

		final String eprS = token.getEnv().getAttribute(getEprParam());

		job.getParameters().put("obsID", getObjectStoreId());

		job.getParameters().put("mime", getContentDescription());

		final Iterator<String> client = resultSetClientFactory.getClient(eprS).iterator();

		final W3CEndpointReference epr = iterableResultSetFactory.createIterableResultSet(new Iterable<String>() {

			@Override

			public Iterator<String> iterator() {

				return new MetadataObjectIterator(client, "//*[local-name()='objIdentifier']", "xml");