D-Net

package eu.dnetlib.msro.openaireplus.workflows.nodes;

import java.util.List;

import java.util.NoSuchElementException;

import javax.annotation.Resource;

import com.google.common.base.Joiner;

import com.google.common.base.Splitter;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.data.hadoop.config.ClusterName;

import eu.dnetlib.data.hadoop.config.ConfigurationEnumerator;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.rmi.MSROException;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.conf.Configuration;

import org.springframework.beans.factory.annotation.Required;

public abstract class PrepareIISParamsV2 extends SimpleJobNode {

	private static final Log log = LogFactory.getLog(PrepareIISParamsV2.class);

	@Resource

	protected ConfigurationEnumerator configurationEnumerator;

	@Resource

	private UniqueServiceLocator serviceLocator;

	private String clusterName;

	private String clusterParam = "cluster";

	private String oozieWfAppPath;

	private String oozieWfAppPathParam = "oozie.wf.application.path";

	private String xqueryMdStoreService;

	private String mdStoreStoreLocationParam = "import_mdstore_service_location";

	private String xqueryObjectStoreService;

	private String objectStoreLocationParam = "import_content_object_store_location";

	private String xqueryIsLookupService;

	private String islookupLocationParam = "import_islookup_service_location";

	private String importProjectConceptsContextIdParam = "import_project_concepts_context_id";

	private String importProjectConceptsContextId;

	private String xqueryDatasetStore;

	private String mdStoreDatasetParam = "import_dataset_mdstore_ids_csv";

	private String objectStoreBlacklistCSV = "";

	private String importHbaseDumpLocationParam = "import_hbase_dump_location";

	private String importHbaseDumpLocation;

	protected void prepare(final NodeToken token) throws Exception {

		token.getEnv().setAttribute(getClusterParam(), getClusterName());

		// Assumes we only have one mdStore service instance

		token.getEnv().setAttribute(getMdStoreStoreLocationParam(), getServiceEndpoint(getXqueryMdStoreService()));

		// Assumes we only have one objectStore service instance

		token.getEnv().setAttribute(getObjectStoreLocationParam(), getServiceEndpoint(getXqueryObjectStoreService()));

		token.getEnv().setAttribute(getIslookupLocationParam(), getServiceEndpoint(getXqueryIsLookupService()));

		token.getEnv().setAttribute(getImportProjectConceptsContextIdParam(), getImportProjectConceptsContextId());

		Configuration dmConf = configurationEnumerator.get(ClusterName.DM);

		String dmNameNode = dmConf.get("fs.defaultFS");

		token.getEnv().setAttribute(getImportHbaseDumpLocationParam(), getURI(dmNameNode, getImportHbaseDumpLocation()));

		Configuration conf = configurationEnumerator.get(ClusterName.valueOf(getClusterName()));

		String nameNode = conf.get("fs.defaultFS");

		token.getEnv().setAttribute("nameNode", nameNode);

		token.getEnv().setAttribute("jobTracker", conf.get("mapred.job.tracker"));

		token.getEnv().setAttribute(getOozieWfAppPathParam(), getURI(nameNode, getOozieWfAppPath()));

		token.getEnv().setAttribute(getMdStoreDatasetParam(), asCSV(getProfileIds(getXqueryDatasetStore())));

	}

	protected String getServiceEndpoint(final String xquery) throws MSROException {

		try {

			return Iterables.getOnlyElement(serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));

		} catch (ISLookUpException e) {

			throw new MSROException("unable to fetch service endpoint", e);

		} catch (NoSuchElementException e) {

			throw new MSROException("unable to find service endpoint, xquery: " + getXqueryMdStoreService(), e);

		} catch (IllegalArgumentException e) {

			throw new MSROException("more than one services found, we assume to have only one available", e);

		}

	}

	protected String getProfileId(final String xquery) throws MSROException {

		try {

			return Iterables.getOnlyElement(serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));

		} catch (ISLookUpException e) {

			throw new MSROException("unable to fetch profile id", e);

		} catch (NoSuchElementException e) {

			throw new MSROException("unable to find profile profile, xquery: " + xquery, e);

		} catch (IllegalArgumentException e) {

			throw new MSROException("more than one profile profiles was found, we assume to have only one available, xquery: " + xquery, e);

		}

	}

	protected List<String> getProfileIds(final String xquery) throws MSROException {

		try {

			List<String> ids = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);

			if (ids.isEmpty()) {

				log.warn("couldn't find any profile, xquery: " + xquery);

			}

			return ids;

		} catch (ISLookUpException e) {

			throw new MSROException("unable to fetch profile ids, x query: " + xquery, e);

		}

	}

	protected String getFilteredObjectStoreCSV(final String xquery) throws MSROException {

		return asCSV(filter(getProfileIds(xquery), asList(getObjectStoreBlacklistCSV())));

	}

	protected List<String> filter(final List<String> list, final List<String> filter) {

		if (filter == null || filter.isEmpty()) { return list; }

		list.removeAll(filter);

		return list;

	}

	protected String asCSV(final List<String> list) {

		return Joiner.on(",").skipNulls().join(list);

	}

	protected List<String> asList(final String csv) {

		return Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings().split(csv));

	}

	private String getURI(final String nameNode, final String relative) {

		// TODO ensure to return a valid URI

		return nameNode + relative;

	}

	private String getZkQuorumCSV(final Configuration conf, final String zkPort) {

		return Joiner.on(":" + zkPort + ",").join(Splitter.on(",").omitEmptyStrings().split(conf.get("hbase.zookeeper.quorum")));

	}

	@Required

	public void setXqueryMdStoreService(final String xqueryMdStoreService) {

		this.xqueryMdStoreService = xqueryMdStoreService;

	}

	public String getXqueryMdStoreService() {

		return xqueryMdStoreService;

	}

	public String getMdStoreStoreLocationParam() {

		return mdStoreStoreLocationParam;

	}

	public void setMdStoreStoreLocationParam(final String mdStoreStoreLocationParam) {

		this.mdStoreStoreLocationParam = mdStoreStoreLocationParam;

	}

	public String getClusterName() {

		return clusterName;

	}

	public void setClusterName(final String clusterName) {

		this.clusterName = clusterName;

	}

	public String getClusterParam() {

		return clusterParam;

	}

	public void setClusterParam(final String clusterParam) {

		this.clusterParam = clusterParam;

	}

	public String getOozieWfAppPathParam() {

		return oozieWfAppPathParam;

	}

	public void setOozieWfAppPathParam(final String oozieWfAppPathParam) {

		this.oozieWfAppPathParam = oozieWfAppPathParam;

	}

	public String getOozieWfAppPath() {

		return oozieWfAppPath;

	}

	public void setOozieWfAppPath(final String oozieWfAppPath) {

		this.oozieWfAppPath = oozieWfAppPath;

	}

	@Required

	public String getXqueryDatasetStore() {

		return xqueryDatasetStore;

	}

	public void setXqueryDatasetStore(final String xqueryDatasetStore) {

		this.xqueryDatasetStore = xqueryDatasetStore;

	}

	public String getMdStoreDatasetParam() {

		return mdStoreDatasetParam;

	}

	public void setMdStoreDatasetParam(final String mdStoreDatasetParam) {

		this.mdStoreDatasetParam = mdStoreDatasetParam;

	}

	public String getXqueryObjectStoreService() {

		return xqueryObjectStoreService;

	}

	@Required

	public void setXqueryObjectStoreService(final String xqueryObjectStoreService) {

		this.xqueryObjectStoreService = xqueryObjectStoreService;

	}

	public String getObjectStoreLocationParam() {

		return objectStoreLocationParam;

	}

	public void setObjectStoreLocationParam(final String objectStoreLocationParam) {

		this.objectStoreLocationParam = objectStoreLocationParam;

	}

	public String getObjectStoreBlacklistCSV() {

		return objectStoreBlacklistCSV;

	}

	public void setObjectStoreBlacklistCSV(final String objectStoreBlacklistCSV) {

		this.objectStoreBlacklistCSV = objectStoreBlacklistCSV;

	}

	public String getXqueryIsLookupService() {

		return xqueryIsLookupService;

	}

	@Required

	public void setXqueryIsLookupService(final String xqueryIsLookupService) {

		this.xqueryIsLookupService = xqueryIsLookupService;

	}

	public String getIslookupLocationParam() {

		return islookupLocationParam;

	}

	public void setIslookupLocationParam(final String islookupLocationParam) {

		this.islookupLocationParam = islookupLocationParam;

	}

	public String getImportProjectConceptsContextIdParam() {

		return importProjectConceptsContextIdParam;

	}

	public void setImportProjectConceptsContextIdParam(final String importProjectConceptsContextIdParam) {

		this.importProjectConceptsContextIdParam = importProjectConceptsContextIdParam;

	}

	public String getImportProjectConceptsContextId() {

		return importProjectConceptsContextId;

	}

	public void setImportProjectConceptsContextId(final String importProjectConceptsContextId) {

		this.importProjectConceptsContextId = importProjectConceptsContextId;

	}

	public String getImportHbaseDumpLocationParam() {

		return importHbaseDumpLocationParam;

	}

	public void setImportHbaseDumpLocationParam(final String importHbaseDumpLocationParam) {

		this.importHbaseDumpLocationParam = importHbaseDumpLocationParam;

	}

	public String getImportHbaseDumpLocation() {

		return importHbaseDumpLocation;

	}

	public void setImportHbaseDumpLocation(final String importHbaseDumpLocation) {

		this.importHbaseDumpLocation = importHbaseDumpLocation;

	}

}

package eu.dnetlib.msro.openaireplus.workflows.nodes;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import org.springframework.beans.factory.annotation.Required;

public class PrepareIISPreprocessingParamsV2JobNode extends PrepareIISParamsV2 {

	private String xqueryWosMDStore;

	private String mdStoreWosParam = "import_wos_mdstore_id";

	private String xqueryDatabaseService;

	private String databaseServiceLocationParam = "import_database_service_location";

	private String xqueryDataciteObjectStore;

	private String dataciteObjectStoreParam = "import_content_datacite_objectstores_csv";

	private String xqueryWosObjectStore;

	private String wosObjectStorePlaintextParam = "import_content_wos_plaintext_objectstores_csv";

	@Override

	protected String execute(final NodeToken token) throws Exception {

		super.prepare(token);

		token.getEnv().setAttribute(getMdStoreWosParam(), getProfileId(getXqueryWosMDStore()));

		token.getEnv().setAttribute(getDatabaseServiceLocationParam(), getServiceEndpoint(getXqueryDatabaseService()));

		token.getEnv().setAttribute(getDataciteObjectStoreParam(), getFilteredObjectStoreCSV(getXqueryDataciteObjectStore()));

		token.getEnv().setAttribute(getWosObjectStorePlaintextParam(), getProfileId(getXqueryWosObjectStore()));

		return Arc.DEFAULT_ARC;

	}

	public String getXqueryWosMDStore() {

		return xqueryWosMDStore;

	}

	@Required

	public void setXqueryWosMDStore(final String xqueryWosMDStore) {

		this.xqueryWosMDStore = xqueryWosMDStore;

	}

	public String getMdStoreWosParam() {

		return mdStoreWosParam;

	}

	public void setMdStoreWosParam(final String mdStoreWosParam) {

		this.mdStoreWosParam = mdStoreWosParam;

	}

	public String getXqueryDatabaseService() {

		return xqueryDatabaseService;

	}

	@Required

	public void setXqueryDatabaseService(final String xqueryDatabaseService) {

		this.xqueryDatabaseService = xqueryDatabaseService;

	}

	public String getDatabaseServiceLocationParam() {

		return databaseServiceLocationParam;

	}

	public void setDatabaseServiceLocationParam(final String databaseServiceLocationParam) {

		this.databaseServiceLocationParam = databaseServiceLocationParam;

	}

	public String getWosObjectStorePlaintextParam() {

		return wosObjectStorePlaintextParam;

	}

	public void setWosObjectStorePlaintextParam(final String wosObjectStorePlaintextParam) {

		this.wosObjectStorePlaintextParam = wosObjectStorePlaintextParam;

	}

	public String getXqueryWosObjectStore() {

		return xqueryWosObjectStore;

	}

	@Required

	public void setXqueryWosObjectStore(final String xqueryWosObjectStore) {

		this.xqueryWosObjectStore = xqueryWosObjectStore;

	}

	public String getXqueryDataciteObjectStore() {

		return xqueryDataciteObjectStore;

	}

	@Required

	public void setXqueryDataciteObjectStore(final String xqueryDataciteObjectStore) {

		this.xqueryDataciteObjectStore = xqueryDataciteObjectStore;

	}

	public String getDataciteObjectStoreParam() {

		return dataciteObjectStoreParam;

	}

	public void setDataciteObjectStoreParam(final String dataciteObjectStoreParam) {

		this.dataciteObjectStoreParam = dataciteObjectStoreParam;

	}

}

package eu.dnetlib.msro.openaireplus.workflows.nodes;

import java.util.List;

import java.util.Map;

import com.google.gson.Gson;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import org.springframework.beans.factory.annotation.Required;

public class PrepareIISMainParamsV2JobNode extends PrepareIISParamsV2 {

	private String xqueryObjectStores;

	private String objectStoreParam = "import_content_objectstores_csv";

	// Enable/Disable inference modules

	@Override

	protected String execute(final NodeToken token) throws Exception {

		super.prepare(token);

		token.getEnv().setAttribute(getObjectStoreParam(), getFilteredObjectStoreCSV(getXqueryObjectStores()));

		@SuppressWarnings("unchecked")

		final List<Map<String, String>> sets = new Gson().fromJson(token.getEnv().getAttribute("sets"), List.class);

		for (Map<String, String> set : sets) {

			token.getEnv().setAttribute(set.get("enablingProperty"), set.get("enabled"));

		}

		return Arc.DEFAULT_ARC;

	}

	public String getXqueryObjectStores() {

		return xqueryObjectStores;

	}

	@Required

	public void setXqueryObjectStores(final String xqueryObjectStores) {

		this.xqueryObjectStores = xqueryObjectStores;

	}

	public String getObjectStoreParam() {

		return objectStoreParam;

	}

	public void setObjectStoreParam(final String objectStoreParam) {

		this.objectStoreParam = objectStoreParam;

	}

}

				<WORKFLOW id="4801c33c-66ca-4ab6-af64-aa812194ec67_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" name="IIS Preprocessing V2"/>

				<WORKFLOW id="dc19a8d1-93a8-4ffb-9fed-950f4156eedc_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" name="IIS Main V2"/>

<?xml version="1.0" encoding="UTF-8"?>

<RESOURCE_PROFILE xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

    <HEADER>

        <RESOURCE_IDENTIFIER value="4801c33c-66ca-4ab6-af64-aa812194ec67_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>

        <RESOURCE_TYPE value="WorkflowDSResourceType"/>

        <RESOURCE_KIND value="WorkflowDSResources"/>

        <RESOURCE_URI value=""/>

        <DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/>

    </HEADER>

    <BODY>

        <WORKFLOW_NAME>IIS preprocessing V2</WORKFLOW_NAME>

        <WORKFLOW_TYPE>IIS</WORKFLOW_TYPE>

        <WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>

        <CONFIGURATION start="manual">

			<NODE name="start" isStart="true">				

				<DESCRIPTION>start</DESCRIPTION>

				<PARAMETERS/>

				<ARCS>

					<ARC to="prepareActionSets" />

					<ARC to="prepareParameters" />

				</ARCS>

			</NODE>

			<NODE name="prepareActionSets" type="PrepareActionSets">				

				<DESCRIPTION>prepare action sets</DESCRIPTION>

				<PARAMETERS>

					<PARAM required="true" type="string" name="sets" managedBy="system">

					[

						{

							'set' : 'iis-referenced-projects-preprocessing',

							'jobProperty' : 'export_action_set_id_document_referencedProjects',

							'enablingProperty' : 'active_referencedProjects_export',

							'enabled' : 'true'

						},

						{

							'set' : 'iis-referenced-datasets-preprocessing',

							'jobProperty' : 'export_action_set_id_document_referencedDatasets',

							'enablingProperty' : 'active_referencedDatasets_export',

							'enabled' : 'true'

						},

						{

							'set' : 'iis-wos-entities',

							'jobProperty' : 'export_action_set_id_entity_wos',

							'enablingProperty' : 'active_entity_wos_export',

							'enabled' : 'true'							

						},

						{

							'set' : 'iis-dataset-entities-preprocessing',

							'jobProperty' : 'export_action_set_id_entity_dataset',

							'enablingProperty' : 'active_entity_dataset_export',

							'enabled' : 'true'								

						}

					]

					</PARAM>

				</PARAMETERS>

				<ARCS>

					<ARC to="preprocessing" />

				</ARCS>

			</NODE>

			<NODE name="prepareParameters" type="PreparePreprocessParamsV2">

				<DESCRIPTION>prepare parameters</DESCRIPTION>

				<PARAMETERS>

					<PARAM required="true" type="string" name="islookupLocationParam" managedBy="system">import_islookup_service_location</PARAM>

					<PARAM required="true" type="string" name="objectStoreLocationParam" managedBy="system">import_content_object_store_location</PARAM>

					<PARAM required="true" type="string" name="mdStoreStoreLocationParam" managedBy="system">import_mdstore_service_location</PARAM>

					<PARAM required="true" type="string" name="mdStoreDatasetParam" managedBy="system">import_dataset_mdstore_ids_csv</PARAM>

					<PARAM required="true" type="string" name="mdStoreWosParam" managedBy="system">import_wos_mdstore_id</PARAM>

					<PARAM required="true" type="string" name="databaseServiceLocationParam" managedBy="system">import_database_service_location</PARAM>

					<PARAM required="true" type="string" name="dataciteObjectStoreParam" managedBy="system">import_content_datacite_objectstores_csv</PARAM>

					<PARAM required="true" type="string" name="wosObjectStorePlaintextParam" managedBy="system">import_content_wos_plaintext_objectstores_csv</PARAM>					

					<PARAM required="true" type="string" name="oozieWfAppPathParam" managedBy="system">oozie.wf.application.path</PARAM>

					<PARAM required="true" type="string" name="oozieWfAppPath" managedBy="user">/tmp/integration/apps/preprocessing</PARAM>

					<PARAM required="true" type="string" name="clusterName" managedBy="user" function="validValues(['IIS','DM'])">DM</PARAM>

					<PARAM required="false" type="string" name="objectStoreBlacklistCSV" managedBy="user"></PARAM>

					<PARAM required="true" type="string" name="importProjectConceptsContextIdParam" managedBy="system">import_project_concepts_context_id</PARAM>

					<PARAM required="true" type="string" name="importProjectConceptsContextId" managedBy="user">fet-fp7</PARAM>

				</PARAMETERS>

				<ARCS>

					<ARC to="preprocessing" />

				</ARCS>

			</NODE>				

			<NODE name="preprocessing" type="SubmitHadoopJob" isJoin="true">

				<DESCRIPTION>IIS preprocessing</DESCRIPTION>

				<PARAMETERS>

					<PARAM required="true" type="string" name="hadoopJob" managedBy="system">iisPreprocessingJob</PARAM>				

					<PARAM required="true" type="string" name="envParams" managedBy="system">

						{

							'cluster' : 'cluster',

							'nameNode' : 'nameNode',

							'jobTracker' : 'jobTracker',

							'oozie.wf.application.path' : 'oozie.wf.application.path',

							'import_content_object_store_location' : 'import_content_object_store_location',

							'import_mdstore_service_location' : 'import_mdstore_service_location',

							'import_islookup_service_location' : 'import_islookup_service_location', 

							'import_project_concepts_context_id' : 'import_project_concepts_context_id',

							'import_dataset_mdstore_ids_csv' : 'import_dataset_mdstore_ids_csv',

							'import_wos_mdstore_id' : 'import_wos_mdstore_id',

							'import_database_service_location' : 'import_database_service_location',

							'import_content_datacite_objectstores_csv' : 'import_content_datacite_objectstores_csv',

							'import_content_wos_plaintext_objectstores_csv' : 'import_content_wos_plaintext_objectstores_csv',

							'export_action_set_id_document_referencedProjects' : 'export_action_set_id_document_referencedProjects',

							'export_action_set_id_document_referencedDatasets' : 'export_action_set_id_document_referencedDatasets',

							'export_action_set_id_entity_wos' : 'export_action_set_id_entity_wos',

							'export_action_set_id_entity_dataset' : 'export_action_set_id_entity_dataset',

							'output_remote_location' : 'actionManagerBasePath'

						}

					</PARAM>

					<PARAM required="true" type="string" name="sysParams" managedBy="system">

						{ 	

							'export_action_hbase_table_name' : 'hbase.actions.table',

							'import_database_dbname' : 'dnet.openaire.db.name'

						}

					</PARAM>

					<PARAM required="true" type="boolean" name="simulation"	managedBy="user">false</PARAM>				

				</PARAMETERS>

				<ARCS>

					<ARC to="updateActionSets" />

				</ARCS>

			</NODE>

			<NODE name="updateActionSets" type="UpdateActionSets">				

				<DESCRIPTION>update action sets</DESCRIPTION>

				<PARAMETERS/>

				<ARCS>

					<ARC to="success" />

				</ARCS>

			</NODE>

        </CONFIGURATION>

        <STATUS />

    </BODY>

</RESOURCE_PROFILE>

<?xml version="1.0" encoding="UTF-8"?>

<RESOURCE_PROFILE xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

	<HEADER>

		<RESOURCE_IDENTIFIER

			value="dc19a8d1-93a8-4ffb-9fed-950f4156eedc_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" />

		<RESOURCE_TYPE value="WorkflowDSResourceType" />

		<RESOURCE_KIND value="WorkflowDSResources" />

		<RESOURCE_URI value="" />

		<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z" />

	</HEADER>

	<BODY>

		<WORKFLOW_NAME>IIS main workflow V2</WORKFLOW_NAME>

		<WORKFLOW_TYPE>IIS</WORKFLOW_TYPE>

		<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>

		<CONFIGURATION start="manual">

			<NODE name="start" isStart="true">

				<DESCRIPTION>start</DESCRIPTION>

				<PARAMETERS />

				<ARCS>

					<ARC to="prepareActionSets" />

				</ARCS>

			</NODE>

			<NODE name="prepareActionSets" type="PrepareActionSets">

				<DESCRIPTION>prepare action sets</DESCRIPTION>

				<PARAMETERS>

					<PARAM required="true" type="string" name="sets" managedBy="system">

						[

						{

						'set' : 'iis-document-affiliation',

						'jobProperty' : 'export_action_set_id_affiliation_matching',

						'enablingProperty' : 'active_document_affiliation',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-referenced-projects-main',

						'jobProperty' : 'export_action_set_id_document_referencedProjects',

						'enablingProperty' : 'active_referenceextraction_project',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-referenced-datasets-main',

						'jobProperty' : 'export_action_set_id_document_referencedDatasets',

						'enablingProperty' : 'active_referenceextraction_dataset',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-dataset-entities-main',

						'jobProperty' : 'export_action_set_id_entity_dataset',

						'enablingProperty' : 'active_referenceextraction_dataset',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-researchinitiative',

						'jobProperty' : 'export_action_set_id_document_research_initiative',

						'enablingProperty' :

						'active_referenceextraction_researchinitiative',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-document-similarities',

						'jobProperty' : 'export_action_set_id_document_similarities_standard',

						'enablingProperty' : 'active_documentssimilarity',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-document-classes',

						'jobProperty' : 'export_action_set_id_document_classes',

						'enablingProperty' : 'active_documentsclassification',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-document-citations',

						'jobProperty' : 'export_action_set_id_document_referencedDocuments',

						'enablingProperty' : 'active_citationmatching',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-referenceextraction-pdb',

						'jobProperty' : 'export_action_set_id_document_pdb',

						'enablingProperty' : 'active_referenceextraction_pdb',

						'enabled' : 'true'

						},

						{

						'set' : 'iis-referenceextraction-software',

						'jobProperty' : 'export_action_set_id_document_software_url',

						'enablingProperty' : 'active_referenceextraction_software_url',

						'enabled' : 'true'

						}

						]

					</PARAM>

				</PARAMETERS>

				<ARCS>

					<ARC to="prepareParameters" />

				</ARCS>

			</NODE>

			<NODE name="prepareParameters" type="PrepareIisMainParamsV2">

				<DESCRIPTION>prepare parameters</DESCRIPTION>

				<PARAMETERS>

					<PARAM required="true" type="string" name="islookupLocationParam" managedBy="system">import_islookup_service_location</PARAM>

					<PARAM required="true" type="string" name="objectStoreParam" managedBy="system">import_content_objectstores_csv</PARAM>

					<PARAM required="true" type="string" name="objectStoreLocationParam" managedBy="system">import_content_object_store_location</PARAM>

					<PARAM required="true" type="string" name="mdStoreStoreLocationParam" managedBy="system">import_mdstore_service_location</PARAM>

					<PARAM required="true" type="string" name="mdStoreDatasetParam" managedBy="system">import_dataset_mdstore_ids_csv</PARAM>

					<PARAM required="true" type="string" name="oozieWfAppPathParam" managedBy="system">oozie.wf.application.path</PARAM>

					<PARAM required="true" type="string" name="oozieWfAppPath" managedBy="user">/tmp/integration/apps/main</PARAM>

					<PARAM required="true" type="string" name="clusterName"	managedBy="user" function="validValues(['IIS','DM'])">IIS</PARAM>

					<PARAM required="true" type="string" name="importHbaseDumpLocation" managedBy="user"></PARAM>

					<PARAM required="true" type="string" name="importHbaseDumpLocationParam" managedBy="system">import_hbase_dump_location</PARAM>

					<PARAM required="false" type="string" name="objectStoreBlacklistCSV" managedBy="user"></PARAM>

					<PARAM required="true" type="string" name="importProjectConceptsContextIdParam" managedBy="system">import_project_concepts_context_id</PARAM>

					<PARAM required="true" type="string" name="importProjectConceptsContextId" managedBy="user">fet-fp7</PARAM>

				</PARAMETERS>

				<ARCS>

					<ARC to="main" />

				</ARCS>

			</NODE>

			<NODE name="main" type="SubmitHadoopJob" isJoin="true">

				<DESCRIPTION>IIS main</DESCRIPTION>

				<PARAMETERS>

					<PARAM required="true" type="string" name="hadoopJob" managedBy="system">iisMainJob</PARAM>

					<PARAM required="true" type="string" name="envParams" managedBy="system">

						{

						'cluster' : 'cluster',

						'nameNode' : 'nameNode',

						'jobTracker' : 'jobTracker',

						'oozie.wf.application.path' : 'oozie.wf.application.path',

						'active_document_affiliation' : 'active_document_affiliation',

						'active_referenceextraction_project' : 'active_referenceextraction_project',

						'active_referenceextraction_dataset' : 'active_referenceextraction_dataset',

						'active_referenceextraction_researchinitiative' : 'active_referenceextraction_researchinitiative',

						'active_documentsclassification' : 'active_documentsclassification',

						'active_documentssimilarity' : 'active_documentssimilarity',

						'active_citationmatching' : 'active_citationmatching',

						'active_referenceextraction_pdb' : 'active_referenceextraction_pdb',

						'active_referenceextraction_software_url' : 'active_referenceextraction_software_url',

						'import_content_objectstores_csv' : 'import_content_objectstores_csv',

						'import_content_object_store_location' : 'import_content_object_store_location',

						'import_mdstore_service_location' : 'import_mdstore_service_location',

						'import_islookup_service_location' : 'import_islookup_service_location', 

						'import_project_concepts_context_id' : 'import_project_concepts_context_id',

						'import_dataset_mdstore_ids_csv' : 'import_dataset_mdstore_ids_csv',

						'export_action_set_id_affiliation_matching' : 'export_action_set_id_affiliation_matching',

						'export_action_set_id_document_referencedDatasets' : 'export_action_set_id_document_referencedDatasets',

						'export_action_set_id_document_referencedProjects' : 'export_action_set_id_document_referencedProjects',

						'export_action_set_id_document_research_initiative' : 'export_action_set_id_document_research_initiative',

						'export_action_set_id_document_similarities_standard' : 'export_action_set_id_document_similarities_standard',

						'export_action_set_id_document_classes' : 'export_action_set_id_document_classes',

						'export_action_set_id_document_referencedDocuments' : 'export_action_set_id_document_referencedDocuments',

						'export_action_set_id_entity_dataset' : 'export_action_set_id_entity_dataset',

						'export_action_set_id_document_pdb' : 'export_action_set_id_document_pdb',

						'export_action_set_id_document_software_url' : 'export_action_set_id_document_software_url',

						'output_remote_location' : 'actionManagerBasePath'

						}

					</PARAM>

					<PARAM required="true" type="boolean" name="simulation"	managedBy="user">false</PARAM>

				</PARAMETERS>

				<ARCS>

					<ARC to="updateActionSets" />

				</ARCS>

			</NODE>

			<NODE name="updateActionSets" type="UpdateActionSets">

				<DESCRIPTION>update action sets</DESCRIPTION>

				<PARAMETERS />

				<ARCS>

					<ARC to="success" />

				</ARCS>

			</NODE>

		</CONFIGURATION>

		<STATUS />

	</BODY>

</RESOURCE_PROFILE>

    <bean id="wfNodePrepareIisMainParamsV2"

          class="eu.dnetlib.msro.openaireplus.workflows.nodes.PrepareIISMainParamsV2JobNode"

          scope="prototype" p:xqueryMdStoreService="${dnet.openaire.iis.mdstore.endpoint.xquery}"

          p:xqueryDatasetStore="${dnet.openaire.iis.mdstore.dataset.xquery}"

          p:xqueryObjectStores="${dnet.openaire.iis.objecstores.xquery}"

          p:xqueryObjectStoreService="${dnet.openaire.iis.objecstore.endpoint.xquery}"

          p:xqueryIsLookupService="${dnet.openaire.iis.islookup.endpoint.xquery}"/>

    <bean id="wfNodePreparePreprocessParamsV2"

          class="eu.dnetlib.msro.openaireplus.workflows.nodes.PrepareIISPreprocessingParamsV2JobNode"

          scope="prototype" p:xqueryDatabaseService="${dnet.openaire.iis.dbservice.endpoint.xquery}"

          p:xqueryMdStoreService="${dnet.openaire.iis.mdstore.endpoint.xquery}"

          p:xqueryObjectStoreService="${dnet.openaire.iis.objecstore.endpoint.xquery}"

          p:xqueryDatasetStore="${dnet.openaire.iis.mdstore.dataset.xquery}"

          p:xqueryWosMDStore="${dnet.openaire.iis.mdstore.wos.xquery}"

          p:xqueryDataciteObjectStore="${dnet.openaire.iis.objecstores.dataset.xquery}"

          p:xqueryWosObjectStore="${dnet.openaire.iis.objecstores.wos.xquery}"

          p:xqueryIsLookupService="${dnet.openaire.iis.islookup.endpoint.xquery}"/>