Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/tara_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/tara_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/tara_contexts.xsl (revision 58540)
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
+
+ taraexp_____::tara
+ TARA
+ Tara Expeditions Foundation
+ Fondation Tara Expéditions
+ FR
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/wt_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/wt_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/wt_contexts.xsl (revision 58540)
@@ -0,0 +1,40 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ WT
+ Wellcome Trust
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ wt:fundingStream
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nwo_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nwo_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nwo_contexts.xsl (revision 58540)
@@ -0,0 +1,21 @@
+
+
+
+
+
+
+
+
+
+ nwo_________::NWO
+ NWO
+ Netherlands Organisation for Scientific Research (NWO)
+ NL
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nhmrc_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nhmrc_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nhmrc_contexts.xsl (revision 58540)
@@ -0,0 +1,45 @@
+
+
+
+
+
+
+
+
+
+ nhmrc_______::NHMRC
+ NHMRC
+ National Health and Medical Research Council (NHMRC)
+ AU
+
+
+
+
+
+ nhmrc_______::NHMRC
+ NHMRC
+ National Health and Medical Research Council (NHMRC)
+ AU
+
+
+
+
+
+
+
+
+
+
+
+
+ nhmrc:fundingStream
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/fct_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/fct_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/fct_contexts.xsl (revision 58540)
@@ -0,0 +1,76 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ FCT
+ Fundação para a Ciência e a Tecnologia, I.P.
+ PT
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fct:program
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fct:program
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/fwf_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/fwf_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/fwf_contexts.xsl (revision 58540)
@@ -0,0 +1,41 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ FWF
+ Austrian Science Fund (FWF)
+ Fonds zur Förderung der Wissenschaftlichen Forschung
+ AT
+
+
+
+
+
+
+
+
+
+
+
+
+
+ fwf:programmes
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nih_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nih_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nih_contexts.xsl (revision 58540)
@@ -0,0 +1,45 @@
+
+
+
+
+
+
+
+
+
+ nih_________::NIH
+ NIH
+ National Institutes of Health
+ US
+
+
+
+
+
+ nih_________::NIH
+ NIH
+ National Institutes of Health
+ US
+
+
+
+
+
+
+
+
+
+
+
+
+
+ nih:fundingStream
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/corda_h2020_substreams_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/corda_h2020_substreams_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/corda_h2020_substreams_contexts.xsl (revision 58540)
@@ -0,0 +1,118 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ EC
+ European Commission
+ EU
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ H2020
+ Horizon 2020 Framework Programme
+
+ ec:h2020fundings
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ec:h2020toas
+
+
+
+
+
+
+
+
+
+
+
+ ec:h2020fundings
+
+
+
+
+
+ H2020
+ Horizon 2020 Framework Programme
+
+ ec:h2020fundings
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ec:h2020toas
+
+
+
+
+
+ H2020
+ Horizon 2020 Framework Programme
+
+ ec:h2020fundings
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/generateFETContext.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/generateFETContext.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/generateFETContext.wf.st (revision 58540)
@@ -0,0 +1,20 @@
+
+ Fetch records from MDStore
+
+ $params.("row_id")$
+ $params.("row_format")$
+ rs_epr
+
+
+
+
+
+
+ Generate FET hierarchy
+
+ rs_epr
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/mestd_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/mestd_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/mestd_contexts.xsl (revision 58540)
@@ -0,0 +1,45 @@
+
+
+
+
+
+
+
+
+
+ mestd_______::MESTD
+ MESTD
+ Ministry of Education, Science and Technological Development of Republic of Serbia
+ Ministarstvo Prosvete, Nauke i Tehnološkog Razvoja
+ RS
+
+
+
+
+ mestd_______::MESTD
+ MESTD
+ Ministry of Education, Science and Technological Development of Republic of Serbia
+ Ministarstvo Prosvete, Nauke i Tehnološkog Razvoja
+ RS
+
+
+
+
+
+
+
+
+
+
+
+
+ mestd:fundingStream
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedBy/fillHostedBy.sql.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedBy/fillHostedBy.sql.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedBy/fillHostedBy.sql.st (revision 58540)
@@ -0,0 +1,25 @@
+INSERT INTO hostedby_map(_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+SELECT d.issn||'@@'||d.collectedfrom, d.collectedfrom, d.issn, d.id
+ FROM dsm_datasources d where
+ d.collectedfrom ='$datasourceId$' and d.issn is not null and d.id like '$datasourceNamespacePrefix$%'
+ AND
+ (d.collectedfrom, d.issn) NOT IN
+ (SELECT oa_source_id, entry from hostedby_map);
+
+
+INSERT INTO hostedby_map(_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+SELECT d.eissn||'@@'||d.collectedfrom, d.collectedfrom, d.eissn, d.id
+ FROM dsm_datasources d where
+ d.collectedfrom ='$datasourceId$' and d.eissn is not null and d.id like '$datasourceNamespacePrefix$%'
+ AND
+ (d.collectedfrom, d.eissn) NOT IN
+ (SELECT oa_source_id, entry from hostedby_map);
+
+
+INSERT INTO hostedby_map(_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+SELECT d.lissn||'@@'||d.collectedfrom, d.collectedfrom, d.lissn, d.id
+ FROM dsm_datasources d where
+ d.collectedfrom ='$datasourceId$' and d.lissn is not null and d.id like '$datasourceNamespacePrefix$%'
+ AND
+ (d.collectedfrom, d.lissn) NOT IN
+ (SELECT oa_source_id, entry from hostedby_map);
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/findContextsFromMDStore.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/findContextsFromMDStore.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/findContextsFromMDStore.wf.st (revision 58540)
@@ -0,0 +1,70 @@
+
+ Fetch records from MDStore
+
+ $params.("orig_id")$
+ $params.("orig_format")$
+ collected_epr
+
+
+
+
+
+
+
+ Find project contexts
+
+
+ collected_epr
+ contexts_epr
+
+
+
+
+
+
+
+ Process project contexts
+
+ contexts_epr
+ contexts
+
+
+ funding
+ all
+ {openaire-guidelines3.0:XXX, FundRef:YYYY, prefix:PPPP}
+
+
+
+
+
+
+
+ Prepare the DB entries
+
+ db_epr
+ contexts
+
+
+
+
+
+
+
+ Update Db
+
+ db_epr
+
+
+
+
+
+
+
+ Save the Context Profile
+
+ contexts
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/transform.wf.st (revision 58540)
@@ -0,0 +1,115 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Start reading Hadoop MD Store
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Prepare a new MdStore Version
+
+ $params.("tran_id")$
+
+
+
+
+
+
+ Retrieve all the parameters needed to run the transformation workflow
+
+
+
+
+
+
+
+
+
+ Start the Hadoop Job
+
+ dnetHadoopTrasnformation
+ DHP
+
+ {
+ "mdstoreInputPath":"mdstoreInputPath",
+ "mdstoreOutputPath":"mdstoreOutputPath",
+ "transformationRule":"transformationRule",
+ "timestamp":"timestamp",
+ "workflowId":"workflowId"
+ }
+
+
+
+
+
+
+
+
+ Commit the mdstore version
+
+
+
+
+
+
+ End reading Hadoop MD Store
+
+
+
+
+
+
+ Abort the mdstore version
+
+
+
+
+
+
+ End reading Hadoop MD Store
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tran_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/repoBye_files.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/repoBye_files.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/repoBye_files.wf.st (revision 58540)
@@ -0,0 +1,21 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedBy/hostedby_manual_entries.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedBy/hostedby_manual_entries.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedBy/hostedby_manual_entries.sql (revision 58540)
@@ -0,0 +1 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid) VALUES ('RICE.KINDER@@openaire____::datacite', 'openaire____::datacite', 'RICE.KINDER', 're3data_____::r3d100012884');
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the oai_dc mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the OAF mdstore
+
+ $params.("tran_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepositoryHadoop/collection.wf.st (revision 58540)
@@ -0,0 +1,92 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Obtain data source params
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+ Prepare a new MdStore Version
+
+ $params.("harv_id")$
+
+
+
+
+
+
+ Put in the environment all the variable needed to the collection oozie job
+
+
+
+
+
+
+
+
+ Start the Hadoop Job
+
+ dnetHadoopCollection
+ DHP
+
+ {
+ "apiDescription":"apiDescription",
+ "mdStorePath":"mdStorePath",
+ "sequenceFilePath":"sequenceFilePath",
+ "dataSourceInfo":"dataSourceInfo" ,
+ "timestamp":"timestamp",
+ "identifierPath":"identifierPath",
+ "workflowId":"workflowId"
+ }
+
+
+
+
+
+
+
+
+
+ Commit the mdstore version
+
+
+
+
+
+
+
+ Abort the mdstore version
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/feedObjectStore.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/feedObjectStore.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/feedObjectStore.st (revision 58540)
@@ -0,0 +1,83 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Set information about current provider
+
+ INCREMENTAL
+
+
+
+
+
+
+
+
+ Refresh content on the object Store
+
+ $params.("objs_id")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+
+ OAF
+ orig_epr
+
+
+
+
+
+
+
+ Store files into objectStore
+
+ //*[local-name()='objIdentifier']
+ $params.("objs_id")$
+ orig_epr
+ true
+
+ incremental
+
+
+
+
+
+
+ Download records
+
+ $params.("objs_id")$
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("objs_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_download_total
+ last_download_date
+ last_download_objId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/collection.wf.st (revision 58540)
@@ -0,0 +1,150 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Set information about current provider
+
+ REFRESH
+
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Find Date Range For Incremental Harvesting
+
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+ Start incremental harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdtore records incrementally
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the GMF mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the OAF mdstore
+
+ $params.("tran_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/transform.wf.st (revision 58540)
@@ -0,0 +1,83 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Decide REFRESH/INCREMENTAL transformation
+
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records
+
+
+ orig_epr
+ dmf_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tran_id")$
+ REFRESH
+ dmf_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tran_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/transform.wf.st (revision 58540)
@@ -0,0 +1,85 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Decide REFRESH/INCREMENTAL transformation
+
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records
+
+
+ orig_epr
+ tran_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tran_id")$
+ REFRESH
+ tran_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tran_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the native mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the cleaned mdstore
+
+ $params.("tran_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the oai_dc mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the DMF mdstore
+
+ $params.("tran_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the oai_dc mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the DMF mdstore
+
+ $params.("tran_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/collection.wf.st (revision 58540)
@@ -0,0 +1,148 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Set information about current provider
+
+ REFRESH
+
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Find Date Range For Incremental Harvesting
+
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+ Start incremental harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdstore records incrementally
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/transform.wf.st (revision 58540)
@@ -0,0 +1,110 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Decide REFRESH/INCREMENTAL transformation
+
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records
+
+
+ orig_epr
+ tran_epr
+
+
+
+
+
+
+
+ Patch oaf:hostedBy fields
+
+ tran_epr
+ patched_epr
+
+ hostedByCounters
+ hostedby_map
+ //*[local-name()='header']/*[local-name()='setSpec']
+
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tran_id")$
+ REFRESH
+ patched_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tran_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
+
+
+ Validate hostedBy Repos
+
+ hostedByCounters
+ $params.("tran_id")$
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/collection.wf.st (revision 58540)
@@ -0,0 +1,150 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Set information about current provider
+
+ REFRESH
+
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Find Date Range For Incremental Harvesting
+
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+ Start incremental harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdtore records incrementally
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/collection.wf.st (revision 58540)
@@ -0,0 +1,150 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Set information about current provider
+
+ REFRESH
+
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Find Date Range For Incremental Harvesting
+
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+ Start incremental harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdtore records incrementally
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/pom.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/pom.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/pom.xml (revision 58540)
@@ -0,0 +1,202 @@
+
+
+
+ eu.dnetlib
+ dnet45-parent
+ 1.0.0
+
+
+ 4.0.0
+ eu.dnetlib
+ dnet-openaireplus-workflows
+ jar
+ 7.0.6
+
+
+ scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6
+
+
+
+ eu.dnetlib
+ cnr-data-flow-monitoring-core
+ [3.0.0, 4.0.0)
+
+
+ wstx-asl
+ org.codehaus.woodstox
+
+
+
+
+ eu.dnetlib
+ cnr-enabling-database-api
+ [2.0.0,3.0.0)
+
+
+ eu.dnetlib
+ cnr-enabling-database-service
+ [3.0.0,4.0.0)
+
+
+ eu.dnetlib
+ dnet-msro-service
+ [3.0.0,4.0.0)
+
+
+ eu.dnetlib
+ cnr-resultset-service
+ [2.0.0,3.0.0)
+
+
+ eu.dnetlib
+ dnet-openaire-datasource-manager
+ [1.0.0-SNAPSHOT,2.0.0)
+
+
+ eu.dnetlib
+ dnet-openaireplus-mapping-utils
+ [6.3.24,7.0.0)
+
+
+ eu.dnetlib
+ dnet-hadoop-service-rmi
+ [1.0.0,2.0.0)
+
+
+
+ eu.dnetlib
+ dnet-collector-plugins
+ [1.0.0,2.0.0)
+
+
+
+ eu.dnetlib
+ dnet-actionmanager-api
+ [4.0.0,5.0.0)
+
+
+
+ eu.dnetlib
+ dnet-oai-common-workflows
+ [5.0.0,6.0.0)
+
+
+
+ eu.dnetlib
+ cnr-mongo-mdstore
+ [6.0.0,7.0.0)
+ provided
+
+
+
+ eu.dnetlib
+ dnet-modular-objectstore-service
+ [4.2.1,5.0.0)
+ provided
+
+
+
+ eu.dnetlib
+ dnet-index-client
+ [2.3.4,3.0.0)
+
+
+
+ eu.dnetlib
+ dnet-validator-workflows
+ [1.0.0,2.0.0)
+
+
+
+ eu.dnetlib
+ dnet-deduplication
+ [1.6.7,2.0.0)
+
+
+
+
+ eu.dnetlib
+ dnet-directindex-api
+ [2.1.13,3.0.0)
+
+
+
+ org.apache.velocity
+ velocity
+ 1.7
+
+
+ antlr
+ antlr
+
+
+
+
+ org.apache.velocity
+ velocity-tools
+ 2.0
+
+
+ antlr
+ antlr
+
+
+
+
+ javax.servlet
+ javax.servlet-api
+ ${javax.servlet.version}
+ provided
+
+
+ junit
+ junit
+ ${junit.version}
+ test
+
+
+ eu.dnetlib
+ dnet-openaireplus-profiles
+ [1.0.0,2.0.0)
+ test
+
+
+ io.springfox
+ springfox-swagger2
+ ${springfox-version}
+
+
+ org.mockito
+ mockito-core
+ 1.9.5
+
+
+ org.springframework
+ spring-jdbc
+ ${spring.version}
+
+
+
+ eu.dnetlib
+ cnr-misc-utils
+ [1.0.5, 2.0.0)
+
+
+
+
+ eu.dnetlib.dhp
+ dhp-common
+ [1.0.0, 2.0.0)
+
+
+ net.sf.saxon
+ Saxon-HE
+
+
+
+
+
+
+ 2.5.0
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/transform.wf.st (revision 58540)
@@ -0,0 +1,113 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Decide REFRESH/INCREMENTAL transformation
+
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records
+
+
+ orig_epr
+ tran_epr
+
+
+
+
+
+
+
+ Patch oaf:hostedBy fields
+
+ tran_epr
+ patched_epr
+
+ hostedByCounters
+ hostedby_map
+ //*[local-name()='header']/*[local-name()='setSpec']
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tran_id")$
+ REFRESH
+ patched_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tran_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
+
+
+ Validate hostedBy Repos
+
+ hostedByCounters
+ $params.("tran_id")$
+ datacite::last_aggregation_total
+ datacite::last_aggregation_date
+ datacite::last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/find_journal_intersection.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/find_journal_intersection.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/find_journal_intersection.sql (revision 58540)
@@ -0,0 +1,9 @@
+SELECT
+ h.entry AS id,
+ h.datasourceid AS datasource,
+ d.officialname AS name
+
+FROM hostedby_map h, datasources d
+WHERE
+ h.oa_source_id = 're3data_____::r3d100010134' AND
+ h.datasourceid = d.id
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/pangaea_preparetable.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/pangaea_preparetable.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/pangaea_preparetable.sql (revision 58540)
@@ -0,0 +1,7 @@
+DROP TABLE IF EXISTS pangaea_temp_journals;
+CREATE TABLE IF NOT EXISTS pangaea_temp_journals (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) DEFAULT 'temp_' || md5(clock_timestamp() :: TEXT) || '_' || md5(random() :: TEXT),
+ id CHARACTER VARYING(255) PRIMARY KEY,
+ issn CHARACTER VARYING(255),
+ oa_source_id CHARACTER VARYING(255) REFERENCES datasources (id)
+);
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/collection.wf.st (revision 58540)
@@ -0,0 +1,86 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Fetch records by Query
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/find_journal_intersection.sql
+ sql_epr
+
+
+
+
+
+
+
+ Start Harvesting Datasets
+
+ $params.("dataprovider:id")$
+ sql_epr
+ collected_epr
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+
+ collected_epr
+ store_epr
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasources.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasources.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasources.sql (revision 58540)
@@ -0,0 +1,149 @@
+SELECT
+ d.id AS datasourceid,
+ d.id || array_agg(distinct di.pid) AS identities,
+ d.officialname AS officialname,
+ d.englishname AS englishname,
+ CASE
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility):: TEXT) @> ARRAY ['openaire-cris_1.1'])
+ THEN
+ 'openaire-cris_1.1@@@OpenAIRE CRIS v1.1@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility):: TEXT) @> ARRAY ['openaire4.0'])
+ THEN
+ 'openaire4.0@@@OpenAIRE 4.0@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility):: TEXT) @> ARRAY ['driver', 'openaire2.0'])
+ THEN
+ 'driver-openaire2.0@@@OpenAIRE 2.0+ (DRIVER OA, EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['driver'])
+ THEN
+ 'driver@@@OpenAIRE Basic (DRIVER OA)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire2.0'])
+ THEN
+ 'openaire2.0@@@OpenAIRE 2.0 (EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire3.0'])
+ THEN
+ 'openaire3.0@@@OpenAIRE 3.0 (OA, funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire2.0_data'])
+ THEN
+ 'openaire2.0_data@@@OpenAIRE Data (funded, referenced datasets)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['native'])
+ THEN
+ 'native@@@proprietary@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['hostedBy'])
+ THEN
+ 'hostedBy@@@collected from a compatible aggregator@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['notCompatible'])
+ THEN
+ 'notCompatible@@@under validation@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ ELSE
+ 'UNKNOWN@@@not available@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
+ END AS openairecompatibility,
+ d.websiteurl AS websiteurl,
+ d.logourl AS logourl,
+ array_agg(DISTINCT CASE WHEN a.protocol = 'oai' and last_aggregation_date is not null THEN a.baseurl ELSE NULL END) AS accessinfopackage,
+ d.latitude AS latitude,
+ d.longitude AS longitude,
+ d.namespaceprefix AS namespaceprefix,
+ NULL AS odnumberofitems,
+ NULL AS odnumberofitemsdate,
+
+ (SELECT array_agg(s|| '###keywords@@@keywords@@@dnet:subject_classification_typologies@@@dnet:subject_classification_typologies')
+ FROM UNNEST(
+ ARRAY(
+ SELECT trim(s)
+ FROM unnest(string_to_array(d.subjects, '@@')) AS s)) AS s) AS subjects,
+
+ d.description AS description,
+ NULL AS odpolicies,
+ ARRAY(SELECT trim(s)
+ FROM unnest(string_to_array(d.languages, ',')) AS s) AS odlanguages,
+ ARRAY(SELECT trim(s)
+ FROM unnest(string_to_array(d.od_contenttypes, '-')) AS s) AS odcontenttypes,
+ false AS inferred,
+ false AS deletedbyinference,
+ 0.9 AS trust,
+ NULL AS inferenceprovenance,
+ d.dateofcollection AS dateofcollection,
+ d.dateofvalidation AS dateofvalidation,
+ -- re3data fields
+ d.releasestartdate AS releasestartdate,
+ d.releaseenddate AS releaseenddate,
+ d.missionstatementurl AS missionstatementurl,
+ d.dataprovider AS dataprovider,
+ d.serviceprovider AS serviceprovider,
+ d.databaseaccesstype AS databaseaccesstype,
+ d.datauploadtype AS datauploadtype,
+ d.databaseaccessrestriction AS databaseaccessrestriction,
+ d.datauploadrestriction AS datauploadrestriction,
+ d.versioning AS versioning,
+ d.citationguidelineurl AS citationguidelineurl,
+ d.qualitymanagementkind AS qualitymanagementkind,
+ d.pidsystems AS pidsystems,
+ d.certificates AS certificates,
+ ARRAY[]::text[] AS policies,
+ dc.id AS collectedfromid,
+ dc.officialname AS collectedfromname,
+ d.typology || '@@@' || CASE
+ WHEN (d.typology = 'crissystem') THEN 'CRIS System'
+ WHEN (d.typology = 'datarepository::unknown') THEN 'Data Repository'
+ WHEN (d.typology = 'aggregator::datarepository') THEN 'Data Repository Aggregator'
+ WHEN (d.typology = 'infospace') THEN 'Information Space'
+ WHEN (d.typology = 'pubsrepository::institutional') THEN 'Institutional Repository'
+ WHEN (d.typology = 'aggregator::pubsrepository::institutional') THEN 'Institutional Repository Aggregator'
+ WHEN (d.typology = 'pubsrepository::journal') THEN 'Journal'
+ WHEN (d.typology = 'aggregator::pubsrepository::journals') THEN 'Journal Aggregator/Publisher'
+ WHEN (d.typology = 'pubsrepository::mock') THEN 'Other'
+ WHEN (d.typology = 'pubscatalogue::unknown') THEN 'Publication Catalogue'
+ WHEN (d.typology = 'pubsrepository::unknown') THEN 'Publication Repository'
+ WHEN (d.typology = 'aggregator::pubsrepository::unknown') THEN 'Publication Repository Aggregator'
+ WHEN (d.typology = 'entityregistry') THEN 'Registry'
+ WHEN (d.typology = 'scholarcomminfra') THEN 'Scholarly Comm. Infrastructure'
+ WHEN (d.typology = 'pubsrepository::thematic') THEN 'Thematic Repository'
+ WHEN (d.typology = 'websource') THEN 'Web Source'
+ WHEN (d.typology = 'entityregistry::projects') THEN 'Funder database'
+ WHEN (d.typology = 'entityregistry::repositories') THEN 'Registry of repositories'
+ WHEN (d.typology = 'softwarerepository') THEN 'Software Repository'
+ WHEN (d.typology = 'aggregator::softwarerepository') THEN 'Software Repository Aggregator'
+ WHEN (d.typology = 'orprepository') THEN 'Repository'
+ ELSE 'Other'
+ END || '@@@dnet:datasource_typologies@@@dnet:datasource_typologies' AS datasourcetype,
+ 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction,
+ CONCAT(d.issn, '@@@', d.eissn, '@@@', d.lissn) AS journal
+
+FROM dsm_datasources d
+
+LEFT OUTER JOIN dsm_datasources dc on (d.collectedfrom = dc.id)
+LEFT OUTER JOIN dsm_api a ON (d.id = a.datasource)
+LEFT OUTER JOIN dsm_datasourcepids di ON (d.id = di.datasource)
+
+GROUP BY
+ d.id,
+ d.officialname,
+ d.englishname,
+ d.websiteurl,
+ d.logourl,
+ d.contactemail,
+ d.namespaceprefix,
+ d.description,
+ d.latitude,
+ d.longitude,
+ d.dateofcollection,
+ d.dateofvalidation,
+ d.releasestartdate,
+ d.releaseenddate,
+ d.missionstatementurl,
+ d.dataprovider,
+ d.serviceprovider,
+ d.databaseaccesstype,
+ d.datauploadtype,
+ d.databaseaccessrestriction,
+ d.datauploadrestriction,
+ d.versioning,
+ d.citationguidelineurl,
+ d.qualitymanagementkind,
+ d.pidsystems,
+ d.certificates,
+ dc.id,
+ dc.officialname,
+ d.issn,
+ d.eissn,
+ d.lissn
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/xsl/jorunal2db.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/xsl/jorunal2db.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/xsl/jorunal2db.xsl (revision 58540)
@@ -0,0 +1,30 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjectOrganization.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjectOrganization.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjectOrganization.sql (revision 58540)
@@ -0,0 +1,17 @@
+SELECT
+ po.project AS project,
+ po.resporganization AS resporganization,
+ po.participantnumber AS participantnumber,
+ po.contribution AS contribution,
+ po.currency AS currency,
+ NULL AS startdate,
+ NULL AS enddate,
+ false AS inferred,
+ false AS deletedbyinference,
+ po.trust AS trust,
+ NULL AS inferenceprovenance,
+
+ po.semanticclass || '@@@' || po.semanticclass || '@@@dnet:project_organization_relations@@@dnet:project_organization_relations' AS semantics,
+ 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction
+
+FROM project_organization po
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/transformPublications.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/transformPublications.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/transformPublications.wf.st (revision 58540)
@@ -0,0 +1,93 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_publication_epr
+
+
+
+
+
+
+
+ Transform original records to Publication
+
+
+ orig_publication_epr
+ packed_publication_epr
+
+
+
+
+
+
+
+ Split Records into datasets and publication
+
+ packed_publication_epr
+ publication_epr
+
+
+
+
+
+
+
+ Patch oaf:hostedBy fields
+
+ publication_epr
+ patched_epr
+ hostedByCounters
+
+
+
+
+
+
+
+ Store records
+
+ $params.("tranOAF_id")$
+ REFRESH
+ patched_epr
+
+
+
+
+
+
+
+ Validate hostedBy Repos
+
+ hostedByCounters
+ $params.("tranOAF_id")$
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizationsFromOpenOrgsDB.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizationsFromOpenOrgsDB.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizationsFromOpenOrgsDB.sql (revision 58540)
@@ -0,0 +1,53 @@
+SELECT
+ o.id AS organizationid,
+ coalesce((array_agg(a.acronym))[1], o.name) AS legalshortname,
+ o.name AS legalname,
+ array_agg(DISTINCT n.name) AS "alternativeNames",
+ (array_agg(u.url))[1] AS websiteurl,
+ o.modification_date AS dateoftransformation,
+ false AS inferred,
+ false AS deletedbyinference,
+ 0.95 AS trust,
+ '' AS inferenceprovenance,
+ 'openaire____::openorgs' AS collectedfromid,
+ 'OpenOrgs Database' AS collectedfromname,
+ o.country || '@@@dnet:countries' AS country,
+ 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction,
+ array_agg(DISTINCT i.otherid || '###' || i.type || '@@@dnet:pid_types') AS pid
+FROM organizations o
+ LEFT OUTER JOIN acronyms a ON (a.id = o.id)
+ LEFT OUTER JOIN urls u ON (u.id = o.id)
+ LEFT OUTER JOIN other_ids i ON (i.id = o.id)
+ LEFT OUTER JOIN other_names n ON (n.id = o.id)
+GROUP BY
+ o.id,
+ o.name,
+ o.modification_date,
+ o.country
+
+UNION ALL
+
+SELECT
+ 'openorgsmesh'||substring(o.id, 13)||'-'||md5(n.name) AS organizationid,
+ n.name AS legalshortname,
+ n.name AS legalname,
+ ARRAY[]::text[] AS "alternativeNames",
+ (array_agg(u.url))[1] AS websiteurl,
+ o.modification_date AS dateoftransformation,
+ false AS inferred,
+ false AS deletedbyinference,
+ 0.88 AS trust,
+ '' AS inferenceprovenance,
+ 'openaire____::openorgs' AS collectedfromid,
+ 'OpenOrgs Database' AS collectedfromname,
+ o.country || '@@@dnet:countries' AS country,
+ 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction,
+ array_agg(DISTINCT i.otherid || '###' || i.type || '@@@dnet:pid_types') AS pid
+FROM other_names n
+ LEFT OUTER JOIN organizations o ON (n.id = o.id)
+ LEFT OUTER JOIN urls u ON (u.id = o.id)
+ LEFT OUTER JOIN other_ids i ON (i.id = o.id)
+GROUP BY
+ o.id, o.modification_date, o.country, n.name
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/transformPublications.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/transformPublications.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/transformPublications.wf.st (revision 58540)
@@ -0,0 +1,69 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_publication_epr
+
+
+
+
+
+
+
+ Transform original records to Publication
+
+
+ orig_publication_epr
+ packed_publication_epr
+
+
+
+
+
+
+
+ Split Records into datasets and publication
+
+ packed_publication_epr
+ publication_epr
+
+
+
+
+
+
+
+ Store records
+
+ $params.("tranOAF_id")$
+ REFRESH
+ publication_epr
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/querySimilarityFromOpenOrgsDB.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/querySimilarityFromOpenOrgsDB.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/querySimilarityFromOpenOrgsDB.sql (revision 58540)
@@ -0,0 +1,17 @@
+SELECT local_id AS id1, oa_original_id AS id2 FROM openaire_simrels WHERE reltype = 'is_similar'
+
+UNION ALL
+
+SELECT
+ o.id AS id1,
+ 'openorgsmesh'||substring(o.id, 13)||'-'||md5(a.acronym) AS id2
+FROM acronyms a
+ LEFT OUTER JOIN organizations o ON (a.id = o.id)
+
+UNION ALL
+
+SELECT
+ o.id AS id1,
+ 'openorgsmesh'||substring(o.id, 13)||'-'||md5(n.name) AS id2
+FROM other_names n
+ LEFT OUTER JOIN organizations o ON (n.id = o.id)
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/repoBye.wf.st (revision 58540)
@@ -0,0 +1,64 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the native mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+
+ Delete the IMEF mdstore
+
+ $params.("tranODF_id")$
+
+
+
+
+
+
+
+ Delete the IMEF mdstore
+
+ $params.("tranOAF_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizations.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizations.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizations.sql (revision 58540)
@@ -0,0 +1,36 @@
+SELECT
+ o.id AS organizationid,
+ o.legalshortname AS legalshortname,
+ o.legalname AS legalname,
+ o.websiteurl AS websiteurl,
+ o.logourl AS logourl,
+ o.ec_legalbody AS eclegalbody,
+ o.ec_legalperson AS eclegalperson,
+ o.ec_nonprofit AS ecnonprofit,
+ o.ec_researchorganization AS ecresearchorganization,
+ o.ec_highereducation AS echighereducation,
+ o.ec_internationalorganizationeurinterests AS ecinternationalorganizationeurinterests,
+ o.ec_internationalorganization AS ecinternationalorganization,
+ o.ec_enterprise AS ecenterprise,
+ o.ec_smevalidated AS ecsmevalidated,
+ o.ec_nutscode AS ecnutscode,
+ o.dateofcollection AS dateofcollection,
+ o.lastupdate AS dateoftransformation,
+ false AS inferred,
+ false AS deletedbyinference,
+ o.trust AS trust,
+ '' AS inferenceprovenance,
+ d.id AS collectedfromid,
+ d.officialname AS collectedfromname,
+
+ o.country || '@@@dnet:countries' AS country,
+ 'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction,
+
+ ARRAY[]::text[] AS pid
+FROM dsm_organizations o
+ LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom)
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/repoBye.wf.st (revision 58540)
@@ -0,0 +1,63 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the oai_datacite mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the IMEF mdstore
+
+ $params.("tranODF_id")$
+
+
+
+
+
+
+
+ Delete the IMEF mdstore
+
+ $params.("tranOAF_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourceOrganization.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourceOrganization.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourceOrganization.sql (revision 58540)
@@ -0,0 +1,16 @@
+SELECT
+ dor.datasource AS datasource,
+ dor.organization AS organization,
+ NULL AS startdate,
+ NULL AS enddate,
+ false AS inferred,
+ false AS deletedbyinference,
+ 0.9 AS trust,
+ NULL AS inferenceprovenance,
+
+ 'providedBy@@@provided by@@@dnet:datasources_organizations_typologies@@@dnet:datasources_organizations_typologies' AS semantics,
+ d.provenanceaction || '@@@' || d.provenanceaction || '@@@dnet:provenanceActions@@@dnet:provenanceActions' AS provenanceaction
+
+FROM dsm_datasource_organization dor
+ LEFT OUTER JOIN dsm_datasources d ON (dor.datasource = d.id)
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourcesForSize.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourcesForSize.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourcesForSize.sql (revision 58540)
@@ -0,0 +1,2 @@
+SELECT count(*)
+FROM dsm_datasources;
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/validate-nostore-openaire.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/validate-nostore-openaire.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/validate-nostore-openaire.wf.st (revision 58540)
@@ -0,0 +1,48 @@
+
+ Retrieve information about current Content Provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+ Get more datasource info
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+ Do we validate the records?
+
+ NO
+
+
+
+
+
+
+
+ Validate from mdStore
+
+ false
+ validatorEpr
+ $params.("harv_id")$
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ValidatorServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+ ALL
+ $params.("dataprovider:interface:compliance")$
+ true
+ blacklist_rules
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/dc_2_oaf.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/dc_2_oaf.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/dc_2_oaf.xsl (revision 58540)
@@ -0,0 +1,53 @@
+
+
+
+
+
+
+
+
+
+ 0001
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ CLOSED
+ RESTRICTED
+ EMBARGO
+ OPEN
+ UNKNOWN
+
+
+
+
+
+
+
+
+
+
+ datasource id taken from the database, es: opendoar____::2659
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/updateDb.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/updateDb.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/updateDb.wf.st (revision 58540)
@@ -0,0 +1,44 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Decide REFRESH/INCREMENTAL Update
+
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("row_id")$
+ $params.("row_format")$
+ rs_epr
+
+
+
+
+
+
+
+
+ Update Db
+
+ rs_epr
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/transform.wf.st (revision 58540)
@@ -0,0 +1,94 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("orig_id")$
+ $params.("orig_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+
+ orig_epr
+ packed_row_epr
+
+
+
+
+
+
+
+ Clean DB rows values according to defined vocabularies
+
+
+ packed_row_epr
+ cleaned_row_epr
+
+
+
+
+
+
+
+ Filter DB rows according to the managed flag state
+
+ cleaned_row_epr
+ filtered_row_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("row_id")$
+ REFRESH
+ filtered_row_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("row_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/collection.wf.st (revision 58540)
@@ -0,0 +1,74 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+
+ collected_epr
+ store_epr
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/openaireMdBuilder.xslt.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/openaireMdBuilder.xslt.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/openaireMdBuilder.xslt.st (revision 58540)
@@ -0,0 +1,119 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ $namespacePrefix$
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ $namespacePrefix$
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ $baseurl$
+
+
+ $metadatanamespace$
+
+
+
+
+ $inferred$
+ $deletedbyinference$
+ $trust$
+ $inferenceprovenance$
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_findMatches.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_findMatches.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_findMatches.sql (revision 58540)
@@ -0,0 +1,15 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+ SELECT
+ s.setspec || '@@' || s.oa_source_id,
+ s.oa_source_id,
+ s.setspec,
+ MIN(d.id) AS id
+ FROM
+ datacite_temp_sets s
+ JOIN datacite_temp_datarepos d ON (s.setname_cleaned = d.name_cleaned)
+ WHERE (s.oa_source_id, s.setspec) NOT IN
+ (SELECT
+ oa_source_id,
+ entry
+ FROM hostedby_map)
+ GROUP BY s.oa_source_id, s.setspec
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/collection.wf.st (revision 58540)
@@ -0,0 +1,126 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Choose collection mode
+
+ REFRESH
+
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("orig_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Find Date Range For Incremental Harvesting
+
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+ Start incremental harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Store mdstore records incrementally
+
+ $params.("orig_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("orig_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_prepareTables.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_prepareTables.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_prepareTables.sql (revision 58540)
@@ -0,0 +1,23 @@
+DROP TABLE IF EXISTS datacite_temp_sets;
+CREATE TABLE IF NOT EXISTS datacite_temp_sets (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) DEFAULT 'temp_' || md5(clock_timestamp() :: TEXT) || '_' || md5(random() :: TEXT),
+ id CHARACTER VARYING(255) PRIMARY KEY,
+ setname CHARACTER VARYING(255),
+ setname_cleaned CHARACTER VARYING(255),
+ setspec CHARACTER VARYING(255),
+ oa_source_id CHARACTER VARYING(255) REFERENCES dsm_datasources (id)
+);
+
+DROP TABLE IF EXISTS datacite_temp_datarepos;
+
+CREATE TABLE IF NOT EXISTS datacite_temp_datarepos (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) DEFAULT 'temp_' || md5(clock_timestamp() :: TEXT) || '_' || md5(random() :: TEXT),
+ id CHARACTER VARYING(255) PRIMARY KEY,
+ name CHARACTER VARYING(512) NOT NULL,
+ name_cleaned CHARACTER VARYING(512) NOT NULL,
+ class CHARACTER VARYING(255)
+);
+
+--CREATE INDEX datacite_sets__dnet_resource_identifier__idx ON datacite_sets (_dnet_resource_identifier_);
+CREATE INDEX datacite_temp_datarepos__dnet_resource_identifier__idx
+ ON datacite_temp_datarepos (_dnet_resource_identifier_);
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryClaimsRel.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryClaimsRel.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryClaimsRel.sql (revision 58540)
@@ -0,0 +1 @@
+SELECT xml, provenance FROM claims WHERE type = 'rels2actions' and set = 'userclaim_result_project' or set = 'userclaim_result_result'
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the GMF mdstore
+
+ $params.("orig_id")$
+
+
+
+
+
+
+
+ Delete the DBMF mdstore
+
+ $params.("row_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/PublicationFromPangaeatransform.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/PublicationFromPangaeatransform.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/PublicationFromPangaeatransform.xsl (revision 58540)
@@ -0,0 +1,170 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 0000
+ und
+
+
+
+ UNKNOWN
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ false
+ false
+ 0.9
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/pangaea_preparetable.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/pangaea_preparetable.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/pangaea_preparetable.sql (revision 58540)
@@ -0,0 +1,7 @@
+DROP TABLE IF EXISTS pangaea_temp_journals;
+CREATE TABLE IF NOT EXISTS pangaea_temp_journals (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) DEFAULT 'temp_' || md5(clock_timestamp() :: TEXT) || '_' || md5(random() :: TEXT),
+ id CHARACTER VARYING(255) PRIMARY KEY,
+ issn CHARACTER VARYING(255),
+ oa_source_id CHARACTER VARYING(255) REFERENCES dsm_datasources (id)
+);
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryClaimsUpdate.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryClaimsUpdate.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryClaimsUpdate.sql (revision 58540)
@@ -0,0 +1,7 @@
+SELECT
+ regexp_replace(xml, '<\?xml version="1\.0" encoding="UTF-8"\?>', '', 'i') AS xml,
+ provenance
+FROM claims
+WHERE type = 'updates2actions' AND set = 'userclaim_dmf'
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/pangaea_journal_findmatches.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/pangaea_journal_findmatches.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/pangaea_journal_findmatches.sql (revision 58540)
@@ -0,0 +1,15 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+ SELECT
+ p.issn || '__' || p.id || '@@' || p.oa_source_id,
+ p.oa_source_id,
+ p.issn || '__' || p.id,
+ d.id
+ FROM dsm_datasources d, pangaea_temp_journals p
+ WHERE
+ d.collectedfrom = 'driver______::1790119e-d281-4b7a-aedf-866d1d853a07' AND d.issn IS NOT NULL AND d.id LIKE 'doajarticles::%' AND p.issn = d.issn
+ AND
+ (p.oa_source_id, p.issn || '__' || p.id) NOT IN
+ (SELECT
+ oa_source_id,
+ entry
+ FROM hostedby_map)
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/DatasetsFromPangaeaIngestion.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/DatasetsFromPangaeaIngestion.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/DatasetsFromPangaeaIngestion.xsl (revision 58540)
@@ -0,0 +1,49 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryPersons.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryPersons.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryPersons.sql (revision 58540)
@@ -0,0 +1,55 @@
+SELECT
+ p.id AS personid,
+ p.firstname AS firstname,
+ p.secondnames AS secondnames,
+ CASE WHEN length(p.secondnames) > 0
+ THEN trim(BOTH FROM p.secondnames || ', ' || p.firstname)
+ ELSE p.firstname END AS fullname,
+ p.fax AS fax,
+ p.email AS email,
+ p.phone AS phone,
+ p.dateofcollection AS dateofcollection,
+ p.inferred AS inferred,
+ p.deletedbyinference AS deletedbyinference,
+ p.trust AS trust,
+ p.inferenceprovenance AS inferenceprovenance,
+ dc.id AS collectedfromid,
+ dc.officialname AS collectedfromname,
+
+ nc.code || '@@@' || nc.name || '@@@' || ns.code || '@@@' || ns.name AS nationality,
+ pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name AS provenanceaction,
+
+ array_agg(DISTINCT i.pid || '###' || i.issuertypeclass) AS pid
+
+FROM persons p
+ LEFT OUTER JOIN class nc ON (nc.code = p.nationalityclass)
+ LEFT OUTER JOIN scheme ns ON (ns.code = p.nationalityscheme)
+
+ LEFT OUTER JOIN class pac ON (pac.code = p.provenanceactionclass)
+ LEFT OUTER JOIN scheme pas ON (pas.code = p.provenanceactionscheme)
+
+ LEFT OUTER JOIN datasources dc ON (dc.id = p.collectedfrom)
+
+ LEFT OUTER JOIN personpids pp ON (pp.person = p.id)
+ LEFT OUTER JOIN identities i ON (i.pid = pp.pid)
+
+ LEFT OUTER JOIN project_organization po ON (po.contactperson = p.id)
+
+WHERE (p.firstname IS NOT NULL OR p.secondnames IS NOT NULL) AND po.resporganization IS NOT NULL
+
+GROUP BY
+ p.id,
+ p.firstname,
+ p.secondnames,
+ p.fax,
+ p.email,
+ p.phone,
+ p.dateofcollection,
+ p.inferred,
+ p.deletedbyinference,
+ p.trust,
+ p.inferenceprovenance,
+ dc.id,
+ dc.officialname,
+ nc.code, nc.name, ns.code, ns.name,
+ pac.code, pac.name, pas.code, pas.name
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/update.db.patch.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/update.db.patch.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/update.db.patch.wf.st (revision 58540)
@@ -0,0 +1,78 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Obtain parameters for current provider
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Create necessary tables
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/pangaea_preparetable.sql
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Transform original records to Publication
+
+
+ collected_epr
+ row_epr
+
+
+
+
+
+
+
+ Update Db
+
+ row_epr
+
+
+
+
+
+
+
+ Add new journal to patch entry tables
+
+ dnet.openaire.db.name
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjects.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjects.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjects.sql (revision 58540)
@@ -0,0 +1,2 @@
+SELECT *
+FROM projects_mv
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/repoBye.wf.st (revision 58540)
@@ -0,0 +1,63 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the oai_datacite mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the IMEF mdstore
+
+ $params.("tranODF_id")$
+
+
+
+
+
+
+
+ Delete the IMEF mdstore
+
+ $params.("tranOAF_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/european_environment_agency_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/european_environment_agency_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/european_environment_agency_context.xsl (revision 58540)
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+ euenvagency_::EEA
+ EEA
+ European Environment Agency
+ European Environment Agency
+ EU
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rsf_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rsf_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rsf_context.xsl (revision 58540)
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+ rsf_________::RSF
+ RSF
+ Russian Science Foundation
+ Russian Science Foundation
+ RU
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/transform.wf.st (revision 58540)
@@ -0,0 +1,83 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Decide REFRESH/INCREMENTAL transformation
+
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("orig_id")$
+ $params.("orig_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+
+ orig_epr
+ tran_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("row_id")$
+ REFRESH
+ tran_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("row_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/transformDatasets.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/transformDatasets.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/transformDatasets.wf.st (revision 58540)
@@ -0,0 +1,85 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_dataset_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records to Datasets
+
+
+ orig_dataset_epr
+ datasets_epr
+
+
+
+
+
+
+
+ Clean dataset records
+
+
+ datasets_epr
+ cleaned_datasets_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tranODF_id")$
+ REFRESH
+ cleaned_datasets_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tranODF_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rif_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rif_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rif_context.xsl (revision 58540)
@@ -0,0 +1,34 @@
+
+
+
+
+
+
+
+
+
+
+
+ RIF
+ Research and Innovation Foundation
+ Research and Innovation Foundation
+ CY
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/collection.wf.st (revision 58540)
@@ -0,0 +1,86 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Fetch records by Query
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/find_journal_intersection.sql
+ sql_epr
+
+
+
+
+
+
+
+ Start Harvesting Datasets
+
+ $params.("dataprovider:id")$
+ sql_epr
+ collected_epr
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+
+ collected_epr
+ store_epr
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/collection.wf.st (revision 58540)
@@ -0,0 +1,63 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("orig_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("orig_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/innoviris_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/innoviris_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/innoviris_context.xsl (revision 58540)
@@ -0,0 +1,38 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ INNOVIRIS
+ INNOVIRIS
+ BE
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt-datacite/datacite_datarepos_2_db.xslt
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt-datacite/datacite_datarepos_2_db.xslt (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt-datacite/datacite_datarepos_2_db.xslt (revision 58540)
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the GMF mdstore
+
+ $params.("orig_id")$
+
+
+
+
+
+
+
+ Delete the DBMF mdstore
+
+ $params.("row_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/update.db.patch.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/update.db.patch.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/update.db.patch.wf.st (revision 58540)
@@ -0,0 +1,22 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Add new journal to patch entry tables
+
+ dnet.openaire.db.name
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/sql/doajJournals_findMatches.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/sql/doajJournals_findMatches.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/sql/doajJournals_findMatches.sql (revision 58540)
@@ -0,0 +1,15 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+ SELECT
+ d.issn || '@@' || d.collectedfrom,
+ d.collectedfrom,
+ d.issn,
+ d.id
+ FROM dsm_datasources d
+ WHERE
+ d.collectedfrom = 'driver______::1790119e-d281-4b7a-aedf-866d1d853a07' AND d.issn IS NOT NULL AND d.id LIKE 'doajarticles::%'
+ AND
+ (d.collectedfrom, d.issn) NOT IN
+ (SELECT
+ oa_source_id,
+ entry
+ FROM hostedby_map)
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/ekt_journal.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/ekt_journal.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/ekt_journal.sql (revision 58540)
@@ -0,0 +1,15 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+ SELECT
+ d.issn || '@@' || d.collectedfrom,
+ d.collectedfrom,
+ d.issn,
+ d.id
+ FROM dsm_datasources d
+ WHERE
+ d.collectedfrom = 'openaire____::ekt_journals' AND d.issn IS NOT NULL AND d.id LIKE 'ekt_journals::%'
+ AND
+ (d.collectedfrom, d.issn) NOT IN
+ (SELECT
+ oa_source_id,
+ entry
+ FROM hostedby_map)
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/repoBye.wf.st (revision 58540)
@@ -0,0 +1,21 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/doajJournals_findMatches.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/doajJournals_findMatches.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/doajJournals_findMatches.sql (revision 58540)
@@ -0,0 +1,15 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+ SELECT
+ d.issn || '@@' || d.collectedfrom,
+ d.collectedfrom,
+ d.issn,
+ d.id
+ FROM dsm_datasources d
+ WHERE
+ d.collectedfrom = 'driver______::1790119e-d281-4b7a-aedf-866d1d853a07' AND d.issn IS NOT NULL AND d.id LIKE 'doajarticles::%'
+ AND
+ (d.collectedfrom, d.issn) NOT IN
+ (SELECT
+ oa_source_id,
+ entry
+ FROM hostedby_map)
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/find_journal_intersection.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/find_journal_intersection.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/find_journal_intersection.sql (revision 58540)
@@ -0,0 +1,9 @@
+SELECT
+ h.entry AS id,
+ h.datasourceid AS datasource,
+ d.officialname AS name
+
+FROM hostedby_map h, dsm_datasources d
+WHERE
+ h.oa_source_id = 're3data_____::r3d100010134' AND
+ h.datasourceid = d.id
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/xsl/jorunal2db.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/xsl/jorunal2db.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/xsl/jorunal2db.xsl (revision 58540)
@@ -0,0 +1,30 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/gsrt_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/gsrt_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/gsrt_contexts.xsl (revision 58540)
@@ -0,0 +1,26 @@
+
+
+
+
+
+
+
+ gsrt________::GSRT
+
+
+
+ GSRT
+ General Secretariat of Research and Technology (GSRT)
+ General Secretariat of Research and Technology
+ GR
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/miur_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/miur_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/miur_context.xsl (revision 58540)
@@ -0,0 +1,38 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ MIUR
+ Ministero dell'Istruzione dell'Università e della Ricerca
+ IT
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rcuk_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rcuk_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/rcuk_contexts.xsl (revision 58540)
@@ -0,0 +1,37 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ RCUK
+ Research Council UK
+ GB
+
+
+
+
+
+
+
+
+
+
+
+
+
+ rcuk:fundingStream
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/aka_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/aka_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/aka_context.xsl (revision 58540)
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
+
+ aka_________::AKA
+ AKA
+ Academy of Finland
+ Academy of Finland
+ FI
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/repoByeGeneric.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/repoByeGeneric.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/repoByeGeneric.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the GMF mdstore
+
+ $params.("orig_id")$
+
+
+
+
+
+
+
+ Delete the DBMF mdstore
+
+ $params.("row_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt-datacite/datacite-filter.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt-datacite/datacite-filter.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt-datacite/datacite-filter.xml (revision 58540)
@@ -0,0 +1,937 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/update.db.patch.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/update.db.patch.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/update.db.patch.wf.st (revision 58540)
@@ -0,0 +1,42 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+ Get more datasource info
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+ Generates the update command
+
+ hostedByUpdateSQL
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedBy/fillHostedBy.sql.st
+
+
+
+
+
+
+
+ Add new journal to patch entry tables
+
+ dnet.openaire.db.name
+ hostedByUpdateSQL
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/pangaea_preparetable.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/pangaea_preparetable.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/pangaea_preparetable.sql (revision 58540)
@@ -0,0 +1,7 @@
+DROP TABLE IF EXISTS pangaea_temp_journals;
+CREATE TABLE IF NOT EXISTS pangaea_temp_journals (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) DEFAULT 'temp_' || md5(clock_timestamp() :: TEXT) || '_' || md5(random() :: TEXT),
+ id CHARACTER VARYING(255) PRIMARY KEY,
+ issn CHARACTER VARYING(255),
+ oa_source_id CHARACTER VARYING(255) REFERENCES dsm_datasources (id)
+);
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/updateDb.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/updateDb.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/updateDb.wf.st (revision 58540)
@@ -0,0 +1,34 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("row_id")$
+ $params.("row_format")$
+ rs_epr
+
+
+
+
+
+
+
+
+ Update Db
+
+ rs_epr
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/update.db.patch.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/update.db.patch.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/update.db.patch.wf.st (revision 58540)
@@ -0,0 +1,78 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Obtain parameters for current provider
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Create necessary tables
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/pangaea_preparetable.sql
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Transform original records to Publication
+
+
+ collected_epr
+ row_epr
+
+
+
+
+
+
+
+ Update Db
+
+ row_epr
+
+
+
+
+
+
+
+ Add new journal to patch entry tables
+
+ dnet.openaire.db.name
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/dfg_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/dfg_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/dfg_context.xsl (revision 58540)
@@ -0,0 +1,42 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ DFG
+ Deutsche Forschungsgemeinschaft
+ DE
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ dfg:fundingStream
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/sgov_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/sgov_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/sgov_context.xsl (revision 58540)
@@ -0,0 +1,75 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ SGOV
+ Gobierno de España
+ ES
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/conicyt_context.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/conicyt_context.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/conicyt_context.xsl (revision 58540)
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ CONICYT
+ Comisión Nacional de Investigación Científica y Tecnológica
+ CL
+
+
+
+
+
+
+
+
+
+
+
+ conicyt:fondecytfundings
+
+
+
+
+
+ FONDECYT
+ Fondecyt fundings
+
+ conicyt:fondecytfundings
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/tubitak_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/tubitak_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/tubitak_contexts.xsl (revision 58540)
@@ -0,0 +1,41 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ TUBITAK
+ Türkiye Bilimsel ve Teknolojik Araştırma Kurumu
+ TR
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ tubitak:fundingStream
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/collection.wf.st (revision 58540)
@@ -0,0 +1,124 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Choose collection mode
+
+ REFRESH
+
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("orig_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Find Date Range For Incremental Harvesting
+
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+ Start incremental harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Store mdstore records incrementally
+
+ $params.("orig_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("orig_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/repoBye.wf.st (revision 58540)
@@ -0,0 +1,21 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/DatasetsFromPangaeaIngestion.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/DatasetsFromPangaeaIngestion.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/DatasetsFromPangaeaIngestion.xsl (revision 58540)
@@ -0,0 +1,50 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nsf_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nsf_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/nsf_contexts.xsl (revision 58540)
@@ -0,0 +1,88 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ NSF
+ National Science Foundation
+ US
+
+
+
+
+
+
+
+
+
+
+
+ NSF
+ National Science Foundation
+ US
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ nsf:fundingStream
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ nsf:fundingStream
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/corda_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/corda_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/corda_contexts.xsl (revision 58540)
@@ -0,0 +1,86 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ EC
+ European Commission
+ EU
+
+
+
+
+
+
+
+
+
+
+
+ ec:program
+
+
+
+
+
+
+
+
+
+
+
+ ec:specificprogram
+
+
+
+
+
+ SEVENTH FRAMEWORK PROGRAMME
+ FP7
+
+ ec:frameworkprogram
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/DatasetfromPangaeaTransform.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/DatasetfromPangaeaTransform.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/DatasetfromPangaeaTransform.xsl (revision 58540)
@@ -0,0 +1,207 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ PANGAEA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ info:eu-repo/semantics/openAccess
+
+
+ European Commission
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ false
+ false
+ 0.9
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/collection.wf.st (revision 58540)
@@ -0,0 +1,74 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+
+ collected_epr
+ store_epr
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/transformDatasets.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/transformDatasets.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/transformDatasets.wf.st (revision 58540)
@@ -0,0 +1,85 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_dataset_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records to Datasets
+
+
+ orig_dataset_epr
+ datasets_epr
+
+
+
+
+
+
+
+ Clean dataset records
+
+
+ datasets_epr
+ cleaned_datasets_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tranODF_id")$
+ REFRESH
+ cleaned_datasets_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tranODF_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/transform.wf.st (revision 58540)
@@ -0,0 +1,72 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records
+
+
+ orig_epr
+ trans_epr
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("tran_id")$
+ REFRESH
+ trans_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tran_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/downloadFilesFromPubsRepository/download.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/downloadFilesFromPubsRepository/download.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/downloadFilesFromPubsRepository/download.st (revision 58540)
@@ -0,0 +1,147 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Should download?
+
+ YES
+
+
+
+
+
+
+
+
+ Set download mode
+
+ INCREMENTAL
+
+
+
+
+
+
+
+
+ Drop content on the object Store
+
+ $params.("objs_id")$
+
+
+
+
+
+
+
+ Find MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ clean_epr
+
+
+
+
+
+
+
+ Retrieve URL To Download from MDStore
+
+ //*[local-name()='metadata']//*[local-name()='fulltext']/text()
+ //*[local-name()='metadata']//*[local-name()='accessrights']/text()
+ //*[local-name()='metadata']//*[local-name()='embargoenddate']/text()
+ clean_epr
+ item_epr
+
+
+
+
+
+
+
+
+ Download records
+
+ item_epr
+ $params.("objs_id")$
+
+
+
+
+
+ 1
+ 10000
+ 10000
+ 0
+
+
+
+
+
+
+
+ Download records
+
+ $params.("objs_id")$
+
+
+
+
+
+
+
+ Update datasource API extra fields
+
+ $params.("objs_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_download_total
+ last_download_date
+ last_download_objId
+
+
+
+
+
+
+
+ Should update IIS cache?
+
+ NO
+
+
+
+
+
+
+
+
+ Update IIS cache with latest objectStore content
+
+ iisCacheBuilderJob
+ $params.("objs_id")$
+ IIS
+ false
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/applicationContext-msro-openaire-message.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/applicationContext-msro-openaire-message.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/applicationContext-msro-openaire-message.xml (revision 58540)
@@ -0,0 +1,16 @@
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/applicationContext-claims.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/applicationContext-claims.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/applicationContext-claims.xml (revision 58540)
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/applicationContext-msro-openaire-nodes.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/applicationContext-msro-openaire-nodes.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/applicationContext-msro-openaire-nodes.xml (revision 58540)
@@ -0,0 +1,427 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listCollectionMdStores.xquery.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listCollectionMdStores.xquery.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listCollectionMdStores.xquery.st (revision 58540)
@@ -0,0 +1,9 @@
+for \$mdId in
+ distinct-values(
+ for \$x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')[.//DATAPROVIDER/@id='$dsId$' and .//DATAPROVIDER/@interface='$apiId$']//WORKFLOW
+ let \$f := concat('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType/', substring-before(\$x/@id/string(), '_'))
+ return doc(\$f)[.//PARAM[@name="extraFieldForTotal" and text() = 'last_collection_total']]//NODE[@type='StoreMDStoreRecords']//PARAM[@category="MDSTORE_ID"]/text()
+ )
+ let \$fmd := concat('/db/DRIVER/MDStoreDSResources/MDStoreDSResourceType/', substring-before(\$mdId, '_'))
+ return doc(\$fmd)
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/transformPublications.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/transformPublications.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/transformPublications.wf.st (revision 58540)
@@ -0,0 +1,93 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_publication_epr
+
+
+
+
+
+
+
+ Transform original records to Publication
+
+
+ orig_publication_epr
+ packed_publication_epr
+
+
+
+
+
+
+
+ Split Records into datasets and publication
+
+ packed_publication_epr
+ publication_epr
+
+
+
+
+
+
+
+ Patch oaf:hostedBy fields
+
+ publication_epr
+ patched_epr
+ hostedByCounters
+
+
+
+
+
+
+
+ Store records
+
+ $params.("tranOAF_id")$
+ REFRESH
+ patched_epr
+
+
+
+
+
+
+
+ Validate hostedBy Repos
+
+ hostedByCounters
+ $params.("tranOAF_id")$
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/decapsule_claims_xml.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/decapsule_claims_xml.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/decapsule_claims_xml.xsl (revision 58540)
@@ -0,0 +1,21 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/collection.wf.st (revision 58540)
@@ -0,0 +1,81 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Obtain data source params
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ user:claim
+ user:claim
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryClaimsDmf.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryClaimsDmf.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryClaimsDmf.sql (revision 58540)
@@ -0,0 +1,5 @@
+SELECT
+ regexp_replace(xml, '&(?!amp;)', '&', 'g') AS xml,
+ provenance
+FROM claims
+WHERE type = 'dmf2actions' AND set = 'userclaim_dmf' AND provenance <> 'user:claim:datacite'
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/applicationContext-nodeValues.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/applicationContext-nodeValues.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/applicationContext-nodeValues.xml (revision 58540)
@@ -0,0 +1,13 @@
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/downloadFilesFromPubsRepository/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/downloadFilesFromPubsRepository/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/downloadFilesFromPubsRepository/repoBye.wf.st (revision 58540)
@@ -0,0 +1,43 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the associated ObjectStore
+
+ $params.("objs_id")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_download_total, last_download_date, last_download_objId
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/vocEntry_2_db.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/vocEntry_2_db.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/vocEntry_2_db.xsl (revision 58540)
@@ -0,0 +1,50 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listTransformationMdStores.xquery.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listTransformationMdStores.xquery.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listTransformationMdStores.xquery.st (revision 58540)
@@ -0,0 +1,9 @@
+for \$mdId in
+ distinct-values(
+ for \$x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')[.//DATAPROVIDER/@id='$dsId$' and .//DATAPROVIDER/@interface='$apiId$']//WORKFLOW
+ let \$f := concat('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType/', substring-before(\$x/@id/string(), '_'))
+ return doc(\$f)[.//PARAM[@name="extraFieldForTotal" and text() = 'last_aggregation_total']]//NODE[@type='StoreMDStoreRecords']//PARAM[@category="MDSTORE_ID"]/text()
+ )
+ let \$fmd := concat('/db/DRIVER/MDStoreDSResources/MDStoreDSResourceType/', substring-before(\$mdId, '_'))
+ return doc(\$fmd)
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.xml (revision 58540)
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/croatian_mses_projects_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/croatian_mses_projects_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/croatian_mses_projects_contexts.xsl (revision 58540)
@@ -0,0 +1,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ MZOS
+ Ministry of Science, Education and Sports of the Republic of Croatia (MSES)
+ Ministarstvo znanosti, obrazovanja i športa
+ HR
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/applicationContext-dnet-openaire-claims.properties
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/applicationContext-dnet-openaire-claims.properties (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/applicationContext-dnet-openaire-claims.properties (revision 58540)
@@ -0,0 +1,4 @@
+dnet.openaire.claims.db.baseurl=jdbc:postgresql://localhost:5432
+dnet.openaire.claims.db.name=dnet_openaireplus
+dnet.openaire.claims.db.username=dnet
+dnet.openaire.claims.db.password=dnetPwd
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/arc_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/arc_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/arc_contexts.xsl (revision 58540)
@@ -0,0 +1,45 @@
+
+
+
+
+
+
+
+
+
+ arc_________::ARC
+ ARC
+ Australian Research Council (ARC)
+ AU
+
+
+
+
+
+ arc_________::ARC
+ ARC
+ Australian Research Council (ARC)
+ AU
+
+
+
+
+
+
+
+
+
+
+
+
+ arc:fundingStream
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/dedup.ocean.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/dedup.ocean.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/dedup.ocean.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+ Deduplication [OCEAN]
+
+ InfoSpace Deduplication
+ alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it,michele.debonis@isti.cnr.it
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.organizations.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.organizations.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.organizations.xml (revision 58540)
@@ -0,0 +1,167 @@
+
+
+
+
+ Dedup organizations (Online)
+ Deduplication
+ 30
+
+
+
+
+ dnet.openaire.model.relclasses.xquery
+ relClasses
+
+
+
+
+
+
+ Load Dedup conf
+
+ organization
+
+
+
+
+
+
+ Do we start from GROUND TRUTH?
+
+ NO
+
+
+
+
+
+
+
+ Dup Scan
+
+ dedupCandidateScanJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'dedup.pace.conf' : 'dedup.pace.conf',
+ 'dedup.wf.conf' : 'dedup.wf.conf'
+ }
+
+
+ {
+ 'entityTypeId' : '20',
+ 'entityType' : 'organization'
+ }
+
+
+
+
+
+
+
+ dedup grouper
+
+ dedupGrouperJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'dedup.pace.conf' : 'dedup.pace.conf',
+ 'dedup.wf.conf' : 'dedup.wf.conf'
+ }
+
+
+ {
+ 'entityTypeId' : '20'
+ }
+
+
+
+
+
+
+
+
+ find roots
+
+ dedupFindRootsJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'dedup.pace.conf' : 'dedup.pace.conf',
+ 'dedup.wf.conf' : 'dedup.wf.conf'
+ }
+
+
+ {
+ 'entityTypeId' : '20',
+ 'entityType' : 'organization'
+ }
+
+
+
+
+
+
+
+ redirect rels
+
+ dedupBuildRootsJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'dedup.pace.conf' : 'dedup.pace.conf',
+ 'dedup.wf.conf' : 'dedup.wf.conf',
+ 'relClasses' : 'relClasses'
+ }
+
+
+ {
+ 'entityTypeId' : '20'
+ }
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.results.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.results.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.results.xml (revision 58540)
@@ -0,0 +1,159 @@
+
+
+
+
+ Dedup results (Online)
+ Deduplication
+ 30
+
+
+
+
+ dnet.openaire.model.relclasses.xquery
+ relClasses
+
+
+
+
+
+
+ Load Dedup conf
+
+ result
+
+
+
+
+
+
+ Dedup M/R job
+
+ dedupCandidateScanJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'dedup.pace.conf' : 'dedup.pace.conf',
+ 'dedup.wf.conf' : 'dedup.wf.conf'
+ }
+
+
+ {
+ 'entityTypeId' : '50',
+ 'entityType' : 'result'
+ }
+
+
+
+
+
+
+
+ dedup grouper
+
+ dedupGrouperJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'dedup.pace.conf' : 'dedup.pace.conf',
+ 'dedup.wf.conf' : 'dedup.wf.conf'
+ }
+
+
+ {
+ 'entityTypeId' : '50'
+ }
+
+
+
+
+
+
+
+
+
+ find roots
+
+ dedupFindRootsJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'dedup.pace.conf' : 'dedup.pace.conf',
+ 'dedup.wf.conf' : 'dedup.wf.conf'
+ }
+
+
+ {
+ 'entityTypeId' : '50',
+ 'entityType' : 'result'
+ }
+
+
+
+
+
+
+
+ redirect rels
+
+ dedupBuildRootsJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'dedup.pace.conf' : 'dedup.pace.conf',
+ 'dedup.wf.conf' : 'dedup.wf.conf',
+ 'relClasses' : 'relClasses'
+ }
+
+
+ {
+ 'entityTypeId' : '50'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/snsf_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/snsf_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/snsf_contexts.xsl (revision 58540)
@@ -0,0 +1,93 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ SNSF
+ Swiss National Science Foundation
+ CH
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ snsf:fundingStream
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ snsf:fundingStream
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ snsf:fundingStream
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryACM.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryACM.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryACM.sql (revision 58540)
@@ -0,0 +1,5 @@
+SELECT
+ regexp_replace(xml, '&(?!amp;)', '&', 'g') AS xml,
+ provenance
+FROM claims
+WHERE type = 'dmf2actions' AND set = 'acm-referenced-projects'
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.properties
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.properties (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.properties (revision 58540)
@@ -0,0 +1,100 @@
+dnet.openaire.db.name=dnet_openaireplus
+hbase.mapred.datatable=db_openaireplus_test
+#services.data.hadoop.hdfs.mdrecords.tmp.directory = hdfs://nmis-hadoop-cluster/tmp
+dnet.openaire.entitygruper.xquery=/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'EntityGrouperConfigurationDSResourceType']//CONFIGURATION/GROUP/text()
+dnet.openaire.context.xquery=for $x in //RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ContextDSResourceType']//*[name()='context' or name()='category' or name()='concept'] return
+dnet.openaire.context.profiles.xquery=for $x in //RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ContextDSResourceType' and .//RESOURCE_KIND/@value='ContextDSResources'] return $x
+dnet.actionmanager.garbage.sets.xupdate=for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'ActionManagerSetDSResourceType'] let $latest:=$x//LATEST return update replace $x//RAW_SETS with {$latest}
+dnet.openaire.iis.objecstores.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'ObjectStoreDSResourceType'] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.iis.objecstore.endpoint.xquery=/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'ObjectStoreServiceResourceType']/HEADER/PROTOCOLS/PROTOCOL[./@name = "SOAP"]/@address/string()
+dnet.openaire.iis.islookup.endpoint.xquery=/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'IS_LookUpServiceResourceType']/HEADER/PROTOCOLS/PROTOCOL[./@name = "SOAP"]/@address/string()
+dnet.openaire.iis.mdstore.endpoint.xquery=/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'MDStoreServiceResourceType']/HEADER/PROTOCOLS/PROTOCOL[./@name = "SOAP"]/@address/string()
+dnet.openaire.iis.dbservice.endpoint.xquery=/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'DatabaseServiceResourceType']/HEADER/PROTOCOLS/PROTOCOL[./@name = "SOAP"]/@address/string()
+dnet.openaire.iis.mdstore.wos.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'MDStoreDSResourceType' and .//METADATA_FORMAT = 'OAF' and .//METADATA_FORMAT_LAYOUT = 'store' and .//METADATA_FORMAT_INTERPRETATION = 'cleaned_noingestion'] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.iis.mdstore.dataset.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'MDStoreDSResourceType' and .//METADATA_FORMAT = 'ODF' and .//METADATA_FORMAT_LAYOUT = 'store' and (.//METADATA_FORMAT_INTERPRETATION = 'cleaned_noingestion' or .//METADATA_FORMAT_INTERPRETATION = 'cleaned')] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.iis.objecstores.dataset.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'ObjectStoreDSResourceType' and .//OBJECTSTORE_INTERPRETATION='pubrepository record'] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.iis.objecstores.wos.xquery=for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value = 'ObjectStoreDSResourceType' and .//OBJECTSTORE_INTERPRETATION='wos record'] return string($x/HEADER/RESOURCE_IDENTIFIER/@value)
+dnet.openaire.model.relclasses.xquery=distinct-values(for $x in /RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value='OntologyDSResourceType']//TERM return concat($x/@code/string(), ":", $x//RELATION[./@type = 'inverseOf']/@code/string()))
+dnet.openaire.service.search.lookup.xquery=/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='SearchServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources' and .//SERVICE_PROPERTIES/PROPERTY[./@key="infrastructure"]/@value="public"]/HEADER/RESOURCE_IDENTIFIER/@value/string()
+dnet.openaire.service.stats.lookup.xquery=/RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+dnet.openaire.similaritygroups.index.format=DEDUP
+dnet.openaire.similaritygroups.indexrecord2html.xsl=classpath:/eu/dnetlib/functionality/modular/ui/xslt/indexRecordsToHtml.xsl
+dnet.openaire.similaritygroups.indexrecord2htmldetails.xsl=classpath:/eu/dnetlib/functionality/modular/ui/xslt/indexRecordsToHtmlDetails.xsl
+oaf.schema.location=http://www.openaire.eu/schema/0.2/oaf-0.2.xsd
+dnet.openaire.lod.jsonrels={ "result": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/openAireId", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/openAireId", "5": "http://www.eurocris.org/ontologies/cerif/1.3#name", "6": "http://purl.org/dc/terms/dateAccepted", "7": "http://purl.org/dc/terms/publisher", "8": "http://purl.org/dc/terms/openAireId", "9": "http://purl.org/dc/terms/language", "10": "http://purl.org/dc/terms/date", "11": "http://lod.openaire.eu/vocab/resultSubject", "12": "http://lod.openaire.eu/vocab/externalReference", "13": "http://purl.org/dc/terms/source", "14": "http://purl.org/dc/terms/format", "15": "http://lod.openaire.eu/vocab/context", "16": "http://dbpedia.org/ontology/country", "17": "http://purl.org/dc/terms/accessRights", "18": "http://purl.org/dc/terms/description", "19": "http://lsdis.cs.uga.edu/projects/semdis/opus#journal_name", "20": "http://lod.openaire.eu/vocab/dataSourceType", "21": "http://lod.openaire.eu/vocab/device", "22": "http://lod.openaire.eu/vocab/size", "23": "http://lod.openaire.eu/vocab/version", "24": "http://lod.openaire.eu/vocab/lastMetadataUpdate", "25": "http://lod.openaire.eu/vocab/metadataVersion", "26": "http://lod.openaire.eu/vocab/year", "27": "http://lod.openaire.eu/vocab/resultType", "28": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://www.eurocris.org/ontologies/cerif/1.3#ResultEntity" }], "person": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/openAireId", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/openAireId", "5": "http://xmlns.com/foaf/0.1/firstName", "6": "http://xmlns.com/foaf/spec/lastName", "7": "http://xmlns.com/foaf/0.1/name", "8": "http://schema.org/faxNumber", "9": "http://xmlns.com/foaf/0.1/mbox", "10": "http://xmlns.com/foaf/0.1/phone", "11": "http://schema.org/nationality", "12": "http://purl.org/dc/terms/openAireId", "13": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http: //xmlns.com/foaf/0.1/Person" }], "datasource": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/openAireId", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/openAireId", "5": "http://lod.openaire.eu/vocab/datasourceType", "6": "http://lod.openaire.eu/vocab/openAIRECompatibility", "7": "http://dbpedia.org/ontology/officialName", "8": "http://lod.openaire.eu/vocab/englishName", "9": "http://schema.org/url", "10": "http://xmlns.com/foaf/0.1/logo", "11": "http://xmlns.com/foaf/0.1/mbox", "12": "http://purl.org/vocab/vann/preferredNamespacePrefix", "13": "http://www.w3.org/2003/01/geo/wgs84_pos#lat", "14": "http://www.w3.org/2003/01/geo/wgs84_pos#long", "15": "http://lod.openaire.eu/vocab/dateOfValidity", "16": "http://purl.org/dc/terms/description", "17": "http://lod.openaire.eu/vocab/subjectList", "18": "http://lod.openaire.eu/numberOfItems", "19": "http://purl.org/dc/terms/date", "20": "http://lod.openaire.eu/vocab/policies", "21": "http://lod.openaire.eu/vocab/languages", "22": "http://lod.openaire.eu/vocab/contentType", "23": "http://lod.openaire.eu/vocab/accessInfoPackage", "24": "http://lod.openaire.eu/vocab/releaseStartDate", "25": "http://lod.openaire.eu/vocab/releaseEndDate", "26": "http://lod.openaire.eu/vocab/missionStatementUrl", "27": "http://www.europeana.eu/schemas/edm/dataProvider", "28": "http://lod.openaire.eu/vocab/serviceProvider", "29": "http://lod.openaire.eu/vocab/databaseAccessType", "30": "http://lod.openaire.eu/vocab/dataUploadType", "31": "http://lod.openaire.eu/vocab/dataUploadRestrictions", "32": "http://lod.openaire.eu/vocab/versioning", "33": "http://lod.openaire.eu/vocab/citationGuidelineUrl", "34": "http://lod.openaire.eu/vocab/qualityManagementKind", "35": "http://lod.openaire.eu/vocab/pidSystems", "36": "http://lod.openaire.eu/vocab/certificates", "37": "http://purl.org/dc/terms/accessRights", "38": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http: //www.w3.org/ns/prov#Entity" }], "organization": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/openAireId", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/openAireId", "5": "http://www.w3.org/2004/02/skos/core#altLabel", "6": "http://www.w3.org/2004/02/skos/core#prefLabel", "7": "http://lod.openaire.eu/vocab/webSiteUrl", "8": "http://xmlns.com/foaf/0.1/logo", "9": "http://dbpedia.org/ontology/country", "10": "http://lod.openaire.eu/vocab/entityType", "11": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http: //xmlns.com/foaf/0.1/Organization" }], "project": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/openAireId", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/openAireId", "5": "http://lod.openaire.eu/vocab/projectCode", "6": "http://schema.org/url", "7": "http://www.eurocris.org/ontologies/cerif/1.3#acronym", "8": "http://www.eurocris.org/ontologies/cerif/1.3#name", "9": "http://www.eurocris.org/ontologies/cerif/1.3#startDate", "10": "http://www.eurocris.org/ontologies/cerif/1.3#endDate", "11": "http://purl.org/cerif/frapo/hasCallIdentifier", "12": "http://www.eurocris.org/ontologies/cerif/1.3#keyword", "13": "http://www.w3.org/2006/time#hasDurationDescription", "14": "http://lod.openaire.eu/vocab/ec_SC39", "15": "http://lod.openaire.eu/vocab/contractType", "16": "http://lod.openaire.eu/vocab/oaMandatePublications", "17": "http://lod.openaire.eu/vocab/projectSubjects", "18": "http://od.openaire.eu/vocab/ec_article29-3", "19": "http://lod.openaire.eu/vocab/funder", "20": "http://lod.openaire.eu/vocab/fundingLevel0", "21": "http://lod.openaire.eu/vocab/fundingLevel1", "22": "http://lod.openaire.eu/vocab/fundingLevel2", "23": "http://lod.openaire.eu/vocab/fundingLevel3", "24": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://www.eurocris.org/ontologies/cerif/1.3#Project" }] }
+dnet.openaire.lod.wfId=42c42eeb-8cf4-4410-bdf5-a20db879d23b_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl
+dnet.openaire.lod.default.date=01/01/1970
+dnet.openaire.bulktag.xquery = for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') \
+ let $subj := $x//CONFIGURATION/context/param[./@name='subject']/text() \
+ let $datasources := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::contentproviders')]/concept \
+ let $organizations := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::resultorganizations')]/concept \
+ let $communities := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::zenodocommunities')]/concept \
+ where $x//CONFIGURATION/context[./@type='community' or ./@type='ri'] \
+ return \
+ \
+ { $x//CONFIGURATION/context/@id} \
+ \
+ {for $y in tokenize($subj,',') \
+ return \
+ {$y} } \
+ \
+ \
+ {for $d in $datasources \
+ where $d/param[./@name='enabled']/text()='true' \
+ return \
+ \
+ \
+ {$d//param[./@name='openaireId']/text()} \
+ \
+ \
+ {$d/param[./@name='selcriteria']/text()} \
+ \
+ }\
+ \
+ \
+ {for $zc in $communities \
+ return \
+ \
+ \
+ {$zc/param[./@name='zenodoid']/text()}\
+ \
+ \
+ {$zc/param[./@name='selcriteria']/text()}\
+ \
+ }\
+ \
+ \
+ {for $org in $organizations \
+ return\
+ \
+ \
+ {$org/param[./@name='organizationId']/text()}\
+ \
+ \
+ {$org/param[./@name='selcriteria']/text()}\
+ \
+ }\
+ \
+
+
+dnet.openaire.index.api.cacheEvictUrl = http://localhost:8980/provision/api/admin/evictCache
+
+dhp.mdstore.manager.url=http://localhost:8080/mdstores
+
+dnet.openaire.messageManager.host=localhost
+dnet.openaire.messageManager.username=
+dnet.openaire.messageManager.password=
+
+dnet.openaire.bulktag.mapping = { "author" : "$['entity']['result']['metadata']['author'][*]['fullname']",\
+ "title" : "$['entity']['result']['metadata']['title'][*]['value']",\
+ "orcid" : "$['entity']['result']['metadata']['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']",\
+ "contributor" : "$['entity']['result']['metadata']['contributor'][*]['value']",\
+ "description" : "$['entity']['result']['metadata']['description'][*]['value']"}
+
+dnet.openaire.propagation.xquery = for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')\
+ where $x//CONFIGURATION/context[./@type='community' or ./@type='ri']\
+ and $x//CONFIGURATION/context/param[./@name='status']/text() != 'hidden'\
+ return $x//CONFIGURATION/context/@id/string()
+
+ dnet.openorgs.db.name=dev_orgs
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.ocean.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.ocean.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.ocean.xml (revision 58540)
@@ -0,0 +1,167 @@
+
+
+
+ Data Provision [OCEAN]
+ Data Provision
+ 30
+
+
+
+
+ Set the target directory to materialize the graph
+
+ graphBasePath
+ /var/lib/dnet/graph_BETA
+
+
+
+
+
+
+
+ create the raw graph
+
+ executeOozieJob
+ IIS
+
+ {
+ 'graphBasePath' : 'graphBasePath'
+ }
+
+
+ {
+ 'oozie.wf.application.path' : '/lib/dnet/migration/wfs/regular_all_steps/oozie_app',
+ 'mongoURL' : 'mongodb://beta.services.openaire.eu',
+ 'mongoDb' : 'mdstore',
+ 'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus',
+ 'postgresUser' : 'dnet',
+ 'postgresPassword' : 'dnetPwd',
+ 'workingPath' : '/tmp/dhp_migration_beta',
+ 'reuseContent' : 'true'
+ }
+
+ build-report
+
+
+
+
+
+
+
+ Set the path containing the raw graph
+
+ rawGraphPath
+ /var/lib/dnet/graph_BETA/graph_raw
+
+
+
+
+
+
+
+ Set the path used by dedup workflows to store intermediate data
+
+ dedupWorkingPath
+ /tmp/beta_dedup
+
+
+
+
+
+
+
+ Set the target path to store the deduped graph
+
+ dedupGraphPath
+ /var/lib/dnet/graph_BETA/graph_dedup
+
+
+
+
+
+
+
+ deduplicate organizations
+
+ executeOozieJob
+ IIS
+
+ {
+ 'graphBasePath' : 'rawGraphPath',
+ 'workingPath' : 'dedupWorkingPath',
+ 'dedupGraphPath': 'dedupGraphPath'
+ }
+
+
+ {
+ 'oozie.wf.application.path' : '/lib/dnet/dedup/scan/oozie_app',
+ 'actionSetId' : 'dedup-similarity-organization'
+ }
+
+ build-report
+
+
+
+
+
+
+
+ deduplicate results
+
+ executeOozieJob
+ IIS
+
+ {
+ 'graphBasePath' : 'rawGraphPath',
+ 'workingPath' : 'dedupWorkingPath',
+ 'dedupGraphPath': 'dedupGraphPath'
+ }
+
+
+ {
+ 'oozie.wf.application.path' : '/lib/dnet/dedup/scan/oozie_app',
+ 'actionSetId' : 'dedup-similarity-result'
+ }
+
+ build-report
+
+
+
+
+
+
+
+ mark duplicates as deleted and redistribute the relationships
+
+ executeOozieJob
+ IIS
+
+ {
+ 'graphBasePath' : 'rawGraphPath',
+ 'workingPath' : 'dedupWorkingPath',
+ 'dedupGraphPath': 'dedupGraphPath'
+ }
+
+
+ {
+ 'oozie.wf.application.path' : '/lib/dnet/dedup/consistency/oozie_app'
+ }
+
+ build-report
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.persons.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.persons.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/online.dedup.persons.xml (revision 58540)
@@ -0,0 +1,102 @@
+
+
+
+ Dedup persons (Online)
+ Deduplication
+ 30
+
+
+
+
+ dnet.openaire.model.relclasses.xquery
+ relClasses
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+ Set Dedup conf
+
+ dedup-similarity-person
+ dedup.conf.queue
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Dedup M/R job
+
+ dedupPersonJob
+ dedup.conf.queue
+
+ {
+ 'entityTypeId' : 'entityTypeId',
+ 'entityType' : 'entityType',
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName'
+ }
+
+
+
+
+
+
+
+ redirect rels
+
+ dedupBuildRootsJob
+
+ {
+ 'dedup.conf' : 'dedup.conf',
+ 'relClasses' : 'relClasses',
+ 'entityTypeId' : 'entityTypeId',
+ 'entityType' : 'entityType',
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName'
+ }
+
+
+
+
+
+
+
+
+ wf_20151020_211958_695
+ 2015-10-20T22:32:39+00:00
+ SUCCESS
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/reset.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/reset.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/reset.xml (revision 58540)
@@ -0,0 +1,36 @@
+
+
+
+
+ Reset Dedup
+ Deduplication
+ 30
+
+
+ Run dedup reset Job
+
+ resetDedupJob
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.retry.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.retry.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.retry.meta.xml (revision 58540)
@@ -0,0 +1,25 @@
+
+
+
+ OpenAIRE Provision Retry
+
+ InfoSpace Provision
+ alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it,gkatsari@di.uoa.gr
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-prepublic-trigger.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-prepublic-trigger.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-prepublic-trigger.meta.xml (revision 58540)
@@ -0,0 +1,24 @@
+
+
+
+
+ Data Flow Monitoring
+
+ InfoSpace Monitoring
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/sfi_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/sfi_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/sfi_contexts.xsl (revision 58540)
@@ -0,0 +1,41 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ SFI
+
+
+
+ IE
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ sfi:fundingStream
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hbase.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hbase.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hbase.xml (revision 58540)
@@ -0,0 +1,166 @@
+
+
+
+
+ Index from HBASE
+ Provision
+ 30
+
+
+
+
+ dnet.openaire.model.relclasses.xquery
+ relClasses
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ DMF
+ index
+ openaire
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Prepare indexing
+
+ outputRecordsPath
+ rottenRecordsPath
+ /eu/dnetlib/msro/openaireplus/workflows/index/openaireLayoutToRecordStylesheet.xsl
+ oaf.schema.location
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'outputRecordsPath'
+ }
+
+
+
+
+
+
+
+ hdfs cleanup (rotten)
+
+ DM
+
+ {
+ 'path' : 'rottenRecordsPath'
+ }
+
+
+
+
+
+
+
+ M/R group entities
+
+ DM
+ prepareIndexDataJob
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'mapred.output.dir' : 'outputRecordsPath',
+ 'index.entity.links' : 'index.entity.links',
+ 'oaf.schema.location' : 'oaf.schema.location',
+ 'contextmap' : 'contextmap',
+ 'relClasses' : 'relClasses'
+ }
+
+
+
+
+
+
+
+ M/O index records
+
+ DM
+ indexFeedJob
+
+ {
+ 'mapred.input.dir' : 'outputRecordsPath',
+ 'mapred.output.dir' : 'rottenRecordsPath',
+ 'index.xslt' : 'index.xslt',
+ 'index.solr.url' : 'index.solr.url',
+ 'index.solr.collection' : 'index.solr.collection',
+ 'index.buffer.flush.threshold' : 'index.buffer.flush.threshold',
+ 'index.shutdown.wait.time' : 'index.shutdown.wait.time',
+ 'index.solr.sim.mode' : 'index.solr.sim.mode',
+ 'index.feed.timestamp' : 'index.feed.timestamp'
+ }
+
+
+
+
+
+
+
+ commit changes
+
+
+
+
+
+
+ update DS
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/croatian_csf_projects_contexts.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/croatian_csf_projects_contexts.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xsl/croatian_csf_projects_contexts.xsl (revision 58540)
@@ -0,0 +1,29 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ HRZZ
+ Croatian Science Foundation (CSF)
+ Hrvatska zaklada za znanost
+ HR
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.backup.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.backup.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.backup.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Stats Cache Backup
+
+ InfoSpace Provision
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/similarity2hbase.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/similarity2hbase.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/similarity2hbase.xml (revision 58540)
@@ -0,0 +1,64 @@
+
+
+
+ Similarity to HBase
+ Deduplication
+ 30
+
+
+ start
+
+
+
+
+
+
+ query similarity
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/dedup/querySimilarities.sql
+ similaritiesEpr
+
+
+
+
+
+
+ build mesh
+
+ similaritiesEpr
+ meshEpr
+
+
+
+
+
+
+ store mesh
+
+ meshEpr
+ hbase.mapred.datatable
+ DM
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/similarity_2_hbase.xsl
+ false
+
+
+
+
+
+
+
+ wf_20140509_125739_830
+ 2014-05-09T13:03:16+02:00
+ SUCCESS
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xml/contextProfile.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xml/contextProfile.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xml/contextProfile.xml (revision 58540)
@@ -0,0 +1,16 @@
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-prepublic-trigger.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-prepublic-trigger.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-prepublic-trigger.xml (revision 58540)
@@ -0,0 +1,29 @@
+
+
+
+
+ Trigger Prepublic Monitoring Scenario
+ Data Provision
+ 30
+
+
+ Trigger
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/oai.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/oai.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/oai.meta.xml (revision 58540)
@@ -0,0 +1,25 @@
+
+
+
+
+ OAI feeding from HDFS
+
+ InfoSpace Provision
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.multiple.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.multiple.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.multiple.xml (revision 58540)
@@ -0,0 +1,61 @@
+
+
+
+
+ Multiple Index switch
+ Provision
+ 30
+
+
+
+
+ DMF
+ index
+ openaire
+
+
+
+
+
+
+
+ find search services
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='SearchServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+
+
+
+
+
+
+
+ switch index
+
+
+
+
+
+
+
+
+ switch index
+
+ index_id
+ IndexId
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/lod.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/lod.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/lod.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ LOD update
+
+ InfoSpace Provision
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/applicationContext-repohi.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/applicationContext-repohi.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/applicationContext-repohi.xml (revision 58540)
@@ -0,0 +1,426 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/fixRelations.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/fixRelations.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/fixRelations.xml (revision 58540)
@@ -0,0 +1,115 @@
+
+
+
+
+ Fix relationships
+ Data Provision
+ 30
+
+
+
+
+
+
+ ontologies
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ set sub-entity type for organizations
+
+ type
+ organization
+
+
+
+
+
+
+
+
+ fix the relationships among deduped and non-deduped objects
+
+ dedupFixRelationsJob
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'ontologies' : 'ontologies',
+ 'type' : 'type'
+ }
+
+
+
+
+
+
+
+
+
+ set sub-entity type for organizations
+
+ type
+ result
+
+
+
+
+
+
+
+
+ fix the relationships among deduped and non-deduped objects
+
+ dedupFixRelationsJob
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'ontologies' : 'ontologies',
+ 'type' : 'type'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/oai.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/oai.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/oai.xml (revision 58540)
@@ -0,0 +1,86 @@
+
+
+
+
+ OAI Store feed
+ Provision
+ 30
+
+
+
+
+ /tmp/indexrecords_db_openaireplus_node6_t_DMF.seq
+ inputRecordsPath
+
+
+
+
+
+
+ Prepare oai feeding
+
+ oaistore
+ oaf
+ index
+ openaire
+
+ true
+ //*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']
+
+
+
+
+
+
+ Drop the store: upserts are too expensive!
+
+
+
+
+
+
+ Create the OAI store
+
+
+
+
+
+
+
+ M/O oai feeding
+
+ DM
+ oaiFeedJob
+
+ {
+ 'mapred.input.dir' : 'inputRecordsPath',
+ 'services.publisher.oai.collection' : 'oai_collection',
+ 'oaiConfiguration' : 'oai_configuration',
+ 'oai.feed.date' : 'oai_actionDate',
+ 'services.publisher.oai.db' : 'oai_dbName',
+ 'services.publisher.oai.skipDuplicates':'oai_skipDuplicates',
+ 'services.publisher.oai.duplicateXPath':'oai_duplicateXPath'
+ }
+
+
+ {
+ 'services.publisher.oai.host' : 'services.publisher.oai.host',
+ 'services.publisher.oai.port' : 'services.publisher.oai.port'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/collection.wf.st (revision 58540)
@@ -0,0 +1,81 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Obtain parameters for current provider
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("harv_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.restore.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.restore.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.restore.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Stats Cache Restore
+
+ InfoSpace Provision
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.validate.shadow.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.validate.shadow.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.validate.shadow.xml (revision 58540)
@@ -0,0 +1,40 @@
+
+
+
+
+ Shadow StatsDB Validation
+ Data Provision
+ 30
+
+
+ find stats service
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+ xqueryForStatsManagerService
+
+
+
+
+
+
+ generates validation report stats/index
+
+ xqueryForStatsManagerService
+ test
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hbase.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hbase.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hbase.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Index (from HBASE)
+
+ InfoSpace Provision
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hdfs.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hdfs.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hdfs.xml (revision 58540)
@@ -0,0 +1,111 @@
+
+
+
+
+ Index from HDFS
+ Provision
+ 30
+
+
+
+
+ /tmp/indexrecords_db_openaireplus_node6_t_DMF.seq
+ inputRecordsPath
+
+
+
+
+
+
+
+
+ DMF
+ index
+ openaire
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Prepare indexing
+
+ rottenRecordsPath
+ /eu/dnetlib/msro/openaireplus/workflows/index/openaireLayoutToRecordStylesheet.xsl
+ oaf.schema.location
+
+
+
+
+
+
+ hdfs cleanup (rotten)
+
+ DM
+
+ {
+ 'path' : 'rottenRecordsPath'
+ }
+
+
+
+
+
+
+
+ M/O index records
+
+ DM
+ indexFeedJob
+
+ {
+ 'mapred.input.dir' : 'inputRecordsPath',
+ 'mapred.output.dir' : 'rottenRecordsPath',
+ 'index.xslt' : 'index.xslt',
+ 'index.solr.url' : 'index.solr.url',
+ 'index.solr.collection' : 'index.solr.collection',
+ 'index.buffer.flush.threshold' : 'index.buffer.flush.threshold',
+ 'index.shutdown.wait.time' : 'index.shutdown.wait.time',
+ 'index.solr.sim.mode' : 'index.solr.sim.mode',
+ 'index.feed.timestamp' : 'index.feed.timestamp'
+ }
+
+
+
+
+
+
+
+ commit changes
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/lod.update.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/lod.update.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/lod.update.xml (revision 58540)
@@ -0,0 +1,65 @@
+
+
+
+
+ Update LOD
+ Provision
+ 30
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Update stats DB
+
+ DM
+ lodExportJob
+
+ {
+ 'lod_hbase_table' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'nameNode' : 'nameNode',
+ 'jobTracker' : 'jobTracker',
+ 'isLookupEndpoint' : 'isLookupEndpoint',
+ 'lod_indexConf' : 'index.entity.links',
+ 'lod_jsonRels' : 'lod_jsonRels',
+ 'lod_lastExecutionDate' : 'lod_lastExecutionDate'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-index-trigger.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-index-trigger.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/dataFlowMonitoring-index-trigger.xml (revision 58540)
@@ -0,0 +1,29 @@
+
+
+
+
+ Trigger Index Monitoring Scenario
+ Data Provision
+ 30
+
+
+ Trigger
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/dataRepository_claim_ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/dataRepository_claim_ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/dataRepository_claim_ingestion.xml (revision 58540)
@@ -0,0 +1,93 @@
+
+
+
+
+ Collect and transform claimed dataset metadata records
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ scholarcomminfra,researchgraph
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Collect and transform claimed dataset metadata records
+
+
+
+
+
+
+
+
+ Create native store
+
+ claim
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create cleaned store
+
+ ODF
+ claim
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfClaimedMdRecords
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/wos/transform.wf.st (revision 58540)
@@ -0,0 +1,73 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records
+
+
+ orig_epr
+ dmf_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tran_id")$
+ REFRESH
+ dmf_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tran_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/collection.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/collection.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/pubsRepository/collection.wf.st (revision 58540)
@@ -0,0 +1,148 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Obtain data source params
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Choose collection mode
+
+ REFRESH
+
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdstore records
+
+ $params.("harv_id")$
+ REFRESH
+ store_epr
+
+
+
+
+
+
+
+ Find Date Range For Incremental Harvesting
+
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+ Start incremental harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+ FROM_DATE
+ UNTIL_DATE
+
+
+
+
+
+
+
+
+
+ Prepare mdstore records
+
+ collected_epr
+ store_epr
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ false
+ false
+
+ 0.9
+ sysimport:crosswalk
+ sysimport:crosswalk
+
+
+
+
+
+
+
+ Store mdstore records incrementally
+
+ $params.("harv_id")$
+ INCREMENTAL
+ store_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("harv_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total
+ last_collection_date
+ last_collection_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_PubsRepository_Hadoop.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_PubsRepository_Hadoop.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_PubsRepository_Hadoop.xml (revision 58540)
@@ -0,0 +1,93 @@
+
+
+
+ Aggregate Metadata (publications) from PubsRepository [Hadoop]
+
+ Aggregate Metadata
+ Support
+ PubsRepository
+ publications
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate Metadata (publications) from PubsRepository [Hadoop]
+
+
+
+
+
+
+
+
+ Create oai_dc store
+
+ oai_dc
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create OAF store
+
+ OAF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfPubsRepositoryHadoop
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/transform.wf.st (revision 58540)
@@ -0,0 +1,110 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Decide REFRESH/INCREMENTAL transformation
+
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records
+
+
+ orig_epr
+ tran_epr
+
+
+
+
+
+
+
+ Patch oaf:hostedBy fields
+
+ tran_epr
+ patched_epr
+
+ hostedByCounters
+ hostedby_map
+ //*[local-name()='journal']/@issn
+
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tran_id")$
+ REFRESH
+ patched_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tran_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
+
+
+ Validate hostedBy Repos
+
+ hostedByCounters
+ $params.("tran_id")$
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Aggregator_dc_Ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Aggregator_dc_Ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Aggregator_dc_Ingestion.xml (revision 58540)
@@ -0,0 +1,100 @@
+
+
+
+
+ Aggregate and Validate Metadata in Dublin Core from Aggregator [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ Aggregator
+ results
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ aggregator,pubscatalogue,infospace
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate and Validate Metadata in Dublin Core from Aggregator [Ingestion]
+
+
+
+
+
+
+
+
+ Create oai_dc store
+
+ oai_dc
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create OAF store
+
+ OAF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfAggregatorPubsRepositoryIngestion
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_PubsRepository_Inference.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_PubsRepository_Inference.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_PubsRepository_Inference.xml (revision 58540)
@@ -0,0 +1,114 @@
+
+
+
+
+ [DEPRECATED] Aggregate Metadata (publications) from PubsRepository [Inference]
+
+ Aggregate Metadata
+ Inference
+ PubsRepository
+ publications
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ websource
+
+ native
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate
+ [DEPRECATED]Metadata (publications) from PubsRepository [Inference]
+
+
+
+
+
+
+
+
+
+ Create store for native records
+
+ GMF
+ native
+
+ store
+ harv_
+
+
+
+
+
+
+
+ Create store for cleaned records
+
+ OAF
+ cleaned_noingestion
+
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfPubsRepository
+
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the native mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the cleaned mdstore
+
+ $params.("tran_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.backup.cache.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.backup.cache.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.backup.cache.xml (revision 58540)
@@ -0,0 +1,40 @@
+
+
+
+
+ Stats Cache Backup
+ Provision
+ 30
+
+
+ find stats service
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+ xqueryForStatsManagerService
+
+
+
+
+
+
+ Backup the current stats cache. If the new generated shadow cache is not good we can rollback with the restore wf.
+
+ xqueryForStatsManagerService
+ test
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/pangaea_journal_findmatches.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/pangaea_journal_findmatches.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/pangaea_journal_findmatches.sql (revision 58540)
@@ -0,0 +1,15 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+ SELECT
+ p.issn || '__' || p.id || '@@' || p.oa_source_id,
+ p.oa_source_id,
+ p.issn || '__' || p.id,
+ d.id
+ FROM datasources d, pangaea_temp_journals p
+ WHERE
+ d.collectedfrom = 'driver______::1790119e-d281-4b7a-aedf-866d1d853a07' AND d.issn IS NOT NULL AND d.id LIKE 'doajarticles::%' AND p.issn = d.issn
+ AND
+ (p.oa_source_id, p.issn || '__' || p.id) NOT IN
+ (SELECT
+ oa_source_id,
+ entry
+ FROM hostedby_map)
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.generate.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.generate.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.generate.meta.xml (revision 58540)
@@ -0,0 +1,25 @@
+
+
+
+
+ Generate Stats
+
+ InfoSpace Provision
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorDataRepository_Ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorDataRepository_Ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorDataRepository_Ingestion.xml (revision 58540)
@@ -0,0 +1,99 @@
+
+
+
+
+ [DEPRECATED] Aggregate Metadata (datasets) from Aggregator::DataRepository [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ aggregator::datarepository,crissystem
+ datasets
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ aggregator::datarepository,crissystem
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ [DEPRECATED] Aggregate Metadata (datasets) from Aggregator::DataRepository [Ingestion]
+
+
+
+
+
+
+
+
+ Create oai_datacite native store
+
+ oai_datacite
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create ODF_datacite cleaned store
+
+ ODF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfAggregatorDataRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_PubsRepository_Ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_PubsRepository_Ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_PubsRepository_Ingestion.xml (revision 58540)
@@ -0,0 +1,102 @@
+
+
+
+
+ [DEPRECATED] Aggregate and Validate Metadata (publications) from PubsRepository [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ PubsRepository
+ publications
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ pubsrepository
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ [DEPRECATED] Aggregate and Validate Metadata (publications) from PubsRepository [Ingestion]
+
+
+
+
+
+
+
+
+
+
+ Create oai_dc store
+
+ oai_dc
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create OAF store
+
+ OAF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfPubsRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_OAF_Intersection.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_OAF_Intersection.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_OAF_Intersection.xml (revision 58540)
@@ -0,0 +1,100 @@
+
+
+
+
+ Aggregate Metadata (OAF results) [Intersection]
+
+ Aggregate Metadata
+ Intersection
+
+ results
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate Metadata (OAF results) [Intersection]
+
+
+
+
+
+
+
+
+ Create oai_dc store
+
+ oai_dc
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create OAF store
+
+ OAF
+ intersection
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfPubsRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Repository_datacite.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Repository_datacite.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Repository_datacite.xml (revision 58540)
@@ -0,0 +1,98 @@
+
+
+
+
+ Aggregate and Validate Metadata in Datacite format from Repository [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ Repository
+ results
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ pubsrepository,datarepository,softwarerepository,orprepository
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate and Validate Metadata in Datacite format from Repository [Ingestion]
+
+
+
+
+
+
+
+
+ Create oai_datacite native store
+
+ oai_datacite
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create ODF cleaned store
+
+ ODF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfDataRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Extract_Contexts_from_EntityRegistry_Native.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Extract_Contexts_from_EntityRegistry_Native.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Extract_Contexts_from_EntityRegistry_Native.xml (revision 58540)
@@ -0,0 +1,73 @@
+
+
+
+
+ [DEPRECATED] Aggregate project contexts from EntityRegistry [Ingestion]
+
+ Aggregate contexts
+ Ingestion
+ EntityRegistry
+ project contexts
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ entityregistry::projects
+ native
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate project contexts from EntityRegistry [Ingestion]
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfContextsEntityReg
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Aggregator_datacite_Ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Aggregator_datacite_Ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Aggregator_datacite_Ingestion.xml (revision 58540)
@@ -0,0 +1,98 @@
+
+
+
+
+ Aggregate metadata in datacite format from Aggregator [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ Aggregator
+ results
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ aggregator,crissystem,pubscatalogue,infospace
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate metadata in datacite format from Aggregator [Ingestion]
+
+
+
+
+
+
+
+
+ Create oai_datacite native store
+
+ oai_datacite
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create ODF_datacite cleaned store
+
+ ODF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfAggregatorDataRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Repository_dc.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Repository_dc.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_Repository_dc.xml (revision 58540)
@@ -0,0 +1,99 @@
+
+
+
+
+ Aggregate and Validate Metadata in Dublin Core format from Repository [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ Repository
+ results
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ pubsrepository,datarepository,softwarerepository,orprepository
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate and Validate Metadata in Dublin Core format from Repository [Ingestion]
+
+
+
+
+
+
+
+
+ Create oai_dc store
+
+ oai_dc
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create OAF store
+
+ OAF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfPubsRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/update.db.patch.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/update.db.patch.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/update.db.patch.wf.st (revision 58540)
@@ -0,0 +1,78 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Obtain parameters for current provider
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Create necessary tables
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_journal/sql/pangaea_preparetable.sql
+
+
+
+
+
+
+
+ Start Harvesting
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ collected_epr
+
+
+
+
+
+
+
+ Transform original records to Publication
+
+
+ collected_epr
+ row_epr
+
+
+
+
+
+
+
+ Update Db
+
+ row_epr
+
+
+
+
+
+
+
+ Add new journal to patch entry tables
+
+ dnet.openaire.db.name
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Download_files_from_PubsRepository.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Download_files_from_PubsRepository.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Download_files_from_PubsRepository.xml (revision 58540)
@@ -0,0 +1,101 @@
+
+
+
+
+ Download Files (files) from PubsRepository [Inference]
+
+ Download Files
+ Inference
+ PubsRepository
+ files
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ pubsrepository, aggregator::pubsrepository
+
+ files
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+ Retreive Interface Info
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Download Files (files) from PubsRepository [Inference]
+ openaire-fulltexts@openaire.eu
+
+
+
+
+
+
+ Create Object store
+
+ pubrepository record
+
+ objs_
+
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfPubRepositoryFiles
+
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_DataRepositorybyJournal.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_DataRepositorybyJournal.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_DataRepositorybyJournal.xml (revision 58540)
@@ -0,0 +1,111 @@
+
+
+
+
+ Aggregate Metadata (datasets and publications by journal) from DataRepository [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ DataRepository
+ datasets and publications by journal
+
+ REPO_HI
+ 20
+
+
+ Retrieve datasets by projects id
+
+ datarepository::unknown
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate Metadata (datasets and publications by journal) from DataRepository [Ingestion]
+
+
+
+
+
+
+
+
+
+ Create dataset native store
+
+ dataset
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create ODF_dataset cleaned store
+
+ ODF
+ cleaned
+ store
+ tranODF_
+
+
+
+
+
+
+ Create OAF_publication cleaned store
+
+ OAF
+ cleaned
+ store
+ tranOAF_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfDatasetbyJournalsOpenaireMdRecords
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_DataRepository_Ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_DataRepository_Ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_DataRepository_Ingestion.xml (revision 58540)
@@ -0,0 +1,99 @@
+
+
+
+
+ [DEPRECATED] Aggregate and Validate Metadata (datasets) from DataRepository [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ DataRepository
+ datasets
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ datarepository
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ [DEPRECATED] Aggregate and Validate Metadata (datasets) from DataRepository [Ingestion]
+
+
+
+
+
+
+
+
+
+ Create oai_datacite native store
+
+ oai_datacite
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create ODF cleaned store
+
+ ODF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfDataRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_oaisets.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_oaisets.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_oaisets.xml (revision 58540)
@@ -0,0 +1,99 @@
+
+
+
+
+ Fill hostedBy-Map (OAI-PMH sets in DataRepositories) from DataCite [Context]
+
+ Fill hostedBy-Map
+ Context
+ DataCite
+ OAI-PMH sets in DataRepositories
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ aggregator::datarepository
+ native
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Fill hostedBy-Map (OAI-PMH sets in DataRepositories) from DataCite [Context]
+
+
+
+
+
+
+
+
+ Create store for native records
+
+ GMF
+ native
+ store
+ orig_
+
+
+
+
+
+
+ Create store for DB rows
+
+ DBMF
+ cleaned
+ store
+ row_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfAggrDatarepoOaiSets
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/transformDatasets.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/transformDatasets.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/dataset_project/transformDatasets.wf.st (revision 58540)
@@ -0,0 +1,85 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_dataset_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records to Datasets
+
+
+ orig_dataset_epr
+ datasets_epr
+
+
+
+
+
+
+
+ Clean dataset records
+
+
+ datasets_epr
+ cleaned_datasets_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("tranODF_id")$
+ REFRESH
+ cleaned_datasets_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("tranODF_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_AggregatorPubsRepository_Support.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_AggregatorPubsRepository_Support.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_AggregatorPubsRepository_Support.xml (revision 58540)
@@ -0,0 +1,102 @@
+
+
+
+
+ Aggregate Metadata (publications) from Aggregator::PubsRepository [Support]
+
+ Aggregate Metadata
+ Support
+ Aggregator::PubsRepository
+ publications
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ aggregator::pubsrepository,pubscatalogue
+
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate Metadata (publications) from Aggregator::PubsRepository [Support]
+
+
+
+
+
+
+
+
+
+ Create oai_dc store
+
+ oai_dc
+ nativesupport
+ store
+ harv_
+
+
+
+
+
+
+ Create OAF store
+
+ OAF
+ cleanedsupport
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfAggregatorPubsRepositoryIngestion
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_EntityRegistryProjects_Native.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_EntityRegistryProjects_Native.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_EntityRegistryProjects_Native.xml (revision 58540)
@@ -0,0 +1,99 @@
+
+
+
+
+ Aggregate Projects, Fundings and Contextes [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ EntityRegistry, Aggregator
+ project, organization, person, funding, context
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ entityregistry::projects
+ native
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate Projects, Fundings and Contextes [Ingestion]
+
+
+
+
+
+
+
+
+ Create store for native records
+
+ GMF
+ native
+ store
+ orig_
+
+
+
+
+
+
+ Create store for DB rows
+
+ DBMF
+ cleaned
+ store
+ row_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfEntityRegProjects
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Copy_files_from_PubsRepository_ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Copy_files_from_PubsRepository_ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Copy_files_from_PubsRepository_ingestion.xml (revision 58540)
@@ -0,0 +1,95 @@
+
+
+
+
+ Copy Metadata as Files (publications) from PubsRepository [Ingestion]
+
+ Copy Metadata as Files
+ Inference
+ PubsRepository
+ files
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ files
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+ Retreive Interface Info
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Copy Metadata as Files (publications) from PubsRepository [Ingestion]
+
+
+
+
+
+
+ Create Object store
+
+ pubrepository record
+
+ objs_
+
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfCopyFiles
+
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/row_2_mdstore.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/row_2_mdstore.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/row_2_mdstore.xsl (revision 58540)
@@ -0,0 +1,28 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorPubsRepository_Ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorPubsRepository_Ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorPubsRepository_Ingestion.xml (revision 58540)
@@ -0,0 +1,101 @@
+
+
+
+
+ [DEPRECATED] Aggregate and Validate Metadata (publications) from Aggregator::PubsRepository [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ Aggregator::PubsRepository
+ publications
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ aggregator::pubsrepository,pubscatalogue,infospace
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ [DEPRECATED] Aggregate and Validate Metadata (publications) from Aggregator::PubsRepository [Ingestion]
+
+
+
+
+
+
+
+
+
+ Create oai_dc store
+
+ oai_dc
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create OAF store
+
+ OAF
+ cleaned
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfAggregatorPubsRepositoryIngestion
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/transformWithTDS.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/transformWithTDS.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg/transformWithTDS.wf.st (revision 58540)
@@ -0,0 +1,105 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Decide REFRESH/INCREMENTAL transformation
+
+
+
+
+
+
+
+
+
+ Obtain params on datasource
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("orig_id")$
+ $params.("orig_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+
+ orig_epr
+ packed_row_epr
+
+
+
+
+
+
+
+ Clean DB rows values according to defined vocabularies
+
+
+ packed_row_epr
+ cleaned_row_epr
+
+
+
+
+
+
+
+ Filter DB rows according to the managed flag state
+
+ cleaned_row_epr
+ filtered_row_epr
+
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("row_id")$
+ REFRESH
+ filtered_row_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("row_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_findDataRepos.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_findDataRepos.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_findDataRepos.sql (revision 58540)
@@ -0,0 +1,9 @@
+SELECT
+ id AS id,
+ officialname AS name,
+ englishname AS englishname,
+ typology AS class
+FROM
+ dsm_datasources
+WHERE
+ typology LIKE '%datarepository%'
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationOrcidThroughSemanticRelation.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationOrcidThroughSemanticRelation.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationOrcidThroughSemanticRelation.xml (revision 58540)
@@ -0,0 +1,101 @@
+
+
+
+ Propagation ORCID 2 result through semantic relation
+ Data Provision
+ 30
+
+
+ Set the CSV file path on HDFS
+
+ csvPath
+ /tmp/db_openaireplus_services_beta.ORCIDPropagationResult
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+ dev_openaire
+
+
+
+
+
+
+ CSV files cleanup
+
+
+ {
+ 'path' : 'csvPath',
+ 'cluster' : 'cluster'
+ }
+
+
+
+
+
+
+
+
+
+ Propagatio of result to community through the affiliation to organizations
+
+ propagationOrcidToResultSaveToFile
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'mapred.output.dir' : 'csvPath'
+ }
+
+
+
+
+
+
+
+
+ wf_20180807_172327_510
+ 2019-06-28T17:24:53+02:00
+ SUCCESS
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationCountryToDatasource.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationCountryToDatasource.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationCountryToDatasource.xml (revision 58540)
@@ -0,0 +1,76 @@
+
+
+
+ Propagate Country from Institutional Repos
+ Data Provision
+ 30
+
+
+ Set the CSV file path on HDFS
+
+ csvPath
+ /tmp/db_openaireplus_services_beta.countryPropagation
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+ db_openaireplus_services_beta
+
+
+
+
+
+
+ CSV files cleanup
+
+
+ {
+ 'path' : 'csvPath',
+ 'cluster' : 'cluster'
+ }
+
+
+
+
+
+
+
+ Propagation of country from institutional repos
+
+ propagationCountryFromInstitutionalRepositories
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'mapred.output.dir' : 'csvPath'
+ }
+
+
+
+
+
+
+
+
+ wf_20190613_071747_187
+ 2019-06-13T07:46:27+00:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_EntityRegistry_Native.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_EntityRegistry_Native.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_EntityRegistry_Native.xml (revision 58540)
@@ -0,0 +1,99 @@
+
+
+
+
+ Aggregate Metadata (datasource and organization entities) [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ EntityRegistry, Aggregator
+ datasource and organization entities
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ entityregistry,entityregistry::repositories,aggregator::pubsrepository
+ native
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate Metadata (datasource and organization entities) [Ingestion]
+
+
+
+
+
+
+
+
+ Create store for native records
+
+ GMF
+ native
+ store
+ orig_
+
+
+
+
+
+
+ Create store for DB rows
+
+ DBMF
+ cleaned
+ store
+ row_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfEntityRegDatasource
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/DatasetfromPangaeaTransform.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/DatasetfromPangaeaTransform.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/xsl/DatasetfromPangaeaTransform.xsl (revision 58540)
@@ -0,0 +1,217 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ PANGAEA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ info:eu-repo/semantics/openAccess
+
+
+ European Commission
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ false
+ false
+ 0.9
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_PubsRepository_Support.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_PubsRepository_Support.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_PubsRepository_Support.xml (revision 58540)
@@ -0,0 +1,101 @@
+
+
+
+
+ Aggregate Metadata (publications) from PubsRepository [Support]
+
+ Aggregate Metadata
+ Support
+ PubsRepository
+ publications
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ pubsrepository
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate Metadata (publications) from PubsRepository [Support]
+
+
+
+
+
+
+
+
+
+ Create oai_dc store
+
+ oai_dc
+ nativesupport
+ store
+ harv_
+
+
+
+
+
+
+ Create OAF store
+
+ OAF
+ cleanedsupport
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfPubsRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/find_journal_intersection.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/find_journal_intersection.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/sql/find_journal_intersection.sql (revision 58540)
@@ -0,0 +1,10 @@
+SELECT
+ h.entry AS id,
+ h.datasourceid AS datasource,
+ d.officialname AS name,
+ p.value AS issn
+
+FROM hostedby_map h, dsm_datasources d, hostedby_props p
+WHERE
+ h.oa_source_id = 're3data_____::r3d100010134' AND
+ h.datasourceid = d.id AND p._dnet_resource_identifier_ = h._dnet_resource_identifier_ AND p.key = 'issn'
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/claims2hbase.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/claims2hbase.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/claims2hbase.xml (revision 58540)
@@ -0,0 +1,183 @@
+
+
+
+ Claim to HBase
+ Data Load
+ 30
+
+
+ set hbase table
+
+ dev_openaire
+
+
+
+
+
+
+
+ Configure export to HDFS for OAF claims
+
+ hdfsPath
+ /tmp/mdstores_oaf-store-claim_dev.seq
+ xslt
+ 43108eba-4b67-4d7a-8928-55ca09f1c8ff_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=
+
+
+
+
+
+
+ Fetch OAF claim records
+
+ OAF
+ store
+ claim
+ records_epr
+
+
+
+
+
+
+ Store OAF claims to HDFS
+
+ records_epr
+ hdfsPath
+ DM
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Run M/R import Job for OAF claims
+
+ DM
+ mdStoreHdfsImportJob
+ false
+
+ {
+ 'mapred.input.dir' : 'hdfsPath',
+ 'hbase.import.xslt' : 'xslt',
+ 'hbase.mapred.outputtable' : 'hbaseTable'
+ }
+
+
+
+
+
+
+
+ Configure export to HDFS for ODF claims
+
+ hdfsPath
+ /tmp/mdstores_odf-store-cleaned_dev.seq
+ xslt
+ a8258f53-3cc0-4353-879f-699883fd1d0c_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU=
+
+
+
+
+
+
+ Fetch ODF claim records
+
+ ODF
+ store
+ claim
+ records_epr
+
+
+
+
+
+
+ Store records to HDFS
+
+ records_epr
+ hdfsPath
+ DM
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Run M/R import Job for ODF claims
+
+ DM
+ mdStoreHdfsImportJob
+ false
+
+ {
+ 'mapred.input.dir' : 'hdfsPath',
+ 'hbase.import.xslt' : 'xslt',
+ 'hbase.mapred.outputtable' : 'hbaseTable'
+ }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Apply Claim Rels
+
+ DM
+ dev_openaire
+ SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type !='context'
+ SELECT count(*) FROM claim WHERE approved=TRUE AND source_type !='context'
+
+
+
+
+
+
+ Apply Claim Updates
+
+ DM
+ dev_openaire
+ SELECT source_type, source_id, target_type, target_id, semantics FROM claim WHERE approved=TRUE AND source_type ='context'
+ SELECT count(*) FROM claim WHERE approved=TRUE AND source_type ='context'
+
+
+
+
+
+
+
+ wf_20191111_170912_747
+ 2019-11-11T17:36:30+01:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/transformPublications.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/transformPublications.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByJournal/transformPublications.wf.st (revision 58540)
@@ -0,0 +1,93 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("harv_id")$
+ $params.("harv_format")$
+ orig_publication_epr
+
+
+
+
+
+
+
+ Transform original records to Publication
+
+
+ orig_publication_epr
+ packed_publication_epr
+
+
+
+
+
+
+
+ Split Records into datasets and publication
+
+ packed_publication_epr
+ publication_epr
+
+
+
+
+
+
+
+ Patch oaf:hostedBy fields
+
+ publication_epr
+ patched_epr
+ hostedByCounters
+
+
+
+
+
+
+
+ Store records
+
+ $params.("tranOAF_id")$
+ REFRESH
+ patched_epr
+
+
+
+
+
+
+
+ Validate hostedBy Repos
+
+ hostedByCounters
+ $params.("tranOAF_id")$
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importScholexplorer.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importScholexplorer.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importScholexplorer.xml (revision 58540)
@@ -0,0 +1,84 @@
+
+
+
+
+ Import ScholExplorer Links and entities
+ Import InfoSpace
+ 30
+
+
+ set the hdfs output path
+
+ /usr/dnet/dli/export/scolixDumpExport
+ inputPath
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'scholexplorer-dump',
+ 'jobProperty' : 'export_action_set_scholexplorer_dump',
+ 'enablingProperty' : 'active_scholexplorer_dump',
+ 'enabled' : 'true'
+ }
+ ]
+
+
+
+
+
+
+
+
+ extract the hdfs output path generated in the previous node
+
+ outputPath
+
+
+
+
+
+
+
+
+ IIS main
+
+ importScholexplorerJob
+ DM
+
+ {
+ 'cluster' : 'cluster',
+ 'mapred.input.dir':'inputPath',
+ 'mapred.output.dir':'outputPath'
+ }
+
+ false
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/updateDb.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/updateDb.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/updateDb.wf.st (revision 58540)
@@ -0,0 +1,90 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Create necessary tables
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_prepareTables.sql
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("row_id")$
+ $params.("row_format")$
+ rs_epr
+
+
+
+
+
+
+
+ Update Db
+
+ rs_epr
+
+
+
+
+
+
+
+ Fetch records by Query
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_findDataRepos.sql
+ sql_epr
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt-datacite/datacite_datarepos_2_db.xslt
+ sql_epr
+ clean_sql_epr
+
+
+
+
+
+
+
+ Update Db with cleaned datarepos
+
+ clean_sql_epr
+
+
+
+
+
+
+
+ Create necessary tables
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/repo-hi/sql-datacite/dataciteSets_findMatches.sql
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/distcp.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/distcp.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/distcp.xml (revision 58540)
@@ -0,0 +1,62 @@
+
+
+
+ distcp
+ Export InfoSpace
+ 30
+
+
+
+ Set the source path on HDFS (DM)
+
+ sourcePath
+ /tmp/db_openaireplus_services_beta.export.2019.08.17
+
+
+
+
+
+
+ Set the target path on HDFS (IIS)
+
+ targetPath
+ /tmp
+
+
+
+
+
+
+ distcp
+
+ distcpJob
+ IIS
+
+ {
+ 'sourcePath' : 'sourcePath',
+ 'targetPath' : 'targetPath'
+ }
+
+ false
+ build-report
+
+
+
+
+
+
+
+
+ wf_20190805_085505_893
+ 2019-08-07T05:19:02+00:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/doaj_journals/repoBye.wf.st (revision 58540)
@@ -0,0 +1,21 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importDOIBoost.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importDOIBoost.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importDOIBoost.xml (revision 58540)
@@ -0,0 +1,83 @@
+
+
+
+ Import DOIboost
+ Import InfoSpace
+ 30
+
+
+ set the hdfs output path
+
+ /usr/dnet/data/doiboost
+ inputPath
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'doiboost',
+ 'jobProperty' : 'export_action_set_doiboost',
+ 'enablingProperty' : 'active_doiboost',
+ 'enabled' : 'true'
+ }
+ ]
+
+
+
+
+
+
+
+ extract the hdfs output path generated in the previous node
+
+ outputPath
+
+
+
+
+
+
+ IIS main
+
+ importDOIBoostJob
+ DM
+
+ {
+ 'mapred.input.dir':'inputPath',
+ 'mapred.output.dir':'outputPath'
+ }
+
+ false
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+ wf_20181114_161523_453
+ 2018-11-14T16:59:55+00:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/odf2hbase.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/odf2hbase.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/odf2hbase.xml (revision 58540)
@@ -0,0 +1,101 @@
+
+
+
+
+ ODF to HBase
+ Data Load
+ 30
+
+
+ set hbase table
+
+
+
+
+
+
+
+
+
+ Configure export to HDFS
+
+ hdfsPath
+ /tmp/mdstores_odf-store-cleaned.seq
+ xslt
+
+
+
+
+
+
+
+ reuse mdstore records
+
+ false
+
+
+
+
+
+
+
+ Fetch mdstore records
+
+ ODF
+ store
+ cleaned
+ records_epr
+
+
+
+
+
+
+ Store records to HDFS
+
+ records_epr
+ hdfsPath
+ DM
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Run M/R import Job
+
+ DM
+ mdStoreHdfsImportJob
+ false
+
+ {
+ 'mapred.input.dir' : 'hdfsPath',
+ 'hbase.import.xslt' : 'xslt',
+ 'hbase.mapred.outputtable' : 'hbaseTable'
+ }
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/oaf2hbase_intersection.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/oaf2hbase_intersection.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/oaf2hbase_intersection.xml (revision 58540)
@@ -0,0 +1,105 @@
+
+
+
+
+ OAF to HBase [intersection]
+ Data Load
+ 30
+
+
+ set hbase table
+
+
+
+
+
+
+
+
+ Configure export to HDFS
+
+ hdfsPath
+
+ xslt
+
+
+
+
+
+
+
+ reuse mdstore records
+
+
+
+
+
+
+
+
+
+ Fetch mdstore records
+
+ OAF
+ store
+ intersection
+ records_epr
+
+
+
+
+
+
+ Store records to HDFS
+
+ records_epr
+ hdfsPath
+ DM
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Run M/R import Job
+
+ DM
+ mdStoreHdfsImportJob
+ false
+
+ {
+ 'mapred.input.dir' : 'hdfsPath',
+ 'hbase.import.xslt' : 'xslt',
+ 'hbase.mapred.outputtable' : 'hbaseTable'
+ }
+
+
+ {
+ 'hbase.import.invisible' : 'true'
+ }
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/openorgs_new_similarities.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/openorgs_new_similarities.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/openorgs_new_similarities.xml (revision 58540)
@@ -0,0 +1,72 @@
+
+
+
+
+ Export simrels to OpenOrgs DB
+ OpenOrgs
+ 30
+
+
+
+
+
+ /tmp/openOrgs.simrels.tmp.seq
+ openOrgsExportPath
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'openOrgsExportPath'
+ }
+
+
+
+
+
+
+
+
+ ExportSimrRels M/R job
+
+ exportOpenOrgsSimilarities
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'mapred.output.dir' : 'openOrgsExportPath'
+ }
+
+
+ {}
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/actions2hbase.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/actions2hbase.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/actions2hbase.xml (revision 58540)
@@ -0,0 +1,27 @@
+
+
+
+
+ Actions to HBase
+ Data Load
+ 30
+
+
+ Promote actions
+
+ ALL SETS
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_CordaH2020Projects_Native.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_CordaH2020Projects_Native.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_CordaH2020Projects_Native.xml (revision 58540)
@@ -0,0 +1,99 @@
+
+
+
+
+ EC H2020 Aggregate Projects, Fundings and Contexts [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ EntityRegistry
+ project, organization, person, funding, context
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ entityregistry::projects
+ native
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ EC H2020 Aggregate Projects, Fundings and Contexts [Ingestion]
+
+
+
+
+
+
+
+
+ Create store for native records
+
+ GMF
+ native
+ store
+ orig_
+
+
+
+
+
+
+ Create store for DB rows
+
+ DBMF
+ cleaned
+ store
+ row_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfH2020Projects
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/tubitak_journal.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/tubitak_journal.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/fillhostedByDoaj/sql/tubitak_journal.sql (revision 58540)
@@ -0,0 +1,15 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+ SELECT
+ d.issn || '@@' || d.collectedfrom,
+ d.collectedfrom,
+ d.issn,
+ d.id
+ FROM dsm_datasources d
+ WHERE
+ d.collectedfrom = 'openaire____::tubitak' AND d.issn IS NOT NULL AND d.id LIKE 'tubitak::%'
+ AND
+ (d.collectedfrom, d.issn) NOT IN
+ (SELECT
+ oa_source_id,
+ entry
+ FROM hostedby_map)
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_Generic_HostedbyMap.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_Generic_HostedbyMap.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_Generic_HostedbyMap.xml (revision 58540)
@@ -0,0 +1,100 @@
+
+
+
+
+ Fill Generic hostedBy-Map [Context]
+
+ Fill hostedBy-Map
+ Context
+ EntityRegistry, Aggregator
+ ISSNs
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ entityregistry,aggregator::pubsrepository
+ native
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Fill hostedBy-Map (ISSNs) [Context]
+
+
+
+
+
+
+
+
+ Create store for native records
+
+ GMF
+ native
+ store
+ orig_
+
+
+
+
+
+
+ Create store for DB rows
+
+ DBMF
+ cleaned
+ store
+ row_
+
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfGenericHostedByMap
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/pangaea_journal_findmatches.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/pangaea_journal_findmatches.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/sql/pangaea_journal_findmatches.sql (revision 58540)
@@ -0,0 +1,32 @@
+INSERT INTO hostedby_map (_dnet_resource_identifier_, oa_source_id, entry, datasourceid)
+ SELECT
+ p.id || '@@' || p.oa_source_id,
+ p.oa_source_id,
+ p.id,
+ d.id
+ FROM dsm_datasources d, pangaea_temp_journals p
+ WHERE
+ d.collectedfrom = 'driver______::1790119e-d281-4b7a-aedf-866d1d853a07' AND d.issn IS NOT NULL AND d.id LIKE 'doajarticles::%' AND p.issn = d.issn
+ AND
+ (p.oa_source_id, p.id) NOT IN
+ (SELECT
+ oa_source_id,
+ entry
+ FROM hostedby_map);
+
+
+INSERT INTO hostedby_props (_dnet_resource_identifier_, key, value)
+ SELECT
+ p.id || '@@' || p.oa_source_id,
+ 'issn',
+ p.issn
+ FROM dsm_datasources d, pangaea_temp_journals p
+ WHERE
+ d.collectedfrom = 'driver______::1790119e-d281-4b7a-aedf-866d1d853a07' AND d.issn IS NOT NULL AND d.id LIKE 'doajarticles::%' AND p.issn = d.issn
+ AND
+ (p.id || '@@' || p.oa_source_id, 'issn', p.issn) NOT IN
+ (SELECT
+ _dnet_resource_identifier_,
+ key,
+ value
+ FROM hostedby_props);
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/openorgs_new_orgs.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/openorgs_new_orgs.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/openorgs_new_orgs.xml (revision 58540)
@@ -0,0 +1,72 @@
+
+
+
+
+ Export organizations to OpenOrgs DB
+ OpenOrgs
+ 30
+
+
+
+
+
+ /tmp/openOrgs.orgs.tmp.seq
+ openOrgsExportPath
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'openOrgsExportPath'
+ }
+
+
+
+
+
+
+
+
+ ExportSimrRels M/R job
+
+ exportOpenOrgsOrganizations
+ DM
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapred.outputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.outputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'mapred.output.dir' : 'openOrgsExportPath'
+ }
+
+
+ {}
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/xsl/irdb2hostedByMap.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/xsl/irdb2hostedByMap.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/xsl/irdb2hostedByMap.xsl (revision 58540)
@@ -0,0 +1,45 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ irdb journal id
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/post-vre-newsfeed.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/post-vre-newsfeed.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/post-vre-newsfeed.xml (revision 58540)
@@ -0,0 +1,34 @@
+
+
+
+ Post on VRE News feed
+ Communication
+ 30
+
+
+ Post news on the VRE
+
+ https
+ socialnetworking1.d4science.org
+ social-networking-library-ws/rest/2/posts/write-post-app
+
+
+
+ Updated content and statistics are available on OpenAIRE #Beta and in your Research Community Dashboard (#RCD)! Check them out and if you have questions click on the Activity Tracker tab and open a new issue. The OpenAIRE team wishes you a good day
+
+ false
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/transform.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/transform.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/hostedbyMapJournalIntersection/transform.wf.st (revision 58540)
@@ -0,0 +1,72 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("orig_id")$
+ $params.("orig_format")$
+ orig_epr
+
+
+
+
+
+
+
+ Fetch records from MDStore
+
+ $params.("dataprovider:id")$
+
+
+
+
+
+
+
+ Transform original records in DB rows
+
+
+ orig_epr
+ cleaned_row_epr
+
+
+
+
+
+
+ Store mdtore records
+
+ $params.("row_id")$
+ REFRESH
+ cleaned_row_epr
+
+
+
+
+
+
+
+ Update datasouce API extra fields
+
+ $params.("row_id")$
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_aggregation_total
+ last_aggregation_date
+ last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/import.infospace.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/import.infospace.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/import.infospace.xml (revision 58540)
@@ -0,0 +1,27 @@
+
+
+
+ Import Infospace
+
+ InfoSpace Provision
+ alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it
+
+
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase.infospace.counts.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase.infospace.counts.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase.infospace.counts.xml (revision 58540)
@@ -0,0 +1,69 @@
+
+
+
+ Data Provision Counts
+ Data Provision
+ 35
+
+
+
+
+ dnet.openaire.model.relclasses.xquery
+ relClasses
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ M/R count entities and relationships
+
+ DM
+ countInfospaceJob
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'index.entity.links' : 'index.entity.links',
+ 'contextmap' : 'contextmap',
+ 'relClasses' : 'relClasses'
+ }
+
+
+
+
+
+
+
+
+ wf_20181107_103348_265
+ 2018-11-08T08:48:56+00:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationResult2community.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationResult2community.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationResult2community.xml (revision 58540)
@@ -0,0 +1,86 @@
+
+
+
+ Propagation result 2 community
+ Data Provision
+ 30
+
+
+ Set the CSV file path on HDFS
+
+ csvPath
+ /tmp/db_openaireplus_services_beta.communityPropagation
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+ dev_openaire
+
+
+
+
+
+
+ CSV files cleanup
+
+
+ {
+ 'path' : 'csvPath',
+ 'cluster' : 'cluster'
+ }
+
+
+
+
+
+
+
+ Fetch the list of the community ids
+
+ community.ids
+
+
+
+
+
+
+ Propagatio of result to community through semantic relation
+
+ propagationResultToCommunityThroughSemanticRelationSaveToFile
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'community.ids' : 'community.ids',
+ 'mapred.output.dir' : 'csvPath'
+ }
+
+
+
+
+
+
+
+
+ wf_20180807_172327_510
+ 2019-06-28T17:24:53+02:00
+ SUCCESS
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/export.native.softwares.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/export.native.softwares.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/export.native.softwares.xml (revision 58540)
@@ -0,0 +1,74 @@
+
+
+
+ Softwares Exporter
+ Export Infospace
+ 30
+
+
+ Set the output file path on HDFS
+
+ outputPath
+ /tmp/organizations.2018.10.31
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+ db_openaireplus_services_beta
+
+
+
+
+
+
+ CSV files cleanup
+
+
+ {
+ 'path' : 'outputPath',
+ 'cluster' : 'cluster'
+ }
+
+
+
+
+
+
+
+ export the information space as sequence file
+
+ informationSpaceSoftwareExportJob
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'mapred.output.dir' : 'outputPath'
+ }
+
+
+
+
+
+
+
+
+ wf_20181031_114608_13
+ 2018-10-31T11:49:33+00:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/db2hbase.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/db2hbase.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/db2hbase.xml (revision 58540)
@@ -0,0 +1,201 @@
+
+
+
+ DB to HBase
+ Data Load
+ 30
+
+
+ start
+
+
+
+
+
+
+ query Datasources
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasources.sql
+
+
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourcesForSize.sql
+
+ dsEpr
+
+
+
+
+
+
+ Store Datasources to HBase
+
+ dsEpr
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+
+ query relation Datasource_Organization
+
+ dnet.openaire.db.name
+
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourceOrganization.sql
+
+ dsOrgEpr
+
+
+
+
+
+
+ Store relation Datasource_Organization
+
+ dsOrgEpr
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+
+ query Organizations
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizations.sql
+
+ orgEpr
+
+
+
+
+
+
+ Store Organizations to HBase
+
+ orgEpr
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+
+ query Open Organizations
+
+ dnet.openorgs.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizationsFromOpenOrgsDB.sql
+
+ openOrgsEpr
+
+
+
+
+
+
+ Store Open Organizations to HBase
+
+ openOrgsEpr
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+
+ query similarities (openorgs)
+
+ dnet.openorgs.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/querySimilarityFromOpenOrgsDB.sql
+ openSimRelsEpr
+
+
+
+
+
+
+ Store similarities to HBase (openorgs)
+
+ openSimRelsEpr
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+
+ query Projects
+
+ dnet.openaire.db.name
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjects.sql
+ projEpr
+
+
+
+
+
+
+ Store Projects to HBase
+
+ projEpr
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+
+ query relation Project_Organization
+
+ dnet.openaire.db.name
+
+ /eu/dnetlib/msro/openaireplus/workflows/hbase/queryProjectOrganization.sql
+
+ projOrgEpr
+
+
+
+
+
+
+ Store relation Project_Organizations
+
+ projOrgEpr
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+
+
+ wf_20140509_125739_830
+ 2014-05-09T13:03:16+02:00
+ SUCCESS
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.provision.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.provision.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.provision.xml (revision 58540)
@@ -0,0 +1,314 @@
+
+
+
+ Data Provision
+ Data Provision
+ 30
+
+
+
+
+ set mdformat, layout, interpretation
+
+ DMF
+ index
+ openaire
+
+
+
+
+
+
+
+
+ dnet.openaire.model.relclasses.xquery
+ relClasses
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Prepare indexing
+
+ hdfsRecordsPath
+ rottenRecordsPath
+ /eu/dnetlib/msro/openaireplus/workflows/index/openaireLayoutToRecordStylesheet.xsl
+ oaf.schema.location
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'hdfsRecordsPath'
+ }
+
+
+
+
+
+
+
+ hdfs cleanup (rotten)
+
+ DM
+
+ {
+ 'path' : 'rottenRecordsPath'
+ }
+
+
+
+
+
+
+
+ M/R group entities
+
+ DM
+ prepareIndexDataJob
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'mapred.output.dir' : 'hdfsRecordsPath',
+ 'index.entity.links' : 'index.entity.links',
+ 'oaf.schema.location' : 'oaf.schema.location',
+ 'contextmap' : 'contextmap',
+ 'relClasses' : 'relClasses'
+ }
+
+ false
+
+
+
+
+
+
+
+
+ M/O index records
+
+ DM
+ indexFeedJob
+
+ {
+ 'mapred.input.dir' : 'hdfsRecordsPath',
+ 'mapred.output.dir' : 'rottenRecordsPath',
+ 'index.xslt' : 'index.xslt',
+ 'index.solr.url' : 'index.solr.url',
+ 'index.solr.collection' : 'index.solr.collection',
+ 'index.buffer.flush.threshold' : 'index.buffer.flush.threshold',
+ 'index.shutdown.wait.time' : 'index.shutdown.wait.time',
+ 'index.solr.sim.mode' : 'index.solr.sim.mode',
+ 'index.feed.timestamp' : 'index.feed.timestamp'
+ }
+
+ false
+
+
+
+
+
+
+ commit changes
+
+
+
+
+
+
+ update DS
+
+
+
+
+
+
+
+
+ set params for stats
+
+
+
+
+
+
+ Update stats DB
+
+ DM
+ StatsExportJob
+
+ {
+ 'Stats_Hbase_Source_Table' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'nameNode' : 'nameNode',
+ 'jobTracker' : 'jobTracker',
+ 'isLookupEndpoint' : 'isLookupEndpoint',
+ 'Stats_indexConf' : 'index.entity.links'
+ }
+
+ false
+
+
+
+
+
+
+
+
+
+ Prepare target OAI store
+
+ oaistore
+ oaf
+ index
+ openaire
+
+ true
+ //*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']
+
+
+
+
+
+
+ Drop the store: upserts are too expensive!
+
+
+
+
+
+
+
+
+
+
+
+
+
+ M/O oai feeding
+
+ DM
+ oaiFeedJob
+
+ {
+ 'mapred.input.dir' : 'hdfsRecordsPath',
+ 'services.publisher.oai.collection' : 'oai_collection',
+ 'oaiConfiguration' : 'oai_configuration',
+ 'oai.feed.date' : 'oai_actionDate',
+ 'services.publisher.oai.db' : 'oai_dbName',
+ 'services.publisher.oai.skipDuplicates':'oai_skipDuplicates',
+ 'services.publisher.oai.duplicateXPath':'oai_duplicateXPath'
+ }
+
+
+ {
+ 'services.publisher.oai.host' : 'services.publisher.oai.host',
+ 'services.publisher.oai.port' : 'services.publisher.oai.port'
+ }
+
+ false
+
+
+
+
+
+
+ Create composite indexes for the OAI store
+
+ set,resulttypeid;set,datestamp;license,oaftype;set,oaftype;oaftype,funder;resulttypeid,oaftype;oaftype,funding;resulttypeid,funder;resulttypeid,funding
+
+
+
+
+
+
+ Ensure an index exists on fields as specified in the configuration profile
+
+
+
+
+
+
+
+ Count records in each OAI set, for each exported metadata format
+
+ false
+
+
+
+
+
+
+ Reads the current OAI configuration and upsert OAI sets accordingly (counts are updated as well)
+
+
+
+
+
+
+
+
+ wait for all the branches to complete
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase-to-hdfs.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase-to-hdfs.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase-to-hdfs.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ HBase to HDFS
+
+ InfoSpace Provision
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.prepare.preview.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.prepare.preview.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.prepare.preview.xml (revision 58540)
@@ -0,0 +1,144 @@
+
+
+
+
+ Prepare Pre-public Information Space
+ InfoSpace preview
+ 30
+
+
+
+
+ DMF
+ index
+ openaire
+
+
+
+
+
+
+
+ find shadow search service
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='SearchServiceResourceType'
+ and .//RESOURCE_KIND/@value='ServiceResources' and
+ .//PROPERTY[@key="infrastructure"]/@value="shadow"]/HEADER/RESOURCE_IDENTIFIER/@value/string()
+
+
+
+
+
+
+
+
+ switch index
+
+
+
+
+
+
+
+ switch index
+
+ index_id
+ IndexId
+
+
+
+
+
+
+ Wait for the search branch to fail or to success
+
+
+
+
+
+
+
+ Do we feed the the most recent claims today?
+
+ NO
+
+
+
+
+
+
+
+
+ Feed the most recent claims
+
+
+
+
+
+
+
+ find stats service
+
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and
+ .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+
+ xqueryForStatsManagerService
+
+
+
+
+
+
+
+ Do we prepare the shadow stats for preview today?
+
+ YES
+
+
+
+
+
+
+
+ set the target portal instance for pre-public preview
+
+ portal
+ test
+
+
+
+
+
+
+ Updates the shadow cache according to the shadow stats db.
+
+ xqueryForStatsManagerService
+ portal
+
+
+
+
+
+
+ Promote the shadow cache.
+
+ xqueryForStatsManagerService
+ portal
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importAffiliationsDOIBoost.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importAffiliationsDOIBoost.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importAffiliationsDOIBoost.xml (revision 58540)
@@ -0,0 +1,94 @@
+
+
+
+ Import Organizations from DOIboost
+ Import InfoSpace
+ 30
+
+
+ set the parameter to drive the import procedure behaviour so that it will import only organizations
+
+ onlyOrganization
+ true
+
+
+
+
+
+
+ set the hdfs output path
+
+ /usr/dnet/data/doiboost
+ inputPath
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'doiboost-organizations',
+ 'jobProperty' : 'export_action_set_doiboost-organizations',
+ 'enablingProperty' : 'active_doiboost-organizations',
+ 'enabled' : 'true'
+ }
+ ]
+
+
+
+
+
+
+
+ extract the hdfs output path generated in the previous node
+
+ outputPath
+
+
+
+
+
+
+ IIS main
+
+ importDOIBoostJob
+ DM
+
+ {
+ 'mapred.input.dir':'inputPath',
+ 'mapred.output.dir':'outputPath',
+ 'onlyOrganization' : 'onlyOrganization'
+ }
+
+ false
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+ wf_20181026_140552_878
+ 2018-10-26T14:21:47+00:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/apply-blacklist.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/apply-blacklist.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/apply-blacklist.xml (revision 58540)
@@ -0,0 +1,43 @@
+
+
+
+
+ Apply relationship blacklist
+ Data Provision
+ 30
+
+
+ Resolve the identifiers in the blacklist
+
+ blacklist
+ DM
+ db_openaireplus_services
+
+
+
+
+
+
+
+ Generates and apply the HBASE DELETE statements.
+
+ blacklist
+ DM
+ db_openaireplus_test
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/oaf2hbase.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/oaf2hbase.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/oaf2hbase.xml (revision 58540)
@@ -0,0 +1,100 @@
+
+
+
+
+ OAF to HBase
+ Data Load
+ 30
+
+
+ set hbase table
+
+
+
+
+
+
+
+
+ Configure export to HDFS
+
+ hdfsPath
+
+ xslt
+
+
+
+
+
+
+
+ reuse mdstore records
+
+
+
+
+
+
+
+
+
+ Fetch mdstore records
+
+ OAF
+ store
+ cleaned
+ records_epr
+
+
+
+
+
+
+ Store records to HDFS
+
+ records_epr
+ hdfsPath
+ DM
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Run M/R import Job
+
+ DM
+ mdStoreHdfsImportJob
+ false
+
+ {
+ 'mapred.input.dir' : 'hdfsPath',
+ 'hbase.import.xslt' : 'xslt',
+ 'hbase.mapred.outputtable' : 'hbaseTable'
+ }
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentProjectEvents.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentProjectEvents.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentProjectEvents.xml (revision 58540)
@@ -0,0 +1,75 @@
+
+
+
+ Calculate enrichment Project events
+ Notification Broker
+ 30
+
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+
+
+ Set Dedup conf
+
+
+ dedup.conf.queue
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'brokerEventsPath'
+ }
+
+
+
+
+
+
+
+
+
+ Run M/R Job
+
+ DM
+ brokerEnrichmentProjectsJob
+ dedup.conf.queue
+
+ {
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentEvents.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentEvents.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentEvents.meta.xml (revision 58540)
@@ -0,0 +1,26 @@
+
+
+
+ Calculate person distribution
+
+ Notification Broker
+ michele.artini@isti.cnr.it
+
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/PublicationFromPangaeatransform.xsl
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/PublicationFromPangaeatransform.xsl (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/xsl/PublicationFromPangaeatransform.xsl (revision 58540)
@@ -0,0 +1,161 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 0000
+ und
+
+
+
+ UNKNOWN
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ false
+ false
+ 0.9
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/resetHbase.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/resetHbase.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/resetHbase.xml (revision 58540)
@@ -0,0 +1,64 @@
+
+
+
+
+ Reset HBase
+ Data Load
+ 30
+
+
+ check hbase table
+
+ hbase.mapred.datatable
+ DM
+ hTableColumns
+ drop
+ define
+
+
+
+
+
+
+
+ drop hbase table
+
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+ define OpenAIRE hbase table
+
+ hTableColumns
+ hbase.mapred.datatable
+ DM
+
+
+
+
+
+
+ create hbase table
+
+ hbase.mapred.datatable
+ DM
+ hTableColumns
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepositoryByProject/repoBye.wf.st (revision 58540)
@@ -0,0 +1,63 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the oai_datacite mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the IMEF mdstore
+
+ $params.("tranODF_id")$
+
+
+
+
+
+
+
+ Delete the IMEF mdstore
+
+ $params.("tranOAF_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/repoBye.wf.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/repoBye.wf.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/claim/repoBye.wf.st (revision 58540)
@@ -0,0 +1,53 @@
+
+ Set information about current provider
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:name")$
+ $params.("dataprovider:interface")$
+
+
+
+
+
+
+
+ Delete the MetaWorkflow
+
+ $params.("META_WORKFLOW_ID")$
+
+
+
+
+
+
+
+ Delete the native mdstore
+
+ $params.("harv_id")$
+
+
+
+
+
+
+
+ Delete the cleaned mdstore
+
+ $params.("tran_id")$
+
+
+
+
+
+
+
+ Reset the extrafields of the api
+
+ $params.("dataprovider:id")$
+ $params.("dataprovider:interface")$
+ last_collection_total, last_collection_date, last_collection_mdId, last_aggregation_total, last_aggregation_date, last_aggregation_mdId
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateAdditionEvents.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateAdditionEvents.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateAdditionEvents.xml (revision 58540)
@@ -0,0 +1,75 @@
+
+
+
+ Calculate addition events
+ Notification Broker
+ 30
+
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+
+
+
+
+ /tmp/brokerEvents.tmp.seq
+ brokerEventsPath
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'brokerEventsPath'
+ }
+
+
+
+
+
+
+
+
+
+ Run M/R Job
+
+ DM
+ brokerAdditionJob
+
+ {
+ 'mapred.output.dir' : 'brokerEventsPath',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculatePersonDistribution.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculatePersonDistribution.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculatePersonDistribution.meta.xml (revision 58540)
@@ -0,0 +1,22 @@
+
+
+
+ Calculate person distribution
+
+ Notification Broker
+ michele.artini@isti.cnr.it
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.meta.xml (revision 58540)
@@ -0,0 +1,39 @@
+
+
+
+ OpenAIRE Provision
+
+ InfoSpace Provision
+ alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/applicationContext-msro-openaire-message.properties
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/applicationContext-msro-openaire-message.properties (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/applicationContext-msro-openaire-message.properties (revision 58540)
@@ -0,0 +1,5 @@
+dnet.openaire.messageManager.host =
+dnet.openaire.messageManager.username =
+dnet.openaire.messageManager.password =
+dnet.openaire.messageManager.ongoingMessageQueueName =
+dnet.openaire.messageManager.reportMessageQueueName =
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/refreshMdstoreSizes.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/refreshMdstoreSizes.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/refreshMdstoreSizes.xml (revision 58540)
@@ -0,0 +1,24 @@
+
+
+
+ Refresh Mdstore Sizes
+ Consistency
+ 30
+
+
+ Refresh Mdstore Sizes
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/content.publishing.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/content.publishing.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/content.publishing.meta.xml (revision 58540)
@@ -0,0 +1,26 @@
+
+
+
+
+ OpenAIRE Content Publishing
+
+ InfoSpace Provision
+
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/reposStoresConsistency.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/reposStoresConsistency.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/reposStoresConsistency.meta.xml (revision 58540)
@@ -0,0 +1,26 @@
+
+
+
+ Datasources and stores
+
+ InfoSpace Consistency
+ michele.artini@isti.cnr.it
+
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/xmlRecords.counts.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/xmlRecords.counts.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/xmlRecords.counts.xml (revision 58540)
@@ -0,0 +1,48 @@
+
+
+
+ XML records counts
+ Data Provision
+ 30
+
+
+ set the input path on hdfs to read xml records
+
+ inputHdfsPath
+ /tmp/indexrecords_db_openaireplus_services_TMF.seq
+
+
+
+
+
+
+ Run M/R count Job
+
+ DM
+ xmlRecordCounterJob
+ false
+
+ {
+ 'mapred.input.dir' : 'inputHdfsPath'
+ }
+
+
+
+
+
+
+
+
+ wf_20181022_103455_527
+ 2018-10-22T12:44:29+00:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listDownloadObjectStores.xquery.st
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listDownloadObjectStores.xquery.st (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/listDownloadObjectStores.xquery.st (revision 58540)
@@ -0,0 +1,9 @@
+for \$mdId in
+ distinct-values(
+ for \$x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')[.//DATAPROVIDER/@id='$dsId$' and .//DATAPROVIDER/@interface='$apiId$']//WORKFLOW
+ let \$f := concat('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType/', substring-before(\$x/@id/string(), '_'))
+ return doc(\$f)[.//PARAM[@name="extraFieldForTotal" and text() = 'last_download_total']]//NODE[@type='DownloadFromMetadataOpenaire' or @type='UpdateObjectStoreSize']//PARAM[@category="OBJECTSTORE_ID"]/text()
+ )
+ let \$fmd := concat('/db/DRIVER/ObjectStoreDSResources/ObjectStoreDSResourceType/', substring-before(\$mdId, '_'))
+ return doc(\$fmd)
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/copyHbaseTable.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/copyHbaseTable.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/copyHbaseTable.xml (revision 58540)
@@ -0,0 +1,139 @@
+
+
+
+
+ Copy HBase Table
+ IIS
+ 30
+
+
+ Set copy parameters
+
+ DM
+
+ IIS
+
+
+
+
+
+
+
+ check hbase source table
+
+ hTableColumns
+
+ {
+ 'hbaseTable' : 'sourceTable',
+ 'cluster' : 'sourceCluster'
+ }
+
+ yes
+ no
+
+
+
+
+
+
+
+ get source table description
+
+ hTableColumns
+
+ {
+ 'hbaseTable' : 'sourceTable',
+ 'cluster' : 'sourceCluster'
+ }
+
+
+
+
+
+
+
+ check hbase target table
+
+ hTableColumns
+
+ {
+ 'hbaseTable' : 'targetTable',
+ 'cluster' : 'targetCluster'
+ }
+
+ yes
+ no
+
+
+
+
+
+
+
+ select copy mode
+
+ REFRESH
+
+
+
+
+
+
+
+ drop hbase target table
+
+
+ {
+ 'hbaseTable' : 'targetTable',
+ 'cluster' : 'targetCluster'
+ }
+
+
+
+
+
+
+
+ create hbase table
+
+ hTableColumns
+
+ {
+ 'hbaseTable' : 'targetTable',
+ 'cluster' : 'targetCluster'
+ }
+
+
+
+
+
+
+
+ Copy table Job
+
+ copytable
+
+ {
+ 'cluster' : 'sourceCluster',
+ 'targetCluster' : 'targetCluster',
+ 'new.name' : 'targetTable',
+ 'hbase.mapreduce.inputtable' : 'sourceTable',
+ 'peer.adr' : 'peer.adr'
+ }
+
+ false
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryClaimsDataset.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryClaimsDataset.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/queryClaimsDataset.sql (revision 58540)
@@ -0,0 +1,5 @@
+SELECT
+ regexp_replace(xml, '&(?!amp;)', '&', 'g') AS xml,
+ provenance
+FROM claims
+WHERE type = 'dmf2actions' AND set = 'userclaim_dmf' AND provenance = 'user:claim:datacite'
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-preprocessingV2.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-preprocessingV2.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-preprocessingV2.xml (revision 58540)
@@ -0,0 +1,126 @@
+
+
+
+
+ IIS preprocessing V2
+ IIS
+ 30
+
+
+ start
+
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'iis-referenced-projects-preprocessing',
+ 'jobProperty' : 'export_action_set_id_document_referencedProjects',
+ 'enablingProperty' : 'active_referencedProjects_export',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenced-datasets-preprocessing',
+ 'jobProperty' : 'export_action_set_id_document_referencedDatasets',
+ 'enablingProperty' : 'active_referencedDatasets_export',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-wos-entities',
+ 'jobProperty' : 'export_action_set_id_entity_wos',
+ 'enablingProperty' : 'active_entity_wos_export',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-dataset-entities-preprocessing',
+ 'jobProperty' : 'export_action_set_id_entity_dataset',
+ 'enablingProperty' : 'active_entity_dataset_export',
+ 'enabled' : 'true'
+ }
+ ]
+
+
+
+
+
+
+
+ prepare parameters
+
+ import_islookup_service_location
+ import_content_object_store_location
+ import_mdstore_service_location
+ import_dataset_mdstore_ids_csv
+ import_wos_mdstore_id
+ import_database_service_location
+ import_content_datacite_objectstores_csv
+ import_content_wos_plaintext_objectstores_csv
+ oozie.wf.application.path
+ /tmp/integration/apps/preprocessing
+ DM
+
+ import_project_concepts_context_ids_csv
+ fet-fp7,fet-h2020
+
+
+
+
+
+
+ IIS preprocessing
+
+ iisPreprocessingJobV2
+
+ {
+ 'cluster' : 'cluster',
+ 'oozie.wf.application.path' : 'oozie.wf.application.path',
+ 'import_content_object_store_location' : 'import_content_object_store_location',
+ 'import_mdstore_service_location' : 'import_mdstore_service_location',
+ 'import_islookup_service_location' : 'import_islookup_service_location',
+ 'import_project_concepts_context_ids_csv' : 'import_project_concepts_context_ids_csv',
+ 'import_dataset_mdstore_ids_csv' : 'import_dataset_mdstore_ids_csv',
+ 'import_wos_mdstore_id' : 'import_wos_mdstore_id',
+ 'import_database_service_location' : 'import_database_service_location',
+ 'import_content_datacite_objectstores_csv' : 'import_content_datacite_objectstores_csv',
+ 'import_content_wos_plaintext_objectstores_csv' : 'import_content_wos_plaintext_objectstores_csv',
+ 'export_action_set_id_document_referencedProjects' : 'export_action_set_id_document_referencedProjects',
+ 'export_action_set_id_document_referencedDatasets' : 'export_action_set_id_document_referencedDatasets',
+ 'export_action_set_id_entity_wos' : 'export_action_set_id_entity_wos',
+ 'export_action_set_id_entity_dataset' : 'export_action_set_id_entity_dataset'
+ }
+
+
+ {
+ 'import_database_dbname' : 'dnet.openaire.db.name'
+ }
+
+ false
+ build-report
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/vocabulary_entries.xquery
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/vocabulary_entries.xquery (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/db/vocabulary_entries.xquery (revision 58540)
@@ -0,0 +1,13 @@
+for $x in collection("/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType")//TERM
+where
+ string-length($x/@english_name) > 0 and string-length($x/@code) > 0
+return
+
+ {$x/@encoding/string()}
+ {$x/@code/string()}
+ {$x/@english_name/string()}
+ {$x/@native_name/string()}
+ {$x/../../VOCABULARY_NAME/@code/string()}
+ {$x/../../VOCABULARY_NAME/text()}
+ {$x/../../VOCABULARY_DESCRIPTION/text()}
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/importHBaseTable.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/importHBaseTable.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/importHBaseTable.xml (revision 58540)
@@ -0,0 +1,65 @@
+
+
+
+
+ Import HBase Table
+ IIS
+ 30
+
+
+ Set the CSV file path on HDFS
+
+ csvPath
+
+
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+
+
+
+ export the information space as sequence file
+
+ informationSpaceImportJob
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'mapred.input.dir' : 'csvPath'
+ }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/dedup.ocean.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/dedup.ocean.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/dedup.ocean.xml (revision 58540)
@@ -0,0 +1,171 @@
+
+
+
+ Offline Deduplication [OCEAN]
+ Deduplication
+ 30
+
+
+
+ Set the target directory to materialize the graph
+
+ graphBasePath
+ /var/lib/dnet/graph_BETA
+
+
+
+
+
+
+
+ Is the raw graph already available?
+
+ YES
+
+
+
+
+
+
+
+
+ create the raw graph
+
+ executeOozieJob
+ IIS
+
+ {
+ 'graphBasePath' : 'graphBasePath'
+ }
+
+
+ {
+ 'oozie.wf.application.path' : '/lib/dnet/migration/wfs/regular_all_steps/oozie_app',
+ 'mongoURL' : 'mongodb://beta.services.openaire.eu',
+ 'mongoDb' : 'mdstore',
+ 'postgresURL' : 'jdbc:postgresql://beta.services.openaire.eu:5432/dnet_openaireplus',
+ 'postgresUser' : 'dnet',
+ 'postgresPassword' : 'xxx',
+ 'workingPath' : '/tmp/dhp_migration_beta',
+ 'reuseContent' : 'true'
+ }
+
+ false
+ build-report
+
+
+
+
+
+
+
+ Set the path containing the raw graph
+
+ rawGraphPath
+ /var/lib/dnet/graph_BETA/graph_raw
+
+
+
+
+
+
+
+ Set the path used by dedup workflows to store intermediate data
+
+ dedupWorkingPath
+ /tmp/beta_dedup
+
+
+
+
+
+
+
+ Set the target path to store the deduped graph
+
+ dedupGraphPath
+ /var/lib/dnet/graph_BETA/graph_dedup
+
+
+
+
+
+
+
+ Set Dedup conf
+
+ dedup-similarity-organization
+ dedup.conf.queue
+
+
+
+
+
+
+
+ create the raw graph
+
+ executeOozieJob
+ IIS
+
+ {
+ 'actionSetId' : 'setActionSetId',
+ 'graphBasePath' : 'rawGraphPath',
+ 'workingPath' : 'dedupWorkingPath',
+ 'dedupGraphPath': 'dedupGraphPath'
+ }
+
+
+ {
+ 'oozie.wf.application.path' : '/lib/dnet/dedup/scan/oozie_app'
+ }
+
+ build-report
+
+
+
+
+
+
+
+ mark duplicates as deleted and redistribute the relationships
+
+ executeOozieJob
+ IIS
+
+ {
+ 'graphBasePath' : 'rawGraphPath',
+ 'dedupGraphPath': 'dedupGraphPath'
+ 'workingPath' : 'dedupWorkingPath',
+ }
+
+
+ {
+ 'oozie.wf.application.path' : '/lib/dnet/dedup/consistency/oozie_app'
+ }
+
+ build-report
+
+
+
+
+
+
+
+
+
+ wf_20200320_224915_420
+ 2020-03-20T23:07:47+00:00
+ SUCCESS
+
+ 0
+ 0
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/copyTable.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/copyTable.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/copyTable.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Copy InfoSpace
+
+ InfoSpace Inference
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/is/update-profiles.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/is/update-profiles.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/is/update-profiles.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Update profiles
+
+ InfoSpace Consistency
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/test-post-meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/test-post-meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/test-post-meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Testing POST VRE
+
+ InfoSpace Provision
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/exportHBaseTable.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/exportHBaseTable.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/exportHBaseTable.xml (revision 58540)
@@ -0,0 +1,79 @@
+
+
+
+
+ Export HBase Table
+ IIS
+ 30
+
+
+ Set the CSV file path on HDFS
+
+ csvPath
+
+
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+
+
+ CSV files cleanup
+
+
+ {
+ 'path' : 'csvPath',
+ 'cluster' : 'cluster'
+ }
+
+
+
+
+
+
+
+
+ export the information space as sequence file
+
+ informationSpaceExportJob
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'mapred.output.dir' : 'csvPath'
+ }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/reset.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/reset.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dedup/reset.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Reset dedup
+
+ InfoSpace Deduplication
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbageActionSets.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbageActionSets.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbageActionSets.xml (revision 58540)
@@ -0,0 +1,32 @@
+
+
+
+
+ Garbase Action Sets
+ GARBAGE_ACTION_SETS
+ 30
+
+
+ Garbage actions
+
+
+
+
+
+
+ Clean Set profiles
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/db/updateDB.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/db/updateDB.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/db/updateDB.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Update DB
+
+ InfoSpace Consistency
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.ocean.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.ocean.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/provision.ocean.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+ OpenAIRE Provision [OCEAN]
+
+ InfoSpace Provision
+ alessia.bardi@isti.cnr.it,claudio.atzori@isti.cnr.it
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_org_countries.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_org_countries.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_org_countries.sql (revision 58540)
@@ -0,0 +1,26 @@
+update dsm_organizations set country='TW' where country='Taiwan';
+
+update dsm_organizations set country='NL' where country='Netherlands';
+
+update dsm_organizations set country='NA' where country='NAM';
+
+update dsm_organizations set country='KZ' where country='KAZ';
+
+update dsm_organizations set country='IL' where country='Israel';
+
+update dsm_organizations set country='US' where country='United States';
+
+update dsm_organizations set country='DE' where country='Germany';
+
+update dsm_organizations set country='CA' where country='Canada';
+
+update dsm_organizations set country='LK' where country='LKA';
+
+update dsm_organizations set country='CD' where country ='Congo, the Democratic Republic of the';
+update dsm_organizations set country='CD' where country like 'Congo Democratic%';
+update dsm_organizations set country='CD' where country = 'Zaire';
+
+update dsm_organizations set country='FI' where country = 'Finland';
+
+update dsm_organizations set country='MX' where country = 'Mexico';
+update dsm_organizations set country='GB' where country = 'United Kingdom';
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbageActionSetsHDFS.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbageActionSetsHDFS.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbageActionSetsHDFS.xml (revision 58540)
@@ -0,0 +1,32 @@
+
+
+
+
+ Garbase Action Sets (HDFS)
+ GARBAGE_ACTION_SETS
+ 30
+
+
+ Garbage actions
+
+
+
+
+
+
+ Clean Set profiles
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/bulktag/bulkTagging.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/bulktag/bulkTagging.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/bulktag/bulkTagging.xml (revision 58540)
@@ -0,0 +1,66 @@
+
+
+
+ Bulk tagging
+ Data Provision
+ 30
+
+
+ Set table name
+
+ DM
+ tableName
+ dev_openaire
+
+
+
+
+
+
+ Fetch the bulk tagging configuration profile
+
+ tagging.conf
+ tagging.enabled
+ proto.mapping
+ false
+
+
+
+
+
+
+ Bulk tag
+
+ bulkTaggingJob
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'tagging.conf' : 'tagging.conf',
+ 'tagging.enabled' : 'tagging.enabled',
+ 'proto.mapping' : 'proto.mapping'
+ }
+
+
+
+
+
+
+
+
+ wf_20180807_172327_510
+ 2018-08-07T17:24:53+02:00
+ SUCCESS
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.promote.test.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.promote.test.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.promote.test.xml (revision 58540)
@@ -0,0 +1,50 @@
+
+
+
+
+ Promote stats on test portal
+ Provision
+ 30
+
+
+ find stats service
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+ xqueryForStatsManagerService
+
+
+
+
+
+
+ set the target portal instance for pre-public preview
+
+ portal
+ test
+
+
+
+
+
+
+ Promote the shadow cache.
+
+ xqueryForStatsManagerService
+ portal
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.restore.cache.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.restore.cache.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.restore.cache.xml (revision 58540)
@@ -0,0 +1,40 @@
+
+
+
+
+ Stats Cache Restore
+ Provision
+ 30
+
+
+ find stats service
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+ xqueryForStatsManagerService
+
+
+
+
+
+
+ Backup the current stats cache. If the new generated shadow cache is not good we can rollback with the restore wf.
+
+ xqueryForStatsManagerService
+ test
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/increase_size_re3data_fields.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/increase_size_re3data_fields.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/increase_size_re3data_fields.sql (revision 58540)
@@ -0,0 +1,9 @@
+ALTER TABLE dsm_datasources ALTER COLUMN databaseaccesstype TYPE varchar(64);
+ALTER TABLE dsm_datasources ALTER COLUMN datauploadtype TYPE varchar(64);
+ALTER TABLE dsm_datasources ALTER COLUMN databaseaccessrestriction TYPE varchar(64);
+ALTER TABLE dsm_datasources ALTER COLUMN datauploadrestriction TYPE varchar(64);
+ALTER TABLE dsm_datasources ALTER COLUMN qualitymanagementkind TYPE varchar(64);
+
+ALTER TABLE dsm_datasources ALTER COLUMN missionstatementurl TYPE text;
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-projects-summary-amount.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-projects-summary-amount.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-projects-summary-amount.sql (revision 58540)
@@ -0,0 +1,11 @@
+alter table projects add column summary text default null ;
+
+alter table projects add column currency text default null ;
+
+alter table projects add column totalcost numeric default null ;
+
+alter table projects add column fundedamount numeric default null ;
+
+alter table project_organization add column contribution numeric default null ;
+
+alter table project_organization add column currency text default null ;
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.meta.xml (revision 58540)
@@ -0,0 +1,24 @@
+
+
+
+
+ Index switch
+
+ InfoSpace Provision
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_dsTypes.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_dsTypes.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_dsTypes.sql (revision 58540)
@@ -0,0 +1,13 @@
+update dsm_api set typology = 'aggregator::pubsrepository::unknown' where id = 'api_________::openaire____::unpaywall::0';
+update dsm_api set typology = 'researchgraph' where id ='api_________::openaire____::microsoft::0';
+update dsm_api set typology = 'entityregistry::organizations' where id ='api_________::openaire____::grid::mock';
+update dsm_api set typology = 'entityregistry::researchers' where id ='api_________::openaire____::orcid::0';
+update dsm_api set typology = 'entityregistry::products' where id like 'api_________::openaire____::datacite%';
+update dsm_api set typology = 'entityregistry::products' where id like 'api_________::openaire____::crossref:%';
+
+update dsm_datasources set typology = 'aggregator::pubsrepository::unknown' where id = 'openaire____::unpaywall';
+update dsm_datasources set typology = 'researchgraph' where id ='openaire____::microsoft';
+update dsm_datasources set typology = 'entityregistry::organizations' where id ='openaire____::grid';
+update dsm_datasources set typology = 'entityregistry::researchers' where id ='openaire____::orcid';
+update dsm_datasources set typology = 'entityregistry::products' where id = 'openaire____::datacite';
+update dsm_datasources set typology = 'entityregistry::products' where id = 'openaire____::crossref';
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentEvents.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentEvents.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentEvents.xml (revision 58540)
@@ -0,0 +1,60 @@
+
+
+
+ Calculate enrichment events
+ Notification Broker
+ 30
+
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+
+
+ Set Dedup conf
+
+
+ dedup.conf.queue
+
+
+
+
+
+
+
+
+ Run M/R Job
+
+ DM
+ brokerEnrichmentJob
+ dedup.conf.queue
+
+ {
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.update.shadow.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.update.shadow.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/stats.update.shadow.xml (revision 58540)
@@ -0,0 +1,57 @@
+
+
+
+
+ Update Shadow Stats
+ Provision
+ 30
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Update stats DB
+
+ DM
+ StatsExportJob
+
+ {
+ 'Stats_Hbase_Source_Table' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'nameNode' : 'nameNode',
+ 'jobTracker' : 'jobTracker',
+ 'isLookupEndpoint' : 'isLookupEndpoint',
+ 'Stats_indexConf' : 'index.entity.links'
+ }
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.switch.xml (revision 58540)
@@ -0,0 +1,54 @@
+
+
+
+
+ Index switch
+ Provision
+ 30
+
+
+
+
+ DMF
+ index
+ openaire
+
+
+
+
+
+
+
+ find search service
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='SearchServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+ xqueryForSearchService
+
+
+
+
+
+
+
+ switch index
+
+ xqueryForSearchService
+ index_id
+ IndexId
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/triggers.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/triggers.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/triggers.sql (revision 58540)
@@ -0,0 +1,23 @@
+-- disable the triggers
+ALTER TABLE dsm_datasources DISABLE TRIGGER ALL;
+ALTER TABLE projectpids DISABLE TRIGGER ALL;
+ALTER TABLE project_subject DISABLE TRIGGER ALL;
+ALTER TABLE fundingpaths DISABLE TRIGGER ALL;
+ALTER TABLE project_fundingpath DISABLE TRIGGER ALL;
+ALTER TABLE project_organization DISABLE TRIGGER ALL;
+ALTER TABLE dsm_organizations DISABLE TRIGGER ALL;
+ALTER TABLE dsm_organizationpids DISABLE TRIGGER ALL;
+ALTER TABLE dsm_datasource_organization DISABLE TRIGGER ALL;
+ALTER TABLE projects DISABLE TRIGGER ALL;
+
+-- enable the triggers
+ALTER TABLE dsm_datasources ENABLE TRIGGER ALL;
+ALTER TABLE projectpids ENABLE TRIGGER ALL;
+ALTER TABLE project_subject ENABLE TRIGGER ALL;
+ALTER TABLE fundingpaths ENABLE TRIGGER ALL;
+ALTER TABLE project_fundingpath ENABLE TRIGGER ALL;
+ALTER TABLE project_organization ENABLE TRIGGER ALL;
+ALTER TABLE dsm_organizations ENABLE TRIGGER ALL;
+ALTER TABLE dsm_organizationpids ENABLE TRIGGER ALL;
+ALTER TABLE dsm_datasource_organization ENABLE TRIGGER ALL;
+ALTER TABLE projects ENABLE TRIGGER ALL;
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hdfs.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hdfs.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/index.hdfs.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Index (from HDFS)
+
+ InfoSpace Provision
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_pangaea_doaj.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_pangaea_doaj.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_pangaea_doaj.sql (revision 58540)
@@ -0,0 +1,85 @@
+CREATE TABLE IF NOT EXISTS hostedby_map (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) DEFAULT 'temp_' || md5(clock_timestamp() :: TEXT) || '_' || md5(random() :: TEXT) UNIQUE,
+ oa_source_id CHARACTER VARYING(255) REFERENCES dsm_datasources (id),
+ entry CHARACTER VARYING(255) NOT NULL,
+ datasourceid CHARACTER VARYING(255) REFERENCES dsm_datasources (id),
+ PRIMARY KEY (oa_source_id, entry)
+);
+
+--DATACITE API
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass)
+VALUES ('api_________::datacite::oaisets', 'api_________::datacite::oaisets', 'oai_sets', 'openaire____::datacite', 'aggregator::datarepository', 'native');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'http://oai.datacite.org/oai', 'api_________::datacite::oaisets');
+INSERT INTO apicollections (param, original, api, accessparam)
+VALUES ('metadata_identifier_path', '//*[local-name() = ''setSpec'']', 'api_________::datacite::oaisets', FALSE);
+
+--PANGAEA BY PROJECT
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass) VALUES
+ ('api_________::re3data_____::r3d100010134::pangaeabyproject', 'api_________::re3data_____::r3d100010134::pangaeabyproject', 'datasetsbyproject',
+ 're3data_____::r3d100010134', 'datarepository::unknown', 'native');
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('metadata_identifier_path', '//*[local-name()=''header'']/*[local-name()=''identifier'']', 'api_________::re3data_____::r3d100010134::pangaeabyproject',
+ 'api_________::re3data_____::r3d100010134::pangaeabyproject@@metadata_identifier_path', FALSE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('baseUrl', 'file:///var/lib/dnet/pangaea_projects.csv', 'api_________::re3data_____::r3d100010134::pangaeabyproject',
+ 'api_________::re3data_____::r3d100010134::pangaeabyproject@@baseUrl', TRUE);
+
+--PANGAEA BY JOURNAL
+
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass) VALUES
+ ('api_________::re3data_____::r3d100010134::pangaeabyjournal', 'api_________::re3data_____::r3d100010134::pangaeabyjournal', 'datasetsbyjournal',
+ 're3data_____::r3d100010134', 'datarepository::unknown', 'native');
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('metadata_identifier_path', '//*[local-name()=''header'']/*[local-name()=''identifier'']', 'api_________::re3data_____::r3d100010134::pangaeabyjournal',
+ 'api_________::re3data_____::r3d100010134::pangaeabyjournal@@metadata_identifier_path', FALSE);
+
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass) VALUES
+ ('api_________::re3data_____::r3d100010134::pangaeabyjournalCSV', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV', 'fileCSV',
+ 're3data_____::r3d100010134', 'datarepository::unknown', 'native');
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('metadata_identifier_path', '//*[local-name()=''column'' and @isID=''true'']', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV',
+ 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV@@metadata_identifier_path', FALSE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('baseUrl', 'file:///var/lib/dnet/pangaea_journals.csv', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV',
+ 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV@@baseUrl', TRUE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('quote', '"', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV@@quote', TRUE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('header', 'true', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV@@header',
+ TRUE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('identifier', '4', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV',
+ 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV@@identifier', TRUE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('separator', '@', 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV',
+ 'api_________::re3data_____::r3d100010134::pangaeabyjournalCSV@@separator', TRUE);
+
+--DOAJ
+
+INSERT INTO api (id, _dnet_resource_identifier_, protocolclass, datasource, contentdescriptionclass, typologyclass, compatibilityclass) VALUES
+ ('api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV', 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV',
+ 'httpCSV', 'driver______::1790119e-d281-4b7a-aedf-866d1d853a07', 'metadata', 'entityregistry', 'native');
+
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('baseUrl', 'http://doaj.org/csv', 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV',
+ 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV@@baseUrl', TRUE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('separator', ',', 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV',
+ 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV@@separator', TRUE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('quote', '"', 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV',
+ 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV@@quote', TRUE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('identifier', 'ISSN', 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV',
+ 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV@@identifier', TRUE);
+INSERT INTO apicollections (param, original, api, _dnet_resource_identifier_, accessparam) VALUES
+ ('metadata_identifier_path', '//column[./@isID=''true'']', 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV',
+ 'api_________::driver______::1790119e-d281-4b7a-aedf-866d1d853a07::httpCSV@@metadata_identifier_path', FALSE);
+
+
+CREATE TABLE IF NOT EXISTS hostedby_props (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) NOT NULL REFERENCES hostedby_map (_dnet_resource_identifier_),
+ key CHARACTER VARYING(255) NOT NULL,
+ value CHARACTER VARYING(255) NOT NULL,
+ PRIMARY KEY (_dnet_resource_identifier_, key, value)
+);
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-unidentifiedProjects.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-unidentifiedProjects.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-unidentifiedProjects.sql (revision 58540)
@@ -0,0 +1,5 @@
+INSERT INTO projects (id, code, title, collectedfrom, provenanceactionclass) VALUES ('wt__________::unidentified', 'unidentified', 'unidentified', 'infrastruct_::openaire', 'sysimport:crosswalk:entityregistry');
+INSERT INTO project_fundingpath (funding, project) VALUES ('wt__________::WT', 'wt__________::unidentified');
+
+INSERT INTO projects (id, code, title, collectedfrom, provenanceactionclass) VALUES ('miur________::unidentified', 'unidentified', 'unidentified', 'infrastruct_::openaire', 'sysimport:crosswalk:entityregistry');
+INSERT INTO project_fundingpath (funding, project) VALUES ('miur________::MIUR', 'miur________::unidentified');
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/pubsRepository_claim_ingestion.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/pubsRepository_claim_ingestion.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/pubsRepository_claim_ingestion.xml (revision 58540)
@@ -0,0 +1,93 @@
+
+
+
+
+ Collect and transform claimed metadata records
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ scholarcomminfra,researchgraph
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Collect and transform claimed metadata records
+
+
+
+
+
+
+
+
+ Create native store
+
+ claim
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create cleaned store
+
+ OAF
+ claim
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfClaimedMdRecords
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_erc_pubsrepository.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_erc_pubsrepository.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_erc_pubsrepository.sql (revision 58540)
@@ -0,0 +1,12 @@
+INSERT INTO datasources (_dnet_resource_identifier_, id, officialname, englishname, websiteurl, logourl, contactemail, latitude, longitude, timezone, namespaceprefix, collectedfrom, datasourceclass)
+VALUES ('openaire____::erc', 'openaire____::erc', 'European Research Council (ERC)', 'European Research Council (ERC)', 'http://erc.europa.eu',
+ 'http://erc.europa.eu/sites/default/files/content/LOGO-ERC.jpg', 'Alexis-Michel.MUGABUSHAKA@ec.europa.eu', 50.85, 4.35, 1,
+ 'erc_________', 'infrastruct_::openaire', 'pubsrepository::unknown');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass)
+VALUES ('api_________::erc::0', 'api_________::erc::0', 'file', 'openaire____::erc', 'pubsrepository::unknown', 'openaire3.0');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'file:///var/lib/dnet/erc/PUB_REPORTED.xml', 'api_________::erc::0');
+INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'RECORD', 'api_________::erc::0');
+INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '//PUB_ID', 'api_________::erc::0', FALSE);
+
+UPDATE apicollections
+SET _dnet_resource_identifier_ = api || '@@' || param;
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-jsonextrainfo.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-jsonextrainfo.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-jsonextrainfo.sql (revision 58540)
@@ -0,0 +1,2 @@
+ALTER TABLE projects
+ ADD COLUMN jsonextrainfo TEXT DEFAULT '{}';
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-fct.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-fct.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-fct.sql (revision 58540)
@@ -0,0 +1,30 @@
+-- REMEMBER TO UPDATE THE CLEANING RULE AND THE FOLLOWING VOCABULARIES:
+-- aecdb60c-e1ed-40bd-a6c1-edc9993f07cb_Vm9jYWJ1bGFyeURTUmVzb3VyY2VzL1ZvY2FidWxhcnlEU1Jlc291cmNlVHlwZQ==
+-- ecdd1e73-cf16-4a8b-8842-d0a7c780ab6e_Vm9jYWJ1bGFyeURTUmVzb3VyY2VzL1ZvY2FidWxhcnlEU1Jlc291cmNlVHlwZQ==
+-- AND TO REGISTER:
+-- 3470afb2-d15f-4046-988e-7a7a2f7c993b_Vm9jYWJ1bGFyeURTUmVzb3VyY2VzL1ZvY2FidWxhcnlEU1Jlc291cmNlVHlwZQ==
+-- 5d11b639-51c3-4dc3-a2aa-da0ad168e768_Vm9jYWJ1bGFyeURTUmVzb3VyY2VzL1ZvY2FidWxhcnlEU1Jlc291cmNlVHlwZQ==
+-- ec216410-9ebc-4547-bb4a-190acb0985ef_Vm9jYWJ1bGFyeURTUmVzb3VyY2VzL1ZvY2FidWxhcnlEU1Jlc291cmNlVHlwZQ==
+
+-- THEN RUN THE CLASS/SCHEME UPDATE FROM THE IS TO THE DB
+
+--FCT ENTITY REGISTRY
+INSERT INTO organizations (_dnet_resource_identifier_, id, legalshortname, legalname, websiteurl, logourl, countryclass, provenanceactionclass) VALUES
+ ('fct_________::FCT', 'fct_________::FCT', 'FCT', 'Fundação para a Ciência e a Tecnologia, I.P.', 'http://www.fct.pt',
+ 'http://www.fct.pt/images/FCT_H_color_v2011_b.png', 'PT', 'sysimport:crosswalk:entityregistry');
+INSERT INTO datasources (_dnet_resource_identifier_, id, officialname, englishname, websiteurl, logourl, contactemail, latitude, longitude, timezone, namespaceprefix, collectedfrom, datasourceclass)
+VALUES
+ ('openaire____::fct', 'openaire____::fct', 'FCT - Fundação para a Ciência e a Tecnologia', 'FCT - Foundation for Science and Technology', 'http://www.fct.pt',
+ 'http://www.fct.pt/images/FCT_H_color_v2011_b.png', 'projectos@fct.pt', 38.71, -9.14, 0.0, 'fct_________', 'infrastruct_::openaire',
+ 'entityregistry');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass)
+VALUES ('api_________::fct::0', 'api_________::fct::0', 'filesystem', 'openaire____::fct', 'entityregistry', 'native');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'file:///var/lib/dnet/fct', 'api_________::fct::0');
+INSERT INTO apicollections (param, original, api) VALUES ('extensions', '*.xml', 'api_________::fct::0');
+INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '/project/id', 'api_________::fct::0', FALSE);
+UPDATE apicollections
+SET _dnet_resource_identifier_ = api || '@@' || param;
+INSERT INTO datasource_organization (_dnet_resource_identifier_, datasource, organization)
+VALUES ('fct_________::fct@@fct_________::FCT', 'openaire____::fct', 'fct_________::FCT');
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_arc_and_ands.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_arc_and_ands.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_arc_and_ands.sql (revision 58540)
@@ -0,0 +1,23 @@
+INSERT INTO datasources (_dnet_resource_identifier_, id, officialname, englishname, websiteurl, logourl, contactemail, latitude, longitude, timezone, namespaceprefix, collectedfrom, datasourceclass)
+VALUES ('openaire____::ands', 'openaire____::ands', 'Australian National Data Service (ANDS) - Grants Registry',
+ 'Australian National Data Service (ANDS) - Grants Registry', 'http://www.ands.org.au/index.html',
+ 'http://www.ands.org.au/assets/images/ands-full-b.png', 'contact@ands.org.au', -34.2333, 140.5667, 10.50, 'ands________',
+ 'infrastruct_::openaire', 'entityregistry');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass)
+VALUES ('api_________::ands::0', 'api_________::ands::0', 'filesystem', 'openaire____::ands', 'entityregistry', 'native');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'file:///var/lib/dnet/ands', 'api_________::ands::0');
+INSERT INTO apicollections (param, original, api) VALUES ('extensions', 'xml', 'api_________::ands::0');
+INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '/project/id', 'api_________::ands::0', FALSE);
+
+INSERT INTO datasources (_dnet_resource_identifier_, id, officialname, englishname, websiteurl, logourl, contactemail, latitude, longitude, timezone, namespaceprefix, collectedfrom, datasourceclass)
+VALUES ('openaire____::arc', 'openaire____::arc', 'Australian Research Council (ARC) - Grants Registry', 'Australian Research Council (ARC) - Grants Registry',
+ 'http://www.arc.gov.au/', 'http://www.arc.gov.au/images/screen_logo.gif', 'info@arc.gov.au', -33.9, 151.1833, 11.0, 'arc_________',
+ 'infrastruct_::openaire', 'entityregistry');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass)
+VALUES ('api_________::arc::0', 'api_________::arc::0', 'filesystem', 'openaire____::arc', 'entityregistry', 'native');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'file:///var/lib/dnet/arc', 'api_________::arc::0');
+INSERT INTO apicollections (param, original, api) VALUES ('extensions', 'xml', 'api_________::arc::0');
+INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '/project/id', 'api_________::arc::0', FALSE);
+
+UPDATE apicollections
+SET _dnet_resource_identifier_ = api || '@@' || param;
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_DataRepository_Inference.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_DataRepository_Inference.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_DataRepository_Inference.xml (revision 58540)
@@ -0,0 +1,97 @@
+
+
+
+
+ [DEPRECATED] Aggregate Metadata (datasets) from DataRepository [Inference]
+
+ Aggregate Metadata
+ Inference
+ DataRepository
+ datasets
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ datarepository
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ [DEPRECATED] Aggregate Metadata (datasets) from DataRepository [Inference]
+
+
+
+
+
+
+
+
+ Create oai_datacite native store
+
+ oai_datacite
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create ODF cleaned store
+
+ ODF
+ cleaned_noingestion
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfDataRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorDataRepository_Inference.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorDataRepository_Inference.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Aggregate_Metadata_from_AggregatorDataRepository_Inference.xml (revision 58540)
@@ -0,0 +1,98 @@
+
+
+
+
+ [DEPRECATED] Aggregate Metadata (datasets) from Aggregator::DataRepository [Inference]
+
+ Aggregate Metadata
+ Inference
+ aggregator::datarepository
+ datasets
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ aggregator::datarepository
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ [DEPRECATED] Aggregate Metadata (datasets) from Aggregator::DataRepository [Inference]
+
+
+
+
+
+
+
+
+ Create oai_datacite native store
+
+ oai_datacite
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create ODF_datacite cleaned store
+
+ ODF
+ cleaned_noingestion
+ store
+ tran_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfAggregatorDataRepository
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Copy_files_from_PubsRepository_inference.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Copy_files_from_PubsRepository_inference.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/DEPRECATED_Copy_files_from_PubsRepository_inference.xml (revision 58540)
@@ -0,0 +1,95 @@
+
+
+
+
+ [DEPRECATED] Copy Metadata as Files (publications) from PubsRepository [Inference]
+
+ Copy Metadata as Files
+ Inference
+ PubsRepository
+ files
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ files
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+ Retreive Interface Info
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ [DEPRECATED] Copy Metadata as Files (publications) from PubsRepository [Inference]
+
+
+
+
+
+
+ Create Object store
+
+ wos record
+
+ objs_
+
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfCopyFiles
+
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FeedMissingClaimsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FeedMissingClaimsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FeedMissingClaimsJobNode.java (revision 58540)
@@ -0,0 +1,127 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.index.CloudIndexClient;
+import eu.dnetlib.data.index.CloudIndexClientFactory;
+import eu.dnetlib.data.index.CloudIndexClientException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;
+import eu.dnetlib.openaire.directindex.api.RecentResultsQueue;
+import eu.dnetlib.openaire.directindex.utils.OafToIndexRecordFactory;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.solr.common.SolrInputDocument;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.ClassPathResource;
+
+/**
+ * Created by michele on 15/12/15.
+ */
+public class FeedMissingClaimsJobNode extends AsyncJobNode {
+
+ private static final Log log = LogFactory.getLog(FeedMissingClaimsJobNode.class);
+ public static final int BATCH_SIZE = 1000;
+ private RecentResultsQueue queue;
+ private OafToIndexRecordFactory oafToIndexRecordFactory;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Value(value = "${openaire.api.directindex.findSolrIndexUrl.xquery}")
+ private ClassPathResource findSolrIndexUrl;
+
+ @Override
+ protected String execute(final NodeToken nodeToken) throws Exception {
+
+ final String format =
+ nodeToken.getEnv().hasAttribute("format") ? nodeToken.getEnv().getAttribute("format") : nodeToken.getFullEnv().getAttribute("format");
+ final String coll = format + "-index-openaire";
+ final String indexDsId = nodeToken.getEnv().getAttribute("index_id");
+ final String baseUrl = calculateIndexBaseUrl();
+
+ CloudIndexClient idxClient = null;
+
+ try {
+ final List toFeed = new ArrayList();
+ final List toDeleteFromCache = new ArrayList();
+
+ final SAXReader reader = new SAXReader();
+ final ApplyXslt xslt = oafToIndexRecordFactory.newTransformer(format);
+
+ idxClient = CloudIndexClientFactory.newIndexClient(baseUrl, coll, false);
+ log.info("Starting to feed claims in index collection "+coll);
+ int count = 1;
+ for (String record : queue) {
+ final String id = reader.read(new StringReader(record)).valueOf("//*[local-name() = 'objIdentifier']");
+ if(log.isDebugEnabled()){
+ log.debug("Processing record "+count++);
+ }
+ if (idxClient.isRecordIndexed(id)) {
+ toDeleteFromCache.add(id);
+ } else {
+ toFeed.add(idxClient.prepareSolrDocument(record, indexDsId, xslt));
+ }
+ if(count % BATCH_SIZE == 0) processLists(idxClient, toFeed, toDeleteFromCache);
+
+ }
+ if(!toFeed.isEmpty() || !toDeleteFromCache.isEmpty()) processLists(idxClient, toFeed, toDeleteFromCache);
+ log.info(String.format("Finished feeding of claims in index collection %s, total: %d", coll, count));
+
+ } catch (Throwable e) {
+ log.error("Error feeding missing claims", e);
+ throw new MSROException("Error feeding missing claims: " + e.getMessage(), e);
+ } finally {
+ if (idxClient != null) {
+ idxClient.close();
+ }
+ }
+
+ return Arc.DEFAULT_ARC;
+ }
+
+
+
+ private void processLists(final CloudIndexClient idxClient, final List toFeed, final List toDeleteFromCache) throws CloudIndexClientException{
+ idxClient.feed(toFeed, null);
+ queue.remove(toDeleteFromCache);
+ log.info(String.format("%d claims fed and cache cleaned of %d records", toFeed.size(), toDeleteFromCache.size()));
+ toFeed.clear();
+ toDeleteFromCache.clear();
+ }
+
+ public RecentResultsQueue getQueue() {
+ return queue;
+ }
+
+ @Required
+ public void setQueue(final RecentResultsQueue queue) {
+ this.queue = queue;
+ }
+
+ public OafToIndexRecordFactory getOafToIndexRecordFactory() {
+ return oafToIndexRecordFactory;
+ }
+
+ @Required
+ public void setOafToIndexRecordFactory(final OafToIndexRecordFactory oafToIndexRecordFactory) {
+ this.oafToIndexRecordFactory = oafToIndexRecordFactory;
+ }
+
+ private String calculateIndexBaseUrl() throws Exception {
+ final String query = IOUtils.toString(findSolrIndexUrl.getInputStream());
+ return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query);
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_DataRepositorybyProject.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_DataRepositorybyProject.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_DataRepositorybyProject.xml (revision 58540)
@@ -0,0 +1,111 @@
+
+
+
+
+ Aggregate Metadata (datasets and publications by project) from PANGAEA [Ingestion]
+
+ Aggregate Metadata
+ Ingestion
+ DataRepository
+ datasets and publications by project
+
+ REPO_HI
+ 20
+
+
+ Retrieve datasets by projects id
+
+ datarepository::unknown
+
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Aggregate Metadata (datasets and publications by project) from PANGAEA [Ingestion]
+
+
+
+
+
+
+
+
+
+ Create dataset native store
+
+ dataset
+ native
+ store
+ harv_
+
+
+
+
+
+
+ Create ODF_dataset cleaned store
+
+ ODF
+ cleaned
+ store
+ tranODF_
+
+
+
+
+
+
+ Create OAF_publication cleaned store
+
+ OAF
+ cleaned
+ store
+ tranOAF_
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfDatasetbyProjectsOpenaireMdRecords
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/PrepareIndexDataJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/PrepareIndexDataJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/PrepareIndexDataJobNode.java (revision 58540)
@@ -0,0 +1,210 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.io.StringWriter;
+import javax.annotation.Resource;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.miscutils.functional.hash.Hashing;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.core.io.ClassPathResource;
+
+public class PrepareIndexDataJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(PrepareIndexDataJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ /**
+ * Stylesheet which transforms a layout to another stylesheet which converts a input record to a index record.
+ */
+ private String layoutToRecordStylesheet;
+
+ private String outputRecordsPathParam;
+
+ private String rottenRecordsPathParam;
+
+ private String hbaseTable;
+
+ private String oafSchemaLocationProperty;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ log.info("start preparing job");
+
+ final String xslt = prepareXslt(env("format", token), env("layout", token));
+
+ token.getEnv().setAttribute("index.xslt", xslt);
+
+ if (!StringUtils.isBlank(getOutputRecordsPathParam())) {
+ token.getEnv().setAttribute(getOutputRecordsPathParam(), "/tmp" + getFileName(token, "indexrecords"));
+ }
+ if (!StringUtils.isBlank(getRottenRecordsPathParam())) {
+ token.getEnv().setAttribute(getRottenRecordsPathParam(), "/tmp" + getFileName(token, "rottenrecords"));
+ }
+
+ token.getEnv().setAttribute("index.solr.url", getIndexSolrUrlZk());
+ token.getEnv().setAttribute("index.solr.collection", getCollectionName(token));
+
+ token.getEnv().setAttribute("index.shutdown.wait.time", getIndexSolrShutdownWait());
+ token.getEnv().setAttribute("index.buffer.flush.threshold", getIndexBufferFlushTreshold());
+ token.getEnv().setAttribute("index.solr.sim.mode", isFeedingSimulationMode());
+
+ token.getEnv().setAttribute("index.feed.timestamp", DateUtils.now_ISO8601());
+
+ token.getEnv().setAttribute(getOafSchemaLocationProperty(), getPropertyFetcher().getProperty(getOafSchemaLocationProperty()));
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ protected String tableName(final NodeToken token) {
+ if (token.getEnv().hasAttribute("hbaseTable")) {
+ String table = token.getEnv().getAttribute("hbaseTable");
+ log.debug("found override value in wfEnv for 'hbaseTable' param: " + table);
+ return table;
+ }
+ return getHbaseTable();
+ }
+
+ public String getIndexSolrUrlZk() throws ISLookUpException {
+ return getServiceConfigValue(
+ "for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//PROTOCOL[./@name='solr']/@address/string()");
+ }
+
+ public String getIndexSolrShutdownWait() throws ISLookUpException {
+ return queryForServiceProperty("solr:feedingShutdownTolerance");
+ }
+
+ public String getIndexBufferFlushTreshold() throws ISLookUpException {
+ return queryForServiceProperty("solr:feedingBufferFlushThreshold");
+ }
+
+ public String isFeedingSimulationMode() throws ISLookUpException {
+ return queryForServiceProperty("solr:feedingSimulationMode");
+ }
+
+ private String queryForServiceProperty(final String key) throws ISLookUpException {
+ return getServiceConfigValue(
+ "for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//SERVICE_PROPERTIES/PROPERTY[./@ key='"
+ + key + "']/@value/string()");
+ }
+
+ private String getServiceConfigValue(final String xquery) throws ISLookUpException {
+ log.debug("quering for service property: " + xquery);
+ final String res = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
+ if (StringUtils.isBlank(res)) { throw new IllegalStateException("unable to find unique service property, xquery: " + xquery); }
+ return res;
+ }
+
+ private String getFileName(final NodeToken token, final String fileNamePrefix) {
+ return "/" + fileNamePrefix + "_" + tableName(token) + "_" + token.getEnv().getAttribute("format") + ".seq";
+ }
+
+ private String getCollectionName(final NodeToken token) {
+ return env("format", token) + "-" + env("layout", token) + "-" + env("interpretation", token);
+ }
+
+ private String env(final String s, final NodeToken token) {
+ return token.getEnv().getAttribute(s);
+ }
+
+ /**
+ * Transforms each OAF record into a index record.
+ *
+ * @param format format
+ * @param layout layout
+ * @return resultset with transformed records
+ * @throws ISLookUpException could happen
+ * @throws IOException could happen
+ * @throws TransformerException could happen
+ */
+ protected String prepareXslt(final String format, final String layout) throws ISLookUpException, IOException, TransformerException {
+
+ final TransformerFactory factory = TransformerFactory.newInstance();
+ final Transformer layoutTransformer = factory.newTransformer(new StreamSource(new StringReader(readXslt(getLayoutToRecordStylesheet()))));
+
+ final StreamResult layoutToXsltXslt = new StreamResult(new StringWriter());
+
+ layoutTransformer.setParameter("format", format);
+ layoutTransformer.transform(new StreamSource(new StringReader(getLayoutSource(format, layout))), layoutToXsltXslt);
+
+ return new String(Hashing.encodeBase64(layoutToXsltXslt.getWriter().toString()));
+ }
+
+ private String readXslt(final String s) throws IOException {
+ ClassPathResource resource = new ClassPathResource(s);
+ InputStream inputStream = resource.getInputStream();
+ return IOUtils.toString(inputStream);
+ }
+
+ private String getLayoutSource(final String format, final String layout) throws ISLookUpDocumentNotFoundException, ISLookUpException {
+ return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
+ "collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + format + "']//LAYOUT[@name='" + layout
+ + "']");
+ }
+
+ public String getLayoutToRecordStylesheet() {
+ return layoutToRecordStylesheet;
+ }
+
+ public void setLayoutToRecordStylesheet(final String layoutToRecordStylesheet) {
+ this.layoutToRecordStylesheet = layoutToRecordStylesheet;
+ }
+
+ public String getHbaseTable() {
+ return hbaseTable;
+ }
+
+ @Required
+ public void setHbaseTable(final String hbaseTable) {
+ this.hbaseTable = hbaseTable;
+ }
+
+ public String getOutputRecordsPathParam() {
+ return outputRecordsPathParam;
+ }
+
+ public void setOutputRecordsPathParam(final String outputRecordsPathParam) {
+ this.outputRecordsPathParam = outputRecordsPathParam;
+ }
+
+ public String getRottenRecordsPathParam() {
+ return rottenRecordsPathParam;
+ }
+
+ public void setRottenRecordsPathParam(final String rottenRecordsPathParam) {
+ this.rottenRecordsPathParam = rottenRecordsPathParam;
+ }
+
+ public String getOafSchemaLocationProperty() {
+ return oafSchemaLocationProperty;
+ }
+
+ public void setOafSchemaLocationProperty(final String oafSchemaLocationProperty) {
+ this.oafSchemaLocationProperty = oafSchemaLocationProperty;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_journal_intersection.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_journal_intersection.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_journal_intersection.xml (revision 58540)
@@ -0,0 +1,72 @@
+
+
+
+
+ Fill hostedBy-Map (ISSNs in DOAJ) from PANGAEA [Context]
+
+ Fill hostedBy-Map
+ Context
+ PANGAEA
+ ISSNs
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ datarepository::unknown
+ native
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Fill hostedBy-Map (ISSNs in DOAJ) from PANGAEA [Context]
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfHostedByMapJournalIntersection
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_journal.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_journal.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Fill_HostedbyMap_journal.xml (revision 58540)
@@ -0,0 +1,74 @@
+
+
+
+
+ Fill hostedBy-Map (ISSNs) [Context]
+
+ Fill hostedBy-Map
+ Context
+ EntityRegistry, Aggregator
+ ISSNs
+
+ REPO_HI
+ 20
+
+
+ Verify if DS is pending
+
+ entityregistry,aggregator::pubsrepository
+ native
+
+
+
+
+
+
+
+
+ Validate DS
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ Fill hostedBy-Map (ISSNs) [Context]
+
+
+
+
+
+
+
+
+ Create MetaWorkflow
+
+ metaWfHostedByMapJournal
+
+
+
+
+
+
+
+ Update MetaWorkflow Status
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FinalizeIndexJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FinalizeIndexJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FinalizeIndexJobNode.java (revision 58540)
@@ -0,0 +1,71 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.provision.index.rmi.IndexService;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import static java.lang.String.format;
+
+public class FinalizeIndexJobNode extends BlackboardJobNode {
+
+ private static final Log log = LogFactory.getLog(FinalizeIndexJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ return getServiceLocator().getServiceId(IndexService.class);
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ final String indexDsId = getEnvParam(token, "index_id");
+
+ log.info("preparing blackboard job DELETE_BY_QUERY index: " + indexDsId);
+
+ final String backendId = getBackendId(indexDsId);
+ if (StringUtils.isBlank(backendId))
+ throw new MSROException("empty index backend Id");
+
+ job.setAction("DELETE_BY_QUERY");
+ job.getParameters().put("id", indexDsId);
+ job.getParameters().put("backend_Id", backendId);
+ job.getParameters().put("query", buildQuery(getEnvParam(token, "index.feed.timestamp")));
+ }
+
+ private String buildQuery(final String version) {
+ final String query =
+ String.format("__dsversion:{* TO %s}", InputDocumentFactory.getParsedDateField(version));
+
+ log.info("delete by query: " + query);
+
+ return query;
+ }
+
+ private String getEnvParam(final NodeToken token, final String name) throws MSROException {
+ final String value = token.getEnv().getAttribute(name);
+
+ if (StringUtils.isBlank(value))
+ throw new MSROException(format("unable to finalize index feeding, cannot find property '%s' in the workflow env.", name));
+
+ return value;
+ }
+
+ public String getBackendId(final String indexDsId) throws ISLookUpDocumentNotFoundException, ISLookUpException {
+ return getServiceLocator().getService(ISLookUpService.class).getResourceProfileByQuery(
+ "//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value='" + indexDsId + "']//BACKEND/text()");
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationResultToCommunityThroughOrganization.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationResultToCommunityThroughOrganization.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationResultToCommunityThroughOrganization.xml (revision 58540)
@@ -0,0 +1,102 @@
+
+
+
+ Propagation result 2 community through organization
+ Data Provision
+ 30
+
+
+ Set the CSV file path on HDFS
+
+ csvPath
+ /tmp/db_openaireplus_services_beta.communityPropagationOrganization
+
+
+
+
+
+
+
+
+
+ organizationCommunityMap
+ {"corda__h2020::3fb05a9524c3f790391261347852f638":["mes"]}
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+ dev_openaire
+
+
+
+
+
+
+ CSV files cleanup
+
+
+ {
+ 'path' : 'csvPath',
+ 'cluster' : 'cluster'
+ }
+
+
+
+
+
+
+
+
+
+ Propagation of ORCIT to result through semantic relation
+
+ propagationResultToCommunityThroughOrganizationSaveToFile
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'organization.community.map' : 'organizationCommunityMap',
+ 'mapred.output.dir' : 'csvPath'
+ }
+
+
+
+
+
+
+
+
+ wf_20180807_172327_510
+ 2019-06-28T17:24:53+02:00
+ SUCCESS
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FindSearchServiceJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FindSearchServiceJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FindSearchServiceJobNode.java (revision 58540)
@@ -0,0 +1,79 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Value;
+
+/**
+ * Superceded by eu.dnetlib.msro.openaireplus.workflows.nodes.index.FindSearchServicesJobNode.
+ *
+ * @author claudio, alessia
+ * @see eu.dnetlib.msro.openaireplus.workflows.nodes.index.FindSearchServicesJobNode
+ * @deprecated
+ */
+@Deprecated
+public class FindSearchServiceJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(FindSearchServiceJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Value(value = "${dnet.openaire.service.search.lookup.xquery}")
+ private String xquery;
+
+ private String xqueryForServiceIdParam;
+
+ /**
+ * {@inheritDoc}
+ *
+ * @throws ISLookUpException
+ * @throws MSROException
+ * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ public String execute(final NodeToken token) throws ISLookUpException, MSROException {
+
+ log.info("lookup for search service: " + getXquery());
+
+ List searchServiceIds = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+
+ if (searchServiceIds.size() > 1) { throw new MSROException("too many search services: " + searchServiceIds); }
+
+ if (searchServiceIds.isEmpty()) {
+ return "notFound";
+ } else {
+ token.getEnv().setAttribute(getXqueryForServiceIdParam(), getXquery());
+ return "found";
+ }
+ }
+
+ public String getXquery() {
+ return xquery;
+ }
+
+ public void setXquery(final String xquery) {
+ this.xquery = xquery;
+ }
+
+ public String getXqueryForServiceIdParam() {
+ return xqueryForServiceIdParam;
+ }
+
+ public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {
+ this.xqueryForServiceIdParam = xqueryForServiceIdParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationProjectResult.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationProjectResult.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/propagation/propagationProjectResult.xml (revision 58540)
@@ -0,0 +1,76 @@
+
+
+
+ Propagation of Project Results
+ Data Provision
+ 30
+
+
+ Set the CSV file path on HDFS
+
+ csvPath
+ /tmp/db_openaireplus_services_beta.projectPropagation
+
+
+
+
+
+
+ Set table name
+
+ DM
+ tableName
+ db_openaireplus_services_beta
+
+
+
+
+
+
+ CSV files cleanup
+
+
+ {
+ 'path' : 'csvPath',
+ 'cluster' : 'cluster'
+ }
+
+
+
+
+
+
+
+ propagation of project link between research result linked by semantic relations
+
+ propagationProjectToResult
+
+ {
+ 'cluster' : 'cluster',
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapred.outputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName',
+ 'hbase.mapreduce.outputtable' : 'tableName',
+ 'mapred.output.dir' : 'csvPath'
+ }
+
+
+
+
+
+
+
+
+ wf_20190613_074630_96
+ 2019-06-13T08:08:06+00:00
+ SUCCESS
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/ContextLoader.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/ContextLoader.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/ContextLoader.java (revision 58540)
@@ -0,0 +1,48 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import javax.annotation.Resource;
+
+import com.google.common.base.Joiner;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.functional.xml.IndentXmlString;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+public class ContextLoader {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(ContextLoader.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ private String xquery;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ public String load() throws ISLookUpException {
+
+ log.info("loading ContextDSResources: " + getXquery());
+
+ StringBuilder sb = new StringBuilder("");
+ Joiner.on("").appendTo(sb, serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery()));
+ sb.append(" ");
+
+ log.debug("got ContextDSResources: \n" + IndentXmlString.apply(sb.toString()));
+
+ return sb.toString();
+ }
+
+ public String getXquery() {
+ return xquery;
+ }
+
+ @Required
+ public void setXquery(final String xquery) {
+ this.xquery = xquery;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importOrcid.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importOrcid.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importOrcid.xml (revision 58540)
@@ -0,0 +1,78 @@
+
+
+
+ Import Orcid
+ Import InfoSpace
+ 30
+
+
+ set the hdfs output path
+
+ /usr/dnet/data/orcid
+ inputPath
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'orcidworks-no-doi',
+ 'jobProperty' : 'export_action_set_orcidworks_no_doi',
+ 'enablingProperty' : 'active_orcidworks_no_doi',
+ 'enabled' : 'true'
+ }
+ ]
+
+
+
+
+
+
+
+ extract the hdfs output path generated in the previous node
+
+ outputPath
+
+
+
+
+
+
+ IIS main
+
+ importOrcidJob
+ DM
+
+ {
+ 'mapred.input.dir':'inputPath',
+ 'mapred.output.dir':'outputPath'
+ }
+
+ false
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextUtils.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextUtils.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextUtils.java (revision 58540)
@@ -0,0 +1,186 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import java.io.StringReader;
+import java.lang.reflect.Type;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Maps;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.*;
+import org.dom4j.io.SAXReader;
+
+/**
+ * Created by claudio on 01/03/16.
+ */
+public class ContextUtils {
+
+ private static final Log log = LogFactory.getLog(ContextUtils.class);
+ private static final String DASHBOARD_VISIBILITY = "status";
+
+ public static ContextDesc getContext(final Iterable it,
+ final String contextId,
+ final String contextLabel,
+ final String contextType,
+ final String params, final String dashboardVisibility) throws
+ DocumentException {
+
+ final Map paramMap = getParamMap(params);
+ paramMap.put(DASHBOARD_VISIBILITY, dashboardVisibility);
+ return new ContextUtils().getContextDesc(it, contextId, contextLabel, contextType, paramMap);
+ }
+
+ private static Map getParamMap(String jsonMap) {
+ Map paramMap = Maps.newHashMap();
+
+ if (StringUtils.isNotBlank(jsonMap)) {
+ Type mapType = new TypeToken>() {
+ }.getType();
+ paramMap = new Gson().fromJson(jsonMap, mapType);
+ }
+ return paramMap;
+ }
+
+ private ContextDesc getContextDesc(final Iterable it,
+ final String contextId,
+ final String contextLabel,
+ final String contextType,
+ final Map params) throws
+ DocumentException {
+ final ContextDesc context = new ContextDesc(contextId, contextLabel, contextType, params);
+ final SAXReader reader = new SAXReader();
+
+ for (String s : it) {
+ populateContext(context, reader, s);
+ }
+ return context;
+ }
+
+ private void populateContext(final ContextDesc context, final SAXReader reader, final String s) throws DocumentException {
+ final Document doc = reader.read(new StringReader(s));
+
+ for (Object o : doc.selectNodes("//fundingtree")) {
+ final Element treeNode = (Element) o;
+
+ final String funder = treeNode.valueOf("./funder/id");
+ if (StringUtils.isBlank(funder)) {
+ log.debug("No funder found, skipping population from the following XML: \n" + s);
+ return;
+ }
+ if (!context.getDbEntries().containsKey(funder)) {
+ log.info("Found funder: " + funder);
+ context.getDbEntries().put(funder, "" + treeNode.selectSingleNode("./funder").asXML() + " ");
+ log.debug("db entry: " + context.getDbEntries().get(funder));
+ }
+
+ final String openaireId = treeNode.valueOf("./*[starts-with(local-name(),'funding_level_')]/id");
+ if (!context.getDbEntries().containsKey(openaireId) && StringUtils.isNotBlank(openaireId)) {
+ log.info("Found funding: " + openaireId);
+ context.getDbEntries().put(openaireId, treeNode.asXML());
+ log.debug("db entry: " + context.getDbEntries().get(openaireId));
+ final Node node0 = treeNode.selectSingleNode(".//funding_level_0");
+ if (node0 != null) {
+ final ContextPart part = calculatePart(node0);
+ if (context.getCategories().containsKey(part.getId())) {
+ for (ContextPart p : part.getParts().values()) {
+ context.getCategories().get(part.getId()).addPart(p);
+ }
+ } else {
+ context.getCategories().put(part.getId(), part);
+ }
+ }
+ }
+ }
+ }
+
+ private ContextPart calculatePart(final Node node) {
+
+ // final String newId = contextId + "::" + StringUtils.substringAfter(node.valueOf("./id"), "::");
+ // ids are built as: nsPrefix :: funderID :: fundingLevel0ID :: etc etc, hence it seems we might not need the contextId parameter.
+ final String newId = StringUtils.substringAfter(node.valueOf("./id"), "::");
+
+ final ContextPart part = new ContextPart(newId, node.valueOf("./description"));
+
+ part.getParams().put("name", node.valueOf("./name"));
+ part.getParams().put("openaireId", node.valueOf("./id"));
+ part.getParams().put("class", node.valueOf("./class"));
+
+ final Element parent = node.getParent() != null ? node.getParent().getParent() : null;
+
+ if (parent != null && parent.getName().startsWith("funding_level_")) {
+ final ContextPart p = calculatePart(parent);
+ part.getParts().put(p.getId(), p);
+ }
+ return part;
+ }
+
+ public static Function, String> getContextRowTransformer() {
+
+ return new Function, String>() {
+
+ private final SAXReader reader = new SAXReader();
+
+ @Override
+ public String apply(final Entry e) {
+ try {
+ final Document docFundingPath = reader.read(new StringReader(e.getValue()));
+
+ final Element root = DocumentHelper.createElement("ROWS");
+
+ final Map fundingFields = Maps.newLinkedHashMap();
+ fundingFields.put("_dnet_resource_identifier_", e.getKey());
+ fundingFields.put("id", e.getKey());
+ fundingFields.put("path", e.getValue());
+ fundingFields.put("jurisdiction", docFundingPath.valueOf("//funder/jurisdiction"));
+ final String desc = docFundingPath.valueOf("(//*[starts-with(local-name(),'funding_level_')])[1]/description");
+ if (StringUtils.isNotBlank(desc)) {
+ fundingFields.put("description", desc);
+ }
+ final Map orgFields = findFunderInfo(docFundingPath);
+ final String orgId = orgFields.get("id");
+ if (StringUtils.isNotBlank(orgId)) {
+ addRow(root, "dsm_organizations", orgFields);
+ fundingFields.put("funder", orgId);
+ }
+
+ addRow(root, "fundingpaths", fundingFields);
+
+ if (log.isDebugEnabled()) {
+ log.debug("Db entries: " + root.asXML());
+ }
+
+ return root.asXML();
+ } catch (DocumentException e1) {
+ log.error("Error parsing xml", e1);
+ throw new RuntimeException("Error parsing xml", e1);
+ }
+ }
+
+ private Map findFunderInfo(final Document doc) {
+ final Map res = Maps.newLinkedHashMap();
+ res.put("_dnet_resource_identifier_", doc.valueOf("//funder/id"));
+ res.put("id", doc.valueOf("//funder/id"));
+ res.put("legalshortname", doc.valueOf("//funder/shortname"));
+ res.put("legalname", doc.valueOf("//funder/name"));
+ res.put("country", doc.valueOf("//funder/jurisdiction"));
+ return res;
+ }
+
+ private void addRow(final Element root, final String table, final Map fields) {
+ final Element row = root.addElement("ROW");
+ row.addAttribute("table", table);
+ for (Map.Entry e : fields.entrySet()) {
+ final Element pathField = row.addElement("FIELD");
+ pathField.addAttribute("name", e.getKey());
+ pathField.setText(e.getValue());
+ }
+ }
+ };
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importGridAC.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importGridAC.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importGridAC.xml (revision 58540)
@@ -0,0 +1,83 @@
+
+
+
+ Import GridAC
+ Import InfoSpace
+ 30
+
+
+ set the hdfs output path
+
+ /usr/dnet/data/gridac
+ inputPath
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'gridac-dump',
+ 'jobProperty' : 'export_action_set_gridac_dump',
+ 'enablingProperty' : 'active_gridac_dump',
+ 'enabled' : 'true'
+ }
+ ]
+
+
+
+
+
+
+
+ extract the hdfs output path generated in the previous node
+
+ outputPath
+
+
+
+
+
+
+ IIS main
+
+ importGridAcJob
+ DM
+
+ {
+ 'mapred.input.dir':'inputPath',
+ 'mapred.output.dir':'outputPath'
+ }
+
+ false
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+ wf_20180807_092640_582
+ 2018-08-07T09:29:23+00:00
+ SUCCESS
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importActionsFromHDFS.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importActionsFromHDFS.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/hbase/importActionsFromHDFS.xml (revision 58540)
@@ -0,0 +1,61 @@
+
+
+
+
+ Actions from HDFS
+ Data Load
+ 30
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+
+ Set the sequence file path on HDFS holding the exported Actions
+
+ actionsPath
+ /tmp/2016-03-25_iis_preprocessing/document_referencedDatasets/rawset_76cfeefd-1139-4a30-a394-f01bf67220bf_1458652954774
+
+
+
+
+
+
+
+ Run M/R import Job
+
+ actionsImportJob
+
+ {
+ 'cluster' : 'cluster',
+ 'mapred.input.dir' : 'actionsPath',
+ 'hbase.mapred.outputtable' : 'tableName'
+ }
+
+ false
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/BuildH2020FETTaxonomy.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/BuildH2020FETTaxonomy.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/BuildH2020FETTaxonomy.java (revision 58540)
@@ -0,0 +1,279 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import java.io.IOException;
+import java.util.*;
+
+import com.google.common.collect.Maps;
+import com.ximpleware.AutoPilot;
+import com.ximpleware.VTDException;
+import com.ximpleware.VTDGen;
+import com.ximpleware.VTDNav;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+import org.joda.time.DateTime;
+
+import static eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions.md5;
+
+public class BuildH2020FETTaxonomy {
+
+ private static final Log log = LogFactory.getLog(BuildH2020FETTaxonomy.class);
+
+ private Iterator projects;
+ private Map>>> dic;
+
+ public BuildH2020FETTaxonomy() {
+ dic = Maps.newHashMap();
+ }
+
+ public void setIterator(Iterator projects) {
+ this.projects = projects;
+ }
+
+ public String parseProjects() throws Exception {
+ while (projects.hasNext())
+ parse(projects.next());
+ return getTaxonomy();
+ }
+
+ public String getTaxonomy() throws Exception {
+ Document taxonomy = DocumentHelper.createDocument();
+ Element root1 = taxonomy.addElement("RESOURCE_PROFILE");
+ createHeader(root1);
+ Element body = root1.addElement("BODY");
+ Element configurationElement = body.addElement("CONFIGURATION");
+ createContext(configurationElement);
+ body.addElement("STATUS");
+ body.addElement("SECURITY_PARAMETERS");
+ return taxonomy.getDocument().asXML();
+
+ }
+
+ private void createContext(final Element configurationElement) {
+ Element context = configurationElement.addElement("context");
+ context.addAttribute("type", "community");
+ context.addAttribute("id", "fet-h2020");
+ context.addAttribute("label", "FET H2020");
+ for (Map.Entry>>> entry : dic.entrySet()) {
+ String type = entry.getKey();
+ Element cat = context.addElement("category");
+ String catid = "fet-h2020::" + type.toLowerCase();
+ cat.addAttribute("id", catid);
+ cat.addAttribute("label", "FET " + type);
+ cat.addAttribute("claim", "false");
+ //int count = 0;
+ for (Map.Entry>> entryst : dic.get(type).entrySet()) {
+ String subtype = entryst.getKey();
+ for (Map.Entry> entryp : dic.get(type).get(subtype).entrySet()) {
+ String opt = entryp.getKey();
+ // count++;
+ Element subcat = cat.addElement("concept");
+ String proj_md5 = md5(opt).substring(0,12);
+ //String subcatid = catid + "::" + count;
+ String subcatid = catid + "::" + proj_md5;
+ subcat.addAttribute("id", subcatid);
+ Element pa = subcat.addElement("param");
+ pa.addAttribute("name", "CALL_ID");
+ List projectRows = dic.get(type).get(subtype).get(opt);
+ if (!projectRows.isEmpty()) {
+ for (Info row : projectRows) {
+ createProjectConcept(row, subcat, subcatid);
+ }
+ subcat.addAttribute("label", projectRows.get(0).getCallname());
+ subcat.addAttribute("claim", "false");
+ pa.setText(projectRows.get(0).getCallID());
+ } else throw new RuntimeException("The lis of projects for " + subcatid + " is empty. This exception should never be thrown.");
+ }
+ }
+ }
+ }
+
+ private void createProjectConcept(Info row, Element father, String prefix) {
+ String code = row.getCode();
+ String acronym = row.getAcronym();
+ String title = row.getTitle();
+ String projcallid = row.getProjcallid();
+ Element concept = father.addElement("concept");
+ concept.addAttribute("id", prefix + "::" + code);
+ concept.addAttribute("label", title);
+ concept.addAttribute("claim", "true");
+ Element p = concept.addElement("param");
+ p.addAttribute("name", "CD_PROJ_ID");
+ p.setText(code);
+ p = concept.addElement("param");
+ p.addAttribute("name", "CD_CALL_ID");
+ p.setText(projcallid);
+ p = concept.addElement("param");
+ p.addAttribute("name", "CD_PROJECT_NUMBER");
+ p.setText(code);
+ p = concept.addElement("param");
+ p.addAttribute("name", "CD_ACRONYM");
+ p.setText(acronym);
+ p = concept.addElement("param");
+ p.addAttribute("name", "CD_FRAMEWORK");
+ p.setText("H2020");
+ p = concept.addElement("param");
+ p.addAttribute("name", "funder");
+ p.setText("EC");
+ }
+
+ private void createHeader(final Element profile) {
+ Element header = profile.addElement("HEADER");
+ Element rs = header.addElement("RESOURCE_IDENTIFIER");
+ rs.addAttribute("value", "");
+ Element rt = header.addElement("RESOURCE_TYPE");
+ rt.addAttribute("value", "ContextDSResourceType");
+ Element rk = header.addElement("RESOURCE_KIND");
+ rk.addAttribute("value", "ContextDSResources");
+ Element ru = header.addElement("RESOURCE_URI");
+ ru.addAttribute("value", "");
+ Element daoc = header.addElement("DATE_OF_CREATION");
+ daoc.addAttribute("value", DateTime.now().toString());
+ }
+
+ private boolean parse(String project) throws VTDException, IOException {
+ boolean fet = false;
+ String callid = "";
+ String projectid = "";
+ VTDGen vg = new VTDGen();
+ vg.setDoc(project.getBytes("UTF-8"));
+ vg.parse(false);
+ VTDNav vn = vg.getNav();
+ AutoPilot ap = new AutoPilot(vn);
+ ap.selectXPath("//metadata/ROWS/ROW[@table=\"projects\"]");
+
+ while (ap.evalXPath() != -1) {
+ AutoPilot ap1 = new AutoPilot(vn);
+ ap1.selectXPath("./FIELD[@name=\"call_identifier\"]");
+ if (ap1.evalXPath() != -1)
+ callid = vn.toNormalizedString(vn.getText());
+ ap1.clearVariableExprs();
+ vn.toElement(VTDNav.PARENT);
+ ap1.selectXPath("./FIELD[@name=\"optional1\"]");
+ if (ap1.evalXPath() != -1)
+ projectid = vn.toNormalizedString(vn.getText());
+ if (callid.contains("FET")) {
+ String type;
+ if (callid.contains("OPEN"))
+ type = "OPEN";
+ else if (callid.contains("PROACT"))
+ type = "PROACT";
+ else if (callid.contains("HPC"))
+ type = "HPC";
+ else
+ type = "FLAG";
+ vn.toElement(VTDNav.ROOT);
+ fet = true;
+ insert(type, callid, projectid, new Info(vn));
+ }
+ }
+ return fet;
+ }
+
+ private void insert(String type, String callid, String projcallid, Info row) {
+ Map>> dopen;
+ Map> entry;
+ List projects;
+ if (dic.containsKey(type)) {
+ dopen = dic.get(type);
+ if (dopen.containsKey(callid)) {
+ entry = dopen.get(callid);
+ if (entry.containsKey(projcallid)) {
+ entry.get(projcallid).add(row);
+ } else {
+ projects = new ArrayList<>();
+ projects.add(row);
+ entry.put(projcallid, projects);
+ }
+ } else {
+ projects = new ArrayList<>();
+ entry = new HashMap<>();
+ projects.add(row);
+ entry.put(projcallid, projects);
+ dopen.put(callid, entry);
+ }
+ } else {
+ projects = new ArrayList<>();
+ entry = new HashMap<>();
+ dopen = new HashMap<>();
+ projects.add(row);
+ entry.put(projcallid, projects);
+ dopen.put(callid, entry);
+ dic.put(type, dopen);
+ }
+ }
+
+ public void parseFETProject(int projects_number) throws VTDException, IOException {
+ int parsed = 0;
+ while (projects.hasNext() && parsed < projects_number) {
+ if (parse(projects.next()))
+ parsed++;
+ }
+ log.debug(parsed);
+ }
+
+ private class Info {
+
+ VTDNav vn;
+ private String code, acronym, title, call_identifier, projcallid, callname;
+
+ public String getCode() {
+ return code;
+ }
+
+ public String getAcronym() {
+ return acronym;
+ }
+
+ public String getTitle() {
+ return title;
+ }
+
+ public String getCallID() {
+ return call_identifier;
+ }
+
+ public String getProjcallid() {
+ return projcallid;
+ }
+
+ public String getCallname() {
+ return callname;
+ }
+
+ public Info(VTDNav vn) {
+ this.vn = vn;
+ parse();
+ }
+
+ private void parse() {
+ try {
+ AutoPilot ap = new AutoPilot(vn);
+ ap.selectXPath("//FIELD");
+ while (ap.evalXPath() != -1) {
+ String aname = vn.toNormalizedString(vn.getAttrVal("name"));
+ String text = "";
+ if (vn.getText() != -1)
+ text = vn.toNormalizedString(vn.getText());
+ if (aname.equals("code"))
+ code = text;
+ else if (aname.equals("acronym"))
+ acronym = text;
+ else if (aname.equals("title"))
+ title = text;
+ else if (aname.equals("call_identifier"))
+ call_identifier = text;
+ else if (aname.equals("optional1"))
+ projcallid = text;
+ else if (aname.equals("optional2"))
+ callname = text;
+ }
+ } catch (Exception e) {
+ }
+ }
+
+ }
+
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextPart.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextPart.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextPart.java (revision 58540)
@@ -0,0 +1,83 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.collect.Maps;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+
+public class ContextPart {
+
+ private String id;
+ private String label;
+ private Map params = Maps.newLinkedHashMap();
+ private Map parts = Maps.newLinkedHashMap();
+
+ public ContextPart() {
+ }
+
+ public ContextPart(final String id, final String label) {
+ this.id = id;
+ this.label = label;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(final String label) {
+ this.label = label;
+ }
+
+ public Map getParams() {
+ return params;
+ }
+
+ public void setParams(final Map params) {
+ this.params = params;
+ }
+
+ public Map getParts() {
+ return parts;
+ }
+
+ public void setParts(final Map parts) {
+ this.parts = parts;
+ }
+
+ public void addPart(final ContextPart part) {
+ if (parts.containsKey(part.getId())) {
+ final ContextPart localChild = getParts().get(part.getId());
+ for (ContextPart child : part.getParts().values()) {
+ localChild.addPart(child);
+ }
+ } else {
+ parts.put(part.getId(), part);
+ }
+ }
+
+ public Element asDomElement(final String name) {
+ final Element elem = DocumentHelper.createElement(name);
+ elem.addAttribute("id", id);
+ elem.addAttribute("label", label);
+ elem.addAttribute("claim", "false");
+ for (Entry e : params.entrySet()) {
+ final Element p = elem.addElement("param");
+ p.addAttribute("name", e.getKey());
+ p.setText(e.getValue());
+ }
+ for (ContextPart child : parts.values()) {
+ elem.add(child.asDomElement("concept"));
+ }
+ return elem;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextDesc.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextDesc.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextDesc.java (revision 58540)
@@ -0,0 +1,110 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.dom4j.DocumentHelper;
+import org.dom4j.Element;
+
+public class ContextDesc {
+
+ private String id;
+ private String label;
+ private String type;
+ private String xml;
+ private Map categories = Maps.newLinkedHashMap();
+ private Map dbEntries = Maps.newLinkedHashMap();
+ private Map params;
+
+ public ContextDesc(final String id, final String label, final String type, final Map params) {
+ this.id = id;
+ this.label = label;
+ this.type = type;
+ this.params = params;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(final String label) {
+ this.label = label;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(final String type) {
+ this.type = type;
+ }
+
+ public String getXml() {
+ return xml;
+ }
+
+ public void setXml(final String xml) {
+ this.xml = xml;
+ }
+
+ public Map getCategories() {
+ return categories;
+ }
+
+ public void setCategories(final Map categories) {
+ this.categories = categories;
+ }
+
+ public Map getDbEntries() {
+ return dbEntries;
+ }
+
+ public void setDbEntries(final Map dbEntries) {
+ this.dbEntries = dbEntries;
+ }
+
+ public Element asDomElement() {
+ final Element ctxElem = DocumentHelper.createElement("context");
+ ctxElem.addAttribute("id", getId());
+ ctxElem.addAttribute("label", getLabel());
+ ctxElem.addAttribute("type", getType());
+ for (Element param : getParamsAsElements()) {
+ ctxElem.add(param);
+ }
+ for (ContextPart part : getCategories().values()) {
+ ctxElem.add(part.asDomElement("category"));
+ }
+
+ return ctxElem;
+ }
+
+ public List getParamsAsElements() {
+ List elements = Lists.newArrayList();
+ for (Entry e : params.entrySet()) {
+ Element param = DocumentHelper.createElement("param");
+ param.addAttribute("name", e.getKey());
+ param.setText(e.getValue());
+ elements.add(param);
+ }
+ return elements;
+ }
+
+ public Map getParams() {
+ return params;
+ }
+
+ public void setParams(final Map params) {
+ this.params = params;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedBy.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedBy.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedBy.java (revision 58540)
@@ -0,0 +1,114 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby;
+
+import java.io.StringReader;
+import java.util.Map;
+
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+
+/**
+ * The Class PatchHostedBy.
+ */
+public class PatchHostedBy implements UnaryFunction {
+
+ /**
+ * The set spec hosted by map.
+ */
+ private Map setSpecHostedByMap;
+
+ /**
+ * The counters.
+ */
+ private HostedByCounters counters;
+
+ /**
+ * The xpath.
+ */
+ private String xpath;
+
+ /**
+ * The reader.
+ */
+ private final SAXReader reader = new SAXReader();
+
+ /**
+ * The Constant log.
+ */
+ private static final Log log = LogFactory.getLog(PatchHostedBy.class);
+
+ /**
+ * Instantiates a new patch hosted by.
+ *
+ * @param setSpecHostedByMap the set spec hosted by map
+ * @param xpath the xpath
+ * @param counters the counters
+ */
+ public PatchHostedBy(final Map setSpecHostedByMap, final String xpath, final HostedByCounters counters) {
+ this.setSpecHostedByMap = setSpecHostedByMap;
+ this.xpath = xpath;
+ this.counters = counters;
+ if (log.isDebugEnabled()) {
+ log.debug("****************************************");
+ log.debug("SetSpec/hostedBy map:");
+ for (Map.Entry e : setSpecHostedByMap.entrySet()) {
+ log.debug(" " + e.getKey() + " -> " + e.getValue());
+ }
+ log.debug("****************************************");
+ }
+ }
+
+ /**
+ * Evaluate.
+ *
+ * @param record the record
+ * @return the string
+ */
+ @Override
+ public String evaluate(final String record) {
+ try {
+ final Document doc = reader.read(new StringReader(record));
+ final Element node = (Element) doc.selectSingleNode("//*[local-name()='metadata']/*[local-name()='hostedBy']");
+ if (node != null) {
+ final HostedByEntry ds = findHostedBy(doc);
+ if (ds != null) {
+ node.addAttribute("id", ds.getId());
+ node.addAttribute("name", ds.getName());
+ counters.increaseCounter(ds.getId());
+ }
+ return doc.asXML();
+ } else if (log.isDebugEnabled()) {
+ log.debug(" -- Missing hostedBy --");
+ }
+ } catch (Throwable e) {
+ log.error("Error adding hosted by to " + record);
+ }
+ return record;
+ }
+
+ /**
+ * Find hosted by.
+ *
+ * @param doc the doc
+ * @return the hosted by entry
+ */
+ private HostedByEntry findHostedBy(final Document doc) {
+ for (Object o : doc.selectNodes(this.xpath)) {
+ final String set = ((Node) o).getText().trim();
+ if (setSpecHostedByMap.containsKey(set)) {
+ if (log.isDebugEnabled()) {
+ log.debug(set + " -> " + setSpecHostedByMap.get(set));
+ }
+ return setSpecHostedByMap.get(set);
+ } else if (log.isDebugEnabled()) {
+ log.debug(set + " -> UNKNOWN REPO");
+ }
+ }
+ return null;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/export.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/export.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/openorgs/export.meta.xml (revision 58540)
@@ -0,0 +1,25 @@
+
+
+
+
+ Export to OpenOrgs DB
+
+ InfoSpace Deduplication
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/HostedByEntry.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/HostedByEntry.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/HostedByEntry.java (revision 58540)
@@ -0,0 +1,34 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby;
+
+public class HostedByEntry {
+
+ private String id;
+ private String name;
+
+ public HostedByEntry(String id, String name) {
+ this.id = id;
+ this.name = name;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return "[ id: " + id + ", name: " + name + " ]";
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/UpsertHostedByApisJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/UpsertHostedByApisJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/UpsertHostedByApisJobNode.java (revision 58540)
@@ -0,0 +1,92 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby;
+
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;
+import eu.dnetlib.enabling.datasources.common.Api;
+import eu.dnetlib.enabling.datasources.common.ApiParam;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class UpsertHostedByApisJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(UpsertHostedByApisJobNode.class);
+
+ private static final String HOSTED_BY_COMPLIANCE = "hostedBy";
+
+ @Autowired
+ private LocalOpenaireDatasourceManager dsManager;
+
+ private String countersParam;
+ private String mdId;
+ public static final String extraFieldsForTotal = "last_aggregation_total";
+ public static final String extraFieldForDate = "last_aggregation_date";
+ public static final String extraFieldForMdId = "last_aggregation_mdId";
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final HostedByCounters counters = (HostedByCounters) token.getEnv().getTransientAttribute(countersParam);
+ final String date = DateUtils.now_ISO8601();
+ final String namespacePrefix = token.getEnv().getAttribute("namespacePrefix");
+
+ log.info(counters);
+
+ for (final Entry e : counters.getCounters().entrySet()) {
+ updateHostedByApi(e.getKey(), namespacePrefix, date, e.getValue());
+ }
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ private void updateHostedByApi(final String dsId,
+ final String namepsacePrefix,
+ final String date,
+ final int size) {
+ log.info("Verifying hostedBy api in ds: " + dsId);
+ try {
+
+ for (final Api iface : dsManager.getApis(dsId)) {
+ if (HOSTED_BY_COMPLIANCE.equals(iface.getCompatibility())) { return; }
+ }
+
+ final Api api = new Api<>();
+ api.setDatasource(dsId);
+ api.setId("api_________::" + dsId + "::hostedBy");
+ api.setTypology(dsManager.getDs(dsId).getTypology());
+ api.setCompatibility(HOSTED_BY_COMPLIANCE);
+ api.setProtocol("UNKNOWN");
+ api.setContentdescription("metadata");
+ api.setBaseurl("");
+ api.setActive(false);
+ api.setRemovable(true);
+ dsManager.addApi(api);
+ } catch (final DsmException e) {
+ log.warn("Error setting hostedBy api of ds: " + dsId, e);
+ }
+ }
+
+ public String getMdId() {
+ return mdId;
+ }
+
+ public void setMdId(final String mdId) {
+ this.mdId = mdId;
+ }
+
+ public String getCountersParam() {
+ return countersParam;
+ }
+
+ public void setCountersParam(final String countersParam) {
+ this.countersParam = countersParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/test-post.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/test-post.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/test-post.xml (revision 58540)
@@ -0,0 +1,37 @@
+
+
+
+
+ Test POST VRE
+ Testing
+ 30
+
+
+ Post news on the VRE
+
+ http
+ XXX
+ XXX
+ XXX
+ Preview title
+ Preview description
+ Post text
+ http image URL
+ false
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PrepareHostedbyUpdateSQL.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PrepareHostedbyUpdateSQL.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PrepareHostedbyUpdateSQL.java (revision 58540)
@@ -0,0 +1,52 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+/**
+ * Created by alessia on 01/02/16.
+ */
+public class PrepareHostedbyUpdateSQL extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(PrepareHostedbyUpdateSQL.class);
+ private String updateSqlTemplatePath;
+ private String theQueryParamName = "hostedByUpdateSQL";
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String datasourceId = token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID);
+ final String datasourceNamespacePrefix = token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_NAMESPACE_PREFIX);
+ final Resource updateSqlTemplate = new ClassPathResource(updateSqlTemplatePath);
+ final StringTemplate st = new StringTemplate(IOUtils.toString(updateSqlTemplate.getInputStream()));
+ st.setAttribute("datasourceId", datasourceId);
+ st.setAttribute("datasourceNamespacePrefix", datasourceNamespacePrefix);
+ final String theQuery = st.toString();
+ log.debug("Generated query: " + theQuery);
+ token.getFullEnv().setAttribute(theQueryParamName, theQuery);
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getUpdateSqlTemplatePath() {
+ return updateSqlTemplatePath;
+ }
+
+ public void setUpdateSqlTemplatePath(final String updateSqlTemplatePath) {
+ this.updateSqlTemplatePath = updateSqlTemplatePath;
+ }
+
+ public String getTheQueryParamName() {
+ return theQueryParamName;
+ }
+
+ public void setTheQueryParamName(final String theQueryParamName) {
+ this.theQueryParamName = theQueryParamName;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/content.publishing.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/content.publishing.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/content.publishing.xml (revision 58540)
@@ -0,0 +1,182 @@
+
+
+
+ Publish Information Space
+ Content Publishing
+ 30
+
+
+ Start content Publishing
+
+
+
+
+
+
+
+
+ Do we promote the shadow stats to public today?
+
+ YES
+
+
+
+
+
+
+
+ find stats service
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+ xqueryForStatsManagerService
+
+
+
+
+
+
+
+ Copy shadow Joomla cache and stats db to public schema
+
+ xqueryForStatsManagerService
+ beta
+
+
+
+
+
+
+ Do we switch the index read by the public Search Service today?
+
+ YES
+
+
+
+
+
+
+
+ Select index mdformat
+
+ TMF
+ index
+ openaire
+
+
+
+
+
+
+
+ find search services
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='SearchServiceResourceType'
+ and .//RESOURCE_KIND/@value='ServiceResources' and
+ .//PROPERTY[@key="infrastructure"]/@value="public"]/HEADER/RESOURCE_IDENTIFIER/@value/string()
+
+
+
+
+
+
+
+
+ switch index
+
+
+
+
+
+
+
+ switch index
+
+ index_id
+ IndexId
+
+
+
+
+
+
+ Do we switch the store used by the OAI Publisher today?
+
+ NO
+
+
+
+
+
+
+
+ Update the currentdb in the OAI configuration profile
+
+ oaistore2
+
+
+
+
+
+
+
+ Wait for the search branch to fail or to success
+
+
+
+
+
+
+ Wait for the stats branch to fail or success
+
+
+
+
+
+
+ Wait for the OAI branch to fail or to success
+
+
+
+
+
+
+
+ wait for all the branches to complete
+
+
+
+
+
+
+ Do we feed the the most recent claims today?
+
+ YES
+
+
+
+
+
+
+
+ Feed the most recent claims
+
+
+
+
+
+
+
+ wf_20170723_163915_443
+ 2017-07-23T16:39:26+00:00
+ SUCCESS
+
+ YES
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/stats.prepare.public.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/stats.prepare.public.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/stats.prepare.public.xml (revision 58540)
@@ -0,0 +1,92 @@
+
+
+
+
+ Prepare Public Information Space Stats
+ Content Publishing
+ 30
+
+
+
+ find stats service
+
+ /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='StatsManagerServiceResourceType' and .//RESOURCE_KIND/@value='ServiceResources']/HEADER/RESOURCE_IDENTIFIER/@value/string()
+ xqueryForStatsManagerService
+
+
+
+
+
+
+ Set the target portal instance
+
+ portal
+
+
+
+
+
+
+
+ A backup of the stats cache first, shall we?
+
+ YES
+
+
+
+
+
+
+
+ Backup the current stats cache. If the new generated shadow cache is not good we can rollback with the restore wf.
+
+ xqueryForStatsManagerService
+ portal
+
+
+
+
+
+
+ Shall we migrate the cache from another portal?
+
+ YES
+
+
+
+
+
+
+
+ Select the cache from another portal
+
+ xqueryForStatsManagerService
+ portal
+
+
+
+
+
+
+
+ Updates the shadow cache according to the shadow stats db.
+
+ xqueryForStatsManagerService
+ portal
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/EndReadingMDStoreJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/EndReadingMDStoreJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/EndReadingMDStoreJobNode.java (revision 58540)
@@ -0,0 +1,53 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class EndReadingMDStoreJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(EndReadingMDStoreJobNode.class);
+
+ /* Spring managed params */
+ private String mdStoreManagerUrl;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String url = getMdStoreManagerUrl() + "/version/{versionId}/endReading";
+
+ final Map params = new HashMap<>();
+ params.put("versionId", token.getEnv().getAttribute("reading_versionId"));
+
+ final URI uri = UriComponentsBuilder.fromUriString(url)
+ .buildAndExpand(params)
+ .toUri();
+
+ final RestTemplate restTemplate = new RestTemplate();
+ restTemplate.getForObject(uri, MDStoreVersion.class);
+
+ log.info("mdstore version read complete ");
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdStoreManagerUrl() {
+ return mdStoreManagerUrl;
+ }
+
+ @Required
+ public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {
+ this.mdStoreManagerUrl = mdStoreManagerUrl;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase2hdfs.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase2hdfs.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/hbase2hdfs.xml (revision 58540)
@@ -0,0 +1,140 @@
+
+
+
+ HBase to HDFS
+ Data Provision
+ 30
+
+
+
+
+ set mdformat, layout, interpretation
+
+ TMF
+ index
+ openaire
+
+
+
+
+
+
+
+
+ dnet.openaire.model.relclasses.xquery
+ relClasses
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Prepare indexing
+
+ hdfsRecordsPath
+ rottenRecordsPath
+ /eu/dnetlib/msro/openaireplus/workflows/index/openaireLayoutToRecordStylesheet.xsl
+
+ oaf.schema.location
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'hdfsRecordsPath'
+ }
+
+
+
+
+
+
+
+ hdfs cleanup (rotten)
+
+ DM
+
+ {
+ 'path' : 'rottenRecordsPath'
+ }
+
+
+
+
+
+
+
+ M/R group entities
+
+ DM
+ prepareIndexDataJob
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable'
+ }
+
+
+ {
+ 'mapred.output.dir' : 'hdfsRecordsPath',
+ 'index.entity.links' : 'index.entity.links',
+ 'oaf.schema.location' : 'oaf.schema.location',
+ 'contextmap' : 'contextmap',
+ 'relClasses' : 'relClasses'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentSoftwareLinksEvents.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentSoftwareLinksEvents.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentSoftwareLinksEvents.xml (revision 58540)
@@ -0,0 +1,75 @@
+
+
+
+ Calculate enrichment Software links events
+ Notification Broker
+ 30
+
+
+
+ Set table name
+
+ DM
+ tableName
+
+
+
+
+
+
+
+
+ Set Dedup conf
+
+
+ dedup.conf.queue
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'brokerEventsPath'
+ }
+
+
+
+
+
+
+
+
+
+ Run M/R Job
+
+ DM
+ brokerEnrichmentSoftwareLinksJob
+ dedup.conf.queue
+
+ {
+ 'hbase.mapred.inputtable' : 'tableName',
+ 'hbase.mapreduce.inputtable' : 'tableName'
+ }
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/DnetMessageManager.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/DnetMessageManager.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/message/DnetMessageManager.java (revision 58540)
@@ -0,0 +1,144 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp.message;
+
+
+import eu.dnetlib.message.Message;
+import eu.dnetlib.message.MessageManager;
+import eu.dnetlib.message.MessageType;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+
+public class DnetMessageManager {
+ private static final Log log = LogFactory.getLog(DnetMessageManager.class);
+
+ private MessageManager manager;
+
+ private LinkedBlockingQueue messages = new LinkedBlockingQueue<>();
+
+
+ private final Map onGonginMessages = new HashMap<>();
+
+ private final Map> reportMessages = new HashMap<>();
+
+ private String messageQueueServer;
+
+ private String username;
+
+ private String password;
+
+ @Value("${dnet.openaire.messageManager.ongoingMessageQueueName}")
+ private String onGoingMessageQueue;
+
+ @Value("${dnet.openaire.messageManager.reportMessageQueueName}")
+ private String reportMessageQueue;
+
+
+ public DnetMessageManager() {
+
+ }
+
+
+ private String createReportId(final String wfId, final String jobName) {
+ return String.format("%s::%s", wfId, jobName);
+ }
+
+ public void startListeningMessage() throws Exception {
+ if (manager == null && StringUtils.isNotBlank(messageQueueServer) && StringUtils.isNotBlank(reportMessageQueue)) {
+ manager = new MessageManager(messageQueueServer, username, password, messages);
+ manager.startConsumingMessage(onGoingMessageQueue, true, false);
+ manager.startConsumingMessage(reportMessageQueue, true, false);
+
+ Runnable r = () -> {
+ while (true) {
+ try {
+ Message currentMessage = messages.take();
+
+ if (currentMessage.getType() == MessageType.ONGOING) {
+ synchronized (onGonginMessages) {
+ onGonginMessages.put(currentMessage.getWorkflowId(), currentMessage);
+ }
+ } else {
+ synchronized (reportMessages) {
+ if (!reportMessages.containsKey(currentMessage.getWorkflowId()))
+ {
+ reportMessages.put(currentMessage.getWorkflowId(), new ArrayList<>());
+ }
+ reportMessages.get(currentMessage.getWorkflowId()).add(currentMessage);
+ }
+ }
+ } catch (InterruptedException e) {
+ log.error("An error occured on retrieving messages from the blocking queue",e);
+ throw new RuntimeException("An error occured on retrieving messages from the blocking queue",e);
+ }
+
+ }
+ };
+ new Thread(r).start();
+ }
+ }
+
+ public List getReport(final String workflowId) {
+
+ return getMessages(reportMessages, workflowId);
+ }
+
+ private List getMessages(final Map> messageMap, String reportId) {
+ if (messageMap.containsKey(reportId)) {
+ List m = messageMap.get(reportId);
+ messageMap.remove(reportId);
+ return m;
+ }
+ return null;
+ }
+
+
+ private Message getMessage(final Map messageMap, String reportId) {
+ if (messageMap.containsKey(reportId)) {
+ Message m = messageMap.get(reportId);
+ messageMap.remove(reportId);
+ return m;
+ }
+ return null;
+ }
+
+
+ public Message getOnGoingMessages(final String workflowId) {
+ return getMessage(onGonginMessages, workflowId);
+ }
+
+
+ public String getMessageQueueServer() {
+ return messageQueueServer;
+ }
+
+ @Required
+ public void setMessageQueueServer(String messageQueueServer) {
+ this.messageQueueServer = messageQueueServer;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ @Required
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ @Required
+ public void setPassword(String password) {
+ this.password = password;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareEnvCollectHadoopJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareEnvCollectHadoopJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareEnvCollectHadoopJobNode.java (revision 58540)
@@ -0,0 +1,100 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import eu.dnetlib.dhp.model.mdstore.Provenance;
+import eu.dnetlib.enabling.datasources.common.Datasource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.google.gson.Gson;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.collector.worker.model.ApiDescriptor;
+import eu.dnetlib.enabling.datasources.common.ApiParam;
+import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+
+public class PrepareEnvCollectHadoopJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(PrepareEnvCollectHadoopJobNode.class);
+
+ @Autowired
+ private LocalDatasourceManager, ?> dsManager;
+
+ private String hdfsBasePath;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ // param 1 : hdfs path
+ // param 2 : api descriptor (json)
+ // param 3 : nameNode
+
+ final String dsId = token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ID);
+ final String apiId = token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE);
+ final String mdId = token.getEnv().getAttribute("mdId");
+ final String versionId = token.getEnv().getAttribute("versionId");
+
+ final Optional opt = dsManager.getApis(dsId)
+ .stream()
+ .filter(a -> a.getId().equals(apiId))
+ .map(a -> {
+ final ApiDescriptor res = new ApiDescriptor();
+ res.setBaseUrl(a.getBaseurl());
+ res.setId(a.getId());
+ res.setProtocol(a.getProtocol());
+ res.getParams().put("metadata_identifier_path", a.getMetadataIdentifierPath());
+ res.getParams().putAll(a.getApiParams()
+ .stream()
+ .map(o -> (ApiParam) o)
+ .collect(Collectors.toMap(ApiParam::getParam, ApiParam::getValue)));
+ return res;
+ })
+ .findFirst();
+
+ if (opt.isPresent()) {
+ final ApiDescriptor api = opt.get();
+ final String hdfsPath = String.format("%s/%s/%s/store", hdfsBasePath, mdId, versionId);
+ final String seqFilePath = String.format("%s/%s/%s/seqFile", hdfsBasePath, mdId, versionId);
+ token.getEnv().setAttribute("apiDescription", new Gson().toJson(api));
+ token.getEnv().setAttribute("mdStorePath", hdfsPath);
+ token.getEnv().setAttribute("sequenceFilePath", seqFilePath);
+ final Provenance provenance = new Provenance();
+ provenance.setDatasourceId(dsId);
+ final Datasource, ?> ds = dsManager.getDs(dsId);
+ provenance.setDatasourceName(ds.getOfficialname());
+ provenance.setNsPrefix(ds.getNamespaceprefix());
+ token.getEnv().setAttribute("dataSourceInfo", new Gson().toJson(provenance));
+ token.getEnv().setAttribute("timestamp", ""+System.currentTimeMillis());
+ token.getEnv().setAttribute("identifierPath",api.getParams().get("metadata_identifier_path"));
+ token.getEnv().setAttribute("workflowId",token.getProcess().getEnv().getAttribute("system:processId"));
+
+ token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE_BASEURL, api.getBaseUrl());
+ token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_PREFIX + "protocol", api.getProtocol());
+ final Map params = api.getParams();
+ if (params != null) {
+ for(Map.Entry e : params.entrySet()) {
+ token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_PREFIX + e.getKey(), e.getValue());
+ }
+ }
+
+ return Arc.DEFAULT_ARC;
+ } else {
+ return "abort";
+ }
+
+ }
+
+ public String getHdfsBasePath() {
+ return hdfsBasePath;
+ }
+
+ public void setHdfsBasePath(String hdfsBasePath) {
+ this.hdfsBasePath = hdfsBasePath;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/DeleteMDStoreHadoopJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/DeleteMDStoreHadoopJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/DeleteMDStoreHadoopJobNode.java (revision 58540)
@@ -0,0 +1,63 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class DeleteMDStoreHadoopJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(DeleteMDStoreHadoopJobNode.class);
+
+ private String mdId;
+
+ /* Spring managed params */
+ private String mdStoreManagerUrl;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String url = getMdStoreManagerUrl() + "/mdstore/{mdId}";
+
+ final Map params = new HashMap<>();
+ params.put("mdId", getMdId());
+
+ final URI uri = UriComponentsBuilder.fromUriString(url)
+ .buildAndExpand(params)
+ .toUri();
+
+ final RestTemplate restTemplate = new RestTemplate();
+ restTemplate.delete(uri);
+
+ log.info("mdstor deleted: " + mdId);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdStoreManagerUrl() {
+ return mdStoreManagerUrl;
+ }
+
+ @Required
+ public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {
+ this.mdStoreManagerUrl = mdStoreManagerUrl;
+ }
+
+ public String getMdId() {
+ return mdId;
+ }
+
+ public void setMdId(final String mdId) {
+ this.mdId = mdId;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculatePersonDistribution.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculatePersonDistribution.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculatePersonDistribution.xml (revision 58540)
@@ -0,0 +1,81 @@
+
+
+
+ Calculate person distribution
+ Notification Broker
+ 30
+
+
+
+
+ /tmp/brokerPersonDistribution.tmp.seq
+ brokerPersonDistributionPath
+
+
+
+
+
+
+
+ hdfs cleanup (xml)
+
+ DM
+
+ {
+ 'path' : 'brokerPersonDistributionPath'
+ }
+
+
+
+
+
+
+
+
+
+ Run M/R Job
+
+ DM
+ calculatePersonDistributionStep1Job
+
+ {
+ 'mapred.output.dir' : 'brokerPersonDistributionPath'
+ }
+
+
+ {
+ 'hbase.mapred.inputtable' : 'hbase.mapred.datatable',
+ 'hbase.mapreduce.inputtable' : 'hbase.mapred.datatable'
+ }
+
+
+
+
+
+
+
+ Run M/R Job
+
+ DM
+ calculatePersonDistributionStep2Job
+
+ {
+ 'mapred.input.dir' : 'brokerPersonDistributionPath'
+ }
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/findInvalidRepoProfiles.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/findInvalidRepoProfiles.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/findInvalidRepoProfiles.xml (revision 58540)
@@ -0,0 +1,57 @@
+
+
+
+ Find Invalid Profiles
+ Consistency
+ 30
+
+
+ Find Invalid Repo Profiles
+
+ false
+ false
+
+
+
+
+
+
+
+ Find Invalid Meta Wf Profiles
+
+ false
+
+
+
+
+
+
+
+ Find Invalid Wf Profiles
+
+ false
+
+
+
+
+
+
+
+ Find Invalid [Md/Obj]Stores
+
+ false
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/reposStoresConsistency.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/reposStoresConsistency.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/reposStoresConsistency.xml (revision 58540)
@@ -0,0 +1,26 @@
+
+
+
+ Repos-Stores consistency
+ Consistency
+ 30
+
+
+ Fix repos-stores sizes
+
+ false
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-mainV2.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-mainV2.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-mainV2.xml (revision 58540)
@@ -0,0 +1,189 @@
+
+
+
+
+ IIS main workflow V2
+ IIS
+ 30
+
+
+ start
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'iis-document-affiliation',
+ 'jobProperty' : 'export_action_set_id_matched_doc_organizations',
+ 'enablingProperty' : 'active_document_affiliation',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenced-projects-main',
+ 'jobProperty' : 'export_action_set_id_document_referencedProjects',
+ 'enablingProperty' : 'active_referenceextraction_project',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenced-datasets-main',
+ 'jobProperty' : 'export_action_set_id_document_referencedDatasets',
+ 'enablingProperty' : 'active_referenceextraction_dataset',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-dataset-entities-main',
+ 'jobProperty' : 'export_action_set_id_entity_dataset',
+ 'enablingProperty' : 'active_referenceextraction_dataset',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-researchinitiative',
+ 'jobProperty' : 'export_action_set_id_document_research_initiative',
+ 'enablingProperty' :
+ 'active_referenceextraction_researchinitiative',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-document-similarities',
+ 'jobProperty' : 'export_action_set_id_document_similarities_standard',
+ 'enablingProperty' : 'active_documentssimilarity',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-document-classes',
+ 'jobProperty' : 'export_action_set_id_document_classes',
+ 'enablingProperty' : 'active_documentsclassification',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-document-citations',
+ 'jobProperty' : 'export_action_set_id_document_referencedDocuments',
+ 'enablingProperty' : 'active_citationmatching',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenceextraction-pdb',
+ 'jobProperty' : 'export_action_set_id_document_pdb',
+ 'enablingProperty' : 'active_referenceextraction_pdb',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenceextraction-software',
+ 'jobProperty' : 'export_action_set_id_document_software_url',
+ 'enablingProperty' : 'active_referenceextraction_software_url',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-entities-software',
+ 'jobProperty' : 'export_action_set_id_entity_software',
+ 'enablingProperty' : 'active_referenceextraction_software_url',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-communities',
+ 'jobProperty' : 'export_action_set_id_document_community',
+ 'enablingProperty' : 'active_referenceextraction_community',
+ 'enabled' : 'true'
+ }
+ ]
+
+
+
+
+
+
+
+ prepare parameters
+
+ import_islookup_service_location
+ import_content_objectstores_csv
+ import_content_object_store_location
+ import_mdstore_service_location
+ import_dataset_mdstore_ids_csv
+ oozie.wf.application.path
+ /tmp/integration/apps/main
+ IIS
+
+ import_hbase_dump_location
+
+ import_project_concepts_context_ids_csv
+ fet-fp7,fet-h2020
+
+
+
+
+
+
+ IIS main
+
+ iisMainJobV2
+
+ {
+ 'cluster' : 'cluster',
+ 'oozie.wf.application.path' : 'oozie.wf.application.path',
+
+ 'active_document_affiliation' : 'active_document_affiliation',
+ 'active_referenceextraction_project' : 'active_referenceextraction_project',
+ 'active_referenceextraction_dataset' : 'active_referenceextraction_dataset',
+ 'active_referenceextraction_researchinitiative' : 'active_referenceextraction_researchinitiative',
+ 'active_documentsclassification' : 'active_documentsclassification',
+ 'active_documentssimilarity' : 'active_documentssimilarity',
+ 'active_citationmatching' : 'active_citationmatching',
+ 'active_referenceextraction_pdb' : 'active_referenceextraction_pdb',
+ 'active_referenceextraction_software_url' : 'active_referenceextraction_software_url',
+ 'active_referenceextraction_community' : 'active_referenceextraction_community',
+
+ 'import_content_objectstores_csv' : 'import_content_objectstores_csv',
+ 'import_content_object_store_location' : 'import_content_object_store_location',
+ 'import_mdstore_service_location' : 'import_mdstore_service_location',
+ 'import_islookup_service_location' : 'import_islookup_service_location',
+ 'import_project_concepts_context_ids_csv' : 'import_project_concepts_context_ids_csv',
+ 'import_dataset_mdstore_ids_csv' : 'import_dataset_mdstore_ids_csv',
+ 'import_hbase_dump_location' : 'import_hbase_dump_location',
+
+ 'export_action_set_id_matched_doc_organizations' : 'export_action_set_id_matched_doc_organizations',
+ 'export_action_set_id_document_referencedDatasets' : 'export_action_set_id_document_referencedDatasets',
+ 'export_action_set_id_document_referencedProjects' : 'export_action_set_id_document_referencedProjects',
+ 'export_action_set_id_document_research_initiative' : 'export_action_set_id_document_research_initiative',
+ 'export_action_set_id_document_similarities_standard' : 'export_action_set_id_document_similarities_standard',
+
+ 'export_action_set_id_document_classes' : 'export_action_set_id_document_classes',
+ 'export_action_set_id_document_referencedDocuments' : 'export_action_set_id_document_referencedDocuments',
+ 'export_action_set_id_entity_dataset' : 'export_action_set_id_entity_dataset',
+ 'export_action_set_id_document_pdb' : 'export_action_set_id_document_pdb',
+ 'export_action_set_id_document_software_url' : 'export_action_set_id_document_software_url',
+ 'export_action_set_id_entity_software' : 'export_action_set_id_entity_software',
+ 'export_action_set_id_document_community' : 'export_action_set_id_document_community'
+ }
+
+ false
+ build-report
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/CreateMDStoreHadoopJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/CreateMDStoreHadoopJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/CreateMDStoreHadoopJobNode.java (revision 58540)
@@ -0,0 +1,102 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.data.mdstore.manager.common.model.MDStoreWithInfo;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+
+public class CreateMDStoreHadoopJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(CreateMDStoreHadoopJobNode.class);
+
+ /* Workflow params */
+ private String format;
+ private String layout;
+ private String interpretation;
+ private String outputPrefix = "mdstore";
+
+ /* Spring managed params */
+ private String mdStoreManagerUrl;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String url = getMdStoreManagerUrl() + "/new/{format}/{layout}/{interpretation}";
+
+ final Map params = new HashMap<>();
+ params.put("format", getFormat());
+ params.put("layout", getLayout());
+ params.put("interpretation", getInterpretation());
+
+ final URI uri = UriComponentsBuilder.fromUriString(url)
+ .queryParam("dsName", token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_NAME))
+ .queryParam("dsId", token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ID))
+ .queryParam("apiId", token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE))
+ .buildAndExpand(params)
+ .toUri();
+
+ final RestTemplate restTemplate = new RestTemplate();
+ final MDStoreWithInfo result = restTemplate.getForObject(uri, MDStoreWithInfo.class);
+
+ log.info("mdstore created " + result.toString());
+
+ token.getEnv().setAttribute(getOutputPrefix() + "format", format);
+ token.getEnv().setAttribute(getOutputPrefix() + "layout", layout);
+ token.getEnv().setAttribute(getOutputPrefix() + "interpretation", interpretation);
+ token.getEnv().setAttribute(getOutputPrefix() + "id", result.getId());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getFormat() {
+ return format;
+ }
+
+ public void setFormat(final String format) {
+ this.format = format;
+ }
+
+ public String getLayout() {
+ return layout;
+ }
+
+ public void setLayout(final String layout) {
+ this.layout = layout;
+ }
+
+ public String getInterpretation() {
+ return interpretation;
+ }
+
+ public void setInterpretation(final String interpretation) {
+ this.interpretation = interpretation;
+ }
+
+ public String getOutputPrefix() {
+ return outputPrefix;
+ }
+
+ public void setOutputPrefix(final String outputPrefix) {
+ this.outputPrefix = outputPrefix;
+ }
+
+ public String getMdStoreManagerUrl() {
+ return mdStoreManagerUrl;
+ }
+
+ @Required
+ public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {
+ this.mdStoreManagerUrl = mdStoreManagerUrl;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IncrementalOperationJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IncrementalOperationJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IncrementalOperationJobNode.java (revision 58540)
@@ -0,0 +1,100 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.common.logging.DnetLogger;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class IncrementalOperationJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(IncrementalOperationJobNode.class);
+
+ //incremental or refresh
+ private String operationType;
+
+ @Autowired
+ private DnetLogger dnetLogger;
+
+ @Override
+ protected String execute(NodeToken nodeToken) throws Exception {
+
+ if ("incremental".equalsIgnoreCase(operationType)) {
+ final String currentWfProfileId = findCurrentWfProfileId(nodeToken);
+ final Long lastSuccessStartDate = findLastSuccessStartDate(currentWfProfileId);
+ if(forceRefresh(nodeToken, lastSuccessStartDate, currentWfProfileId)) return Arc.DEFAULT_ARC;
+ log.info("Last success start date "+ lastSuccessStartDate);
+
+ nodeToken.getFullEnv().setAttribute("OperationTypeInfo", "Operation type set to INCREMENTAL with date "+DateUtils.calculate_ISO8601(lastSuccessStartDate));
+ nodeToken.getFullEnv().setAttribute("operationType", "INCREMENTAL");
+ nodeToken.getFullEnv().setAttribute("DateFromFilter", lastSuccessStartDate);
+ return Arc.DEFAULT_ARC;
+ }
+ nodeToken.getFullEnv().setAttribute("operationType", "REFRESH");
+ nodeToken.getFullEnv().setAttribute("OperationTypeInfo", "Operation type manually set to REFRESH");
+ return Arc.DEFAULT_ARC;
+ }
+
+ protected boolean forceRefresh(final NodeToken nodeToken, final Long lastSuccessStartDate, final String currentWfProfileId) throws Exception {
+ if (lastSuccessStartDate < 0) {
+ nodeToken.getFullEnv().setAttribute("OperationTypeInfo", "Last success start date < 0, operation forced to REFRESH");
+ nodeToken.getFullEnv().setAttribute("operationType", "REFRESH");
+ return true;
+ }
+ return false;
+ }
+
+ private Long findLastSuccessStartDate(String profId) {
+ long res = -1;
+
+ final Iterator> iter = dnetLogger.find(WorkflowsConstants.SYSTEM_WF_PROFILE_ID, profId);
+ while (iter.hasNext()) {
+ final Map map = iter.next();
+ if ("true".equalsIgnoreCase(map.get(WorkflowsConstants.SYSTEM_COMPLETED_SUCCESSFULLY))) {
+ final long curr = NumberUtils.toLong(map.get(WorkflowsConstants.SYSTEM_START_DATE), -1);
+ if (curr > res) {
+ res = curr;
+ }
+ }
+ }
+ log.debug("D-Net logger says the last success end date was on "+res);
+ return res;
+ }
+
+ private String findCurrentWfProfileId(NodeToken token) throws MSROException {
+ log.debug("Start to find the current profile Id");
+ final String p1 = token.getEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID);
+ if (p1 != null && !p1.isEmpty()) {
+ log.debug("The profile Id found is "+p1);
+ return p1;
+ }
+ final String p2 = token.getFullEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID);
+ if (p2 != null && !p2.isEmpty()) {
+ log.debug("The profile Id found is "+p2);
+ return p2;
+ }
+ final String p3 = token.getProcess().getEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID);
+ if (p3 != null && !p3.isEmpty()) {
+ log.debug("The profile Id found is "+p3);
+ return p3;
+ }
+ throw new MSROException("Missing property in env: " + WorkflowsConstants.SYSTEM_WF_PROFILE_ID);
+ }
+
+ public String getOperationType() {
+ return operationType;
+ }
+
+ public void setOperationType(final String operationType) {
+ this.operationType = operationType;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-preprocessing.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-preprocessing.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-preprocessing.xml (revision 58540)
@@ -0,0 +1,133 @@
+
+
+
+
+ IIS preprocessing
+ IIS
+ 30
+
+
+ start
+
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'iis-referenced-projects-preprocessing',
+ 'jobProperty' : 'export_action_set_id_document_referencedProjects',
+ 'enablingProperty' : 'active_referencedProjects_export',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenced-datasets-preprocessing',
+ 'jobProperty' : 'export_action_set_id_document_referencedDatasets',
+ 'enablingProperty' : 'active_referencedDatasets_export',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-wos-entities',
+ 'jobProperty' : 'export_action_set_id_entity_wos',
+ 'enablingProperty' : 'active_entity_wos_export',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-dataset-entities-preprocessing',
+ 'jobProperty' : 'export_action_set_id_entity_dataset',
+ 'enablingProperty' : 'active_entity_dataset_export',
+ 'enabled' : 'true'
+ }
+ ]
+
+
+
+
+
+
+
+ prepare parameters
+
+ import_islookup_service_location
+ import_content_object_store_location
+ import_mdstore_service_location
+ import_dataset_mdstore_ids_csv
+ import_wos_mdstore_id
+ import_database_service_location
+ import_content_datacite_objectstores_csv
+ import_content_wos_plaintext_objectstores_csv
+ export_action_hbase_remote_zookeeper_quorum
+ export_action_hbase_remote_zookeeper_clientport
+ oozie.wf.application.path
+ /tmp/integration/apps/preprocessing
+ DM
+
+ import_project_concepts_context_id
+ fet-fp7
+
+
+
+
+
+
+ IIS preprocessing
+
+ iisPreprocessingJob
+
+ {
+ 'cluster' : 'cluster',
+ 'nameNode' : 'nameNode',
+ 'jobTracker' : 'jobTracker',
+ 'oozie.wf.application.path' : 'oozie.wf.application.path',
+ 'import_content_object_store_location' : 'import_content_object_store_location',
+ 'import_mdstore_service_location' : 'import_mdstore_service_location',
+ 'import_islookup_service_location' : 'import_islookup_service_location',
+ 'import_project_concepts_context_id' : 'import_project_concepts_context_id',
+ 'import_dataset_mdstore_ids_csv' : 'import_dataset_mdstore_ids_csv',
+ 'import_wos_mdstore_id' : 'import_wos_mdstore_id',
+ 'import_database_service_location' : 'import_database_service_location',
+ 'import_content_datacite_objectstores_csv' : 'import_content_datacite_objectstores_csv',
+ 'import_content_wos_plaintext_objectstores_csv' : 'import_content_wos_plaintext_objectstores_csv',
+ 'export_action_hbase_remote_zookeeper_quorum' : 'export_action_hbase_remote_zookeeper_quorum',
+ 'export_action_hbase_remote_zookeeper_clientport' : 'export_action_hbase_remote_zookeeper_clientport',
+ 'export_action_set_id_document_referencedProjects' : 'export_action_set_id_document_referencedProjects',
+ 'export_action_set_id_document_referencedDatasets' : 'export_action_set_id_document_referencedDatasets',
+ 'export_action_set_id_entity_wos' : 'export_action_set_id_entity_wos',
+ 'export_action_set_id_entity_dataset' : 'export_action_set_id_entity_dataset',
+ 'output_remote_location' : 'actionManagerBasePath'
+ }
+
+
+ {
+ 'export_action_hbase_table_name' : 'hbase.actions.table',
+ 'import_database_dbname' : 'dnet.openaire.db.name'
+ }
+
+ false
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/MDStoreToApiExtraFieldHadoopJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/MDStoreToApiExtraFieldHadoopJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/MDStoreToApiExtraFieldHadoopJobNode.java (revision 58540)
@@ -0,0 +1,123 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import java.net.URI;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.data.mdstore.manager.common.model.MDStoreWithInfo;
+import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class MDStoreToApiExtraFieldHadoopJobNode extends SimpleJobNode {
+
+ private String mdId;
+ private String datasourceId;
+ private String datasourceInterface;
+ private String extraFieldForTotal;
+ private String extraFieldForDate;
+ private String extraFieldForMdId;
+
+ /* Spring managed params */
+ private String mdStoreManagerUrl;
+
+ @Autowired
+ private LocalDatasourceManager, ?> dsManager;
+
+ private static final Log log = LogFactory.getLog(MDStoreToApiExtraFieldHadoopJobNode.class);
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final String url = getMdStoreManagerUrl() + "/mdstore/{mdId}";
+
+ final Map params = new HashMap<>();
+ params.put("mdId", getMdId());
+
+ final URI uri = UriComponentsBuilder.fromUriString(url)
+ .buildAndExpand(params)
+ .toUri();
+
+ final RestTemplate restTemplate = new RestTemplate();
+ final MDStoreWithInfo mdstore = restTemplate.getForObject(uri, MDStoreWithInfo.class);
+
+ final int size = Math.toIntExact(mdstore.getSize());
+ final Date date = mdstore.getLastUpdate();
+
+ if (extraFieldForTotal.equals("last_collection_total")) {
+ dsManager.setLastCollectionInfo(datasourceId, datasourceInterface, mdId, size, date);
+ } else if (extraFieldForTotal.equals("last_aggregation_total")) {
+ dsManager.setLastAggregationInfo(datasourceId, datasourceInterface, mdId, size, date);
+ } else {
+ log.warn("Invalid field for total: " + extraFieldForTotal);
+ }
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdId() {
+ return mdId;
+ }
+
+ public void setMdId(final String mdId) {
+ this.mdId = mdId;
+ }
+
+ public String getDatasourceId() {
+ return datasourceId;
+ }
+
+ public void setDatasourceId(final String datasourceId) {
+ this.datasourceId = datasourceId;
+ }
+
+ public String getDatasourceInterface() {
+ return datasourceInterface;
+ }
+
+ public void setDatasourceInterface(final String datasourceInterface) {
+ this.datasourceInterface = datasourceInterface;
+ }
+
+ public String getExtraFieldForTotal() {
+ return extraFieldForTotal;
+ }
+
+ public void setExtraFieldForTotal(final String extraFieldForTotal) {
+ this.extraFieldForTotal = extraFieldForTotal;
+ }
+
+ public String getExtraFieldForDate() {
+ return extraFieldForDate;
+ }
+
+ public void setExtraFieldForDate(final String extraFieldForDate) {
+ this.extraFieldForDate = extraFieldForDate;
+ }
+
+ public String getExtraFieldForMdId() {
+ return extraFieldForMdId;
+ }
+
+ public void setExtraFieldForMdId(final String extraFieldForMdId) {
+ this.extraFieldForMdId = extraFieldForMdId;
+ }
+
+ public String getMdStoreManagerUrl() {
+ return mdStoreManagerUrl;
+ }
+
+ public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {
+ this.mdStoreManagerUrl = mdStoreManagerUrl;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis.meta.xml (revision 58540)
@@ -0,0 +1,27 @@
+
+
+
+
+ Inference
+
+ InfoSpace Inference
+
+
+
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/exportHBaseTable.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/exportHBaseTable.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/exportHBaseTable.meta.xml (revision 58540)
@@ -0,0 +1,24 @@
+
+
+
+
+ Export InfoSpace
+ Export infospace as json over HDFS
+ InfoSpace Inference
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/db/update-class-scheme.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/db/update-class-scheme.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/db/update-class-scheme.xml (revision 58540)
@@ -0,0 +1,28 @@
+
+
+
+
+ Update class and scheme tables
+ UpdateDB
+ 30
+
+
+ Update vocabulary entries
+
+ /eu/dnetlib/msro/openaireplus/workflows/db/vocEntry_2_db.xsl
+ dnet_openaireplus
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/bulktag/LoadBulkTaggingConfigurationJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/bulktag/LoadBulkTaggingConfigurationJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/bulktag/LoadBulkTaggingConfigurationJobNode.java (revision 58540)
@@ -0,0 +1,104 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.bulktag;
+
+import com.google.common.base.Joiner;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.bulktag.CommunityConfigurationFactory;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
+
+public class LoadBulkTaggingConfigurationJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(LoadBulkTaggingConfigurationJobNode.class);
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ private String xquery;
+
+ private String bulkTaggingConfParam;
+
+ private String enabledConfParam;
+
+ private boolean enabled;
+
+ private String mappingProtoParam;
+
+ private String mapping;
+
+
+
+ public void setMappingProtoParam(String mappingProtoParam) {
+ this.mappingProtoParam = mappingProtoParam;
+
+ }
+
+ public String getMappingProtoParam() {
+ return mappingProtoParam;
+ }
+
+ public String getMapping() {
+ return mapping;
+ }
+
+ public void setMapping(String mapping) {
+ this.mapping = mapping;
+
+ }
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final List res = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+
+ final String xmlConf = "" + Joiner.on(" ").join(res) + " ";
+
+ token.getEnv().setAttribute(getBulkTaggingConfParam(), CommunityConfigurationFactory.newInstance(xmlConf).toJson());
+ token.getEnv().setAttribute(getEnabledConfParam(),isEnabled());
+ token.getEnv().setAttribute(getMappingProtoParam(),getMapping());
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getXquery() {
+ log.debug(xquery);
+ return xquery;
+ }
+
+ public void setXquery(final String xquery) {
+ this.xquery = xquery;
+
+ }
+
+ public String getBulkTaggingConfParam() {
+ return bulkTaggingConfParam;
+ }
+
+ public void setBulkTaggingConfParam(final String bulkTaggingConfParam) {
+ this.bulkTaggingConfParam = bulkTaggingConfParam;
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ public String getEnabledConfParam() {
+ return enabledConfParam;
+ }
+
+ public void setEnabledConfParam(String enabledConfParam) {
+ this.enabledConfParam = enabledConfParam;
+ }
+
+
+
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/AbstractClaimsToHBASE.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/AbstractClaimsToHBASE.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/AbstractClaimsToHBASE.java (revision 58540)
@@ -0,0 +1,155 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.claims;
+
+import eu.dnetlib.data.mapreduce.util.OafRowKeyDecoder;
+import eu.dnetlib.data.proto.FieldTypeProtos.DataInfo;
+import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;
+import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.ProgressProvider;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Created by Alessia Bardi on 28/06/2017.
+ *
+ * @author Alessia Bardi
+ */
+public abstract class AbstractClaimsToHBASE extends SimpleJobNode implements ProgressJobNode {
+
+ private static final Log log = LogFactory.getLog(AbstractClaimsToHBASE.class);
+
+ private String sql;
+ private String countQuery;
+
+ private int total = 0;
+ private int processed = 0;
+
+ private String clusterName;
+ private String tableName;
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Autowired
+ private ClaimDatabaseUtils claimDatabaseUtils;
+
+ @Override
+ public ProgressProvider getProgressProvider() {
+ return new ProgressProvider() {
+
+ @Override
+ public boolean isInaccurate() {
+ return false;
+ }
+
+ @Override
+ public int getTotalValue() {
+ return total;
+ }
+
+ @Override
+ public int getCurrentValue() {
+ return processed;
+ }
+ };
+ }
+
+ protected String getOpenAIREType(final String type) {
+ switch (type) {
+ case "publication":
+ case "dataset":
+ case "software":
+ case "other":
+ return "result";
+ default:
+ return type;
+ }
+ }
+
+ protected String getFullId(final String type, final String id) {
+ final String fullId = AbstractDNetXsltFunctions.oafSimpleId(type, id);
+ return OafRowKeyDecoder.decode(fullId).getKey();
+ }
+
+ protected static DataInfo getDataInfo() {
+ return DataInfo.newBuilder().setTrust("0.91").setInferred(false)
+ .setProvenanceaction(
+ Qualifier.newBuilder()
+ .setClassid("user:claim")
+ .setClassname("user:claim")
+ .setSchemeid("dnet:provenanceActions")
+ .setSchemename("dnet:provenanceActions")).build();
+
+ }
+
+ public String getSql() {
+ return sql;
+ }
+
+ public void setSql(final String sql) {
+ this.sql = sql;
+ }
+
+ public String getCountQuery() {
+ return countQuery;
+ }
+
+ public void setCountQuery(final String countQuery) {
+ this.countQuery = countQuery;
+ }
+
+ public int getTotal() {
+ return total;
+ }
+
+ public void setTotal(final int total) {
+ this.total = total;
+ }
+
+ public int getProcessed() {
+ return processed;
+ }
+
+ public void setProcessed(final int processed) {
+ this.processed = processed;
+ }
+
+ public void incrementProcessed() {
+ processed++;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(final String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(final String tableName) {
+ this.tableName = tableName;
+ }
+
+ public UniqueServiceLocator getServiceLocator() {
+ return serviceLocator;
+ }
+
+ public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
+ this.serviceLocator = serviceLocator;
+ }
+
+ public ClaimDatabaseUtils getClaimDatabaseUtils() {
+ return claimDatabaseUtils;
+ }
+
+ public void setClaimDatabaseUtils(final ClaimDatabaseUtils claimDatabaseUtils) {
+ this.claimDatabaseUtils = claimDatabaseUtils;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-main.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-main.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/iis/iis-main.xml (revision 58540)
@@ -0,0 +1,181 @@
+
+
+
+
+ IIS main workflow
+ IIS
+ 30
+
+
+ start
+
+
+
+
+
+
+ prepare action sets
+
+
+ [
+ {
+ 'set' : 'iis-extracted-metadata',
+ 'jobProperty' : 'export_action_set_id_document_extractedMetadata',
+ 'enablingProperty' : 'active_metadataextraction_export',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenced-projects-main',
+ 'jobProperty' : 'export_action_set_id_document_referencedProjects',
+ 'enablingProperty' : 'active_referenceextraction_project',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenced-datasets-main',
+ 'jobProperty' : 'export_action_set_id_document_referencedDatasets',
+ 'enablingProperty' : 'active_referenceextraction_dataset',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-dataset-entities-main',
+ 'jobProperty' : 'export_action_set_id_entity_dataset',
+ 'enablingProperty' : 'active_referenceextraction_dataset',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-researchinitiative',
+ 'jobProperty' : 'export_action_set_id_document_research_initiative',
+ 'enablingProperty' :
+ 'active_referenceextraction_researchinitiative',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-document-similarities',
+ 'jobProperty' : 'export_action_set_id_document_similarities_standard',
+ 'enablingProperty' : 'active_documentssimilarity',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-document-statistics',
+ 'jobProperty' : 'export_action_set_id_document_statistics',
+ 'enablingProperty' : 'active_statistics',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-document-classes',
+ 'jobProperty' : 'export_action_set_id_document_classes',
+ 'enablingProperty' : 'active_documentsclassification',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-document-citations',
+ 'jobProperty' : 'export_action_set_id_document_referencedDocuments',
+ 'enablingProperty' : 'active_citationmatching',
+ 'enabled' : 'true'
+ },
+ {
+ 'set' : 'iis-referenceextraction-pdb',
+ 'jobProperty' : 'export_action_set_id_document_pdb',
+ 'enablingProperty' : 'active_referenceextraction_pdb',
+ 'enabled' : 'false'
+ }
+ ]
+
+
+
+
+
+
+
+ prepare parameters
+
+ import_islookup_service_location
+ import_content_objectstores_csv
+ import_content_object_store_location
+ import_mdstore_service_location
+ import_dataset_mdstore_ids_csv
+ export_action_hbase_remote_zookeeper_quorum
+ export_action_hbase_remote_zookeeper_clientport
+ oozie.wf.application.path
+
+ /tmp/integration/apps/main
+ IIS
+
+
+
+ import_project_concepts_context_id
+ fet-fp7
+
+
+
+
+
+
+ IIS main
+
+ iisMainJob
+
+ {
+ 'cluster' : 'cluster',
+ 'nameNode' : 'nameNode',
+ 'jobTracker' : 'jobTracker',
+ 'oozie.wf.application.path' : 'oozie.wf.application.path',
+ 'active_metadataextraction_export' : 'active_metadataextraction_export',
+ 'active_referenceextraction_project' : 'active_referenceextraction_project',
+ 'active_referenceextraction_dataset' : 'active_referenceextraction_dataset',
+ 'active_referenceextraction_researchinitiative' : 'active_referenceextraction_researchinitiative',
+ 'active_documentsclassification' : 'active_documentsclassification',
+ 'active_documentssimilarity' : 'active_documentssimilarity',
+ 'active_citationmatching' : 'active_citationmatching',
+ 'active_statistics' : 'active_statistics',
+ 'active_referenceextraction_pdb' : 'active_referenceextraction_pdb',
+ 'import_content_objectstores_csv' : 'import_content_objectstores_csv',
+ 'import_content_object_store_location' : 'import_content_object_store_location',
+ 'import_mdstore_service_location' : 'import_mdstore_service_location',
+ 'import_islookup_service_location' : 'import_islookup_service_location',
+ 'import_project_concepts_context_id' : 'import_project_concepts_context_id',
+ 'import_dataset_mdstore_ids_csv' : 'import_dataset_mdstore_ids_csv',
+ 'export_action_hbase_remote_zookeeper_quorum' : 'export_action_hbase_remote_zookeeper_quorum',
+ 'export_action_hbase_remote_zookeeper_clientport' : 'export_action_hbase_remote_zookeeper_clientport',
+ 'export_action_set_id_document_extractedMetadata' : 'export_action_set_id_document_extractedMetadata',
+ 'export_action_set_id_document_referencedDatasets' : 'export_action_set_id_document_referencedDatasets',
+ 'export_action_set_id_document_referencedProjects' : 'export_action_set_id_document_referencedProjects',
+ 'export_action_set_id_document_research_initiative' : 'export_action_set_id_document_research_initiative',
+ 'export_action_set_id_document_similarities_standard' : 'export_action_set_id_document_similarities_standard',
+ 'export_action_set_id_document_statistics' : 'export_action_set_id_document_statistics',
+ 'export_action_set_id_document_classes' : 'export_action_set_id_document_classes',
+ 'export_action_set_id_document_referencedDocuments' : 'export_action_set_id_document_referencedDocuments',
+ 'export_action_set_id_entity_dataset' : 'export_action_set_id_entity_dataset',
+ 'export_action_set_id_document_pdb' : 'export_action_set_id_document_pdb',
+ 'output_remote_location' : 'actionManagerBasePath'
+ }
+
+
+ {
+ 'export_action_hbase_table_name' : 'hbase.actions.table'
+ }
+
+ false
+
+
+
+
+
+
+ update action sets
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/vre/VREPostJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/vre/VREPostJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/vre/VREPostJobNode.java (revision 58540)
@@ -0,0 +1,176 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.vre;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.json.JSONObject;
+
+/**
+ * Created by alessia on 24/03/16.
+ */
+public class VREPostJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(VREPostJobNode.class);
+ private String scheme;
+ private String host;
+ private String postPath;
+ private int followRedirect = 10;
+
+ //POST parameters
+ private String appToken;
+ private String previewTitle;
+ private String previewDescription;
+ private String text;
+ private String httpImageUrl;
+ private boolean enableNotification;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ log.debug("Posting text: "+getText());
+
+ CloseableHttpClient httpclient = HttpClients.createDefault();
+ try {
+ URI postUri = new URIBuilder().setScheme(scheme).setHost(host).setPath(postPath).build();
+ HttpPost postAction = new HttpPost(postUri);
+
+ JSONObject obj = createPostBody();
+
+ postAction.addHeader("gcube-token", appToken);
+ postAction.addHeader("Content-type", ContentType.APPLICATION_JSON.toString());
+
+ StringEntity params = new StringEntity(obj.toString(), ContentType.APPLICATION_JSON);
+ postAction.setEntity(params);
+
+ return doPost(httpclient, postAction, followRedirect);
+ } finally {
+ httpclient.close();
+ }
+
+ }
+
+ protected String doPost(HttpClient httpclient, HttpPost postAction, int followRedirect) throws IOException, URISyntaxException {
+ HttpResponse response = httpclient.execute(postAction);
+ log.debug(" " + response.getStatusLine().getStatusCode() + " and response message is " + response.getStatusLine().getReasonPhrase());
+ int status = response.getStatusLine().getStatusCode();
+ switch (status) {
+ case HttpURLConnection.HTTP_OK:
+ case HttpURLConnection.HTTP_CREATED:
+ return Arc.DEFAULT_ARC;
+ case HttpURLConnection.HTTP_MOVED_TEMP:
+ case HttpURLConnection.HTTP_MOVED_PERM:
+ case HttpURLConnection.HTTP_SEE_OTHER:
+ if (followRedirect > 0) {
+ Header[] locations = response.getHeaders("Location");
+ Header lastLocation = locations[locations.length - 1];
+ String realLocation = lastLocation.getValue();
+ log.info("New location after redirect for posting is " + realLocation);
+
+ // perform again the same request
+ postAction.setURI(new URI(realLocation));
+ return doPost(httpclient, postAction, followRedirect--);
+ } else
+ throw new RuntimeException("Too many redirects to post on the VRE");
+ }
+ throw new RuntimeException("HTTP error code " + status);
+ }
+
+ protected JSONObject createPostBody() {
+ JSONObject obj = new JSONObject();
+ obj.put("text", text.replace("\\n", "\n"));
+ obj.put("preview_title", previewTitle);
+ obj.put("preview_description", previewDescription);
+ obj.put("image_url", httpImageUrl);
+ obj.put("enable_notification", Boolean.toString(enableNotification));
+ log.debug("POST body: " + obj.toString());
+ return obj;
+ }
+
+ public String getScheme() {
+ return scheme;
+ }
+
+ public void setScheme(final String scheme) {
+ this.scheme = scheme;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public void setHost(final String host) {
+ this.host = host;
+ }
+
+ public String getPostPath() {
+ return postPath;
+ }
+
+ public void setPostPath(final String postPath) {
+ this.postPath = postPath;
+ }
+
+ public String getAppToken() {
+ return appToken;
+ }
+
+ public void setAppToken(final String token) {
+ this.appToken = token;
+ }
+
+ public String getPreviewTitle() {
+ return previewTitle;
+ }
+
+ public void setPreviewTitle(final String previewTitle) {
+ this.previewTitle = previewTitle;
+ }
+
+ public String getPreviewDescription() {
+ return previewDescription;
+ }
+
+ public void setPreviewDescription(final String previewDescription) {
+ this.previewDescription = previewDescription;
+ }
+
+ public String getText() {
+ return text;
+ }
+
+ public void setText(final String text) {
+ this.text = text;
+ }
+
+ public String getHttpImageUrl() {
+ return httpImageUrl;
+ }
+
+ public void setHttpImageUrl(final String httpImageUrl) {
+ this.httpImageUrl = httpImageUrl;
+ }
+
+ public boolean isEnableNotification() {
+ return enableNotification;
+ }
+
+ public void setEnableNotification(final boolean enablenotification) {
+ this.enableNotification = enablenotification;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/is/update-repositories.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/is/update-repositories.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/is/update-repositories.xml (revision 58540)
@@ -0,0 +1,25 @@
+
+
+
+
+ Update datasource profiles
+ UpdateIS
+ 30
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/clean.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/clean.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/clean.meta.xml (revision 58540)
@@ -0,0 +1,23 @@
+
+
+
+
+ Clean Profiles
+
+ Garbage Collection
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/clean-profiles.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/clean-profiles.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/clean-profiles.xml (revision 58540)
@@ -0,0 +1,25 @@
+
+
+
+
+ Clean profiles
+ CLEAN
+ 30
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimUpdatesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimUpdatesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimUpdatesJobNode.java (revision 58540)
@@ -0,0 +1,73 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.claims;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.hadoop.rmi.HadoopService;
+import eu.dnetlib.data.proto.KindProtos.Kind;
+import eu.dnetlib.data.proto.OafProtos.Oaf;
+import eu.dnetlib.data.proto.OafProtos.OafEntity;
+import eu.dnetlib.data.proto.ResultProtos.Result;
+import eu.dnetlib.data.proto.ResultProtos.Result.Context;
+import eu.dnetlib.data.proto.TypeProtos.Type;
+import eu.dnetlib.msro.rmi.MSROException;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Created by alessia on 23/10/15.
+ */
+public class ApplyClaimUpdatesJobNode extends AbstractClaimsToHBASE {
+
+ private static final Log log = LogFactory.getLog(ApplyClaimUpdatesJobNode.class);
+
+ @Override
+ protected String execute(NodeToken token) throws Exception {
+ //TODO: use claim.claim_date from the claim db
+ long timestamp = System.currentTimeMillis();
+ setTotal(getClaimDatabaseUtils().count(getCountQuery()));
+
+ final AtomicInteger discardedClaims = new AtomicInteger(0);
+ final HadoopService hadoopService = getServiceLocator().getService(HadoopService.class);
+
+ getClaimDatabaseUtils().query(getSql()).forEach(claim -> {
+ try {
+ log.debug(claim);
+ String contextId = claim.getSource();
+ String rowKey = getFullId(getOpenAIREType(claim.getTargetType()), claim.getTarget());
+
+ String value = getValue(rowKey, contextId, timestamp);
+ hadoopService.addHBaseColumn(getClusterName(), getTableName(), rowKey, "result", "update_" + System.nanoTime(), value);
+ incrementProcessed();
+ } catch (Exception e) {
+ log.error("Discarding claim " + claim + ". Cause: " + e.getMessage());
+ discardedClaims.incrementAndGet();
+ }
+ });
+ log.info("Total Claim Updates: " + getTotal());
+ token.getEnv().setAttribute("claimUpdatesSize", getTotal());
+ log.info("Claim updates: " + getProcessed());
+ token.getEnv().setAttribute("claimUpdates", getProcessed());
+ log.info("Discarded Claim Updates: " + discardedClaims.intValue());
+ token.getEnv().setAttribute("discardedClaimUpdates", discardedClaims.intValue());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ protected String getValue(final String rowkey, final String contextid, final long timestamp) throws MSROException {
+ log.debug(StringUtils.format("%s -- %s", rowkey, contextid));
+
+ Result.Builder resultBuilder = Result.newBuilder().setMetadata(Result.Metadata.newBuilder().addContext(getContext(contextid)));
+ OafEntity.Builder entityBuilder = OafEntity.newBuilder().setId(rowkey).setType(Type.result).setResult(resultBuilder);
+ Oaf.Builder builder = Oaf.newBuilder().setKind(Kind.entity).setLastupdatetimestamp(timestamp).setEntity(entityBuilder);
+
+ return Base64.encodeBase64String(builder.build().toByteArray());
+ }
+
+ private Context getContext(final String sourceId) {
+ return Context.newBuilder().addDataInfo(getDataInfo()).setId(sourceId).build();
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbage.actions.meta.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbage.actions.meta.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/garbage/garbage.actions.meta.xml (revision 58540)
@@ -0,0 +1,24 @@
+
+
+
+
+ Garbage Action Sets
+
+ Garbage Collection
+
+
+
+
+
+
+ 29 5 22 ? * *
+ 10080
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidWorkflowsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidWorkflowsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidWorkflowsJobNode.java (revision 58540)
@@ -0,0 +1,72 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+
+public class FindInvalidWorkflowsJobNode extends SimpleJobNode {
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ private static final Log log = LogFactory.getLog(FindInvalidWorkflowsJobNode.class);
+
+ private boolean deleteInvalidProfiles = false;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final ISLookUpService lookup = serviceLocator.getService(ISLookUpService.class);
+ final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class);
+
+ final String q1 =
+ "for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')//WORKFLOW/@id|//CONFIGURATION/@destroyWorkflow return $x/string()";
+
+ final String q2 = "for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
+
+ final Set valids = new HashSet<>(lookup.quickSearchProfile(q1));
+
+ int count = 0;
+ int countInvalids = 0;
+ int countDeleted = 0;
+ for (final String wfId : lookup.quickSearchProfile(q2)) {
+ count++;
+ if (!valids.contains(wfId)) {
+ log.warn("Invalid wf: " + wfId);
+ countInvalids++;
+ if (deleteInvalidProfiles) {
+ registry.deleteProfile(wfId);
+ log.info(" -- Deleted");
+ countDeleted++;
+ }
+
+ }
+ }
+
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", count);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalid", countInvalids);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "deleted", countDeleted);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public boolean isDeleteInvalidProfiles() {
+ return deleteInvalidProfiles;
+ }
+
+ public void setDeleteInvalidProfiles(final boolean deleteInvalidProfiles) {
+ this.deleteInvalidProfiles = deleteInvalidProfiles;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/DecapsuleClaimsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/DecapsuleClaimsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/DecapsuleClaimsJobNode.java (revision 58540)
@@ -0,0 +1,104 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.claims;
+
+import java.io.StringReader;
+import javax.annotation.Resource;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.resultset.MappedResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+import eu.dnetlib.msro.openaireplus.workflows.nodes.SetHdfsFileJobNode;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.Namespace;
+import org.dom4j.QName;
+import org.dom4j.io.SAXReader;
+
+/**
+ * TODO: this class can be deleted when we are using the new claim database
+ */
+@Deprecated
+public class DecapsuleClaimsJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(SetHdfsFileJobNode.class);
+
+ private static final Namespace OAF_NS = new Namespace("oaf", "http://namespace.openaire.eu/oaf");
+ private static final double DEFAULT_TRUST = 0.9;
+
+ private String inputEprParam;
+ private String outputEprParam;
+
+ @Resource
+ private MappedResultSetFactory mappedResultSetFactory;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final UnaryFunction mapper = new UnaryFunction() {
+
+ final SAXReader reader = new SAXReader();
+
+ @Override
+ public String evaluate(final String row) {
+ try {
+ final Document docRow = reader.read(new StringReader(row));
+
+ final String xml = docRow.valueOf("//ROW/FIELD[@name = 'xml']");
+ final String provenance = docRow.valueOf("//ROW/FIELD[@name = 'provenance']");
+
+ final Document doc = reader.read(new StringReader(xml));
+ final Element record = (Element) doc.selectSingleNode("//*[local-name() = 'record']");
+ final Element about = record.addElement(new QName("about"));
+ final Element dataInfo = about.addElement(new QName("datainfo", OAF_NS));
+
+ dataInfo.addElement(new QName("inferred", OAF_NS)).setText("false");
+ dataInfo.addElement(new QName("deletedbyinference", OAF_NS)).setText("false");
+ dataInfo.addElement(new QName("trust", OAF_NS)).setText(Double.toString(DEFAULT_TRUST));
+ dataInfo.addElement(new QName("inferenceprovenance", OAF_NS)).setText("");
+
+ final Element action = dataInfo.addElement(new QName("provenanceaction", OAF_NS));
+ action.addAttribute("schemename", "dnet:provenanceActions");
+ action.addAttribute("schemeid", "dnet:provenanceActions");
+ action.addAttribute("classname", provenance);
+ action.addAttribute("classid", provenance);
+
+ return doc.asXML();
+ } catch (Throwable e) {
+ log.error("Error parsing row: " + row, e);
+ throw new IllegalArgumentException("Error parsing row: " + row, e);
+ }
+ }
+ };
+
+ final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(getInputEprParam()));
+ final W3CEndpointReference epr = mappedResultSetFactory.createMappedResultSet(inputEpr, mapper);
+
+ token.getEnv().setAttribute(getOutputEprParam(), epr.toString());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getInputEprParam() {
+ return inputEprParam;
+ }
+
+ public void setInputEprParam(final String inputEprParam) {
+ this.inputEprParam = inputEprParam;
+ }
+
+ public String getOutputEprParam() {
+ return outputEprParam;
+ }
+
+ public void setOutputEprParam(final String outputEprParam) {
+ this.outputEprParam = outputEprParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidMetaWorkflowsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidMetaWorkflowsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidMetaWorkflowsJobNode.java (revision 58540)
@@ -0,0 +1,79 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+
+public class FindInvalidMetaWorkflowsJobNode extends SimpleJobNode {
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ private static final Log log = LogFactory.getLog(FindInvalidMetaWorkflowsJobNode.class);
+
+ private boolean deleteInvalidProfiles = false;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final ISLookUpService lookup = serviceLocator.getService(ISLookUpService.class);
+ final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class);
+
+ final String q1 = "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')\n" +
+ "let $id := $x//RESOURCE_IDENTIFIER/@value\n" +
+ "for $api in $x//INTERFACE/@id\n" +
+ "return concat($id,'|',$api)";
+
+ final String q2 = "for $x in collection(' /db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType') \n" +
+ "where $x//DATAPROVIDER/@id and $x//DATAPROVIDER/@interface\n" +
+ "return concat($x//RESOURCE_IDENTIFIER/@value,'@@@',$x//DATAPROVIDER/@id,'|',$x//DATAPROVIDER/@interface)";
+
+ final Set valids = new HashSet<>(lookup.quickSearchProfile(q1));
+
+ int count = 0;
+ int countInvalids = 0;
+ int countDeleted = 0;
+ for (final String s : lookup.quickSearchProfile(q2)) {
+ count++;
+ final String metaWfId = StringUtils.substringBefore(s, "@@@");
+ final String dsapi = StringUtils.substringAfter(s, "@@@");
+ if (!valids.contains(dsapi)) {
+ log.warn("Invalid metaWf: " + metaWfId);
+ countInvalids++;
+ if (deleteInvalidProfiles) {
+ registry.deleteProfile(metaWfId);
+ log.info(" -- Deleted");
+ countDeleted++;
+ }
+
+ }
+ }
+
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", count);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalid", countInvalids);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "deleted", countDeleted);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public boolean isDeleteInvalidProfiles() {
+ return deleteInvalidProfiles;
+ }
+
+ public void setDeleteInvalidProfiles(final boolean deleteInvalidProfiles) {
+ this.deleteInvalidProfiles = deleteInvalidProfiles;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/materialised_views.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/materialised_views.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/materialised_views.sql (revision 58540)
@@ -0,0 +1,188 @@
+-- query performed by IIS to bulk load project details
+DROP TABLE project_details;
+CREATE TABLE project_details AS (
+ SELECT
+ p.id AS projectid,
+ p.acronym AS acronym,
+ p.code AS code,
+ p.jsonextrainfo AS jsonextrainfo,
+ array_agg(fp.path) AS fundingpath
+ FROM projects p
+ LEFT OUTER JOIN project_fundingpath pf ON (pf.project = p.id)
+ LEFT OUTER JOIN fundingpaths fp ON (fp.id = pf.funding)
+ GROUP BY p.id
+);
+
+GRANT SELECT ON project_details TO dnetapi;
+
+
+-- used to serve queries from api tsv exporter
+DROP TABLE projects_tsv;
+CREATE TABLE projects_tsv AS (
+ SELECT
+ nextval('projects_tsv_ids') AS rowid,
+ p.code,
+ p.acronym,
+ p.title,
+ p.call_identifier,
+ p.startdate,
+ p.enddate,
+ p.ec_sc39,
+ p.oa_mandate_for_publications,
+ p.ec_article29_3 AS oa_mandate_for_datasets,
+ p.contactfullname AS contactfullname,
+ p.contactemail AS contactemail,
+ org.legalname AS legalname,
+ org.country AS country,
+ (array_agg(f.id))[1] AS fundingpathid,
+ (array_agg(f.description))[1] AS description,
+ (array_agg(CASE WHEN po.participantnumber = 1
+ THEN 'coordinator'
+ ELSE '' END))[1] AS role
+
+ FROM projects p
+ LEFT OUTER JOIN project_organization po ON (p.id = po.project)
+ LEFT OUTER JOIN dsm_organizations org ON (po.resporganization = org.id)
+ LEFT OUTER JOIN project_fundingpath pf ON (p.id = pf.project)
+ LEFT OUTER JOIN fundingpaths f ON (pf.funding = f.id)
+ GROUP BY
+ p.code,
+ p.acronym,
+ p.title,
+ p.call_identifier,
+ p.startdate,
+ p.enddate,
+ p.ec_sc39,
+ p.oa_mandate_for_publications,
+ p.ec_article29_3,
+ p.contactfullname,
+ p.contactemail,
+ org.legalname,
+ org.country
+ ORDER BY p.acronym
+);
+
+CREATE INDEX projects_tsv_fundingpathid_idx ON projects_tsv (fundingpathid);
+CREATE INDEX projects_tsv_oa_mandate_for_datasets_idx ON projects_tsv (oa_mandate_for_datasets);
+GRANT SELECT ON projects_tsv TO dnetapi;
+
+-- used to serve queries from dspace and eprints exporter
+DROP TABLE projects_api;
+CREATE TABLE projects_api AS (
+ SELECT
+ p.id,
+ p.acronym AS acronym,
+ p.title AS title,
+ p.code AS code,
+ p.startdate AS startdate,
+ p.enddate AS enddate,
+ (array_agg(org.legalshortname))[1] AS funder,
+ (array_agg(fp.jurisdiction))[1] AS jurisdiction,
+ (array_agg(fp.id))[1] AS fundingpathid
+ FROM projects p
+ LEFT OUTER JOIN project_fundingpath pfp ON (p.id = pfp.project)
+ LEFT OUTER JOIN fundingpaths fp ON (pfp.funding = fp.id)
+ LEFT OUTER JOIN dsm_organizations org ON (org.id = fp.funder)
+ GROUP BY p.id);
+
+CREATE INDEX projects_api_fundingpathid_idx ON projects_api (fundingpathid);
+CREATE INDEX projects_api_startdate_idx ON projects_api (startdate);
+CREATE INDEX projects_api_enddate_idx ON projects_api (enddate);
+GRANT SELECT ON projects_api TO dnetapi;
+
+
+DROP MATERIALIZED VIEW projects_mv;
+CREATE MATERIALIZED VIEW projects_mv AS (
+ SELECT
+ p.id AS projectid,
+ p.code AS code,
+ p.websiteurl AS websiteurl,
+ p.acronym AS acronym,
+ p.title AS title,
+ p.startdate AS startdate,
+ p.enddate AS enddate,
+ p.call_identifier AS callidentifier,
+ p.keywords AS keywords,
+ p.duration AS duration,
+ p.ec_sc39 AS ecsc39,
+ p.oa_mandate_for_publications AS oamandatepublications,
+ p.ec_article29_3 AS ecarticle29_3,
+ p.dateofcollection AS dateofcollection,
+ p.lastupdate AS dateoftransformation,
+ p.inferred AS inferred,
+ p.deletedbyinference AS deletedbyinference,
+ p.trust AS trust,
+ p.inferenceprovenance AS inferenceprovenance,
+ p.optional1 AS optional1,
+ p.optional2 AS optional2,
+ p.jsonextrainfo AS jsonextrainfo,
+ p.contactfullname AS contactfullname,
+ p.contactfax AS contactfax,
+ p.contactphone AS contactphone,
+ p.contactemail AS contactemail,
+ p.summary AS summary,
+ p.currency AS currency,
+ p.totalcost AS totalcost,
+ p.fundedamount AS fundedamount,
+ dc.id AS collectedfromid,
+ dc.officialname AS collectedfromname,
+ p.contracttype || '@@@' || p.contracttypename || '@@@' || p.contracttypescheme || '@@@' || p.contracttypescheme AS contracttype,
+ pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name AS provenanceaction,
+ array_agg(DISTINCT i.pid || '###' || i.issuertype) AS pid,
+ array_agg(DISTINCT s.name || '###' || sc.code || '@@@' || sc.name || '@@@' || ss.code || '@@@' || ss.name) AS subjects,
+ array_agg(DISTINCT fp.path) AS fundingtree
+ FROM projects p
+ LEFT OUTER JOIN class pac ON (pac.code = p.provenanceactionclass)
+ LEFT OUTER JOIN scheme pas ON (pas.code = p.provenanceactionscheme)
+
+ LEFT OUTER JOIN projectpids pp ON (pp.project = p.id)
+ LEFT OUTER JOIN dsm_identities i ON (i.pid = pp.pid)
+
+ LEFT OUTER JOIN dsm_datasources dc ON (dc.id = p.collectedfrom)
+
+ LEFT OUTER JOIN project_fundingpath pf ON (pf.project = p.id)
+ LEFT OUTER JOIN fundingpaths fp ON (fp.id = pf.funding)
+
+ LEFT OUTER JOIN project_subject ps ON (ps.project = p.id)
+ LEFT OUTER JOIN subjects s ON (s.id = ps.subject)
+
+ LEFT OUTER JOIN class sc ON (sc.code = s.semanticclass)
+ LEFT OUTER JOIN scheme ss ON (ss.code = s.semanticscheme)
+
+ GROUP BY
+ p.id,
+ p.code,
+ p.websiteurl,
+ p.acronym,
+ p.title,
+ p.startdate,
+ p.enddate,
+ p.call_identifier,
+ p.keywords,
+ p.duration,
+ p.ec_sc39,
+ p.oa_mandate_for_publications,
+ p.ec_article29_3,
+ p.dateofcollection,
+ p.inferred,
+ p.deletedbyinference,
+ p.trust,
+ p.inferenceprovenance,
+ p.contactfullname,
+ p.contactfax,
+ p.contactphone,
+ p.contactemail,
+ p.contracttype,
+ p.summary,
+ p.currency,
+ p.totalcost,
+ p.fundedamount,
+ dc.id,
+ dc.officialname,
+ pac.code, pac.name, pas.code, pas.name
+);
+CREATE INDEX projects_mv_id_idx ON projects_mv (projectid);
+
+GRANT SELECT ON fundingpaths TO dnetapi;
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FixRepoMdstoreSizesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FixRepoMdstoreSizesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FixRepoMdstoreSizesJobNode.java (revision 58540)
@@ -0,0 +1,231 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.annotation.Resource;
+
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;
+import eu.dnetlib.enabling.datasources.common.Api;
+import eu.dnetlib.enabling.datasources.common.ApiParam;
+import eu.dnetlib.enabling.datasources.common.DsmException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.ProgressProvider;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+
+public class FixRepoMdstoreSizesJobNode extends SimpleJobNode implements ProgressJobNode {
+
+ @Autowired
+ private LocalOpenaireDatasourceManager dsManager;
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Resource(name = "mongodbMDStoreDao")
+ private MDStoreDao mdstoreDao;
+
+ @Autowired
+ private ObjectStoreDao objectStoreDao;
+
+ private final DateUtils dateUtils = new DateUtils();
+
+ private int current = 0;
+ private int total = 0;
+
+ private ISRegistryService registry;
+ private ISLookUpService lookup;
+
+ private final Map openaireIds = new HashMap<>();
+ private boolean alwaysUpdate = false;
+
+ private static final Log log = LogFactory.getLog(FixRepoMdstoreSizesJobNode.class);
+
+ public void init(final int total) {
+ this.current = 0;
+ this.total = total;
+ this.lookup = serviceLocator.getService(ISLookUpService.class);
+ this.registry = serviceLocator.getService(ISRegistryService.class);
+ try {
+ openaireIds.putAll(lookup.quickSearchProfile(
+ "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') return concat($x//DATASOURCE_ORIGINAL_ID, ' @@@ ', $x//RESOURCE_IDENTIFIER/@value)")
+ .stream()
+ .collect(Collectors.toMap(
+ s -> StringUtils.substringBefore(s, "@@@").trim(),
+ s -> StringUtils.substringAfter(s, "@@@").trim())));
+ } catch (final ISLookUpException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ }
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final Set list = dsManager.listManagedDatasourceIds();
+
+ init(list.size());
+
+ for (final String dsId : list) {
+ log.info("Processing ds: " + dsId);
+
+ current++;
+
+ try {
+ for (final Api api : dsManager.getApis(dsId)) {
+ verifyApi(dsId, api);
+ }
+ } catch (final Throwable e) {
+ log.error("Error processing ds: " + dsId, e);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + dsId, e.getMessage());
+ }
+ }
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ private void verifyApi(final String dsId, final Api api)
+ throws DsmException, ISRegistryException, IOException, ISLookUpException, MDStoreServiceException, ObjectStoreServiceException {
+
+ for (final Document doc : listCollectionMdStores(dsId, api.getId())) {
+ final String mdId = doc.valueOf("//RESOURCE_IDENTIFIER/@value");
+ final int actualSize = NumberUtils.toInt(doc.valueOf("//NUMBER_OF_RECORDS"), 0);
+ final int size = mdstoreDao.getMDStore(mdId).getSize();
+ if (alwaysUpdate || size != actualSize) {
+ log.info(" -- Updating size of api " + api.getId() + ", new value = " + size);
+ updateMdStoreProfile(mdId, doc, size);
+ dsManager.setLastCollectionInfo(dsId, api.getId(), mdId, size, calculateLastDate(doc));
+ }
+ }
+
+ for (final Document doc : listTransformationMdStores(dsId, api.getId())) {
+ final String mdId = doc.valueOf("//RESOURCE_IDENTIFIER/@value");
+ final int actualSize = NumberUtils.toInt(doc.valueOf("//NUMBER_OF_RECORDS"), 0);
+ final int size = mdstoreDao.getMDStore(mdId).getSize();
+ if (alwaysUpdate || size != actualSize) {
+ updateMdStoreProfile(mdId, doc, size);
+ dsManager.setLastAggregationInfo(dsId, api.getId(), mdId, size, calculateLastDate(doc));
+ }
+ }
+
+ for (final Document doc : listDownloadObjectStores(dsId, api.getId())) {
+ final String objId = doc.valueOf("//RESOURCE_IDENTIFIER/@value");
+ final int actualSize = NumberUtils.toInt(doc.valueOf("//STORE_SIZE"), 0);
+ final int size = objectStoreDao.getObjectStore(objId).getSize();
+ if (alwaysUpdate || size != actualSize) {
+ updateObjStoreProfile(objId, doc, size);
+ dsManager.setLastDownloadInfo(dsId, api.getId(), objId, size, calculateLastDate(doc));
+ }
+ }
+ }
+
+ private Date calculateLastDate(final Document doc) {
+ final String dateS = doc.valueOf("//LAST_STORAGE_DATE");
+ final Date date = StringUtils.isNoneBlank(dateS) ? dateUtils.parse(dateS) : new Date();
+ return date;
+ }
+
+ private List listCollectionMdStores(final String dsId, final String apiId) throws ISLookUpException, IOException {
+ return executeXquery("listCollectionMdStores.xquery.st", dsId, apiId);
+ }
+
+ private List listTransformationMdStores(final String dsId, final String apiId) throws ISLookUpException, IOException {
+ return executeXquery("listTransformationMdStores.xquery.st", dsId, apiId);
+ }
+
+ private List listDownloadObjectStores(final String dsId, final String apiId) throws ISLookUpException, IOException {
+ return executeXquery("listDownloadObjectStores.xquery.st", dsId, apiId);
+ }
+
+ private List executeXquery(final String template, final String dsId, final String apiId) throws ISLookUpException, IOException {
+ final StringTemplate st = new StringTemplate(IOUtils.toString(getClass().getResourceAsStream(template)));
+ st.setAttribute("dsId", openaireIds.get(dsId));
+ st.setAttribute("apiId", apiId);
+
+ final SAXReader reader = new SAXReader();
+
+ return lookup.quickSearchProfile(st.toString())
+ .stream()
+ .map(s -> {
+ try {
+ return reader.read(new StringReader(s));
+ } catch (final DocumentException e) {
+ return null;
+ }
+ })
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+ }
+
+ private void updateMdStoreProfile(final String mdId, final Document doc, final int size) throws ISRegistryException {
+ doc.selectSingleNode("//NUMBER_OF_RECORDS").setText(Integer.toString(size));
+ registry.updateProfile(mdId, doc.asXML(), "MDStoreDSResourceType");
+ }
+
+ private void updateObjStoreProfile(final String objId, final Document doc, final int size) throws ISRegistryException {
+ doc.selectSingleNode("//COUNT_STORE").setText(Integer.toString(size));
+ doc.selectSingleNode("//STORE_SIZE").setText(Integer.toString(size));
+ registry.updateProfile(objId, doc.asXML(), "ObjectStoreDSResourceType");
+ }
+
+ public boolean isAlwaysUpdate() {
+ return alwaysUpdate;
+ }
+
+ public void setAlwaysUpdate(final boolean alwaysUpdate) {
+ this.alwaysUpdate = alwaysUpdate;
+ }
+
+ @Override
+ public ProgressProvider getProgressProvider() {
+ return new ProgressProvider() {
+
+ @Override
+ public int getTotalValue() {
+ return total;
+ }
+
+ @Override
+ public int getCurrentValue() {
+ return current;
+ }
+
+ @Override
+ public boolean isInaccurate() {
+ return false;
+ }
+ };
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_canadian_funder.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_canadian_funder.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_canadian_funder.sql (revision 58540)
@@ -0,0 +1,49 @@
+INSERT INTO dsm_organizations (id, legalshortname,legalname)
+VALUES
+ ('nserc_______::NSERC','NSERC','Natural Sciences and Engineering Research Council of Canada'),
+ ('sshrc_______::SSHRC','SSHRC','Social Sciences and Humanities Research Council'),
+ ('cihr________::CIHR','CIHR','Canadian Institutes of Health Research');
+
+INSERT INTO fundingpaths (id,path,funder,jurisdiction)
+VALUES
+ ('nserc_______::NSERC','
+
+ nserc_______::NSERC
+ NSERC
+ Natural Sciences and Engineering Research Council of Canada
+ Natural Sciences and Engineering Research Council of Canada
+ CA
+
+ ', 'nserc_______::NSERC','CA'),
+ ('sshrc_______::SSHRC','
+
+ sshrc_______::SSHRC
+ SSHRC
+ Social Sciences and Humanities Research Council
+ Social Sciences and Humanities Research Council
+ CA
+
+ ', 'sshrc_______::SSHRC','CA'),
+ ('cihr________::CIHR','
+
+ cihr________::CIHR
+ CIHR
+ Canadian Institutes of Health Research
+ Canadian Institutes of Health Research
+ CA
+
+ ', 'cihr________::CIHR','CA');
+
+INSERT INTO projects (id, code, title, collectedfrom, provenanceactionclass)
+VALUES
+ ('nserc_______::unidentified', 'unidentified', 'unidentified', 'infrastruct_::openaire', 'sysimport:crosswalk:entityregistry'),
+ ('sshrc_______::unidentified', 'unidentified', 'unidentified', 'infrastruct_::openaire', 'sysimport:crosswalk:entityregistry'),
+ ('cihr________::unidentified', 'unidentified', 'unidentified', 'infrastruct_::openaire', 'sysimport:crosswalk:entityregistry');
+
+INSERT INTO project_fundingpath (funding, project)
+VALUES
+ ('nserc_______::NSERC', 'nserc_______::unidentified'),
+ ('sshrc_______::SSHRC', 'sshrc_______::unidentified'),
+ ('cihr________::CIHR', 'cihr________::unidentified');
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PrepareActionSetsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PrepareActionSetsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PrepareActionSetsJobNode.java (revision 58540)
@@ -0,0 +1,104 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.actions;
+
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.actionmanager.set.RawSet;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class PrepareActionSetsJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(PrepareActionSetsJobNode.class);
+
+ private String ACTIONSET_PATH_XQUERY_TEMPLATE =
+ "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') "
+ + "return $x/RESOURCE_PROFILE/BODY/SET[@id = '%s']/@directory/string()";
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ private String sets;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final List> setList = getSetList();
+ final String now = DateUtils.now_ISO8601();
+
+ final ISLookUpService lookUpService = serviceLocator.getService(ISLookUpService.class);
+ final String basePath = lookUpService.getResourceProfileByQuery(
+ "/RESOURCE_PROFILE[./HEADER/RESOURCE_TYPE/@value='ActionManagerServiceResourceType']//SERVICE_PROPERTIES/PROPERTY[@key='basePath']/@value/string()");
+
+ for (Map set : setList) {
+
+ set.put("rawset", RawSet.newInstance().getId());
+ set.put("creationDate", now);
+ set.put("path", getFullPath(basePath, set, lookUpService));
+
+ if (set.get("enabled").equals("true")) {
+ log.info("preparing set: " + simplifySetInfo(set));
+ }
+ // setting the job properties needed to name the rawsets
+ token.getEnv().setAttribute(set.get("jobProperty"), set.get("rawset"));
+ }
+
+ token.getEnv().setAttribute("sets", new Gson().toJson(setList));
+ token.getEnv().setAttribute("actionManagerBasePath", basePath);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ private String getFullPath(final String basePath, final Map set, final ISLookUpService lookUpService) throws MSROException {
+ return new Path(basePath + "/" + getPath(set.get("set"), lookUpService) + "/" + set.get("rawset")).toString();
+ }
+
+ private String getPath(final String setId, final ISLookUpService lookUpService) throws MSROException {
+ try {
+ return lookUpService.getResourceProfileByQuery(String.format(ACTIONSET_PATH_XQUERY_TEMPLATE, setId));
+ } catch (ISLookUpException e) {
+ throw new MSROException(String.format("Error obtaining directory from ActionSet profile %s", setId), e);
+ }
+ }
+
+ private Map simplifySetInfo(final Map set) {
+ return Maps.filterKeys(set, new Predicate() {
+
+ @Override
+ public boolean apply(final String k) {
+
+ return k.equals("set") || k.equals("rawset");
+ }
+ });
+ }
+
+ @SuppressWarnings("unchecked")
+ protected List> getSetList() {
+ return new Gson().fromJson(getSets(), List.class);
+ }
+
+ public String getSets() {
+ return sets;
+ }
+
+ public void setSets(final String sets) {
+ this.sets = sets;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/incrementalTransformation_migration.xqueries
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/incrementalTransformation_migration.xqueries (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/incrementalTransformation_migration.xqueries (revision 58540)
@@ -0,0 +1,37 @@
+#0. READ EVERYTHING BEFORE RUNNING ANY XUPDATE
+#1. Take all transformation workflows and change the arc to select the currently non existing node "selectIncrementalTransformation"
+
+for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType')
+let $arc := $x[.//WORKFLOW_NAME='transform']/RESOURCE_PROFILE/BODY/CONFIGURATION/NODE[./@type='SetProviderInfo' and ./@isStart='true']/ARCS/ARC[./@to ='fetchOriginals']
+return update value $arc/@to with 'selectIncrementalTransformation'
+
+#2. Now we add the missing node just after the start node
+
+for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType')
+let $startnode := $x[.//WORKFLOW_NAME='transform']/RESOURCE_PROFILE/BODY/CONFIGURATION/NODE[./@type='SetProviderInfo' and ./@isStart='true']
+return update insert
+
+ Decide REFRESH/INCREMENTAL transformation
+
+ INCREMENTAL
+
+
+
+
+
+following $startnode
+
+In case there are nodes with the wrong parameter name (transformationType instead of operationType), do not worry.
+It is a left over from previous migration and you can fix it running this xquery:
+
+for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType')
+let $transNode := $x[.//WORKFLOW_NAME='transform']/RESOURCE_PROFILE/BODY/CONFIGURATION/NODE[./@type='IncrementalTransformation']
+let $param := $transNode/PARAMETERS/PARAM[@name='transformationType']
+return update value $param/@name with 'operationType'
+
+NOTE: if there are already incremental workflows, the xupdate in #2. will screw them up
+creating a duplicate IncrementalTransformation node that must be deleted.
+You can look for this workflows with the following xquery:
+
+for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType')
+where (count($x[.//NODE/@name='selectIncrementalTransformation']) > 1) return $x
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/ExtractOutputPathJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/ExtractOutputPathJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/ExtractOutputPathJobNode.java (revision 58540)
@@ -0,0 +1,46 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.actions;
+
+import com.google.gson.Gson;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
+import java.util.Map;
+
+public class ExtractOutputPathJobNode extends SimpleJobNode {
+
+ private String hdfsOutputPathParam;
+
+
+ @Override
+ protected String execute(NodeToken token) throws Exception {
+ final String sets = token.getEnv().getAttribute("sets");
+ if (StringUtils.isBlank(sets))
+ throw new MSROException("¯\\\\_(ツ)_/¯ cannot find sets on env");
+
+ final List> setsMap = new Gson().fromJson(sets, List.class);
+ if (setsMap== null || setsMap.size() !=1) {
+ throw new MSROException("¯\\\\_(ツ)_/¯ Sets map from json is wrong!");
+ }
+
+
+ final String path = setsMap.get(0).get("path");
+ if (StringUtils.isEmpty(path)){
+ throw new MSROException("Path is empty");
+ }
+ token.getEnv().setAttribute(getHdfsOutputPathParam(), path);
+ return Arc.DEFAULT_ARC;
+ }
+
+
+ public String getHdfsOutputPathParam() {
+ return hdfsOutputPathParam;
+ }
+
+ public void setHdfsOutputPathParam(String hdfsOutputPathParam) {
+ this.hdfsOutputPathParam = hdfsOutputPathParam;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-constraints.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-constraints.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-constraints.sql (revision 58540)
@@ -0,0 +1,54 @@
+-- THE AIM OF THIS FILE IS TO COLLECT SQL COMMANDS TO ADD "ON DELETE CASCADE" CONSTRAINTS
+-- SOME OF THE STATEMENTS MAY NOT MAKE SENSE IN THIS BRANCH, AS WE ARE SUPPOSED NOT TO USE THE funding_funding TABLE ANYMORE
+
+-- when we delete a funding we want to the delete all the related stuff: subfundings, projects, organisations
+ALTER TABLE funding_funding
+ DROP CONSTRAINT funding_funding_funding1_fkey;
+ALTER TABLE funding_funding
+ ADD CONSTRAINT funding_funding_funding1_fkey FOREIGN KEY (funding1) REFERENCES fundings (id) ON DELETE CASCADE;
+ALTER TABLE funding_funding
+ DROP CONSTRAINT funding_funding_funding2_fkey;
+ALTER TABLE funding_funding
+ ADD CONSTRAINT funding_funding_funding2_fkey FOREIGN KEY (funding2) REFERENCES fundings (id) ON DELETE CASCADE;
+
+ALTER TABLE project_funding
+ DROP CONSTRAINT project_funding_funding_fkey;
+ALTER TABLE project_funding
+ ADD CONSTRAINT project_funding_funding_fkey FOREIGN KEY (funding) REFERENCES fundings (id) ON DELETE CASCADE;
+ALTER TABLE project_funding
+ DROP CONSTRAINT project_funding_project_fkey;
+ALTER TABLE project_funding
+ ADD CONSTRAINT project_funding_project_fkey FOREIGN KEY (project) REFERENCES projects (id) ON DELETE CASCADE;
+
+ALTER TABLE organization_funding
+ DROP CONSTRAINT organization_funding_funding_fkey;
+ALTER TABLE organization_funding
+ ADD CONSTRAINT organization_funding_funding_fkey FOREIGN KEY (funding) REFERENCES fundings (id) ON DELETE CASCADE;
+
+-- when we delete an organisation we want to delete all the related stuff
+ALTER TABLE datasource_organization
+ DROP CONSTRAINT datasource_organization_organization_fkey;
+ALTER TABLE datasource_organization
+ ADD CONSTRAINT datasource_organization_organization_fkey FOREIGN KEY (organization) REFERENCES organizations (id) ON DELETE CASCADE;
+ALTER TABLE project_organization
+ DROP CONSTRAINT project_organization_resporganization_fkey;
+ALTER TABLE project_organization
+ ADD CONSTRAINT project_organization_resporganization_fkey FOREIGN KEY (resporganization) REFERENCES organizations (id) ON DELETE CASCADE;
+
+-- when we delete a project, also its link to organisations must be deleted
+ALTER TABLE project_organization
+ DROP CONSTRAINT project_organization_project_fkey;
+ALTER TABLE project_organization
+ ADD CONSTRAINT project_organization_project_fkey FOREIGN KEY (project) REFERENCES projects (id) ON DELETE CASCADE;
+
+-- when we delete a fundingpath, also the corresponding entries in the many-to-many tabel to projects must be deleted
+ALTER TABLE project_fundingpath
+ DROP CONSTRAINT project_fundingpath_funding_fkey;
+ALTER TABLE project_fundingpath
+ ADD CONSTRAINT project_fundingpath_funding_fkey FOREIGN KEY (funding) REFERENCES fundingpaths (id) ON DELETE CASCADE;
+
+-- when we delete a project, also its links to fundingpaths must be deleted
+ALTER TABLE project_fundingpath
+ DROP CONSTRAINT project_fundingpath_project_fkey;
+ALTER TABLE project_fundingpath
+ ADD CONSTRAINT project_fundingpath_project_fkey FOREIGN KEY (project) REFERENCES projects (id) ON DELETE CASCADE;
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PromoteActionsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PromoteActionsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PromoteActionsJobNode.java (revision 58540)
@@ -0,0 +1,37 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.actions;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.actionmanager.rmi.ActionManagerService;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.lang.StringUtils;
+
+public class PromoteActionsJobNode extends BlackboardJobNode {
+
+ public static final String ALL_SETS = "ALL SETS";
+
+ private String set;
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ return getServiceLocator().getServiceId(ActionManagerService.class);
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction("PROMOTE");
+ if (!StringUtils.isBlank(getSet()) && !getSet().equals(ALL_SETS)) {
+ job.getParameters().put("set", getSet());
+ token.getEnv().setAttribute("set", getSet());
+ }
+ }
+
+ public String getSet() {
+ return set;
+ }
+
+ public void setSet(final String set) {
+ this.set = set;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_funder_countries.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_funder_countries.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_funder_countries.sql (revision 58540)
@@ -0,0 +1 @@
+update dsm_organizations dor set country = f.jurisdiction from fundingpaths f where f.funder = dor.id ;
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PromoteActionsHDFSJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PromoteActionsHDFSJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/PromoteActionsHDFSJobNode.java (revision 58540)
@@ -0,0 +1,40 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.actions;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.actionmanager.rmi.ActionManagerService;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Created by claudio on 14/04/16.
+ */
+public class PromoteActionsHDFSJobNode extends BlackboardJobNode {
+
+ public static final String ALL_SETS = "ALL SETS";
+
+ private String set;
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ return getServiceLocator().getServiceId(ActionManagerService.class);
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction("PROMOTE_FROM_HDFS");
+ if (!StringUtils.isBlank(getSet()) && !getSet().equals(ALL_SETS)) {
+ job.getParameters().put("set", getSet());
+ token.getEnv().setAttribute("set", getSet());
+ }
+ }
+
+ public String getSet() {
+ return set;
+ }
+
+ public void setSet(final String set) {
+ this.set = set;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repohi/UpdateOpenaireMetaWfStatusJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repohi/UpdateOpenaireMetaWfStatusJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repohi/UpdateOpenaireMetaWfStatusJobNode.java (revision 58540)
@@ -0,0 +1,35 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.repohi;
+
+import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.repohi.UpdateMetaWfStatusJobNode;
+import org.springframework.beans.factory.annotation.Required;
+
+public class UpdateOpenaireMetaWfStatusJobNode extends UpdateMetaWfStatusJobNode {
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Autowired
+ private LocalDatasourceManager dsManager;
+
+ @Override
+ protected void updateDatasource(final String dsId, final String ifaceId) throws Exception {
+
+ final String openaireDsId = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
+ "/*[.//RESOURCE_IDENTIFIER/@value = '" + dsId + "']//FIELD/value[../key='OpenAireDataSourceId']/text()");
+
+ if (openaireDsId.equals("openaire____::bootstrap")) {
+ super.updateDatasource(dsId, ifaceId);
+ } else {
+ dsManager.setManaged(openaireDsId, true);
+ dsManager.setActive(openaireDsId, ifaceId, true);
+ }
+ }
+
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_h2020.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_h2020.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_h2020.sql (revision 58540)
@@ -0,0 +1,146 @@
+-- REMEMBER TO TO REGISTER THE FOLLOWING VOCABULARY (the TOAS):
+-- f9b99dfb-ddd3-4fe1-8ee5-5c7acc5fc901_Vm9jYWJ1bGFyeURTUmVzb3VyY2VzL1ZvY2FidWxhcnlEU1Jlc291cmNlVHlwZQ==
+-- and then launch the consistency workflow to update class/scheme into the database
+
+INSERT INTO datasources (
+ id,
+ officialname,
+ englishname,
+ websiteurl,
+ logourl,
+ contactemail,
+ namespaceprefix,
+ collectedfrom,
+ datasourceclass,
+ _dnet_resource_identifier_
+) VALUES (
+ 'openaire____::corda_h2020',
+ 'CORDA - COmmon Research DAta Warehouse - Horizon 2020',
+ 'CORDA - COmmon Research DAta Warehouse - Horizon 2020',
+ 'http://ec.europa.eu/programmes/horizon2020/',
+ 'http://ec.europa.eu/programmes/horizon2020/sites/horizon2020/themes/h2020/images/bannerEmptyH2020.jpg',
+ 'ec-fp7-it-helpdesk@ec.europa.eu',
+ 'corda__h2020',
+ 'infrastruct_::openaire',
+ 'entityregistry',
+ 'openaire____::corda_h2020'
+);
+
+INSERT INTO api (
+ id,
+ protocolclass,
+ datasource,
+ _dnet_resource_identifier_,
+ typologyclass,
+ compatibilityclass
+) VALUES (
+ 'openaire____::corda_h2020::0',
+ 'filesystem',
+ 'openaire____::corda_h2020',
+ 'openaire____::corda_h2020::0',
+ 'entityregistry',
+ 'native'
+);
+
+INSERT INTO apicollections (
+ param,
+ original,
+ api,
+ _dnet_resource_identifier_
+) VALUES (
+ 'extensions',
+ 'xml',
+ 'openaire____::corda_h2020::0',
+ 'openaire____::corda_h2020::0@@extensions'
+);
+
+INSERT INTO apicollections (
+ param,
+ original,
+ api,
+ _dnet_resource_identifier_
+) VALUES (
+ 'baseUrl',
+ 'file:///var/lib/dnet/corda_h2020',
+ 'openaire____::corda_h2020::0',
+ 'openaire____::corda_h2020::0@@baseUrl'
+);
+
+INSERT INTO apicollections (
+ param,
+ original,
+ api,
+ _dnet_resource_identifier_,
+ accessparam
+) VALUES (
+ 'metadata_identifier_path',
+ '//project/@id',
+ 'openaire____::corda_h2020::0',
+ 'openaire____::corda_h2020::0@@metadata_identifier_path',
+ FALSE
+);
+
+--
+INSERT INTO datasources (
+ id,
+ officialname,
+ englishname,
+ websiteurl,
+ logourl,
+ contactemail,
+ namespaceprefix,
+ collectedfrom,
+ datasourceclass,
+ _dnet_resource_identifier_
+) VALUES (
+ 'openaire____::corda_h2020_fundings',
+ 'CORDA - COmmon Research DAta Warehouse - Horizon 2020 - Fundings',
+ 'CORDA - COmmon Research DAta Warehouse - Horizon 2020 - Fundings',
+ 'http://ec.europa.eu/programmes/horizon2020/',
+ 'http://ec.europa.eu/programmes/horizon2020/sites/horizon2020/themes/h2020/images/bannerEmptyH2020.jpg',
+ 'ec-fp7-it-helpdesk@ec.europa.eu',
+ 'ec__________',
+ 'infrastruct_::openaire',
+ 'entityregistry',
+ 'openaire____::corda_h2020'
+);
+
+INSERT INTO api (
+ id,
+ protocolclass,
+ datasource,
+ _dnet_resource_identifier_,
+ typologyclass,
+ compatibilityclass
+) VALUES (
+ 'openaire____::corda_h2020_fundings::0',
+ 'filesystem',
+ 'openaire____::corda_h2020_fundings',
+ 'openaire____::corda_h2020_fundings::0',
+ 'entityregistry',
+ 'native'
+);
+
+INSERT INTO apicollections (
+ param,
+ original,
+ api,
+ _dnet_resource_identifier_
+) VALUES (
+ 'extensions',
+ 'xml',
+ 'openaire____::corda_h2020_fundings::0',
+ 'openaire____::corda_h2020_fundings::0@@extensions'
+);
+
+INSERT INTO apicollections (
+ param,
+ original,
+ api,
+ _dnet_resource_identifier_
+) VALUES (
+ 'baseUrl',
+ 'file:///var/lib/dnet/corda_h2020_fundings',
+ 'openaire____::corda_h2020_fundings::0',
+ 'openaire____::corda_h2020_fundings::0@@baseUrl'
+);
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch.sql (revision 58540)
@@ -0,0 +1,343 @@
+ALTER TABLE datasources
+ ADD COLUMN issn CHARACTER VARYING(20);
+ALTER TABLE datasources
+ ADD COLUMN eissn CHARACTER VARYING(20);
+ALTER TABLE datasources
+ ADD COLUMN lissn CHARACTER VARYING(20);
+ALTER TABLE datasources
+ ALTER COLUMN databaseaccesstype TYPE VARCHAR(32);
+ALTER TABLE datasources
+ ALTER COLUMN datauploadtype TYPE VARCHAR(32);
+ALTER TABLE datasources
+ ALTER COLUMN databaseaccessrestriction TYPE VARCHAR(32);
+ALTER TABLE datasources
+ ALTER COLUMN datauploadrestriction TYPE VARCHAR(32);
+ALTER TABLE datasources
+ ALTER COLUMN qualitymanagementkind TYPE VARCHAR(32);
+
+ALTER TABLE datasources
+ ADD COLUMN registeredby CHARACTER VARYING(255);
+
+DELETE FROM apicollections
+WHERE api NOT LIKE 'api_________::%';
+DELETE FROM api
+WHERE id NOT LIKE 'api_________::%';
+
+ALTER TABLE apicollections
+ DROP COLUMN name;
+ALTER TABLE apicollections
+ RENAME COLUMN accessparam TO param;
+ALTER TABLE apicollections
+ RENAME COLUMN accessvalue TO original;
+ALTER TABLE apicollections
+ ADD COLUMN edited VARCHAR(255) DEFAULT NULL;
+ALTER TABLE apicollections
+ ADD COLUMN accessparam BOOLEAN DEFAULT TRUE;
+ALTER TABLE apicollections
+ DROP CONSTRAINT apicollection_pkey;
+ALTER TABLE apicollections
+ ALTER COLUMN original SET DEFAULT '';
+
+UPDATE apicollections
+SET param = 'set'
+WHERE param = 'SET';
+UPDATE apicollections
+SET param = 'format'
+WHERE param = 'FORMAT';
+UPDATE apicollections
+SET param = 'baseUrl'
+WHERE param = 'BASEURL';
+
+-- FIX A PROBLEM WITH MULTIPLE SET
+DELETE FROM apicollections
+WHERE api = 'api_________::opendoar____::1560::1' AND param = 'set';
+INSERT INTO apicollections (param, original, api) VALUES
+ ('set', 'hdl_10251_3829, hdl_10251_321, hdl_10251_11256, hdl_10251_11206, hdl_10251_8702, hdl_10251_3026, hdl_10251_11076',
+ 'api_________::opendoar____::1560::1');
+DELETE FROM apicollections
+WHERE api = 'api_________::driver______::d0b68287-678d-4efc-b354-c47cbacaed3c::0' AND param = 'set';
+INSERT INTO apicollections (param, original, api) VALUES
+ ('set', 'pub-type:article, pub-type:conf-proceeding, pub-type:lecture, pub-type:paper, pub-type:report',
+ 'api_________::driver______::d0b68287-678d-4efc-b354-c47cbacaed3c::0');
+DELETE FROM apicollections
+WHERE api = 'api_________::driver______::79476713-c428-4f23-bf97-8f27500f754a::0' AND param = 'set';
+INSERT INTO apicollections (param, original, api)
+VALUES ('set', 'dad:SIQG, jhs,sal:CNF+PA', 'api_________::driver______::79476713-c428-4f23-bf97-8f27500f754a::0');
+DELETE FROM apicollections
+WHERE api = 'api_________::opendoar____::202::0' AND param = 'set';
+INSERT INTO apicollections (param, original, api) VALUES ('set', 'hdl_1887_4540, hdl_1887_4951', 'api_________::opendoar____::202::0');
+DELETE FROM apicollections
+WHERE api = 'api_________::driver______::c75b9f23-25a8-4e45-aadd-ba449211e2a8::0' AND param = 'set';
+INSERT INTO apicollections (param, original, api) VALUES ('set', 'ijdc:ART, ijdc:PAP', 'api_________::driver______::c75b9f23-25a8-4e45-aadd-ba449211e2a8::0');
+-- AND FIX
+
+ALTER TABLE apicollections
+ ADD CONSTRAINT apicollection_pkey PRIMARY KEY (api, param);
+
+INSERT INTO apicollections (param, original, edited, api) SELECT
+ 'baseUrl',
+ COALESCE(original, ''),
+ COALESCE(edited, ''),
+ id
+ FROM api
+ WHERE originalprotocolclass = editedprotocolclass;
+INSERT INTO apicollections (param, original, api) SELECT
+ 'baseUrl',
+ COALESCE(original, ''),
+ id
+ FROM api
+ WHERE originalprotocolclass != editedprotocolclass OR editedprotocolclass IS NULL;
+UPDATE apicollections
+SET edited = NULL
+WHERE edited = '';
+
+UPDATE datasources
+SET datasourceclass = 'websource'
+WHERE id = 'openaire____::webcrawl';
+
+INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'repository', 'api_________::opendoar::0');
+INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'ROW', 'api_________::re3data::0');
+INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'Record', 'api_________::wellcometrust::0');
+INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'ROW', 'api_________::corda::0');
+
+UPDATE apicollections
+SET _dnet_resource_identifier_ = api || '@@' || param;
+
+INSERT INTO scheme (_dnet_resource_identifier_, code, name)
+VALUES ('dnet:content_description_typologies', 'dnet:content_description_typologies', 'D-Net Content Description Typologies');
+
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('UNKNOWN', 'UNKNOWN', 'UNKNOWN');
+
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('native', 'native', 'native');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('native@@dnet:compatibilityLevel', 'native', 'dnet:compatibilityLevel');
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('files', 'files', 'files');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('files@@dnet:compatibilityLevel', 'files', 'dnet:compatibilityLevel');
+
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('metadata', 'metadata', 'metadata');
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('file::WoS', 'file::WoS', 'file::WoS');
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('file::PDF', 'file::PDF', 'file::PDF');
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('file::EuropePMC', 'file::EuropePMC', 'file::EuropePMC');
+
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme)
+VALUES ('metadata@@dnet:content_description_typologies', 'metadata', 'dnet:content_description_typologies');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme)
+VALUES ('file::WoS@@dnet:content_description_typologies', 'file::WoS', 'dnet:content_description_typologies');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme)
+VALUES ('file::PDF@@dnet:content_description_typologies', 'file::PDF', 'dnet:content_description_typologies');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme)
+VALUES ('file::EuropePMC@@dnet:content_description_typologies', 'file::EuropePMC', 'dnet:content_description_typologies');
+
+ALTER TABLE api
+ ADD COLUMN contentdescriptionclass VARCHAR(255) DEFAULT 'metadata';
+ALTER TABLE api
+ ADD COLUMN contentdescriptionscheme VARCHAR(255) DEFAULT 'dnet:content_description_typologies';
+ALTER TABLE api
+ DROP COLUMN editedprotocolclass;
+ALTER TABLE api
+ DROP COLUMN editedprotocolscheme;
+ALTER TABLE api
+ DROP COLUMN original;
+ALTER TABLE api
+ DROP COLUMN edited;
+ALTER TABLE api
+ DROP COLUMN useedited;
+ALTER TABLE api
+ RENAME COLUMN originalprotocolclass TO protocolclass;
+ALTER TABLE api
+ RENAME COLUMN originalprotocolscheme TO protocolscheme;
+ALTER TABLE api
+ ADD COLUMN active BOOLEAN DEFAULT FALSE;
+ALTER TABLE api
+ ADD COLUMN removable BOOLEAN DEFAULT FALSE;
+ALTER TABLE api
+ ADD COLUMN typologyclass VARCHAR(255) DEFAULT 'UNKNOWN';
+ALTER TABLE api
+ ADD COLUMN typologyscheme VARCHAR(255) DEFAULT 'dnet:datasource_typologies';
+ALTER TABLE api
+ ADD CONSTRAINT api_typologyclass_fkey FOREIGN KEY (typologyclass) REFERENCES class (code);
+ALTER TABLE api
+ ADD CONSTRAINT api_typologyscheme_fkey FOREIGN KEY (typologyscheme) REFERENCES scheme (code);
+ALTER TABLE api
+ ADD COLUMN compatibilityclass VARCHAR(255) DEFAULT 'UNKNOWN';
+ALTER TABLE api
+ ADD COLUMN compatibilityscheme VARCHAR(255) DEFAULT 'dnet:compatibilityLevel';
+ALTER TABLE api
+ ADD CONSTRAINT api_compatibilityclass_fkey FOREIGN KEY (compatibilityclass) REFERENCES class (code);
+ALTER TABLE api
+ ADD CONSTRAINT api_compatibilityscheme_fkey FOREIGN KEY (compatibilityscheme) REFERENCES scheme (code);
+
+ALTER TABLE api
+ ADD CONSTRAINT api_contentdescriptionclass_fkey FOREIGN KEY (contentdescriptionclass) REFERENCES class (code);
+ALTER TABLE api
+ ADD CONSTRAINT api_contentdescriptionscheme_fkey FOREIGN KEY (contentdescriptionscheme) REFERENCES scheme (code);
+
+UPDATE api
+SET typologyclass = d.datasourceclass FROM datasources d
+WHERE datasource = d.id;
+--UPDATE api SET compatibilityclass = 'driver' where typology = 'driver';
+--UPDATE api SET compatibilityclass = 'openaire2.0' FROM apicollections ac WHERE typology = 'openaire' AND id = ac.api AND ac.param = 'set' AND ac.original like '%ec_fundedresources%';
+--UPDATE api SET compatibilityclass = 'openaire3.0' FROM apicollections ac WHERE typology = 'openaire' AND id = ac.api AND ac.param = 'set' AND ac.original like '%openaire%';
+
+-- Jochen version
+UPDATE api
+SET compatibilityclass = ds.openairecompatibilityclass FROM datasources ds
+WHERE ds.id = api.datasource;
+UPDATE api
+SET compatibilityclass = 'driver'
+WHERE compatibilityclass = 'driver-openaire2.0' AND typology = 'driver';
+UPDATE api
+SET compatibilityclass = 'openaire2.0'
+WHERE compatibilityclass = 'driver-openaire2.0' AND typology = 'openaire';
+UPDATE api
+SET compatibilityclass = 'UNKNOWN'
+WHERE compatibilityclass = 'openaire2.0-openaire3.0'
+-- END Jochen
+
+UPDATE api
+SET (typologyclass, compatibilityclass) = ('entityregistry', 'native')
+WHERE id = 'api_________::opendoar::0';
+UPDATE api
+SET (typologyclass, compatibilityclass) = ('entityregistry', 'native')
+WHERE id = 'api_________::re3data::0';
+UPDATE api
+SET (typologyclass, compatibilityclass) = ('entityregistry', 'native')
+WHERE id = 'api_________::wellcometrust::0';
+UPDATE api
+SET (typologyclass, compatibilityclass) = ('entityregistry', 'native')
+WHERE id = 'api_________::corda::0';
+
+ALTER TABLE api
+ DROP COLUMN typology;
+ALTER TABLE datasources
+ DROP COLUMN openairecompatibilityclass;
+ALTER TABLE datasources
+ DROP COLUMN openairecompatibilityscheme;
+
+DELETE FROM class_scheme
+WHERE scheme = 'dnet:compatibilityLevel' AND class = 'driver-openaire2.0';
+DELETE FROM class_scheme
+WHERE scheme = 'dnet:compatibilityLevel' AND class = 'driver-openaire2.0-openaire3.0';
+DELETE FROM class_scheme
+WHERE scheme = 'dnet:compatibilityLevel' AND class = 'driver-openaire3.0';
+DELETE FROM class_scheme
+WHERE scheme = 'dnet:compatibilityLevel' AND class = 'openaire2.0-openaire3.0';
+DELETE FROM class
+WHERE code = 'driver-openaire2.0';
+DELETE FROM class
+WHERE code = 'driver-openaire2.0-openaire3.0';
+DELETE FROM class
+WHERE code = 'driver-openaire3.0';
+DELETE FROM class
+WHERE code = 'openaire2.0-openaire3.0';
+
+DELETE FROM apicollections
+WHERE api = 'api_________::openaire____::webcrawl::0';
+DELETE FROM api
+WHERE id = 'api_________::openaire____::webcrawl::0';
+
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('fileCSV', 'fileCSV', 'fileCSV');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('fileCSV@@dnet:protocols', 'fileCSV', 'dnet:protocols');
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('files_from_mdstore', 'files_from_mdstore', 'files_from_mdstore');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('files_from_mdstore@@dnet:protocols', 'files_from_mdstore', 'dnet:protocols');
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('files_from_metadata', 'files_from_metadata', 'files_from_metadata');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('files_from_metadata@@dnet:protocols', 'files_from_metadata', 'dnet:protocols');
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('files_by_rpc', 'files_by_rpc', 'files_by_rpc');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('files_by_rpc@@dnet:protocols', 'files_by_rpc', 'dnet:protocols');
+INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('filesystem', 'filesystem', 'filesystem');
+INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('filesystem@@dnet:protocols', 'filesystem', 'dnet:protocols');
+
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass)
+VALUES ('api_________::openaire____::webcrawl::csv', 'api_________::openaire____::webcrawl::csv', 'fileCSV', 'openaire____::webcrawl', 'websource', 'native');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES
+ ('api_________::openaire____::webcrawl::files', 'api_________::openaire____::webcrawl::files', 'files_from_mdstore', 'openaire____::webcrawl', 'websource',
+ 'files', 'file::WoS');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES
+ ('api_________::opendoar____::18::files', 'api_________::opendoar____::18::files', 'files_from_metadata', 'opendoar____::18', 'pubsrepository::unknown',
+ 'files', 'file::PDF');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES
+ ('api_________::opendoar____::908::files', 'api_________::opendoar____::908::files', 'files_from_metadata', 'opendoar____::908', 'pubsrepository::unknown',
+ 'files', 'file::PDF');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES
+ ('api_________::opendoar____::2367::files', 'api_________::opendoar____::2367::files', 'files_from_metadata', 'opendoar____::2367', 'pubsrepository::unknown',
+ 'files', 'file::PDF');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES
+ ('api_________::opendoar____::165::files', 'api_________::opendoar____::165::files', 'files_from_metadata', 'opendoar____::165', 'pubsrepository::unknown',
+ 'files', 'file::PDF');
+INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES
+ ('api_________::infrastruct_::openaire::jdbc', 'api_________::infrastruct_::openaire::jdbc', 'jdbc', 'infrastruct_::openaire', 'scholarcomminfra',
+ 'openaire2.0', 'metadata');
+
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'jdbc:postgresql://localhost:5432', 'api_________::infrastruct_::openaire::jdbc');
+INSERT INTO apicollections (param, original, api) VALUES ('dbName', 'dnet_openaireplus', 'api_________::infrastruct_::openaire::jdbc');
+INSERT INTO apicollections (param, original, api) VALUES
+ ('sql', 'SELECT regexp_replace(xml, ''&(?!amp;)'', ''&'', ''g'') as xml FROM claims WHERE type = ''dmf2actions'' and set = ''userclaim_dmf''',
+ 'api_________::infrastruct_::openaire::jdbc');
+INSERT INTO apicollections (param, original, api) VALUES ('format', 'OAF', 'api_________::infrastruct_::openaire::jdbc');
+
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'file:///var/lib/dnet/wos/metadata.tsv', 'api_________::openaire____::webcrawl::csv');
+INSERT INTO apicollections (param, original, api) VALUES ('separator', E'\\t', 'api_________::openaire____::webcrawl::csv');
+INSERT INTO apicollections (param, original, api) VALUES ('identifier', '56', 'api_________::openaire____::webcrawl::csv');
+INSERT INTO apicollections (param, original, api) VALUES ('header', 'true', 'api_________::openaire____::webcrawl::csv');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::openaire____::webcrawl::files');
+INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::openaire____::webcrawl::files');
+INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//column[./@isID=''true'']', 'api_________::openaire____::webcrawl::files');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::opendoar____::18::files');
+INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::opendoar____::18::files');
+INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//dc:identifier', 'api_________::opendoar____::18::files');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::opendoar____::908::files');
+INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::opendoar____::908::files');
+INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//dc:identifier', 'api_________::opendoar____::908::files');
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::opendoar____::2367::files');
+INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::opendoar____::2367::files');
+INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//dc:identifier', 'api_________::opendoar____::2367::files');
+
+INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::opendoar____::165::files');
+INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::opendoar____::165::files');
+INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//dc:identifier', 'api_________::opendoar____::165::files');
+
+INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '//repository/@rID', 'api_________::opendoar::0', FALSE);
+INSERT INTO apicollections (param, original, api, accessparam)
+VALUES ('metadata_identifier_path', '//*[local-name()=''repository'']/*[local-name()=''identifier'']', 'api_________::re3data::0', FALSE);
+INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '//Position', 'api_________::wellcometrust::0', FALSE);
+INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '//ProjectId', 'api_________::corda::0', FALSE);
+
+DELETE FROM apicollections
+WHERE param = 'splitOnElement' AND api = 'api_________::corda::0';
+INSERT INTO apicollections (param, original, api) VALUES ('filter', '*.xml', 'api_________::corda::0');
+INSERT INTO apicollections (param, original, api) VALUES ('username', '***', 'api_________::corda::0');
+INSERT INTO apicollections (param, original, api) VALUES ('password', '***', 'api_________::corda::0');
+INSERT INTO apicollections (param, original, api) VALUES ('recursive', 'false', 'api_________::corda::0');
+
+UPDATE apicollections
+SET original = 'ftp://fts.ec.europa.eu'
+WHERE api = 'api_________::corda::0' AND param = 'baseUrl';
+UPDATE api
+SET protocolclass = 'ftp'
+WHERE id = 'api_________::corda::0';
+UPDATE api
+SET protocolclass = 'filesystem'
+WHERE id = 'api_________::re3data::0';
+UPDATE apicollections
+SET original = '/var/lib/dnet/re3data'
+WHERE api = 'api_________::re3data::0' AND param = 'baseUrl';
+UPDATE api
+SET compatibilityclass = 'native'
+WHERE id = 'api_________::datacite::0';
+
+-- metadata_identifier_path for Datacite and WoS
+INSERT INTO apicollections (param, original, api, accessparam)
+VALUES ('metadata_identifier_path', '//*[local-name()=''header'']/*[local-name()=''identifier'']', 'api_________::datacite::0', FALSE);
+INSERT INTO apicollections (param, original, api, accessparam)
+VALUES ('metadata_identifier_path', '//*[local-name()=''column'' and @isID=''true'']', 'api_________::openaire____::webcrawl::csv', FALSE);
+
+UPDATE apicollections
+SET _dnet_resource_identifier_ = api || '@@' || param;
+
+-- get the orgs whose legalshortname is in upper case (probably good sort name, i.e. acronym)
+select id, legalshortname from dsm_organizations where id like 're3data%' and legalshortname ~ '^[^a-z]*$';
+-- removed legalshortnames of re3data orgs when the legalshortname is not all uppercase
+update dsm_organizations set legalshortname = '' where id like 're3data%' and legalshortname !~ '^[^a-z]*$';
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repobye/DeleteOpenaireMetaWfJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repobye/DeleteOpenaireMetaWfJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repobye/DeleteOpenaireMetaWfJobNode.java (revision 58540)
@@ -0,0 +1,29 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.repobye;
+
+import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.repobye.DeleteMetaWfJobNode;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class DeleteOpenaireMetaWfJobNode extends DeleteMetaWfJobNode {
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Autowired
+ private LocalDatasourceManager dsManager;
+
+ @Override
+ protected void setActivationStatus(final String dsId, final String ifaceId, final boolean active) throws Exception {
+
+ final String openaireDsId = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
+ "/*[.//RESOURCE_IDENTIFIER/@value = '" + dsId + "']//FIELD/value[../key='OpenAireDataSourceId']/text()");
+
+ if (openaireDsId.equals("openaire____::bootstrap")) {
+ dsManager.setActive(dsId, ifaceId, active);
+ } else {
+ dsManager.setActive(openaireDsId, ifaceId, active);
+ }
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/UpdateRepositoryProfilesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/UpdateRepositoryProfilesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/UpdateRepositoryProfilesJobNode.java (revision 58540)
@@ -0,0 +1,25 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.enabling.datasources.common.Api;
+import eu.dnetlib.enabling.datasources.common.Datasource;
+import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class UpdateRepositoryProfilesJobNode extends SimpleJobNode {
+
+ @Autowired
+ private LocalDatasourceManager, Api>> dsManager;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ dsManager.regenerateProfiles();
+
+ return Arc.DEFAULT_ARC;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/migrate_dsm.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/migrate_dsm.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/migrate_dsm.sql (revision 58540)
@@ -0,0 +1,206 @@
+-- noinspection SqlNoDataSourceInspectionForFile
+
+-- cleanup the unnecessary functions: the following SQL produces the deletion statement list
+SELECT 'DROP FUNCTION ' || ns.nspname || '.' || proname || '(' || oidvectortypes(proargtypes) || ') CASCADE;'
+FROM pg_proc
+ INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid)
+WHERE ns.nspname = 'public'
+ORDER BY proname;
+
+-- cleanup the unnecessary triggers: the following SQL produces the deletion statement list
+SELECT 'DROP TRIGGER ' || trg.tgname || ' ON ' || tbl.relname || ';'
+FROM pg_trigger trg
+ JOIN pg_class tbl on trg.tgrelid = tbl.oid
+ JOIN pg_namespace ns ON ns.oid = tbl.relnamespace
+WHERE trg.tgname like 'notify_%';
+
+-- export the new dnet_dsm schema using
+-- pg_dump --schema-only -h localhost -U dnet dnet_dsm | gzip -c > /tmp/dnet_dsm.sql.gz
+-- rename tables and index names in order to include prefix "dsm_"
+-- import the dsm table schema to the existing openaire database:
+-- psql -h localhost -U dnet dnet_openaireplus -f dnet_dsm.sql
+
+ALTER TABLE dsm_datasources ALTER COLUMN officialname TYPE VARCHAR(512);
+ALTER TABLE dsm_datasources ALTER COLUMN englishname TYPE VARCHAR(512);
+ALTER TABLE dsm_datasources ALTER COLUMN missionstatementurl TYPE VARCHAR(512);
+ALTER TABLE dsm_datasources ALTER COLUMN citationguidelineurl TYPE VARCHAR(512);
+ALTER TABLE dsm_datasources ALTER COLUMN languages TYPE text;
+ALTER TABLE dsm_datasources ALTER COLUMN od_contenttypes TYPE text;
+ALTER TABLE dsm_datasources ALTER COLUMN subjects TYPE text;
+ALTER TABLE dsm_datasources ALTER COLUMN description TYPE text;
+ALTER TABLE dsm_datasources ALTER COLUMN certificates TYPE text;
+ALTER TABLE dsm_datasources ALTER COLUMN pidsystems TYPE text;
+
+-- insert into the new dsm_* tables picking the information from the old tables
+insert into dsm_datasources (id, officialname, englishname, websiteurl, logourl, contactemail, latitude, longitude, timezone, namespaceprefix, languages, od_contenttypes, collectedfrom, dateofvalidation,
+ typology, provenanceaction, dateofcollection, platform, activationid, description,releasestartdate, releaseenddate, missionstatementurl, dataprovider, serviceprovider,
+ databaseaccesstype, datauploadtype,databaseaccessrestriction, datauploadrestriction, versioning, citationguidelineurl, qualitymanagementkind,pidsystems, certificates,
+ aggregator, issn, eissn, lissn, registeredby, subjects)
+select d.id,
+ d.officialname,
+ d.englishname,
+ d.websiteurl,
+ d.logourl,
+ d.contactemail,
+ d.latitude,
+ d.longitude,
+ d.timezone,
+ d.namespaceprefix as namespaceprefix,
+ d.od_languages as languages,
+ d.od_contenttypes,
+ d.collectedfrom,
+ d.dateofvalidation,
+ d.datasourceclass as typology,
+ d.provenanceactionclass as provenanceaction,
+ d.dateofcollection,
+ d.typology as platform,
+ d.activationid,
+ d.description,
+ d.releasestartdate,
+ d.releaseenddate,
+ d.missionstatementurl,
+ d.dataprovider as dataprovider,
+ d.serviceprovider as serviceprovider,
+ d.databaseaccesstype,
+ d.datauploadtype,
+ d.databaseaccessrestriction,
+ d.datauploadrestriction,
+ d.versioning,
+ d.citationguidelineurl,
+ d.qualitymanagementkind,
+ d.pidsystems,
+ d.certificates,
+ d.aggregator,
+ d.issn,
+ d.eissn,
+ d.lissn,
+ d.registeredby,
+ ARRAY_AGG(s.name) as subjects
+from datasources d
+ left outer join datasource_subject sd on (d.id = sd.datasource)
+ left outer join subjects s on (sd.subject = s.id)
+group by
+ d.id;
+
+insert into dsm_api (id, protocol, datasource, contentdescription, active, removable, typology, compatibility)
+select a.id,
+ a.protocolclass as protocol,
+ a.datasource,
+ a.contentdescriptionclass as contentdescription,
+ a.active,
+ a.removable,
+ a.typologyclass as typology,
+ a.compatibilityclass as compatibility
+from api a;
+
+update dsm_api da set metadata_identifier_path = ac.original from apicollections ac where da.id = ac.api and ac.param = 'metadata_identifier_path' ;
+update dsm_api da set last_collection_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_collection_total' ;
+update dsm_api da set last_collection_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_collection_date' ;
+update dsm_api da set last_collection_mdid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_collection_mdId' ;
+update dsm_api da set last_aggregation_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_total' ;
+update dsm_api da set last_aggregation_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_date' ;
+update dsm_api da set last_aggregation_mdid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_mdId' ;
+update dsm_api da set last_download_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_download_total' ;
+update dsm_api da set last_download_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_download_date' ;
+update dsm_api da set last_download_objid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_download_objId' ;
+update dsm_api da set last_validation_job = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_validation_job' ;
+update dsm_api da set baseurl = CASE WHEN (ac.edited is not null and ac.edited != '') THEN ac.edited ELSE ac.original END from apicollections ac where da.id = ac.api and ac.param = 'baseUrl' ;
+
+insert into dsm_apiparams (param,value, api)
+select
+ a.param,
+ CASE
+ WHEN (a.edited is not null and a.edited != '') THEN a.edited ELSE a.original
+ END AS value,
+ a.api
+from apicollections a
+where param != 'baseUrl' AND accessparam is true;
+
+
+INSERT INTO dsm_organizations (id, legalshortname,legalname,websiteurl, logourl,ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests,
+ ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, country, collectedfrom, dateofcollection, provenanceaction)
+ SELECT
+ o.id,
+ o.legalshortname,
+ o.legalname,
+ o.websiteurl,
+ o.logourl,
+ o.ec_legalbody,
+ o.ec_legalperson,
+ o.ec_nonprofit,
+ o.ec_researchorganization,
+ o.ec_highereducation,
+ o.ec_internationalorganizationeurinterests,
+ o.ec_internationalorganization,
+ o.ec_enterprise,
+ o.ec_smevalidated,
+ o.ec_nutscode,
+ o.countryclass AS country,
+ o.collectedfrom,
+ o.dateofcollection,
+ o.provenanceactionclass AS provenanceaction
+ FROM organizations o;
+
+INSERT INTO dsm_datasource_organization (datasource, organization)
+ SELECT
+ dorg.datasource,
+ dorg.organization
+ FROM datasource_organization dorg;
+
+
+INSERT INTO dsm_identities (pid, issuertype)
+ SELECT
+ i.pid,
+ i.issuertypeclass AS issuertype
+ FROM identities i;
+
+INSERT INTO dsm_datasourcepids (datasource, pid)
+ SELECT
+ dp.datasource,
+ dp.pid
+ FROM datasourcepids dp;
+
+-- is this table needed? --
+INSERT INTO dsm_organizationpids (organization, pid)
+ SELECT
+ op.organization,
+ op.pid
+ FROM organizationpids op;
+
+
+-- Migrate the managed status
+update dsm_datasources d set managed = true where d.id in (
+ SELECT d.id
+ FROM dsm_datasources d
+ LEFT OUTER JOIN dsm_api a ON (d.id = a.datasource)
+ LEFT OUTER JOIN apicollections ap ON (ap.api = a.id)
+ WHERE a.active = TRUE
+ OR a.removable = TRUE
+ OR ap.edited IS NOT NULL
+ GROUP BY d.id );
+
+
+-- !!!!! data insert complete !!!!!
+
+
+delete from organizations where id like 'opendoar%' ;
+delete from organizations where id like 're3data%' ;
+delete from organizations where collectedfrom is null ;
+
+delete from dsm_organizations where collectedfrom = 'openaire____::corda' ;
+delete from dsm_organizations where collectedfrom = 'openaire____::snsf' ;
+delete from dsm_organizations where collectedfrom = 'openaire____::conicytf' ;
+delete from dsm_organizations where collectedfrom = 'openaire____::irb_hr' ;
+delete from dsm_organizations where collectedfrom = 'openaire____::fwf' ;
+delete from dsm_organizations where collectedfrom = 'openaire____::sfi' ;
+delete from dsm_organizations where collectedfrom = 'openaire____::wellcometrust' ;
+delete from dsm_organizations where collectedfrom = 'openaire____::nsf' ;
+delete from dsm_organizations where collectedfrom = 'openaire____::corda_h2020' ;
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_hostedbymap.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_hostedbymap.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch_hostedbymap.sql (revision 58540)
@@ -0,0 +1 @@
+INSERT INTO hostedby_map(oa_source_id , entry, datasourceid) VALUES ('openaire____::datacite', 'BL.MENDELEY', 're3data_____::r3d100011868');
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-fundingpaths.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-fundingpaths.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-fundingpaths.sql (revision 58540)
@@ -0,0 +1,27 @@
+CREATE TABLE IF NOT EXISTS fundingpaths (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) DEFAULT 'temp_' || md5(clock_timestamp() :: TEXT) || '_' || md5(random() :: TEXT),
+ id CHARACTER VARYING(512) PRIMARY KEY,
+ path TEXT NOT NULL,
+ funder CHARACTER VARYING(512) REFERENCES organizations (id),
+ jurisdiction CHARACTER VARYING(16),
+ description CHARACTER VARYING(512),
+ optional1 CHARACTER VARYING(255),
+ optional2 CHARACTER VARYING(255)
+);
+
+CREATE TABLE IF NOT EXISTS project_fundingpath (
+ _dnet_resource_identifier_ CHARACTER VARYING(2048) DEFAULT 'temp_' || md5(clock_timestamp() :: TEXT) || '_' || md5(random() :: TEXT),
+ funding CHARACTER VARYING(255) NOT NULL REFERENCES fundingpaths (id),
+ project CHARACTER VARYING(255) NOT NULL REFERENCES projects (id),
+ startdate DATE,
+ enddate DATE,
+ semanticclass CHARACTER VARYING(255) REFERENCES class (code),
+ semanticscheme CHARACTER VARYING(255) REFERENCES scheme (code),
+ optional1 CHARACTER VARYING(255),
+ optional2 CHARACTER VARYING(255),
+ PRIMARY KEY (funding, project)
+);
+
+UPDATE datasources
+SET namespaceprefix = 'wt__________'
+WHERE id = 'openaire____::wellcometrust';
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/IteratorOnQueue.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/IteratorOnQueue.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/IteratorOnQueue.java (revision 58540)
@@ -0,0 +1,90 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.datacite;
+
+import java.util.Iterator;
+import java.util.concurrent.BlockingQueue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class IteratorOnQueue.
+ *
+ * @param the generic type
+ */
+public class IteratorOnQueue implements Iterator {
+
+ /**
+ * The Constant log.
+ */
+ private static final Log log = LogFactory.getLog(IteratorOnQueue.class);
+
+ /**
+ * The input queue.
+ */
+ private final BlockingQueue inputQueue;
+
+ /**
+ * The current item.
+ */
+ private T currentItem;
+
+ /**
+ * The end queue.
+ */
+ private T endQueue;
+
+ /**
+ * Instantiates a new iterator on queue.
+ *
+ * @param inputQueue the input queue
+ * @param endQueue the end queue
+ */
+ public IteratorOnQueue(final BlockingQueue inputQueue, final T endQueue) {
+ this.inputQueue = inputQueue;
+ this.endQueue = endQueue;
+
+ try {
+ currentItem = this.inputQueue.take();
+ } catch (InterruptedException e) {
+ log.error(e);
+ }
+ }
+
+ /**
+ * Checks for next.
+ *
+ * @return true, if successful
+ */
+ @Override
+ public boolean hasNext() {
+
+ return currentItem != this.endQueue;
+ }
+
+ /**
+ * Next.
+ *
+ * @return the t
+ */
+ @Override
+ public T next() {
+
+ T previous = currentItem;
+ try {
+ currentItem = this.inputQueue.take();
+ } catch (Exception e) {
+ log.error(e);
+ }
+ return previous;
+ }
+
+ /**
+ * Removes the.
+ */
+ @Override
+ public void remove() {
+ // TODO Auto-generated method stub
+
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-identities.sql
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-identities.sql (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/resources/eu/dnetlib/patch-db/patch-identities.sql (revision 58540)
@@ -0,0 +1,99 @@
+ALTER TABLE identities
+ DROP COLUMN IF EXISTS issuertypescheme;
+ALTER TABLE identities
+ DROP CONSTRAINT IF EXISTS identities_issuertypeclass_fkey;
+
+-- piwik ids from https://docs.google.com/spreadsheets/d/1_tanv8pCiupPxenxqhhKip6D_676-tOTi0RjkAYmUeY/edit#gid=0
+
+BEGIN;
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:13', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::307', 'piwik:13');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:14', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1255', 'piwik:14');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:15', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1619', 'piwik:15');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:16', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1271', 'piwik:16');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:19', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1714', 'piwik:19');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:20', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::7', 'piwik:20');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:21', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1689', 'piwik:21');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:22', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1445', 'piwik:22');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:23', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1621', 'piwik:23');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:24', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1564', 'piwik:24');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:25', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1501', 'piwik:25');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:26', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2346', 'piwik:26');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:27', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2016', 'piwik:27');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:28', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1716', 'piwik:28');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:30', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2017', 'piwik:30');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:31', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1620', 'piwik:31');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:32', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1502', 'piwik:32');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:33', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1503', 'piwik:33');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:34', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2340', 'piwik:34');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:35', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1715', 'piwik:35');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:36', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2018', 'piwik:36');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:37', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2019', 'piwik:37');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:38', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1504', 'piwik:38');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:39', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2187', 'piwik:39');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:40', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::1505', 'piwik:40');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:42', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2020', 'piwik:42');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:43', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2594', 'piwik:43');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:44', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2595', 'piwik:44');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:45', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::3162', 'piwik:45');
+
+INSERT INTO identities (pid, issuertypeclass) VALUES ('piwik:48', 'piwik');
+INSERT INTO datasourcepids (datasource, pid) VALUES ('opendoar____::2584', 'piwik:48');
+
+END;
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/SplitDatasetRecord.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/SplitDatasetRecord.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/SplitDatasetRecord.java (revision 58540)
@@ -0,0 +1,134 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.datacite;
+
+import java.util.Iterator;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import javax.annotation.Resource;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.resultset.IterableResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+/**
+ * The Class SplitDatasetRecord.
+ */
+public class SplitDatasetRecord extends SimpleJobNode {
+
+ /**
+ * The input epr parm.
+ */
+ private String inputEprParm;
+
+ /**
+ * The output epr parm.
+ */
+ private String outputEprParm;
+
+ /**
+ * The result set client factory.
+ */
+ private ResultSetClientFactory resultSetClientFactory;
+
+ /**
+ * The result set factory.
+ */
+ @Resource(name = "iterableResultSetFactory")
+ private IterableResultSetFactory resultSetFactory;
+
+ /**
+ * The executor.
+ */
+ private Executor executor = Executors.newSingleThreadExecutor();
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(inputEprParm));
+ Iterable input = resultSetClientFactory.getClient(inputEpr);
+ final LinkedBlockingQueue publicationsQueue = new LinkedBlockingQueue();
+ final SplitterDatasetsIterator splitterIterator = new SplitterDatasetsIterator(publicationsQueue, input, "publications");
+ executor.execute(new Runnable() {
+
+ @Override
+ public void run() {
+ splitterIterator.populateQueues();
+ }
+ });
+
+ W3CEndpointReference eprUrls_publications = resultSetFactory.createIterableResultSet(new Iterable() {
+
+ @Override
+ public Iterator iterator() {
+ return new IteratorOnQueue(publicationsQueue, SplitterDatasetsIterator.END_QUEUE);
+ }
+ });
+
+ token.getEnv().setAttribute(getOutputEprParm(), eprUrls_publications.toString());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ /**
+ * Gets the input epr parm.
+ *
+ * @return the inputEprParm
+ */
+ public String getInputEprParm() {
+ return inputEprParm;
+ }
+
+ /**
+ * Sets the input epr parm.
+ *
+ * @param inputEprParm the inputEprParm to set
+ */
+ public void setInputEprParm(final String inputEprParm) {
+ this.inputEprParm = inputEprParm;
+ }
+
+ /**
+ * Gets the result set client factory.
+ *
+ * @return the resultSetClientFactory
+ */
+ public ResultSetClientFactory getResultSetClientFactory() {
+ return resultSetClientFactory;
+ }
+
+ /**
+ * Sets the result set client factory.
+ *
+ * @param resultSetClientFactory the resultSetClientFactory to set
+ */
+ public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+ this.resultSetClientFactory = resultSetClientFactory;
+ }
+
+ /**
+ * Gets the output epr parm.
+ *
+ * @return the output epr parm
+ */
+ public String getOutputEprParm() {
+ return outputEprParm;
+ }
+
+ /**
+ * Sets the output epr parm.
+ *
+ * @param outputEprParm the new output epr parm
+ */
+ public void setOutputEprParm(final String outputEprParm) {
+ this.outputEprParm = outputEprParm;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/EntityGrouperConfigurationLoader.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/EntityGrouperConfigurationLoader.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/EntityGrouperConfigurationLoader.java (revision 58540)
@@ -0,0 +1,46 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Iterables;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+public class EntityGrouperConfigurationLoader {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(EntityGrouperConfigurationLoader.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ private String xquery;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ public String load() throws ISLookUpException {
+
+ log.info("loading EntityGrouperConfigurationDSResourceType: " + getXquery());
+
+ List conf = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+ if (conf == null || conf.isEmpty()) { throw new IllegalStateException("unable to read entity grouper configuration profile"); }
+ if (conf.size() > 1) { throw new IllegalStateException("found more than one EntityGrouperConfigurationDSResourceType"); }
+
+ return Iterables.getOnlyElement(conf).replaceAll("\\s+", " ").trim();
+ }
+
+ public String getXquery() {
+ return xquery;
+ }
+
+ @Required
+ public void setXquery(final String xquery) {
+ this.xquery = xquery;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISParams.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISParams.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISParams.java (revision 58540)
@@ -0,0 +1,304 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Resource;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.hadoop.config.ClusterName;
+import eu.dnetlib.data.hadoop.config.ConfigurationEnumerator;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.springframework.beans.factory.annotation.Required;
+
+@Deprecated
+public abstract class PrepareIISParams extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(PrepareIISParams.class);
+
+ @Resource
+ protected ConfigurationEnumerator configurationEnumerator;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ private String clusterName;
+
+ private String clusterParam = "cluster";
+
+ private String zkQuorumParam = "export_action_hbase_remote_zookeeper_quorum";
+
+ private String zkPortParam = "export_action_hbase_remote_zookeeper_clientport";
+
+ private String oozieWfAppPath;
+
+ private String oozieWfAppPathParam = "oozie.wf.application.path";
+
+ private String xqueryMdStoreService;
+
+ private String mdStoreStoreLocationParam = "import_mdstore_service_location";
+
+ private String xqueryObjectStoreService;
+
+ private String objectStoreLocationParam = "import_content_object_store_location";
+
+ private String xqueryIsLookupService;
+
+ private String islookupLocationParam = "import_islookup_service_location";
+
+ private String importProjectConceptsContextIdParam = "import_project_concepts_context_id";
+
+ private String importProjectConceptsContextId;
+
+ private String xqueryDatasetStore;
+
+ private String mdStoreDatasetParam = "import_dataset_mdstore_ids_csv";
+
+ private String objectStoreBlacklistCSV = "";
+
+ protected void prepare(final NodeToken token) throws Exception {
+
+ token.getEnv().setAttribute(getClusterParam(), getClusterName());
+
+ // Assumes we only have one mdStore service instance
+ token.getEnv().setAttribute(getMdStoreStoreLocationParam(), getServiceEndpoint(getXqueryMdStoreService()));
+ // Assumes we only have one objectStore service instance
+ token.getEnv().setAttribute(getObjectStoreLocationParam(), getServiceEndpoint(getXqueryObjectStoreService()));
+
+ token.getEnv().setAttribute(getIslookupLocationParam(), getServiceEndpoint(getXqueryIsLookupService()));
+
+ token.getEnv().setAttribute(getImportProjectConceptsContextIdParam(), getImportProjectConceptsContextId());
+
+ Configuration conf = configurationEnumerator.get(ClusterName.valueOf(getClusterName()));
+
+ Configuration exportConf = configurationEnumerator.get(ClusterName.DM);
+ String zkPort = exportConf.get("hbase.zookeeper.property.clientPort");
+ String zkQuorum = getZkQuorumCSV(exportConf, zkPort);
+
+ token.getEnv().setAttribute(getZkQuorumParam(), zkQuorum);
+ token.getEnv().setAttribute(getZkPortParam(), zkPort);
+
+ String nameNode = conf.get("fs.defaultFS");
+
+ token.getEnv().setAttribute("nameNode", nameNode);
+ token.getEnv().setAttribute("jobTracker", conf.get("mapred.job.tracker"));
+ token.getEnv().setAttribute(getOozieWfAppPathParam(), getURI(nameNode, getOozieWfAppPath()));
+ token.getEnv().setAttribute(getMdStoreDatasetParam(), asCSV(getProfileIds(getXqueryDatasetStore())));
+ }
+
+ protected String getServiceEndpoint(final String xquery) throws MSROException {
+ try {
+ return Iterables.getOnlyElement(serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));
+ } catch (ISLookUpException e) {
+ throw new MSROException("unable to fetch service endpoint", e);
+ } catch (NoSuchElementException e) {
+ throw new MSROException("unable to find service endpoint, xquery: " + getXqueryMdStoreService(), e);
+ } catch (IllegalArgumentException e) {
+ throw new MSROException("more than one services found, we assume to have only one available", e);
+ }
+ }
+
+ protected String getProfileId(final String xquery) throws MSROException {
+ try {
+ return Iterables.getOnlyElement(serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));
+ } catch (ISLookUpException e) {
+ throw new MSROException("unable to fetch profile id", e);
+ } catch (NoSuchElementException e) {
+ throw new MSROException("unable to find profile profile, xquery: " + xquery, e);
+ } catch (IllegalArgumentException e) {
+ throw new MSROException("more than one profile profiles was found, we assume to have only one available, xquery: " + xquery, e);
+ }
+ }
+
+ protected List getProfileIds(final String xquery) throws MSROException {
+ try {
+ List ids = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+
+ if (ids.isEmpty()) {
+ log.warn("couldn't find any profile, xquery: " + xquery);
+ }
+
+ return ids;
+ } catch (ISLookUpException e) {
+ throw new MSROException("unable to fetch profile ids, x query: " + xquery, e);
+ }
+ }
+
+ protected String getFilteredObjectStoreCSV(final String xquery) throws MSROException {
+ return asCSV(filter(getProfileIds(xquery), asList(getObjectStoreBlacklistCSV())));
+ }
+
+ protected List filter(final List list, final List filter) {
+ if (filter == null || filter.isEmpty()) { return list; }
+ list.removeAll(filter);
+ return list;
+ }
+
+ protected String asCSV(final List list) {
+ return Joiner.on(",").skipNulls().join(list);
+ }
+
+ protected List asList(final String csv) {
+ return Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings().split(csv));
+ }
+
+ private String getURI(final String nameNode, final String relative) {
+ // TODO ensure to return a valid URI
+ return nameNode + relative;
+ }
+
+ private String getZkQuorumCSV(final Configuration conf, final String zkPort) {
+ return Joiner.on(":" + zkPort + ",").join(Splitter.on(",").omitEmptyStrings().split(conf.get("hbase.zookeeper.quorum")));
+ }
+
+ @Required
+ public void setXqueryMdStoreService(final String xqueryMdStoreService) {
+ this.xqueryMdStoreService = xqueryMdStoreService;
+ }
+
+ public String getXqueryMdStoreService() {
+ return xqueryMdStoreService;
+ }
+
+ public String getMdStoreStoreLocationParam() {
+ return mdStoreStoreLocationParam;
+ }
+
+ public void setMdStoreStoreLocationParam(final String mdStoreStoreLocationParam) {
+ this.mdStoreStoreLocationParam = mdStoreStoreLocationParam;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(final String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ public String getZkQuorumParam() {
+ return zkQuorumParam;
+ }
+
+ public void setZkQuorumParam(final String zkQuorumParam) {
+ this.zkQuorumParam = zkQuorumParam;
+ }
+
+ public String getZkPortParam() {
+ return zkPortParam;
+ }
+
+ public void setZkPortParam(final String zkPortParam) {
+ this.zkPortParam = zkPortParam;
+ }
+
+ public String getClusterParam() {
+ return clusterParam;
+ }
+
+ public void setClusterParam(final String clusterParam) {
+ this.clusterParam = clusterParam;
+ }
+
+ public String getOozieWfAppPathParam() {
+ return oozieWfAppPathParam;
+ }
+
+ public void setOozieWfAppPathParam(final String oozieWfAppPathParam) {
+ this.oozieWfAppPathParam = oozieWfAppPathParam;
+ }
+
+ public String getOozieWfAppPath() {
+ return oozieWfAppPath;
+ }
+
+ public void setOozieWfAppPath(final String oozieWfAppPath) {
+ this.oozieWfAppPath = oozieWfAppPath;
+ }
+
+ @Required
+ public String getXqueryDatasetStore() {
+ return xqueryDatasetStore;
+ }
+
+ public void setXqueryDatasetStore(final String xqueryDatasetStore) {
+ this.xqueryDatasetStore = xqueryDatasetStore;
+ }
+
+ public String getMdStoreDatasetParam() {
+ return mdStoreDatasetParam;
+ }
+
+ public void setMdStoreDatasetParam(final String mdStoreDatasetParam) {
+ this.mdStoreDatasetParam = mdStoreDatasetParam;
+ }
+
+ public String getXqueryObjectStoreService() {
+ return xqueryObjectStoreService;
+ }
+
+ @Required
+ public void setXqueryObjectStoreService(final String xqueryObjectStoreService) {
+ this.xqueryObjectStoreService = xqueryObjectStoreService;
+ }
+
+ public String getObjectStoreLocationParam() {
+ return objectStoreLocationParam;
+ }
+
+ public void setObjectStoreLocationParam(final String objectStoreLocationParam) {
+ this.objectStoreLocationParam = objectStoreLocationParam;
+ }
+
+ public String getObjectStoreBlacklistCSV() {
+ return objectStoreBlacklistCSV;
+ }
+
+ public void setObjectStoreBlacklistCSV(final String objectStoreBlacklistCSV) {
+ this.objectStoreBlacklistCSV = objectStoreBlacklistCSV;
+ }
+
+ public String getXqueryIsLookupService() {
+ return xqueryIsLookupService;
+ }
+
+ @Required
+ public void setXqueryIsLookupService(final String xqueryIsLookupService) {
+ this.xqueryIsLookupService = xqueryIsLookupService;
+ }
+
+ public String getIslookupLocationParam() {
+ return islookupLocationParam;
+ }
+
+ public void setIslookupLocationParam(final String islookupLocationParam) {
+ this.islookupLocationParam = islookupLocationParam;
+ }
+
+ public String getImportProjectConceptsContextIdParam() {
+ return importProjectConceptsContextIdParam;
+ }
+
+ public void setImportProjectConceptsContextIdParam(final String importProjectConceptsContextIdParam) {
+ this.importProjectConceptsContextIdParam = importProjectConceptsContextIdParam;
+ }
+
+ public String getImportProjectConceptsContextId() {
+ return importProjectConceptsContextId;
+ }
+
+ public void setImportProjectConceptsContextId(final String importProjectConceptsContextId) {
+ this.importProjectConceptsContextId = importProjectConceptsContextId;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchIndexJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchIndexJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchIndexJobNode.java (revision 58540)
@@ -0,0 +1,88 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import java.util.List;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * SwitchIndexJobNode performs index switch.
+ *
+ * @author claudio
+ * @see eu.dnetlib.msro.openaireplus.workflows.nodes.index.SwitchSearchServiceJobNode
+ * @see eu.dnetlib.msro.openaireplus.workflows.nodes.index.SwitchIndexesJobNode
+ * @deprecated
+ */
+@Deprecated
+public class SwitchIndexJobNode extends BlackboardJobNode {
+
+ private static final Log log = LogFactory.getLog(SwitchIndexJobNode.class);
+ private static final String BB_ACTION_SWITCH_INDEX = "UpdateIndex";
+
+ private String inputIndexIdParam;
+
+ private String outputIndexIdParam;
+
+ private String xqueryForServiceIdParam;
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ final String xquery = token.getEnv().getAttribute(getXqueryForServiceIdParam());
+ List searchServiceIds;
+ try {
+ searchServiceIds = getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(xquery);
+ if (searchServiceIds.size() > 1) throw new RuntimeException("Too many SearchService ids found using query: " + xquery);
+ if (searchServiceIds.size() < 1) throw new RuntimeException("SearchService id not found using query: " + xquery);
+ return searchServiceIds.get(0);
+ } catch (ISLookUpException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(BB_ACTION_SWITCH_INDEX);
+ String indexId = token.getEnv().getAttribute(getInputIndexIdParam());
+
+ checkParam(getInputIndexIdParam(), "output indexId param is missing");
+ checkParam(indexId, "indexId is required to perform switch");
+ log.info("Switching " + obtainServiceId(token) + " to index " + indexId);
+ job.getParameters().put(getOutputIndexIdParam(), indexId);
+ }
+
+ private void checkParam(final String param, final String msg) throws MSROException {
+ if (StringUtils.isBlank(param)) throw new MSROException(msg);
+ }
+
+ public String getInputIndexIdParam() {
+ return inputIndexIdParam;
+ }
+
+ public void setInputIndexIdParam(final String inputIndexIdParam) {
+ this.inputIndexIdParam = inputIndexIdParam;
+ }
+
+ public String getOutputIndexIdParam() {
+ return outputIndexIdParam;
+ }
+
+ public void setOutputIndexIdParam(final String outputIndexIdParam) {
+ this.outputIndexIdParam = outputIndexIdParam;
+ }
+
+ public String getXqueryForServiceIdParam() {
+ return xqueryForServiceIdParam;
+ }
+
+ public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {
+ this.xqueryForServiceIdParam = xqueryForServiceIdParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISParamsV2.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISParamsV2.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISParamsV2.java (revision 58540)
@@ -0,0 +1,271 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Resource;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.hadoop.config.ClusterName;
+import eu.dnetlib.data.hadoop.config.ConfigurationEnumerator;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.springframework.beans.factory.annotation.Required;
+
+public abstract class PrepareIISParamsV2 extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(PrepareIISParamsV2.class);
+
+ @Resource
+ protected ConfigurationEnumerator configurationEnumerator;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ private String clusterName;
+
+ private String clusterParam = "cluster";
+
+ private String oozieWfAppPath;
+
+ private String oozieWfAppPathParam = "oozie.wf.application.path";
+
+ private String xqueryMdStoreService;
+
+ private String mdStoreStoreLocationParam = "import_mdstore_service_location";
+
+ private String xqueryObjectStoreService;
+
+ private String objectStoreLocationParam = "import_content_object_store_location";
+
+ private String xqueryIsLookupService;
+
+ private String islookupLocationParam = "import_islookup_service_location";
+
+ private String importProjectConceptsContextCSVParam = "import_project_concepts_context_ids_csv";
+
+ private String importProjectConceptsContextCSV;
+
+ private String xqueryDatasetStore;
+
+ private String mdStoreDatasetParam = "import_dataset_mdstore_ids_csv";
+
+ private String objectStoreBlacklistCSV = "";
+
+ protected void prepare(final NodeToken token) throws Exception {
+
+ token.getEnv().setAttribute(getClusterParam(), getClusterName());
+
+ // Assumes we only have one mdStore service instance
+ token.getEnv().setAttribute(getMdStoreStoreLocationParam(), getServiceEndpoint(getXqueryMdStoreService()));
+ // Assumes we only have one objectStore service instance
+ token.getEnv().setAttribute(getObjectStoreLocationParam(), getServiceEndpoint(getXqueryObjectStoreService()));
+
+ token.getEnv().setAttribute(getIslookupLocationParam(), getServiceEndpoint(getXqueryIsLookupService()));
+ token.getEnv().setAttribute(getImportProjectConceptsContextCSVParam(), getImportProjectConceptsContextCSV());
+
+ Configuration conf = configurationEnumerator.get(ClusterName.valueOf(getClusterName()));
+ String nameNode = conf.get("fs.defaultFS");
+
+ token.getEnv().setAttribute(getOozieWfAppPathParam(), getURI(nameNode, getOozieWfAppPath()));
+ token.getEnv().setAttribute(getMdStoreDatasetParam(), asCSV(getProfileIds(getXqueryDatasetStore())));
+ }
+
+ protected String getServiceEndpoint(final String xquery) throws MSROException {
+ try {
+ return Iterables.getOnlyElement(serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));
+ } catch (ISLookUpException e) {
+ throw new MSROException("unable to fetch service endpoint", e);
+ } catch (NoSuchElementException e) {
+ throw new MSROException("unable to find service endpoint, xquery: " + getXqueryMdStoreService(), e);
+ } catch (IllegalArgumentException e) {
+ throw new MSROException("more than one services found, we assume to have only one available", e);
+ }
+ }
+
+ protected String getProfileId(final String xquery) throws MSROException {
+ try {
+ return Iterables.getOnlyElement(serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));
+ } catch (ISLookUpException e) {
+ throw new MSROException("unable to fetch profile id", e);
+ } catch (NoSuchElementException e) {
+ throw new MSROException("unable to find profile profile, xquery: " + xquery, e);
+ } catch (IllegalArgumentException e) {
+ throw new MSROException("more than one profile profiles was found, we assume to have only one available, xquery: " + xquery, e);
+ }
+ }
+
+ protected List getProfileIds(final String xquery) throws MSROException {
+ try {
+ List ids = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+
+ if (ids.isEmpty()) {
+ log.warn("couldn't find any profile, xquery: " + xquery);
+ }
+
+ return ids;
+ } catch (ISLookUpException e) {
+ throw new MSROException("unable to fetch profile ids, x query: " + xquery, e);
+ }
+ }
+
+ protected String getFilteredObjectStoreCSV(final String xquery) throws MSROException {
+ return asCSV(filter(getProfileIds(xquery), asList(getObjectStoreBlacklistCSV())));
+ }
+
+ protected List filter(final List list, final List filter) {
+ if (filter == null || filter.isEmpty()) { return list; }
+ list.removeAll(filter);
+ return list;
+ }
+
+ protected String asCSV(final List list) {
+ return Joiner.on(",").skipNulls().join(list);
+ }
+
+ protected List asList(final String csv) {
+ return Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings().split(csv));
+ }
+
+ private String getURI(final String nameNode, final String relative) {
+ // TODO ensure to return a valid URI
+ return nameNode + relative;
+ }
+
+ private String getZkQuorumCSV(final Configuration conf, final String zkPort) {
+ return Joiner.on(":" + zkPort + ",").join(Splitter.on(",").omitEmptyStrings().split(conf.get("hbase.zookeeper.quorum")));
+ }
+
+ @Required
+ public void setXqueryMdStoreService(final String xqueryMdStoreService) {
+ this.xqueryMdStoreService = xqueryMdStoreService;
+ }
+
+ public String getXqueryMdStoreService() {
+ return xqueryMdStoreService;
+ }
+
+ public String getMdStoreStoreLocationParam() {
+ return mdStoreStoreLocationParam;
+ }
+
+ public void setMdStoreStoreLocationParam(final String mdStoreStoreLocationParam) {
+ this.mdStoreStoreLocationParam = mdStoreStoreLocationParam;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(final String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ public String getClusterParam() {
+ return clusterParam;
+ }
+
+ public void setClusterParam(final String clusterParam) {
+ this.clusterParam = clusterParam;
+ }
+
+ public String getOozieWfAppPathParam() {
+ return oozieWfAppPathParam;
+ }
+
+ public void setOozieWfAppPathParam(final String oozieWfAppPathParam) {
+ this.oozieWfAppPathParam = oozieWfAppPathParam;
+ }
+
+ public String getOozieWfAppPath() {
+ return oozieWfAppPath;
+ }
+
+ public void setOozieWfAppPath(final String oozieWfAppPath) {
+ this.oozieWfAppPath = oozieWfAppPath;
+ }
+
+ @Required
+ public String getXqueryDatasetStore() {
+ return xqueryDatasetStore;
+ }
+
+ public void setXqueryDatasetStore(final String xqueryDatasetStore) {
+ this.xqueryDatasetStore = xqueryDatasetStore;
+ }
+
+ public String getMdStoreDatasetParam() {
+ return mdStoreDatasetParam;
+ }
+
+ public void setMdStoreDatasetParam(final String mdStoreDatasetParam) {
+ this.mdStoreDatasetParam = mdStoreDatasetParam;
+ }
+
+ public String getXqueryObjectStoreService() {
+ return xqueryObjectStoreService;
+ }
+
+ @Required
+ public void setXqueryObjectStoreService(final String xqueryObjectStoreService) {
+ this.xqueryObjectStoreService = xqueryObjectStoreService;
+ }
+
+ public String getObjectStoreLocationParam() {
+ return objectStoreLocationParam;
+ }
+
+ public void setObjectStoreLocationParam(final String objectStoreLocationParam) {
+ this.objectStoreLocationParam = objectStoreLocationParam;
+ }
+
+ public String getObjectStoreBlacklistCSV() {
+ return objectStoreBlacklistCSV;
+ }
+
+ public void setObjectStoreBlacklistCSV(final String objectStoreBlacklistCSV) {
+ this.objectStoreBlacklistCSV = objectStoreBlacklistCSV;
+ }
+
+ public String getXqueryIsLookupService() {
+ return xqueryIsLookupService;
+ }
+
+ @Required
+ public void setXqueryIsLookupService(final String xqueryIsLookupService) {
+ this.xqueryIsLookupService = xqueryIsLookupService;
+ }
+
+ public String getIslookupLocationParam() {
+ return islookupLocationParam;
+ }
+
+ public void setIslookupLocationParam(final String islookupLocationParam) {
+ this.islookupLocationParam = islookupLocationParam;
+ }
+
+ public String getImportProjectConceptsContextCSVParam() {
+ return importProjectConceptsContextCSVParam;
+ }
+
+ public void setImportProjectConceptsContextCSVParam(final String importProjectConceptsContextCSVParam) {
+ this.importProjectConceptsContextCSVParam = importProjectConceptsContextCSVParam;
+ }
+
+ public String getImportProjectConceptsContextCSV() {
+ return importProjectConceptsContextCSV;
+ }
+
+ public void setImportProjectConceptsContextCSV(final String importProjectConceptsContextCSV) {
+ this.importProjectConceptsContextCSV = importProjectConceptsContextCSV;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ProcessContextsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ProcessContextsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ProcessContextsJobNode.java (revision 58540)
@@ -0,0 +1,112 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;
+import eu.dnetlib.msro.workflows.util.ProgressProvider;
+import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class ProcessContextsJobNode extends SimpleJobNode implements ProgressJobNode {
+
+ private static final Log log = LogFactory.getLog(ProcessContextsJobNode.class);
+
+ private String eprParam;
+ private String contextObj;
+ private ResultsetProgressProvider progressProvider;
+ private String contextId;
+ private String contextLabel;
+ private String contextType;
+
+ private String contextParams;
+ private String dashboardVisibility;
+
+ @Resource
+ private ResultSetClientFactory resultSetClientFactory;
+
+ @Resource
+ private ProcessCountingResultSetFactory processCountingResultSetFactory;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final String epr = token.getEnv().getAttribute(eprParam);
+
+ this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);
+
+ final Iterable iter = resultSetClientFactory.getClient(progressProvider.getEpr());
+
+ final ContextDesc context = ContextUtils.getContext(iter, getContextId(), getContextLabel(), getContextType(), getContextParams(), getDashboardVisibility());
+
+ token.getEnv().setTransientAttribute(contextObj, context);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getEprParam() {
+ return eprParam;
+ }
+
+ public void setEprParam(final String eprParam) {
+ this.eprParam = eprParam;
+ }
+
+ public String getContextObj() {
+ return contextObj;
+ }
+
+ public void setContextObj(final String contextObj) {
+ this.contextObj = contextObj;
+ }
+
+ @Override
+ public ProgressProvider getProgressProvider() {
+ return progressProvider;
+ }
+
+ public String getContextId() {
+ return contextId;
+ }
+
+ public void setContextId(final String contextId) {
+ this.contextId = contextId;
+ }
+
+ public String getContextLabel() {
+ return contextLabel;
+ }
+
+ public void setContextLabel(final String contextLabel) {
+ this.contextLabel = contextLabel;
+ }
+
+ public String getContextType() {
+ return contextType;
+ }
+
+ public void setContextType(final String contextType) {
+ this.contextType = contextType;
+ }
+
+ public String getContextParams() {
+ return contextParams;
+ }
+
+ public void setContextParams(final String contextParams) {
+ this.contextParams = contextParams;
+ }
+
+ public String getDashboardVisibility() {
+ return dashboardVisibility;
+ }
+
+ public void setDashboardVisibility(final String dashboardVisibility) {
+ this.dashboardVisibility = dashboardVisibility;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/CheckHDFSCountJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/CheckHDFSCountJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/CheckHDFSCountJobNode.java (revision 58540)
@@ -0,0 +1,75 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class CheckHDFSCountJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(CheckHDFSCountJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ /**
+ * Name of the env param where to find the number to check.
+ **/
+ private String numberToVerifyParamName;
+ /**
+ * Name of the enc param where to find the number of records stored to hdfs.
+ **/
+ private String hdfsCounterParamName = "mainlog:storeHdfsRecords:count";
+
+ /**
+ * {@inheritDoc}
+ *
+ * @throws MSROException
+ * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ public String execute(final NodeToken token) throws MSROException {
+ int hdfsCount = 0;
+ int hbaseCount = 0;
+ String numberToVerifyInEnvParam = WorkflowsConstants.BLACKBOARD_PARAM_PREFIX + numberToVerifyParamName;
+ hbaseCount = getValueFromEnv(token, numberToVerifyInEnvParam);
+ hdfsCount = getValueFromEnv(token, hdfsCounterParamName);
+
+ if (hdfsCount != hbaseCount) {
+ log.warn("Numbers are not the same. Number to verify (hbase count): " + hbaseCount + ". From hdfs: " + hdfsCount);
+ log.warn((hdfsCount - hbaseCount) + " records come without a 'metadata' field");
+ token.getEnv().setAttribute("noMetadataRecords", hdfsCount - hbaseCount);
+ }
+ return Arc.DEFAULT_ARC;
+
+ }
+
+ private int getValueFromEnv(final NodeToken token, final String paramName) throws MSROException {
+ String valueString = token.getEnv().getAttribute(paramName);
+ int number = 0;
+ try {
+ number = Integer.parseInt(valueString);
+ } catch (NumberFormatException e) {
+ log.error("Env param name: " + paramName + " is " + valueString + " and cannot be parsed as integer");
+ throw new MSROException(e);
+ }
+ return number;
+ }
+
+ public String getNumberToVerifyParamName() {
+ return numberToVerifyParamName;
+ }
+
+ public void setNumberToVerifyParamName(final String numberToVerifyParamName) {
+ this.numberToVerifyParamName = numberToVerifyParamName;
+ }
+
+ public String getHdfsCounterParamName() {
+ return hdfsCounterParamName;
+ }
+
+ public void setHdfsCounterParamName(final String hdfsCounterParamName) {
+ this.hdfsCounterParamName = hdfsCounterParamName;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dataFlowMonitoring/TriggerPrepublicMonitoringScenarioJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dataFlowMonitoring/TriggerPrepublicMonitoringScenarioJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dataFlowMonitoring/TriggerPrepublicMonitoringScenarioJobNode.java (revision 58540)
@@ -0,0 +1,67 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dataFlowMonitoring;
+
+import java.io.IOException;
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.monitoring.core.sensors.SensorCreationException;
+import eu.dnetlib.monitoring.core.sensors.SensorFactory;
+import eu.dnetlib.monitoring.core.sensors.collection.SensorWrapper;
+import eu.dnetlib.monitoring.core.sensors.collection.index.solr.SolrSensor;
+import eu.dnetlib.monitoring.core.sensors.collection.kv.redis.RedisSensor;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class TriggerPrepublicMonitoringScenarioJobNode extends SimpleJobNode {
+
+ private final Log log = LogFactory.getLog(TriggerPrepublicMonitoringScenarioJobNode.class);
+
+ private final static String SOLR_COLLECTION_POSTFIX = "-index-openaire";
+ private String mdFormat;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ private SolrSensor solrSensor;
+ private RedisSensor redisSensor;
+ private SensorWrapper wrapper;
+
+ @Override
+ protected String execute(final NodeToken token) throws IOException, InstantiationException, IllegalAccessException, InterruptedException,
+ ISLookUpException, SensorCreationException {
+ log.info("Execution of TriggerPrepublicMonitoringScenarioJobNode");
+
+ /* Configuring SOLR sensor */
+ String actualCollection = mdFormat + SOLR_COLLECTION_POSTFIX;
+ log.info("At this round, the Pre-public monitoring scenario will check against " + actualCollection + " SOLR collection");
+ solrSensor = (SolrSensor) SensorFactory.createSensorWithConfiguration(SolrSensor.class, "prepublic", "solr");
+ solrSensor.getConfiguration().setParam("collection", actualCollection);
+ solrSensor.reinitialize();
+
+ /* Configuring Redis sensor */
+ redisSensor = (RedisSensor) SensorFactory.createSensorWithConfiguration(RedisSensor.class, "prepublic", "redis");
+
+ /* Wrapping the two sensors together */
+ wrapper = (SensorWrapper) SensorFactory.createSensor(SensorWrapper.class);
+ wrapper.registerSensor(solrSensor);
+ wrapper.registerSensor(redisSensor);
+
+ log.info("Measuring prepublic scenario");
+ wrapper.measure();
+ log.info("DONE!");
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdFormat() {
+ return mdFormat;
+ }
+
+ public void setMdFormat(final String mdFormat) {
+ this.mdFormat = mdFormat;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/PrepareContextDbEntriesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/PrepareContextDbEntriesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/PrepareContextDbEntriesJobNode.java (revision 58540)
@@ -0,0 +1,53 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import javax.annotation.Resource;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.google.common.collect.Iterables;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.resultset.IterableResultSetFactory;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class PrepareContextDbEntriesJobNode extends SimpleJobNode {
+
+ private String contextObj;
+ private String eprParam;
+
+ private static final Log log = LogFactory.getLog(PrepareContextDbEntriesJobNode.class);
+
+ @Resource
+ private IterableResultSetFactory iterableResultSetFactory;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final ContextDesc desc = (ContextDesc) token.getEnv().getTransientAttribute(contextObj);
+
+ final Iterable iter = Iterables.transform(desc.getDbEntries().entrySet(), ContextUtils.getContextRowTransformer());
+ final W3CEndpointReference epr = iterableResultSetFactory.createIterableResultSet(iter);
+
+ token.getEnv().setAttribute(eprParam, epr.toString());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getContextObj() {
+ return contextObj;
+ }
+
+ public void setContextObj(final String contextObj) {
+ this.contextObj = contextObj;
+ }
+
+ public String getEprParam() {
+ return eprParam;
+ }
+
+ public void setEprParam(final String eprParam) {
+ this.eprParam = eprParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hbase/PrepareCopyTableJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hbase/PrepareCopyTableJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hbase/PrepareCopyTableJobNode.java (revision 58540)
@@ -0,0 +1,174 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hbase;
+
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Sets;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.hadoop.rmi.HadoopService;
+import eu.dnetlib.data.hadoop.rmi.HadoopServiceException;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The PrepareCopyTableJobNode prepares the parameters needed to run the CopyTable job.
+ */
+public class PrepareCopyTableJobNode extends SimpleJobNode {
+
+ /**
+ * The Constant ZOOKEEPER_ZNODE_PARENT.
+ */
+ private static final String ZOOKEEPER_ZNODE_PARENT = "zookeeper.znode.parent";
+
+ /**
+ * The Constant HBASE_ZOOKEEPER_CLIENT_PORT.
+ */
+ private static final String HBASE_ZOOKEEPER_CLIENT_PORT = "hbase.zookeeper.client.port";
+
+ /**
+ * The Constant HBASE_ZOOKEEPER_QUORUM.
+ */
+ private static final String HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(PrepareCopyTableJobNode.class);
+
+ /**
+ * The source table.
+ */
+ private String sourceCluster;
+
+ /**
+ * The target cluster.
+ */
+ private String targetCluster;
+
+ private String sourceTable;
+
+ private String targetTable;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ checkNodeParams();
+
+ final String outputQuorum = getOutputQuorum();
+ log.info("build hbase quorum: " + outputQuorum);
+ token.getEnv().setAttribute("peer.adr", outputQuorum);
+ token.getEnv().setAttribute("sourceCluster", getSourceCluster());
+ token.getEnv().setAttribute("sourceTable", getSourceTable());
+
+ token.getEnv().setAttribute("targetCluster", getTargetCluster());
+ token.getEnv().setAttribute("targetTable", getTargetTable());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ /**
+ * Builds the output quorum.
+ *
+ * @return the output quorum
+ * @throws HadoopServiceException when cannot retrieve the clustr configuration
+ * @throws MSROException when some of the needed properties is missing in the cluster configuration
+ */
+ private String getOutputQuorum() throws HadoopServiceException, MSROException {
+ Map conf = serviceLocator.getService(HadoopService.class).getClusterConfiguration(getTargetCluster());
+ log.debug(conf);
+
+ String hbaseQuorum = conf.get(HBASE_ZOOKEEPER_QUORUM);
+ String hbasePort = conf.get(HBASE_ZOOKEEPER_CLIENT_PORT);
+ String znodeParent = conf.get(ZOOKEEPER_ZNODE_PARENT);
+
+ checkParamExist(hbaseQuorum, String.format("unable to find property '%s' in cluster configuration: %s", HBASE_ZOOKEEPER_QUORUM, hbaseQuorum));
+ checkParamExist(hbasePort, String.format("unable to find property '%s' in cluster configuration: %s", HBASE_ZOOKEEPER_CLIENT_PORT, hbasePort));
+ checkParamExist(znodeParent, String.format("unable to find property '%s' in cluster configuration: %s", ZOOKEEPER_ZNODE_PARENT, znodeParent));
+
+ String outputQuorum = String.format("%s:%s:%s", hbaseQuorum, hbasePort, znodeParent);
+ return outputQuorum;
+ }
+
+ /**
+ * Checks the wf params.
+ *
+ * @throws MSROException the MSRO exception
+ * @throws HadoopServiceException
+ */
+ private void checkNodeParams() throws MSROException, HadoopServiceException {
+
+ checkParamExist(getSourceCluster(), "source cluster must be set");
+ checkParamExist(getTargetCluster(), "target cluster must be set");
+ checkParamExist(getSourceTable(), "source table must be set");
+ checkParamExist(getTargetTable(), "target table must be set");
+
+ final HadoopService hadoop = serviceLocator.getService(HadoopService.class);
+
+ Set clusters = Sets.newHashSet(hadoop.listClusters());
+ if (!clusters.contains(getSourceCluster())) { throw new MSROException(String.format("source cluster '%s' doesn not exists", getSourceCluster())); }
+ if (!clusters.contains(getTargetCluster())) { throw new MSROException(String.format("target cluster '%s' doesn not exists", getTargetCluster())); }
+
+ if (!hadoop.existHbaseTable(getSourceCluster(), getSourceTable())) {
+ throw new MSROException(String.format(
+ "source table '%s' doesn not exists on cluster '%s'", getSourceTable(), getSourceCluster()));
+ }
+ }
+
+ /**
+ * Check parameter existence.
+ *
+ * @param param the param
+ * @param msg the msg
+ * @throws MSROException the MSRO exception
+ */
+ private void checkParamExist(final String param, final String msg) throws MSROException {
+ if (StringUtils.isBlank(param)) { throw new MSROException(msg); }
+ }
+
+ public String getSourceCluster() {
+ return sourceCluster;
+ }
+
+ public void setSourceCluster(final String sourceCluster) {
+ this.sourceCluster = sourceCluster;
+ }
+
+ public String getTargetCluster() {
+ return targetCluster;
+ }
+
+ public void setTargetCluster(final String targetCluster) {
+ this.targetCluster = targetCluster;
+ }
+
+ public String getSourceTable() {
+ return sourceTable;
+ }
+
+ public void setSourceTable(final String sourceTable) {
+ this.sourceTable = sourceTable;
+ }
+
+ public String getTargetTable() {
+ return targetTable;
+ }
+
+ public void setTargetTable(final String targetTable) {
+ this.targetTable = targetTable;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchIndexesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchIndexesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchIndexesJobNode.java (revision 58540)
@@ -0,0 +1,64 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import java.util.Queue;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.springframework.beans.factory.annotation.Value;
+
+/**
+ * SwitchIndexesJobNode guides the index switches on all search services whose ids are in the queue available in the parameter named 'searchService_ids'.
+ *
+ * @author claudio, alessia
+ * @see eu.dnetlib.msro.openaireplus.workflows.nodes.index.FindSearchServicesJobNode
+ * @see eu.dnetlib.msro.openaireplus.workflows.nodes.index.SwitchIndexJobNode
+ */
+public class SwitchIndexesJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(SwitchIndexesJobNode.class);
+
+ @Value("${dnet.openaire.index.api.cacheEvictUrl}")
+ private String indexApiCacheEvictUrl;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ Queue q = (Queue) token.getEnv().getTransientAttribute("searchService_ids");
+ log.debug("Got the searchService_ids queue: " + q.toString());
+ if (q == null) throw new MSROException("Transient param 'searchService_ids' with queue of string could not be found");
+ if (q.isEmpty()) {
+ log.info("searchService_ids queue consumed, now ending cycle and evicting direct indexing api cache on " + indexApiCacheEvictUrl);
+ final HttpGet get = new HttpGet(indexApiCacheEvictUrl);
+
+ try(CloseableHttpClient client = HttpClients.createDefault()) {
+ try (CloseableHttpResponse response = client.execute(get)) {
+ int statusCode = response.getStatusLine().getStatusCode();
+ log.info(String.format("GET on %s returned %s", indexApiCacheEvictUrl, statusCode));
+ if (statusCode == HttpStatus.SC_OK) {
+ return Arc.DEFAULT_ARC;
+ } else
+ throw new MSROException("Cannot evict cache of direct indexing API at: " + indexApiCacheEvictUrl);
+ }
+ }
+ } else {
+ //we have something to do: setting the xqueryForServiceIdParam for the SwitchIndexJobNode
+ String id = q.poll();
+ log.debug("Polled id: " + id);
+ log.debug("And now the queue is " + q);
+ token.getEnv().setAttribute("search_service_ID", id);
+ log.debug("Asking to switch on profile with id: " + id);
+ //updating the queue for next iteration
+ token.getEnv().setTransientAttribute("searchService_ids", q);
+ return "switch";
+ }
+
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/ImportFilesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/ImportFilesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/ImportFilesJobNode.java (revision 58540)
@@ -0,0 +1,125 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.objectStore;
+
+import java.io.File;
+import java.util.List;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.enabling.resultset.IterableResultSetFactory;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
+import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;
+import eu.dnetlib.msro.workflows.util.ProgressProvider;
+import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+
+public class ImportFilesJobNode extends BlackboardJobNode implements ProgressJobNode {
+
+ private String basePath;
+ private String repositoryPrefix;
+ private String oaiPrefix;
+ private String objectStoreId;
+ private String contentDescription;
+
+ private ResultsetProgressProvider progressProvider;
+ private ProcessCountingResultSetFactory processCountingResultSetFactory;
+ private IterableResultSetFactory iterableResultSetFactory;
+
+ public String getRepositoryPrefix() {
+ return repositoryPrefix;
+ }
+
+ public void setRepositoryPrefix(final String repositoryPrefix) {
+ this.repositoryPrefix = repositoryPrefix;
+ }
+
+ public String getBasePath() {
+ return basePath;
+ }
+
+ public void setBasePath(final String basePath) {
+ this.basePath = basePath;
+ }
+
+ @Override
+ public ProgressProvider getProgressProvider() {
+ return progressProvider;
+ }
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ return getServiceLocator().getServiceId(ObjectStoreService.class);
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ File dir = new File(basePath);
+
+ System.out.println("Getting all files in " + dir.getCanonicalPath() + " including those in subdirectories");
+ List files = (List) FileUtils.listFiles(dir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
+ IteratorRepository irepo;
+ if (repositoryPrefix.equals("od________18")) {
+ irepo = new ArxivIteratorRepository(files, repositoryPrefix, oaiPrefix);
+ } else if (repositoryPrefix.equals("od______2367")) {
+ irepo = new PUMAIteratorRepository(files, repositoryPrefix, oaiPrefix);
+ } else {
+ irepo = new RepositoryNoURLIterator(files, repositoryPrefix, oaiPrefix);
+ }
+ final W3CEndpointReference epr = iterableResultSetFactory.createIterableResultSet(irepo);
+ this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);
+ job.setAction("FEED");
+ job.getParameters().put("obsID", getObjectStoreId());
+ job.getParameters().put("mime", getContentDescription());
+ job.getParameters().put("protocol", Protocols.File_System.toString());
+ job.getParameters().put("epr", progressProvider.getEpr().toString());
+
+ }
+
+ public IterableResultSetFactory getIterableResultSetFactory() {
+ return iterableResultSetFactory;
+ }
+
+ public void setIterableResultSetFactory(final IterableResultSetFactory iterableResultSetFactory) {
+ this.iterableResultSetFactory = iterableResultSetFactory;
+ }
+
+ public void setProgressProvider(final ResultsetProgressProvider progressProvider) {
+ this.progressProvider = progressProvider;
+ }
+
+ public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {
+ return processCountingResultSetFactory;
+ }
+
+ public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {
+ this.processCountingResultSetFactory = processCountingResultSetFactory;
+ }
+
+ public String getObjectStoreId() {
+ return objectStoreId;
+ }
+
+ public void setObjectStoreId(final String objectStoreId) {
+ this.objectStoreId = objectStoreId;
+ }
+
+ public String getContentDescription() {
+ return contentDescription;
+ }
+
+ public void setContentDescription(final String contentDescription) {
+ this.contentDescription = contentDescription;
+ }
+
+ public String getOaiPrefix() {
+ return oaiPrefix;
+ }
+
+ public void setOaiPrefix(final String oaiPrefix) {
+ this.oaiPrefix = oaiPrefix;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/RepositoryNoURLIterator.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/RepositoryNoURLIterator.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/RepositoryNoURLIterator.java (revision 58540)
@@ -0,0 +1,38 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.objectStore;
+
+import java.io.File;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
+
+public class RepositoryNoURLIterator extends IteratorRepository {
+
+ public RepositoryNoURLIterator(final Iterable input, final String repositoryPrefix, final String oaiPrefix) {
+ super(input, repositoryPrefix, oaiPrefix);
+ }
+
+ @Override
+ String generateNextElement() {
+ try {
+ String inputname = input.next().getCanonicalPath();
+ String[] values = inputname.split("/");
+ if (values.length > 1) {
+ ObjectStoreFile info = new ObjectStoreFile();
+ String fileName = values[values.length - 1].replace(".xml", "");
+ info.setAccessProtocol(Protocols.None);
+ // String value = "oai:europepmc.org:" + fileName;
+ String value = oaiPrefix + fileName;
+ String resultID = repositoryPrefix + "::" + DnetXsltFunctions.md5(value);
+ info.setObjectID(resultID);
+ info.setMimeType("pdf");
+ info.setURI(inputname);
+ return info.toJSON();
+ }
+ } catch (Exception e) {
+ return null;
+ }
+ return null;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/IteratorRepository.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/IteratorRepository.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/IteratorRepository.java (revision 58540)
@@ -0,0 +1,42 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.objectStore;
+
+import java.io.File;
+import java.util.Iterator;
+
+public abstract class IteratorRepository implements Iterator, Iterable {
+
+ protected Iterator input;
+ protected Iterable inputIterable;
+ protected String repositoryPrefix;
+ protected String oaiPrefix;
+
+ abstract String generateNextElement();
+
+ public IteratorRepository(final Iterable input, final String repositoryPrefix, final String oaiPrefix) {
+ this.inputIterable = input;
+ this.repositoryPrefix = repositoryPrefix;
+ this.oaiPrefix = oaiPrefix;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return input.hasNext();
+ }
+
+ @Override
+ public String next() {
+ return generateNextElement();
+ }
+
+ @Override
+ public void remove() {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public Iterator iterator() {
+ this.input = inputIterable.iterator();
+ return this;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchSearchServiceJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchSearchServiceJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/SwitchSearchServiceJobNode.java (revision 58540)
@@ -0,0 +1,63 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * SwitchSearchServiceJobNode switches the index used by a SearchService whose profile id is in 'search_service_ID' env param.
+ *
+ * @author claudio, alessia
+ */
+public class SwitchSearchServiceJobNode extends BlackboardJobNode {
+
+ private static final Log log = LogFactory.getLog(SwitchSearchServiceJobNode.class);
+ private static final String BB_ACTION_SWITCH_INDEX = "UpdateIndex";
+
+ private String inputIndexIdParam;
+
+ private String outputIndexIdParam;
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ final String id = token.getEnv().getAttribute("search_service_ID");
+ if (StringUtils.isBlank(id)) throw new RuntimeException("No id found in env attribute 'search_service_ID'");
+ else return id;
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(BB_ACTION_SWITCH_INDEX);
+ String indexId = token.getEnv().getAttribute(getInputIndexIdParam());
+
+ checkParam(getInputIndexIdParam(), "output indexId param is missing");
+ checkParam(indexId, "indexId is required to perform switch");
+ log.info("Switching " + obtainServiceId(token) + " to index " + indexId);
+ job.getParameters().put(getOutputIndexIdParam(), indexId);
+ }
+
+ private void checkParam(final String param, final String msg) throws MSROException {
+ if (StringUtils.isBlank(param)) throw new MSROException(msg);
+ }
+
+ public String getInputIndexIdParam() {
+ return inputIndexIdParam;
+ }
+
+ public void setInputIndexIdParam(final String inputIndexIdParam) {
+ this.inputIndexIdParam = inputIndexIdParam;
+ }
+
+ public String getOutputIndexIdParam() {
+ return outputIndexIdParam;
+ }
+
+ public void setOutputIndexIdParam(final String outputIndexIdParam) {
+ this.outputIndexIdParam = outputIndexIdParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISPreprocessingParamsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISPreprocessingParamsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISPreprocessingParamsJobNode.java (revision 58540)
@@ -0,0 +1,107 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import org.springframework.beans.factory.annotation.Required;
+
+@Deprecated
+public class PrepareIISPreprocessingParamsJobNode extends PrepareIISParams {
+
+ private String xqueryWosMDStore;
+
+ private String mdStoreWosParam = "import_wos_mdstore_id";
+
+ private String xqueryDatabaseService;
+
+ private String databaseServiceLocationParam = "import_database_service_location";
+
+ private String xqueryDataciteObjectStore;
+
+ private String dataciteObjectStoreParam = "import_content_datacite_objectstores_csv";
+
+ private String xqueryWosObjectStore;
+
+ private String wosObjectStorePlaintextParam = "import_content_wos_plaintext_objectstores_csv";
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ super.prepare(token);
+
+ token.getEnv().setAttribute(getMdStoreWosParam(), getProfileId(getXqueryWosMDStore()));
+ token.getEnv().setAttribute(getDatabaseServiceLocationParam(), getServiceEndpoint(getXqueryDatabaseService()));
+
+ token.getEnv().setAttribute(getDataciteObjectStoreParam(), getFilteredObjectStoreCSV(getXqueryDataciteObjectStore()));
+ token.getEnv().setAttribute(getWosObjectStorePlaintextParam(), getProfileId(getXqueryWosObjectStore()));
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getXqueryWosMDStore() {
+ return xqueryWosMDStore;
+ }
+
+ @Required
+ public void setXqueryWosMDStore(final String xqueryWosMDStore) {
+ this.xqueryWosMDStore = xqueryWosMDStore;
+ }
+
+ public String getMdStoreWosParam() {
+ return mdStoreWosParam;
+ }
+
+ public void setMdStoreWosParam(final String mdStoreWosParam) {
+ this.mdStoreWosParam = mdStoreWosParam;
+ }
+
+ public String getXqueryDatabaseService() {
+ return xqueryDatabaseService;
+ }
+
+ @Required
+ public void setXqueryDatabaseService(final String xqueryDatabaseService) {
+ this.xqueryDatabaseService = xqueryDatabaseService;
+ }
+
+ public String getDatabaseServiceLocationParam() {
+ return databaseServiceLocationParam;
+ }
+
+ public void setDatabaseServiceLocationParam(final String databaseServiceLocationParam) {
+ this.databaseServiceLocationParam = databaseServiceLocationParam;
+ }
+
+ public String getWosObjectStorePlaintextParam() {
+ return wosObjectStorePlaintextParam;
+ }
+
+ public void setWosObjectStorePlaintextParam(final String wosObjectStorePlaintextParam) {
+ this.wosObjectStorePlaintextParam = wosObjectStorePlaintextParam;
+ }
+
+ public String getXqueryWosObjectStore() {
+ return xqueryWosObjectStore;
+ }
+
+ @Required
+ public void setXqueryWosObjectStore(final String xqueryWosObjectStore) {
+ this.xqueryWosObjectStore = xqueryWosObjectStore;
+ }
+
+ public String getXqueryDataciteObjectStore() {
+ return xqueryDataciteObjectStore;
+ }
+
+ @Required
+ public void setXqueryDataciteObjectStore(final String xqueryDataciteObjectStore) {
+ this.xqueryDataciteObjectStore = xqueryDataciteObjectStore;
+ }
+
+ public String getDataciteObjectStoreParam() {
+ return dataciteObjectStoreParam;
+ }
+
+ public void setDataciteObjectStoreParam(final String dataciteObjectStoreParam) {
+ this.dataciteObjectStoreParam = dataciteObjectStoreParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedByJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedByJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedByJobNode.java (revision 58540)
@@ -0,0 +1,266 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby;
+
+import java.io.StringReader;
+import java.util.Map;
+
+import javax.annotation.Resource;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.beans.factory.annotation.Value;
+
+import com.google.common.collect.Maps;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.enabling.database.rmi.DatabaseException;
+import eu.dnetlib.enabling.database.rmi.DatabaseService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.MappedResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class PatchHostedByJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(PatchHostedByJobNode.class);
+ private String inputEprParam;
+ private String outputEprParam;
+ @Value("${dnet.openaire.db.name}")
+ private String dbName;
+ private String countersParam;
+ private String hostedbyMapTable;
+ private String xpathEntry;
+ private String overrideDataSourceId;
+ private String keyTypeFilter;
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+ private MappedResultSetFactory mappedResultSetFactory;
+ private ResultSetClientFactory resultSetClientFactory;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(inputEprParam));
+ final HostedByCounters counters = new HostedByCounters();
+ String datasourceId;
+ if (StringUtils.isEmpty(overrideDataSourceId)) {
+ datasourceId = token.getEnv().getAttribute("parentDatasourceId");
+ } else {
+ datasourceId = getOverrideDataSourceId();
+ }
+
+ Map hostedByEntryMap;
+
+ if (!StringUtils.isBlank(keyTypeFilter)) {
+ hostedByEntryMap = loadHostedByMapFromType(keyTypeFilter);
+ } else {
+ hostedByEntryMap = loadHostedByMap(datasourceId);
+ }
+
+ final W3CEndpointReference epr = mappedResultSetFactory.createMappedResultSet(inputEpr, new PatchHostedBy(hostedByEntryMap,
+ getXpathEntry(), counters));
+
+ token.getEnv().setAttribute(outputEprParam, epr.toString());
+ token.getEnv().setTransientAttribute(countersParam, counters);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ private Map loadHostedByMap(final String datasourceId) throws DocumentException, DatabaseException {
+ final String sql = "SELECT d.id, d.officialname, p.entry from %s p JOIN dsm_datasources d ON (p.datasourceid = d.id) WHERE p.oa_source_id= '%s'";
+ return getHostedByEntryMapByQuery(datasourceId, sql);
+ }
+
+ private Map loadHostedByMapFromType(final String keyType) throws DocumentException, DatabaseException {
+ final String sql = "SELECT d.id, d.officialname, p.entry from %s p JOIN dsm_datasources d ON (p.datasourceid = d.id) WHERE p.key_type= '%s'";
+ return getHostedByEntryMapByQuery(keyType, sql);
+ }
+
+ private Map getHostedByEntryMapByQuery(final String datasourceId, final String sql)
+ throws DatabaseException, DocumentException {
+ final Map map = Maps.newHashMap();
+ final W3CEndpointReference epr = serviceLocator.getService(DatabaseService.class).searchSQL(getDbName(),
+ String.format(sql, getHostedbyMapTable(), datasourceId));
+ final SAXReader reader = new SAXReader();
+ for (final String s : resultSetClientFactory.getClient(epr)) {
+ final Document doc = reader.read(new StringReader(s));
+ final String entry = doc.valueOf("//FIELD[@name='entry']");
+ final String dsId = doc.valueOf("//FIELD[@name='id']");
+ final String dsName = doc.valueOf("//FIELD[@name='officialname']");
+ map.put(entry, new HostedByEntry(dsId, dsName));
+ }
+ log.info(String.format("built hostedByMap from dsId '%s', size: '%s'", datasourceId, map.size()));
+ return map;
+ }
+
+ /**
+ * Getter for property 'inputEprParam'.
+ *
+ * @return Value for property 'inputEprParam'.
+ */
+ public String getInputEprParam() {
+ return inputEprParam;
+ }
+
+ /**
+ * Setter for property 'inputEprParam'.
+ *
+ * @param inputEprParam
+ * Value to set for property 'inputEprParam'.
+ */
+ public void setInputEprParam(final String inputEprParam) {
+ this.inputEprParam = inputEprParam;
+ }
+
+ /**
+ * Getter for property 'outputEprParam'.
+ *
+ * @return Value for property 'outputEprParam'.
+ */
+ public String getOutputEprParam() {
+ return outputEprParam;
+ }
+
+ /**
+ * Setter for property 'outputEprParam'.
+ *
+ * @param outputEprParam
+ * Value to set for property 'outputEprParam'.
+ */
+ public void setOutputEprParam(final String outputEprParam) {
+ this.outputEprParam = outputEprParam;
+ }
+
+ /**
+ * Getter for property 'dbName'.
+ *
+ * @return Value for property 'dbName'.
+ */
+ public String getDbName() {
+ return dbName;
+ }
+
+ /**
+ * Setter for property 'dbName'.
+ *
+ * @param dbName
+ * Value to set for property 'dbName'.
+ */
+ public void setDbName(final String dbName) {
+ this.dbName = dbName;
+ }
+
+ public String getKeyTypeFilter() {
+ return keyTypeFilter;
+ }
+
+ public void setKeyTypeFilter(final String keyTypeFilter) {
+ this.keyTypeFilter = keyTypeFilter;
+ }
+
+ /**
+ * Getter for property 'mappedResultSetFactory'.
+ *
+ * @return Value for property 'mappedResultSetFactory'.
+ */
+ public MappedResultSetFactory getMappedResultSetFactory() {
+ return mappedResultSetFactory;
+ }
+
+ /**
+ * Setter for property 'mappedResultSetFactory'.
+ *
+ * @param mappedResultSetFactory
+ * Value to set for property 'mappedResultSetFactory'.
+ */
+ @Required
+ public void setMappedResultSetFactory(final MappedResultSetFactory mappedResultSetFactory) {
+ this.mappedResultSetFactory = mappedResultSetFactory;
+ }
+
+ /**
+ * Getter for property 'resultSetClientFactory'.
+ *
+ * @return Value for property 'resultSetClientFactory'.
+ */
+ public ResultSetClientFactory getResultSetClientFactory() {
+ return resultSetClientFactory;
+ }
+
+ /**
+ * Setter for property 'resultSetClientFactory'.
+ *
+ * @param resultSetClientFactory
+ * Value to set for property 'resultSetClientFactory'.
+ */
+ @Required
+ public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+ this.resultSetClientFactory = resultSetClientFactory;
+ }
+
+ /**
+ * Getter for property 'countersParam'.
+ *
+ * @return Value for property 'countersParam'.
+ */
+ public String getCountersParam() {
+ return countersParam;
+ }
+
+ /**
+ * Setter for property 'countersParam'.
+ *
+ * @param countersParam
+ * Value to set for property 'countersParam'.
+ */
+ public void setCountersParam(final String countersParam) {
+ this.countersParam = countersParam;
+ }
+
+ /**
+ * @return the hostedbyMapTable
+ */
+ public String getHostedbyMapTable() {
+ return hostedbyMapTable;
+ }
+
+ /**
+ * @param hostedbyMapTable
+ * the hostedbyMapTable to set
+ */
+ public void setHostedbyMapTable(final String hostedbyMapTable) {
+ this.hostedbyMapTable = hostedbyMapTable;
+ }
+
+ public String getXpathEntry() {
+ return xpathEntry;
+ }
+
+ public void setXpathEntry(final String xpathEntry) {
+ this.xpathEntry = xpathEntry;
+ }
+
+ /**
+ * @return the overrideDataSourceId
+ */
+ public String getOverrideDataSourceId() {
+ return overrideDataSourceId;
+ }
+
+ /**
+ * @param overrideDataSourceId
+ * the overrideDataSourceId to set
+ */
+ public void setOverrideDataSourceId(final String overrideDataSourceId) {
+ this.overrideDataSourceId = overrideDataSourceId;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/HostedByCounters.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/HostedByCounters.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/HostedByCounters.java (revision 58540)
@@ -0,0 +1,36 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.collect.Maps;
+
+public class HostedByCounters {
+
+ private Map counters = Maps.newHashMap();
+
+ public void increaseCounter(final String dsId) {
+ int val = counters.containsKey(dsId) ? counters.get(dsId) + 1 : 1;
+ counters.put(dsId, val);
+ }
+
+ public Map getCounters() {
+ return counters;
+ }
+
+ @Override
+ public String toString() {
+ final StringWriter sw = new StringWriter();
+ sw.append("\n");
+ sw.append("**************************************************\n");
+ for (Entry e : counters.entrySet()) {
+ sw.append(e.getKey());
+ sw.append(" : ");
+ sw.append(Integer.toString(e.getValue()));
+ sw.append("\n");
+ }
+ sw.append("**************************************************\n");
+ return sw.toString();
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareMDStoreVersionJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareMDStoreVersionJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareMDStoreVersionJobNode.java (revision 58540)
@@ -0,0 +1,67 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class PrepareMDStoreVersionJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(PrepareMDStoreVersionJobNode.class);
+
+ private String mdId;
+
+ /* Spring managed params */
+ private String mdStoreManagerUrl;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String url = getMdStoreManagerUrl() + "/mdstore/{mdId}/newVersion";
+
+ final Map params = new HashMap<>();
+ params.put("mdId", getMdId());
+
+ final URI uri = UriComponentsBuilder.fromUriString(url)
+ .buildAndExpand(params)
+ .toUri();
+
+ final RestTemplate restTemplate = new RestTemplate();
+ final MDStoreVersion version = restTemplate.getForObject(uri, MDStoreVersion.class);
+
+ log.info("mdstore version prepared " + version.getId());
+
+ token.getEnv().setAttribute("mdId", mdId);
+ token.getEnv().setAttribute("versionId", version.getId());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdStoreManagerUrl() {
+ return mdStoreManagerUrl;
+ }
+
+ @Required
+ public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {
+ this.mdStoreManagerUrl = mdStoreManagerUrl;
+ }
+
+ public String getMdId() {
+ return mdId;
+ }
+
+ public void setMdId(final String mdId) {
+ this.mdId = mdId;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/SubmitDnetHadoopJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/SubmitDnetHadoopJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/SubmitDnetHadoopJobNode.java (revision 58540)
@@ -0,0 +1,115 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import com.googlecode.sarasvati.Engine;
+import com.googlecode.sarasvati.NodeToken;
+import com.googlecode.sarasvati.env.Env;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.message.Message;
+import eu.dnetlib.msro.openaireplus.workflows.nodes.dhp.message.DnetMessageManager;
+import eu.dnetlib.msro.workflows.hadoop.SubmitHadoopJobNode;
+import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
+import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;
+import eu.dnetlib.msro.workflows.util.ProgressProvider;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
+import java.util.Map;
+
+public class SubmitDnetHadoopJobNode extends SubmitHadoopJobNode implements ProgressProvider, ProgressJobNode {
+
+ private static final Log log = LogFactory.getLog(SubmitDnetHadoopJobNode.class);
+
+ @Autowired
+ DnetMessageManager dnetMessageManager;
+
+ private boolean ongoing = true;
+
+ private int currentValue;
+
+ private String wfId;
+
+
+ @Override
+ protected void prepareJob(BlackboardJob job, NodeToken token) throws Exception {
+ this.wfId = token.getProcess().getEnv().getAttribute("system:processId");
+
+
+ Runnable r = () -> {
+ while (ongoing) {
+ Message mess = dnetMessageManager.getOnGoingMessages(wfId);
+ if (mess != null && mess.getBody() != null && mess.getBody().containsKey("ongoing")) {
+ try {
+ this.currentValue = Integer.parseInt(mess.getBody().get("ongoing"));
+ Thread.sleep(1000);
+ } catch (Throwable e) {
+ log.error("Error ono receiving messages ", e);
+ }
+ }
+ }
+ };
+ new Thread(r).start();
+ super.prepareJob(job, token);
+ }
+
+
+ @Override
+ protected BlackboardWorkflowJobListener generateBlackboardListener(Engine engine, NodeToken token) {
+ return new BlackboardWorkflowJobListener(engine, token) {
+ @Override
+ protected void onFailed(final BlackboardJob job) {
+ ongoing = false;
+ log.warn("Blackboard workflow node FAILED: " + job.getError());
+ token.getEnv().setAttribute(WorkflowsConstants.SYSTEM_HAS_FAILED, true);
+ token.getEnv().setAttribute(WorkflowsConstants.SYSTEM_ERROR, job.getError());
+ complete(job, "abort");
+ }
+ @Override
+ protected void populateEnv(Env env, Map responseParams) {
+ ongoing = false;
+
+ List reports = dnetMessageManager.getReport(wfId);
+ if (reports == null) {
+ int numberOftries = 0;
+ try {
+ while (reports == null && numberOftries < 3) {
+ reports = dnetMessageManager.getReport(wfId);
+ Thread.sleep(3000 * numberOftries++);
+ }
+ } catch (InterruptedException e) {
+ log.error("Error on waiting report", e);
+ }
+ }
+
+ if (reports == null) {
+ throw new RuntimeException("Unable to get report for WorklowId " + wfId);
+ }
+
+
+ reports.forEach(it -> it.getBody().forEach(env::setAttribute));
+ }
+ };
+ }
+
+ @Override
+ public int getTotalValue() {
+ return 0;
+ }
+
+ @Override
+ public int getCurrentValue() {
+ return currentValue;
+ }
+
+ @Override
+ public boolean isInaccurate() {
+ return false;
+ }
+
+ @Override
+ public ProgressProvider getProgressProvider() {
+ return this;
+ }
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FindSearchServicesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FindSearchServicesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FindSearchServicesJobNode.java (revision 58540)
@@ -0,0 +1,69 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.index;
+
+import java.util.List;
+import java.util.Queue;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Queues;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Value;
+
+/**
+ * Supercedes eu.dnetlib.msro.openaireplus.workflows.nodes.index.FindSearchServiceJobNode.
+ * Returns a queue of SearchService ids that match the xquery and saves it in the env with the name 'searchService_ids'.
+ * Use with eu.dnetlib.msro.openaireplus.workflows.nodes.index.SwitchIndexesJobNode
+ *
+ * @author claudio, alessia
+ */
+public class FindSearchServicesJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(FindSearchServicesJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Value(value = "${dnet.openaire.service.search.lookup.xquery}")
+ private String xquery;
+
+ /**
+ * {@inheritDoc}
+ *
+ * @throws ISLookUpException
+ * @throws MSROException
+ * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, NodeToken)
+ */
+ @Override
+ public String execute(final NodeToken token) throws ISLookUpException, MSROException {
+
+ log.info("lookup for search service: " + getXquery());
+
+ List searchServiceIds = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+ log.info("SearchService found: " + searchServiceIds.size());
+ Queue q = Queues.newLinkedBlockingQueue(searchServiceIds);
+
+ if (searchServiceIds.isEmpty()) {
+ return "notFound";
+ } else {
+ token.getEnv().setTransientAttribute("searchService_ids", q);
+ return "found";
+ }
+ }
+
+ public String getXquery() {
+ return xquery;
+ }
+
+ public void setXquery(final String xquery) {
+ this.xquery = xquery;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/CommitMDStoreVersionJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/CommitMDStoreVersionJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/CommitMDStoreVersionJobNode.java (revision 58540)
@@ -0,0 +1,53 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class CommitMDStoreVersionJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(CommitMDStoreVersionJobNode.class);
+
+ /* Spring managed params */
+ private String mdStoreManagerUrl;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String url = getMdStoreManagerUrl() + "/version/{versionId}/commit/{size}";
+
+ final Map params = new HashMap<>();
+ params.put("versionId", token.getEnv().getAttribute("versionId"));
+ params.put("size", token.getEnv().getAttribute("mdStoreSize"));
+
+ final URI uri = UriComponentsBuilder.fromUriString(url)
+ .buildAndExpand(params)
+ .toUri();
+
+ final RestTemplate restTemplate = new RestTemplate();
+ restTemplate.getForObject(uri, Void.class);
+
+ log.info("mdstore version committed ");
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdStoreManagerUrl() {
+ return mdStoreManagerUrl;
+ }
+
+ @Required
+ public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {
+ this.mdStoreManagerUrl = mdStoreManagerUrl;
+ }
+}
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/SaveContextProfileJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/SaveContextProfileJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/SaveContextProfileJobNode.java (revision 58540)
@@ -0,0 +1,95 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import java.io.StringReader;
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+
+public class SaveContextProfileJobNode extends SimpleJobNode {
+
+ private String contextObj;
+
+ private static final Log log = LogFactory.getLog(SaveContextProfileJobNode.class);
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final ContextDesc desc = (ContextDesc) token.getEnv().getTransientAttribute(contextObj);
+
+ final String xquery = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') " + "where $x//CONFIGURATION/context[@id='"
+ + desc.getId() + "' " + "and @type='" + desc.getType() + "'] " + "return $x";
+ List list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+
+ if (list.isEmpty()) {
+ registerNewProfile(desc);
+ } else {
+ updateProfile(list.get(0), desc);
+ }
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ private boolean updateProfile(final String profile, final ContextDesc desc) throws DocumentException, ISRegistryException {
+ final SAXReader reader = new SAXReader();
+ final Document doc = reader.read(new StringReader(profile));
+ final String profId = doc.valueOf("//HEADER/RESOURCE_IDENTIFIER/@value");
+ final Element ctxElem = (Element) doc.selectSingleNode("//CONFIGURATION/context[@id='" + desc.getId() + "' and @type='" + desc.getType() + "']");
+ updateContextParams(ctxElem, desc);
+
+ for (ContextPart cat : desc.getCategories().values()) {
+ final Node catElem = ctxElem.selectSingleNode("./category[@id='" + cat.getId() + "']");
+ if (catElem != null) {
+ catElem.detach();
+ }
+ ctxElem.add(cat.asDomElement("category"));
+ }
+ log.info("registering profile context " + desc.getId());
+ return serviceLocator.getService(ISRegistryService.class).updateProfile(profId, doc.asXML(), "ContextDSResourceType");
+ }
+
+ private void updateContextParams(final Element ctxElement, ContextDesc desc) {
+ //removing old PARAMs
+ List oldParams = ctxElement.selectNodes("param");
+ for (Node n : oldParams) {
+ n.detach();
+ }
+ //adding new params
+ for (Element param : desc.getParamsAsElements()) {
+ ctxElement.add(param);
+ }
+ }
+
+ private String registerNewProfile(final ContextDesc desc) throws DocumentException, ISRegistryException {
+ final SAXReader reader = new SAXReader();
+ final Document doc = reader.read(getClass().getResourceAsStream(
+ "/eu/dnetlib/msro/openaireplus/workflows/repo-hi/entityreg-contexts/xml/contextProfile.xml"));
+ ((Element) doc.selectSingleNode("//CONFIGURATION")).add(desc.asDomElement());
+ log.info("updating profile context " + desc.getId());
+ return serviceLocator.getService(ISRegistryService.class).registerProfile(doc.asXML());
+ }
+
+ public String getContextObj() {
+ return contextObj;
+ }
+
+ public void setContextObj(final String contextObj) {
+ this.contextObj = contextObj;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/OpenaireMdBuilderJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/OpenaireMdBuilderJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/OpenaireMdBuilderJobNode.java (revision 58540)
@@ -0,0 +1,223 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import javax.annotation.Resource;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.XSLTMappedResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+
+public class OpenaireMdBuilderJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(OpenaireMdBuilderJobNode.class);
+
+ private StringTemplate mdBuilderTemplateXslt;
+
+ private String inputEprParam;
+ private String outputEprParam;
+ private String datasourceId;
+ private String datasourceInterface;
+
+ private boolean inferred = false;
+ private boolean deletedbyinference = false;
+ private String inferenceprovenance = "";
+ private String trust = "";
+ private String provenanceactionclassname = "";
+ private String provenanceactionclassid = "";
+
+ private String overridingMetadataIdentifierXPath = "";
+
+ private XSLTMappedResultSetFactory xsltMappedResultSetFactory;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected String execute(final NodeToken token) throws MSROException {
+ final String inputEpr = token.getEnv().getAttribute(inputEprParam);
+
+ if ((inputEpr == null) || inputEpr.isEmpty()) throw new MSROException("InputEprParam (" + inputEprParam + ") not found in ENV");
+ final StringTemplate st = new StringTemplate(getMdBuilderTemplateXslt().getTemplate());
+ try {
+ st.setAttribute("xpath", getMetadataIdentifierXPath());
+ st.setAttribute("datasourceId", datasourceId);
+ if (token.getEnv().hasAttribute("namespacePrefix")) {
+ st.setAttribute("namespacePrefix", token.getEnv().getAttribute("namespacePrefix"));
+ } else {
+ st.setAttribute("namespacePrefix", datasourceId);
+ }
+
+ st.setAttribute("inferred", String.valueOf(getInferred()));
+ st.setAttribute("deletedbyinference", String.valueOf(getDeletedbyinference()));
+ st.setAttribute("inferenceprovenance", getInferenceprovenance());
+ st.setAttribute("trust", getTrust());
+ st.setAttribute("provenanceactionclassname", getProvenanceactionclassname());
+ st.setAttribute("provenanceactionclassid", getProvenanceactionclassid());
+ st.setAttribute("baseurl", URLEncoder.encode(getBaseUrl(), "UTF-8"));
+ st.setAttribute("metadatanamespace", getMetadataNamespace());
+
+ final W3CEndpointReference epr = xsltMappedResultSetFactory.createMappedResultSet(new EPRUtils().getEpr(inputEpr), st.toString());
+
+ token.getEnv().setAttribute(outputEprParam, epr.toString());
+
+ return Arc.DEFAULT_ARC;
+ } catch (ISLookUpException e) {
+ throw new MSROException("Error while initializing mdBuilder template for datasource " + datasourceId, e);
+ } catch (UnsupportedEncodingException e) {
+ log.error("Cannot encode baseUrl in UTF-8");
+ throw new MSROException(e);
+ }
+ }
+
+ private String getMetadataIdentifierXPath() throws ISLookUpException {
+ String xQuery =
+ String.format("for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') "
+ + "return $x//INTERFACE[@id='%s']/INTERFACE_EXTRA_FIELD[@name='metadata_identifier_path']/string()", getDatasourceInterface());
+
+ return hasOverridingMetadataIdentifierXPath() ? getOverridingMetadataIdentifierXPath() : serviceLocator.getService(ISLookUpService.class)
+ .getResourceProfileByQuery(xQuery);
+ }
+
+ private String getBaseUrl() throws ISLookUpException {
+ String xQuery = "/*[.//RESOURCE_IDENTIFIER/@value='{datasourceId}']//INTERFACE[@id='{interfaceId}']//BASE_URL/string()";
+ xQuery = xQuery.replace("{interfaceId}", datasourceInterface).replace("{datasourceId}", datasourceId);
+ return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xQuery);
+ }
+
+ private String getMetadataNamespace() {
+ try {
+ String xQuery =
+ "let $x := /*[.//RESOURCE_IDENTIFIER/@value='{datasourceId}']//INTERFACE[@id='{interfaceId}']/ACCESS_PROTOCOL/@format/string() "
+ + "return /*[.//RESOURCE_TYPE/@value='MetadataFormatDSResourceType']//METADATAFORMAT[@Prefix=$x]/@NameSpace/string()";
+ xQuery = xQuery.replace("{interfaceId}", datasourceInterface).replace("{datasourceId}", datasourceId);
+ return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xQuery);
+ } catch (ISLookUpException e) {
+ log.error("The interface is not OAI or the format is not found in the MetadataFormatDSResourceType, thus metadata format in the section "
+ + "cannot managed here and it will be leaved empty (for the time being)");
+ return "";
+ }
+ }
+
+ private boolean hasOverridingMetadataIdentifierXPath() {
+ return !getOverridingMetadataIdentifierXPath().isEmpty();
+ }
+
+ public String getInputEprParam() {
+ return inputEprParam;
+ }
+
+ public void setInputEprParam(final String inputEprParam) {
+ this.inputEprParam = inputEprParam;
+ }
+
+ public String getOutputEprParam() {
+ return outputEprParam;
+ }
+
+ public void setOutputEprParam(final String outputEprParam) {
+ this.outputEprParam = outputEprParam;
+ }
+
+ public XSLTMappedResultSetFactory getXsltMappedResultSetFactory() {
+ return xsltMappedResultSetFactory;
+ }
+
+ public String getDatasourceId() {
+ return datasourceId;
+ }
+
+ public void setDatasourceId(final String datasourceId) {
+ this.datasourceId = datasourceId;
+ }
+
+ public String getDatasourceInterface() {
+ return datasourceInterface;
+ }
+
+ public void setDatasourceInterface(final String datasourceInterface) {
+ this.datasourceInterface = datasourceInterface;
+ }
+
+ @Required
+ public void setXsltMappedResultSetFactory(final XSLTMappedResultSetFactory xsltMappedResultSetFactory) {
+ this.xsltMappedResultSetFactory = xsltMappedResultSetFactory;
+ }
+
+ public StringTemplate getMdBuilderTemplateXslt() {
+ return mdBuilderTemplateXslt;
+ }
+
+ public void setMdBuilderTemplateXslt(final StringTemplate mdBuilderTemplateXslt) {
+ this.mdBuilderTemplateXslt = mdBuilderTemplateXslt;
+ }
+
+ public boolean getInferred() {
+ return inferred;
+ }
+
+ public void setInferred(final boolean inferred) {
+ this.inferred = inferred;
+ }
+
+ public boolean getDeletedbyinference() {
+ return deletedbyinference;
+ }
+
+ public void setDeletedbyinference(final boolean deletedbyinference) {
+ this.deletedbyinference = deletedbyinference;
+ }
+
+ public String getInferenceprovenance() {
+ return inferenceprovenance;
+ }
+
+ public void setInferenceprovenance(final String inferenceprovenance) {
+ this.inferenceprovenance = inferenceprovenance;
+ }
+
+ public String getTrust() {
+ return trust;
+ }
+
+ public void setTrust(final String trust) {
+ this.trust = trust;
+ }
+
+ public String getProvenanceactionclassname() {
+ return provenanceactionclassname;
+ }
+
+ public void setProvenanceactionclassname(final String provenanceactionclassname) {
+ this.provenanceactionclassname = provenanceactionclassname;
+ }
+
+ public String getProvenanceactionclassid() {
+ return provenanceactionclassid;
+ }
+
+ public void setProvenanceactionclassid(final String provenanceactionclassid) {
+ this.provenanceactionclassid = provenanceactionclassid;
+ }
+
+ public String getOverridingMetadataIdentifierXPath() {
+ return overridingMetadataIdentifierXPath;
+ }
+
+ public void setOverridingMetadataIdentifierXPath(final String overridingMetadataIdentifierXPath) {
+ this.overridingMetadataIdentifierXPath = overridingMetadataIdentifierXPath;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/GenerateFETH2020ContextJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/GenerateFETH2020ContextJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/GenerateFETH2020ContextJobNode.java (revision 58540)
@@ -0,0 +1,110 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;
+import eu.dnetlib.msro.workflows.util.ProgressProvider;
+import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class GenerateFETH2020ContextJobNode extends SimpleJobNode implements ProgressJobNode {
+
+ private static final Log log = LogFactory.getLog(GenerateFETH2020ContextJobNode.class);
+ private String eprParam;
+ private ResultsetProgressProvider progressProvider;
+ @Resource
+ private ResultSetClientFactory resultSetClientFactory;
+ @Resource
+ private ProcessCountingResultSetFactory processCountingResultSetFactory;
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ public ProgressProvider getProgressProvider() {
+ return this.progressProvider;
+ }
+
+ @Override
+ protected String execute(NodeToken token) throws Exception {
+ final String epr = token.getEnv().getAttribute(eprParam);
+
+ this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);
+
+ final Iterable iter = resultSetClientFactory.getClient(progressProvider.getEpr());
+
+ BuildH2020FETTaxonomy builder = new BuildH2020FETTaxonomy();
+ builder.setIterator(iter.iterator());
+ String taxonomy = builder.parseProjects();
+ final String xquery = "for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') "
+ + "where $x//CONFIGURATION/context[@id='fet-h2020'] return $x//RESOURCE_IDENTIFIER/@value/string()";
+ List list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+ if (list.isEmpty()) {
+ registerProfile(taxonomy);
+ } else {
+ deleteProfile(list.get(0));
+ registerProfile(taxonomy);
+ }
+ return Arc.DEFAULT_ARC;
+ }
+
+ private void registerProfile(String profile) throws ISRegistryException {
+ log.info("registering fet-h2020 profile");
+ ISRegistryService is = serviceLocator.getService(ISRegistryService.class);
+ String id = is.registerProfile(profile);
+ log.info("Generating profile with id " + id);
+ }
+
+ private void deleteProfile(String profId) throws ISRegistryException {
+ log.info("deleting fet-h2020 profile with id " + profId);
+ ISRegistryService is = serviceLocator.getService(ISRegistryService.class);
+ is.deleteProfile(profId);
+ }
+
+ public String getEprParam() {
+ return eprParam;
+ }
+
+ public void setEprParam(final String eprParam) {
+ this.eprParam = eprParam;
+ }
+
+ public void setProgressProvider(final ResultsetProgressProvider progressProvider) {
+ this.progressProvider = progressProvider;
+ }
+
+ public ResultSetClientFactory getResultSetClientFactory() {
+ return resultSetClientFactory;
+ }
+
+ public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+ this.resultSetClientFactory = resultSetClientFactory;
+ }
+
+ public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {
+ return processCountingResultSetFactory;
+ }
+
+ public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {
+ this.processCountingResultSetFactory = processCountingResultSetFactory;
+ }
+
+ public UniqueServiceLocator getServiceLocator() {
+ return serviceLocator;
+ }
+
+ public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
+ this.serviceLocator = serviceLocator;
+ }
+
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FetchOntologiesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FetchOntologiesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FetchOntologiesJobNode.java (revision 58540)
@@ -0,0 +1,36 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.utils.ontologies.Ontologies;
+import eu.dnetlib.utils.ontologies.OntologyLoader;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class FetchOntologiesJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(FetchOntologiesJobNode.class);
+
+ private String paramName;
+
+ @Override
+ protected String execute(NodeToken token) throws Exception {
+
+ final Ontologies ontologies = OntologyLoader.loadOntologiesFromIS();
+
+ log.info(String.format("loaded %s ontology profiles", ontologies.size()));
+
+ token.getEnv().setAttribute(getParamName(), ontologies.toJson());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getParamName() {
+ return paramName;
+ }
+
+ public void setParamName(String paramName) {
+ this.paramName = paramName;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/FindHostedByJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/FindHostedByJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/FindHostedByJobNode.java (revision 58540)
@@ -0,0 +1,161 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby;
+
+import java.io.StringReader;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.resultset.MappedResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.io.SAXReader;
+
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class FindHostedByJonbNode.
+ */
+public class FindHostedByJobNode extends SimpleJobNode {
+
+ /**
+ * The input epr param.
+ */
+ private String inputEprParam;
+
+ /**
+ * The output epr param.
+ */
+ private String outputEprParam;
+
+ /**
+ * The counters param.
+ */
+ private String countersParam;
+
+ /**
+ * The result set client factory.
+ */
+ private ResultSetClientFactory resultSetClientFactory;
+
+ /**
+ * The mapped result set factory.
+ */
+ private MappedResultSetFactory mappedResultSetFactory;
+
+ private final String unknown_repo_id = "openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18";
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(inputEprParam));
+ final HostedByCounters counters = new HostedByCounters();
+
+ final SAXReader reader = new SAXReader();
+
+ final UnaryFunction hostedByMapFunction = new UnaryFunction() {
+
+ @Override
+ public String evaluate(final String input) {
+ try {
+ final Document doc = reader.read(new StringReader(input));
+ final Element node = (Element) doc.selectSingleNode("//*[local-name()='hostedBy']");
+ if (node != null) {
+ String hostedById = node.attributeValue("id");
+ if (!hostedById.equals(unknown_repo_id)) {
+ counters.increaseCounter(hostedById);
+ }
+ }
+ } catch (Exception e) {
+
+ }
+
+ return input;
+ }
+ };
+
+ final W3CEndpointReference epr = mappedResultSetFactory.createMappedResultSet(inputEpr, hostedByMapFunction);
+ token.getEnv().setAttribute(outputEprParam, epr.toString());
+ token.getEnv().setTransientAttribute(getCountersParam(), counters);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ /**
+ * @return the inputEprParam
+ */
+ public String getInputEprParam() {
+ return inputEprParam;
+ }
+
+ /**
+ * @param inputEprParam the inputEprParam to set
+ */
+ public void setInputEprParam(final String inputEprParam) {
+ this.inputEprParam = inputEprParam;
+ }
+
+ /**
+ * @return the outputEprParam
+ */
+ public String getOutputEprParam() {
+ return outputEprParam;
+ }
+
+ /**
+ * @param outputEprParam the outputEprParam to set
+ */
+ public void setOutputEprParam(final String outputEprParam) {
+ this.outputEprParam = outputEprParam;
+ }
+
+ /**
+ * @return the resultSetClientFactory
+ */
+ public ResultSetClientFactory getResultSetClientFactory() {
+ return resultSetClientFactory;
+ }
+
+ /**
+ * @param resultSetClientFactory the resultSetClientFactory to set
+ */
+ public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+ this.resultSetClientFactory = resultSetClientFactory;
+ }
+
+ /**
+ * @return the mappedResultSetFactory
+ */
+ public MappedResultSetFactory getMappedResultSetFactory() {
+ return mappedResultSetFactory;
+ }
+
+ /**
+ * @param mappedResultSetFactory the mappedResultSetFactory to set
+ */
+ public void setMappedResultSetFactory(final MappedResultSetFactory mappedResultSetFactory) {
+ this.mappedResultSetFactory = mappedResultSetFactory;
+ }
+
+ /**
+ * @return the countersParam
+ */
+ public String getCountersParam() {
+ return countersParam;
+ }
+
+ /**
+ * @param countersParam the countersParam to set
+ */
+ public void setCountersParam(final String countersParam) {
+ this.countersParam = countersParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareEnvTransformHadoopJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareEnvTransformHadoopJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareEnvTransformHadoopJobNode.java (revision 58540)
@@ -0,0 +1,73 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import eu.dnetlib.dhp.utils.DHPUtils;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class PrepareEnvTransformHadoopJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(PrepareEnvTransformHadoopJobNode.class);
+
+ private String hdfsBasePath;
+
+ private String ruleId;
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final String reading_mdId = token.getEnv().getAttribute("reading_mdId");
+ final String reading_versionId = token.getEnv().getAttribute("reading_versionId");
+
+ final String writing_mdId = token.getEnv().getAttribute("mdId");
+ final String writing_versionId = token.getEnv().getAttribute("versionId");
+
+ final String hdfsNativePath = String.format("%s/%s/%s/store", hdfsBasePath, reading_mdId, reading_versionId);
+ final String hdfsTransformedPath = String.format("%s/%s/%s/store", hdfsBasePath, writing_mdId, writing_versionId);
+
+
+ final String trRule = serviceLocator.getService(ISLookUpService.class).getResourceProfile(getRuleId());
+
+ token.getEnv().setAttribute("timestamp", "" + System.currentTimeMillis());
+ token.getEnv().setAttribute("workflowId", token.getProcess().getEnv().getAttribute("system:processId"));
+ token.getEnv().setAttribute("mdstoreInputPath", hdfsNativePath);
+ token.getEnv().setAttribute("mdstoreOutputPath", hdfsTransformedPath);
+ token.getEnv().setAttribute("transformationRule", DHPUtils.compressString(trRule));
+ return Arc.DEFAULT_ARC;
+
+ }
+
+ public String getHdfsBasePath() {
+ return hdfsBasePath;
+ }
+
+ public void setHdfsBasePath(final String hdfsBasePath) {
+ this.hdfsBasePath = hdfsBasePath;
+ }
+
+ public String getRuleId() {
+ return ruleId;
+ }
+
+ public void setRuleId(String ruleId) {
+ this.ruleId = ruleId;
+ }
+
+ public UniqueServiceLocator getServiceLocator() {
+ return serviceLocator;
+ }
+
+ public void setServiceLocator(UniqueServiceLocator serviceLocator) {
+ this.serviceLocator = serviceLocator;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ClaimDatabaseUtils.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ClaimDatabaseUtils.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ClaimDatabaseUtils.java (revision 58540)
@@ -0,0 +1,76 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.claims;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import javax.annotation.PostConstruct;
+
+import eu.dnetlib.enabling.database.DataSourceFactory;
+import eu.dnetlib.enabling.database.utils.JdbcTemplateFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.support.rowset.SqlRowSet;
+import org.springframework.stereotype.Component;
+
+/**
+ * Created by Alessia Bardi on 23/06/2017.
+ *
+ * @author Alessia Bardi
+ */
+@Component
+public class ClaimDatabaseUtils {
+
+ @Autowired
+ private JdbcTemplateFactory claimsJdbcTemplateFactory;
+ @Autowired
+ private DataSourceFactory claimsDataSourceFactory;
+
+ private JdbcTemplate jdbcTemplate;
+
+ @Value("${dnet.openaire.claims.db.name}")
+ private String dbName;
+
+ @PostConstruct
+ public void init() {
+ claimsJdbcTemplateFactory.setDataSourceFactory(claimsDataSourceFactory);
+ this.jdbcTemplate = claimsJdbcTemplateFactory.createJdbcTemplate(dbName);
+ }
+
+ public int count(final String sqlCountQuery) {
+ return jdbcTemplate.queryForObject(sqlCountQuery, Integer.class);
+ }
+
+ public Stream query(final String sqlQuery) {
+ final SqlRowSet rowSet = jdbcTemplate.queryForRowSet(sqlQuery);
+ return StreamSupport.stream(
+ Spliterators.spliteratorUnknownSize(
+ new Iterator() {
+
+ @Override
+ public boolean hasNext() {
+ return !rowSet.isLast();
+ }
+
+ @Override
+ public Claim next() {
+ if (!rowSet.next()) {
+ throw new NoSuchElementException();
+ }
+ return asClaim(rowSet);
+ }
+ },
+ Spliterator.IMMUTABLE), false);
+ }
+
+ private Claim asClaim(SqlRowSet rs) {
+ return new Claim()
+ .setSemantics(rs.getString("semantics"))
+ .setSource(rs.getString("source_id")).setSourceType(rs.getString("source_type"))
+ .setTarget(rs.getString("target_id")).setTargetType(rs.getString("target_type"));
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/StartReadingMDStoreJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/StartReadingMDStoreJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/StartReadingMDStoreJobNode.java (revision 58540)
@@ -0,0 +1,67 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.data.mdstore.manager.common.model.MDStoreVersion;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class StartReadingMDStoreJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(StartReadingMDStoreJobNode.class);
+
+ private String mdId;
+
+ /* Spring managed params */
+ private String mdStoreManagerUrl;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String url = getMdStoreManagerUrl() + "/mdstore/{mdId}/startReading";
+
+ final Map params = new HashMap<>();
+ params.put("mdId", getMdId());
+
+ final URI uri = UriComponentsBuilder.fromUriString(url)
+ .buildAndExpand(params)
+ .toUri();
+
+ final RestTemplate restTemplate = new RestTemplate();
+ final MDStoreVersion version = restTemplate.getForObject(uri, MDStoreVersion.class);
+
+ log.info("mdstore version ready to read " + version.getId());
+
+ token.getEnv().setAttribute("reading_mdId", mdId);
+ token.getEnv().setAttribute("reading_versionId", version.getId());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdStoreManagerUrl() {
+ return mdStoreManagerUrl;
+ }
+
+ @Required
+ public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {
+ this.mdStoreManagerUrl = mdStoreManagerUrl;
+ }
+
+ public String getMdId() {
+ return mdId;
+ }
+
+ public void setMdId(final String mdId) {
+ this.mdId = mdId;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/FindStatsServiceJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/FindStatsServiceJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/FindStatsServiceJobNode.java (revision 58540)
@@ -0,0 +1,71 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Value;
+
+public class FindStatsServiceJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(FindStatsServiceJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Value(value = "${dnet.openaire.service.stats.lookup.xquery}")
+ private String xquery;
+
+ private String xqueryForServiceIdParam;
+
+ /**
+ * {@inheritDoc}
+ *
+ * @throws ISLookUpException
+ * @throws MSROException
+ * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ public String execute(final NodeToken token) throws ISLookUpException, MSROException {
+
+ log.info("lookup for stats service: " + getXquery());
+
+ List statsServiceIds = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+
+ if (statsServiceIds.size() > 1) { throw new MSROException("too many stats services: " + statsServiceIds); }
+
+ if (statsServiceIds.isEmpty()) {
+ return "notFound";
+ } else {
+ token.getEnv().setAttribute(getXqueryForServiceIdParam(), getXquery());
+ return "found";
+ }
+ }
+
+ public String getXquery() {
+ return xquery;
+ }
+
+ public void setXquery(final String xquery) {
+ this.xquery = xquery;
+ }
+
+ public String getXqueryForServiceIdParam() {
+ return xqueryForServiceIdParam;
+ }
+
+ public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {
+ this.xqueryForServiceIdParam = xqueryForServiceIdParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/PromoteShadowCacheStatsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/PromoteShadowCacheStatsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/PromoteShadowCacheStatsJobNode.java (revision 58540)
@@ -0,0 +1,30 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to promote the shadow cache so that its entries can be used by the portal specified by the additional BB
+ * parameter StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName()
.
+ *
+ * @author alessia
+ */
+public class PromoteShadowCacheStatsJobNode extends AbstractStatsJobNode {
+
+ private static final Log log = LogFactory.getLog(PromoteShadowCacheStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.action());
+ String portal = getPortalName(token);
+ if (StringUtils.isNotBlank(portal)) {
+ job.getParameters().put(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName(), portal);
+ } else {
+ log.warn(StatsManagerServiceBBAction.PROMOTE_SHADOW_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+ }
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/RefreshShadowCacheJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/RefreshShadowCacheJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/RefreshShadowCacheJobNode.java (revision 58540)
@@ -0,0 +1,34 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to refresh the shadow cache according to the content of the shadow stats db for the portal specified by the
+ * additional BB parameter StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName()
.
+ *
+ * In order for the given portal to actually use the generated cache, we must promote the cache with the BB message 'promoteCache'.
+ *
+ *
+ * @author alessia
+ * @see eu.dnetlib.msro.openaireplus.workflows.nodes.stats.PromoteShadowCacheStatsJobNode
+ */
+public class RefreshShadowCacheJobNode extends AbstractStatsJobNode {
+
+ private static final Log log = LogFactory.getLog(RefreshShadowCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.action());
+ String portal = getPortalName(token);
+ if (StringUtils.isNotBlank(portal)) {
+ job.getParameters().put(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName(), portal);
+ } else {
+ log.warn(StatsManagerServiceBBAction.REFRESH_SHADOW_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+ }
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/RestoreStatsCacheJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/RestoreStatsCacheJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/RestoreStatsCacheJobNode.java (revision 58540)
@@ -0,0 +1,34 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to restore the cache of the portal specified by the additional BB parameter
+ * StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName()
to the last available backup.
+ *
+ * To create a backup you must send a "backup" BB message.
+ *
+ *
+ * @author alessia
+ * @see eu.dnetlib.msro.openaireplus.workflows.nodes.stats.BackupStatsCacheJobNode
+ */
+public class RestoreStatsCacheJobNode extends AbstractStatsJobNode {
+
+ private static final Log log = LogFactory.getLog(RestoreStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(StatsManagerServiceBBAction.RESTORE_CACHE.action());
+ String portal = getPortalName(token);
+ if (StringUtils.isNotBlank(portal)) {
+ job.getParameters().put(StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName(), portal);
+ } else {
+ log.warn(StatsManagerServiceBBAction.RESTORE_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+ }
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/SetFormatInfoJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/SetFormatInfoJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/SetFormatInfoJobNode.java (revision 58540)
@@ -0,0 +1,92 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import static java.lang.String.format;
+
+public class SetFormatInfoJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(SetFormatInfoJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ private String format;
+ private String formatParam = "format";
+
+ private String layout;
+ private String layoutParam = "layout";
+
+ private String interpretation;
+ private String interpretationParam = "interpretation";
+
+ /**
+ * {@inheritDoc}
+ *
+ * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ public String execute(final NodeToken token) {
+
+ log.info(format("setting mdFromat: %s, layout: %s, interpretation: %s", getFormat(), getLayout(), getInterpretation()));
+
+ token.getEnv().setAttribute(getFormatParam(), getFormat());
+ token.getEnv().setAttribute(getLayoutParam(), getLayout());
+ token.getEnv().setAttribute(getInterpretationParam(), getInterpretation());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getFormat() {
+ return format;
+ }
+
+ public void setFormat(final String format) {
+ this.format = format;
+ }
+
+ public String getLayout() {
+ return layout;
+ }
+
+ public void setLayout(final String layout) {
+ this.layout = layout;
+ }
+
+ public String getInterpretation() {
+ return interpretation;
+ }
+
+ public void setInterpretation(final String interpretation) {
+ this.interpretation = interpretation;
+ }
+
+ public String getFormatParam() {
+ return formatParam;
+ }
+
+ public void setFormatParam(final String formatParam) {
+ this.formatParam = formatParam;
+ }
+
+ public String getLayoutParam() {
+ return layoutParam;
+ }
+
+ public void setLayoutParam(final String layoutParam) {
+ this.layoutParam = layoutParam;
+ }
+
+ public String getInterpretationParam() {
+ return interpretationParam;
+ }
+
+ public void setInterpretationParam(final String interpretationParam) {
+ this.interpretationParam = interpretationParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/MigrateStatsCacheJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/MigrateStatsCacheJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/MigrateStatsCacheJobNode.java (revision 58540)
@@ -0,0 +1,51 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.rmi.MSROException;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to copy a cache of a portal to another portal cache.
+ *
+ * The source portal is specified in the BB parameter StatsManagerServiceBBAction.MIGRATE_CACHE.getSourcePortalParamName()
.
+ *
+ *
+ * The target portal is specified in the BB parameter StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName()
+ *
+ *
+ * @author alessia
+ */
+public class MigrateStatsCacheJobNode extends AbstractStatsJobNode {
+
+ private static final Log log = LogFactory.getLog(MigrateStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ /**
+ * BB parameter defining which is the portal with the source cache to migrate.
+ */
+ private String sourcePortal;
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(StatsManagerServiceBBAction.MIGRATE_CACHE.action());
+ if (StringUtils.isBlank(sourcePortal)) throw new MSROException("Parameter sourcePortal required to migrate the cache");
+ job.getParameters().put(StatsManagerServiceBBAction.MIGRATE_CACHE.getSourcePortalParamName(), sourcePortal);
+ String targetPortal = getPortalName(token);
+ if (StringUtils.isNotBlank(targetPortal)) {
+ job.getParameters().put(StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName(), targetPortal);
+ } else {
+ log.warn(StatsManagerServiceBBAction.MIGRATE_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+ }
+
+ }
+
+ public String getSourcePortal() {
+ return sourcePortal;
+ }
+
+ public void setSourcePortal(final String sourcePortal) {
+ this.sourcePortal = sourcePortal;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FindVocabularyEntriesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FindVocabularyEntriesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FindVocabularyEntriesJobNode.java (revision 58540)
@@ -0,0 +1,49 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.List;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.io.IOUtils;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.core.io.Resource;
+
+public class FindVocabularyEntriesJobNode extends SimpleJobNode {
+
+ private String outputEprParam = "epr";
+
+ @javax.annotation.Resource
+ private UniqueServiceLocator serviceLocator;
+
+ private Resource xquery;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String query = IOUtils.toString(xquery.getInputStream());
+ final List list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query);
+ token.getEnv().setTransientAttribute(getOutputEprParam(), list);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getOutputEprParam() {
+ return outputEprParam;
+ }
+
+ public void setOutputEprParam(final String outputEprParam) {
+ this.outputEprParam = outputEprParam;
+ }
+
+ public Resource getXquery() {
+ return xquery;
+ }
+
+ @Required
+ public void setXquery(final Resource xquery) {
+ this.xquery = xquery;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/AbortMDStoreVersionJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/AbortMDStoreVersionJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/AbortMDStoreVersionJobNode.java (revision 58540)
@@ -0,0 +1,52 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class AbortMDStoreVersionJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(AbortMDStoreVersionJobNode.class);
+
+ /* Spring managed params */
+ private String mdStoreManagerUrl;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String url = getMdStoreManagerUrl() + "/version/{versionId}?force=true";
+
+ final Map params = new HashMap<>();
+ params.put("versionId", token.getEnv().getAttribute("versionId"));
+
+ final URI uri = UriComponentsBuilder.fromUriString(url)
+ .buildAndExpand(params)
+ .toUri();
+
+ final RestTemplate restTemplate = new RestTemplate();
+ restTemplate.delete(uri);
+
+ log.info("mdstore version deleted ");
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdStoreManagerUrl() {
+ return mdStoreManagerUrl;
+ }
+
+ @Required
+ public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {
+ this.mdStoreManagerUrl = mdStoreManagerUrl;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IISCacheBuilderJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IISCacheBuilderJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IISCacheBuilderJobNode.java (revision 58540)
@@ -0,0 +1,86 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.NoSuchElementException;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Iterables;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.hadoop.config.ClusterName;
+import eu.dnetlib.data.hadoop.config.ConfigurationEnumerator;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.hadoop.SubmitHadoopJobNode;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Created by claudio on 07/09/16.
+ */
+public class IISCacheBuilderJobNode extends SubmitHadoopJobNode {
+
+ @Resource
+ protected ConfigurationEnumerator configurationEnumerator;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ private String xqueryObjectStoreService;
+
+ private String oozieWfAppPath;
+
+ private String objectStoreId;
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+
+ final Configuration conf = configurationEnumerator.get(ClusterName.valueOf(getCluster()));
+
+ String nameNode = conf.get("fs.defaultFS");
+
+ job.getParameters().put("nameNode", nameNode);
+ job.getParameters().put("jobTracker", conf.get("mapred.job.tracker"));
+ job.getParameters().put("objectstore_service_location", getServiceEndpoint(getXqueryObjectStoreService()));
+ job.getParameters().put("approved_objectstores_csv", getObjectStoreId());
+ job.getParameters().put("execution_environment", "cache_builder_" + getObjectStoreId());
+
+ super.prepareJob(job, token);
+ }
+
+ protected String getServiceEndpoint(final String xquery) throws MSROException {
+ try {
+ return Iterables.getOnlyElement(serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery));
+ } catch (ISLookUpException e) {
+ throw new MSROException("unable to fetch service endpoint", e);
+ } catch (NoSuchElementException e) {
+ throw new MSROException("unable to find service endpoint, xquery: " + xquery, e);
+ } catch (IllegalArgumentException e) {
+ throw new MSROException("more than one services found, we assume to have only one available", e);
+ }
+ }
+
+ public String getOozieWfAppPath() {
+ return oozieWfAppPath;
+ }
+
+ public void setOozieWfAppPath(final String oozieWfAppPath) {
+ this.oozieWfAppPath = oozieWfAppPath;
+ }
+
+ public String getObjectStoreId() {
+ return objectStoreId;
+ }
+
+ public void setObjectStoreId(final String objectStoreId) {
+ this.objectStoreId = objectStoreId;
+ }
+
+ public String getXqueryObjectStoreService() {
+ return xqueryObjectStoreService;
+ }
+
+ public void setXqueryObjectStoreService(final String xqueryObjectStoreService) {
+ this.xqueryObjectStoreService = xqueryObjectStoreService;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/RunMDStorePluginJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/RunMDStorePluginJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/RunMDStorePluginJobNode.java (revision 58540)
@@ -0,0 +1,55 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.mdstore.MDStoreService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Created by claudio on 21/03/16.
+ */
+public class RunMDStorePluginJobNode extends BlackboardJobNode {
+
+ private static final Log log = LogFactory.getLog(RunMDStorePluginJobNode.class);
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ private String mdId;
+
+ private String pluginName;
+
+ @Override
+ protected String obtainServiceId(final NodeToken nodeToken) {
+ return serviceLocator.getServiceId(MDStoreService.class);
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken nodeToken) throws Exception {
+ job.setAction("RUN_PLUGIN");
+ job.getParameters().put("plugin.name", getPluginName());
+ job.getParameters().put("mdId", getMdId());
+
+ log.debug(job.getParameters());
+ }
+
+ public String getMdId() {
+ return mdId;
+ }
+
+ public void setMdId(final String mdId) {
+ this.mdId = mdId;
+ }
+
+ public String getPluginName() {
+ return pluginName;
+ }
+
+ public void setPluginName(final String pluginName) {
+ this.pluginName = pluginName;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/profiles/RepositoryXsltFunctions.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/profiles/RepositoryXsltFunctions.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/profiles/RepositoryXsltFunctions.java (revision 58540)
@@ -0,0 +1,132 @@
+package eu.dnetlib.msro.openaireplus.workflows.profiles;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+public class RepositoryXsltFunctions {
+
+ private static final Log log = LogFactory.getLog(RepositoryXsltFunctions.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ private static Resource ifaceTemplate = new ClassPathResource("/eu/dnetlib/msro/openaireplus/workflows/profiles/repo_interface.st");
+
+ private static Resource paramTemplate = new ClassPathResource("/eu/dnetlib/msro/openaireplus/workflows/profiles/repo_param.st");
+
+ private enum Params {
+ FORMAT, SET
+ }
+
+ ;
+
+ public static String buildInterface(final String infopackage) {
+
+ final List formats = Lists.newArrayList();
+ final List sets = Lists.newArrayList();
+ final Map otherParams = Maps.newHashMap();
+
+ for (String param : parseParams(getValueBetween(infopackage, "===6===", null))) {
+ StringTemplate p = getTemplate(paramTemplate);
+ String paramName = StringUtils.substringBefore(param, "=").trim();
+ String paramValue = StringUtils.substringAfter(param, "=").trim();
+
+ p.setAttribute("param", paramName.toUpperCase());
+ p.setAttribute("value", paramValue);
+
+ try {
+ switch (Params.valueOf(paramName.toUpperCase())) {
+ case FORMAT:
+ formats.add(p.toString());
+ break;
+ case SET:
+ sets.add(p.toString());
+ break;
+ }
+ } catch (Exception e) {
+ if ((paramName != null) && !paramName.isEmpty()) {
+ otherParams.put(paramName, StringEscapeUtils.escapeXml(paramValue));
+ }
+ }
+ }
+
+ final StringTemplate i = getTemplate(ifaceTemplate);
+ i.setAttribute("id", getValueBetween(infopackage, null, "===1==="));
+ i.setAttribute("fileMode", getValueBetween(infopackage, "===1===", "===2==="));
+ i.setAttribute("fileDesc", getValueBetween(infopackage, "===2===", "===3==="));
+ i.setAttribute("iisWf", getValueBetween(infopackage, "===3===", "===4==="));
+ i.setAttribute("baseUrl", getBaseUrl(getValueBetween(infopackage, "===4===", "===5===")));
+ i.setAttribute("accessProtocol", getAccessProtocol(getValueBetween(infopackage, "===5===", "===6==="), otherParams));
+ i.setAttribute("formats", ensureMinOccurs(formats, " "));
+ i.setAttribute("sets", ensureMinOccurs(sets, " "));
+ if (!otherParams.isEmpty()) {
+ i.setAttribute("otherParams", otherParams);
+ }
+
+ final String iface = i.toString();
+ return iface;
+ }
+
+ private static String getValueBetween(final String s, final String pre, final String post) {
+ if ((pre == null) && (post == null)) {
+ return s;
+ } else if (pre == null) {
+ return StringUtils.substringBefore(s, post);
+ } else if (post == null) {
+ return StringUtils.substringAfter(s, pre);
+ } else {
+ return StringUtils.substringBetween(s, pre, post);
+ }
+ }
+
+ private static List ensureMinOccurs(final List list, final String empty) {
+ if (list.isEmpty()) {
+ list.add(empty);
+ }
+ return list;
+ }
+
+ private static Iterable parseParams(final String s) {
+ return Splitter.on("***").omitEmptyStrings().trimResults().split(s);
+ }
+
+ private static String getBaseUrl(final String url) {
+ final StringTemplate b = getTemplate(paramTemplate);
+ b.setAttribute("param", "BASE_URL");
+ b.setAttribute("value", url);
+ return b.toString();
+ }
+
+ private static String getAccessProtocol(final String protocol, final Map otherParams) {
+ final StringTemplate a = getTemplate(paramTemplate);
+ a.setAttribute("param", "ACCESS_PROTOCOL");
+ a.setAttribute("value", protocol);
+ if ((otherParams != null) && !otherParams.isEmpty()) {
+ a.setAttribute("attrs", otherParams);
+ }
+ return a.toString();
+ }
+
+ private static StringTemplate getTemplate(final Resource res) {
+ final StringWriter body = new StringWriter();
+ try {
+ IOUtils.copy(res.getInputStream(), body);
+ return new StringTemplate(body.toString());
+ } catch (IOException e) {
+ log.error("unable to get template", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaire/ui/DownloadPluginValues.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaire/ui/DownloadPluginValues.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaire/ui/DownloadPluginValues.java (revision 58540)
@@ -0,0 +1,51 @@
+package eu.dnetlib.msro.openaire.ui;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Resource;
+
+import eu.dnetlib.data.download.rmi.DownloadPlugin;
+import eu.dnetlib.data.download.rmi.DownloadService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.util.ValidNodeValuesFetcher;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class DownloadPluginValues extends ValidNodeValuesFetcher {
+
+ /**
+ * The download plugin enumerator.
+ */
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Autowired(required = false)
+ private List plugins;
+
+ @Override
+ protected List obtainValues(final Map params) throws Exception {
+
+ List otherPlugins = serviceLocator.getService(DownloadService.class).listPlugins();
+
+ ArrayList output = new ArrayList();
+
+ // if (plugins != null) {
+ // output.addAll(Lists.newArrayList(Iterables.transform(plugins, new Function() {
+ //
+ // @Override
+ // public DnetParamValue apply(final DownloadPlugin input) {
+ // return new DnetParamValue(input.getPluginName(), input.getPluginName());
+ // }
+ // })));
+ // }
+
+ if (otherPlugins != null) {
+ for (String pluginName : otherPlugins) {
+ output.add(new DnetParamValue(pluginName, pluginName));
+ }
+ }
+
+ return output;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/functionality/modular/ui/workflows/values/ListActionManagerSetsValues.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/functionality/modular/ui/workflows/values/ListActionManagerSetsValues.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/functionality/modular/ui/workflows/values/ListActionManagerSetsValues.java (revision 58540)
@@ -0,0 +1,35 @@
+package eu.dnetlib.functionality.modular.ui.workflows.values;
+
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Resource;
+
+import com.google.common.collect.Lists;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.openaireplus.workflows.nodes.actions.PromoteActionsJobNode;
+import eu.dnetlib.msro.workflows.util.ValidNodeValuesFetcher;
+
+public class ListActionManagerSetsValues extends ValidNodeValuesFetcher {
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected List obtainValues(final Map params) throws Exception {
+
+ final String xquery =
+ "for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ActionManagerSetDSResourceType'] return concat($x//SET/@id, ' @@@ ', $x//SET) ";
+
+ final List result = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
+ final List values = Lists.newArrayList(new DnetParamValue("", PromoteActionsJobNode.ALL_SETS));
+
+ for (final String s : result) {
+ final String[] arr = s.split("@@@");
+ values.add(new DnetParamValue(arr[0].trim(), arr[1].trim()));
+ }
+
+ return values;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidStoresJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidStoresJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidStoresJobNode.java (revision 58540)
@@ -0,0 +1,103 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+
+public class FindInvalidStoresJobNode extends SimpleJobNode {
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Autowired
+ private MDStoreDao mdstoreDao;
+
+ @Autowired
+ private ObjectStoreDao objDao;
+
+ private static final Log log = LogFactory.getLog(FindInvalidStoresJobNode.class);
+
+ private boolean deleteInvalidStores = false;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final ISLookUpService lookup = serviceLocator.getService(ISLookUpService.class);
+ final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class);
+
+ final String q1 = "distinct-values("
+ + "for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType')//PARAM[@category='MDSTORE_ID' or @category='OBJECTSTORE_ID'] "
+ + "return $x/text())";
+
+ final String q2 = "for $x in collection('/db/DRIVER/MDStoreDSResources/MDStoreDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
+ final String q3 = "for $x in collection('/db/DRIVER/ObjectStoreDSResources/ObjectStoreDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
+
+ final Set valids = new HashSet<>(lookup.quickSearchProfile(q1));
+
+ int countMdstores = 0;
+ int countInvalidMdStores = 0;
+ int countDeletedMdStores = 0;
+ int countObjstores = 0;
+ int countInvalidObjStores = 0;
+ int countDeletedObjStores = 0;
+
+ for (final String id : lookup.quickSearchProfile(q2)) {
+ countMdstores++;
+ if (!valids.contains(id)) {
+ log.warn("Invalid mdstore: " + id);
+ countInvalidMdStores++;
+ if (deleteInvalidStores) {
+ registry.deleteProfile(id);
+ mdstoreDao.deleteMDStore(id);
+ log.info(" -- Deleted");
+ countDeletedMdStores++;
+ }
+
+ }
+ }
+
+ for (final String id : lookup.quickSearchProfile(q3)) {
+ countObjstores++;
+ if (!valids.contains(id)) {
+ log.warn("Invalid objstore: " + id);
+ countInvalidObjStores++;
+ if (deleteInvalidStores) {
+ registry.deleteProfile(id);
+ objDao.deleteObjectStore(id);
+ log.info(" -- Deleted");
+ countDeletedObjStores++;
+ }
+
+ }
+ }
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "totalMdStores", countMdstores);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalidMdStores", countInvalidMdStores);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "deletedMdStores", countDeletedMdStores);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "totalObjStores", countObjstores);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalidObjStores", countInvalidObjStores);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "deletedObjStores", countDeletedObjStores);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public boolean isDeleteInvalidStores() {
+ return deleteInvalidStores;
+ }
+
+ public void setDeleteInvalidStores(final boolean deleteInvalidStores) {
+ this.deleteInvalidStores = deleteInvalidStores;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/VerifyMdStoreSizesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/VerifyMdStoreSizesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/VerifyMdStoreSizesJobNode.java (revision 58540)
@@ -0,0 +1,22 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency;
+
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+
+public class VerifyMdStoreSizesJobNode extends SimpleJobNode {
+
+ @Resource(name = "mongodbMDStoreDao")
+ private MDStoreDao dao;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ dao.refreshSizes();
+ return Arc.DEFAULT_ARC;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/FETTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/FETTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/FETTest.java (revision 58540)
@@ -0,0 +1,52 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+
+public class FETTest {
+
+ private static final String PATTERN = "classpath*:/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/*.xml";
+ //class under test
+ private BuildH2020FETTaxonomy fetBuilder;
+ private Iterator iterator;
+
+ //String outputCheck = " H2020-FETOPEN-2014-2015-RIA 713140 FETOPEN-RIA-2014-2015 713140 MESO_BRAIN H2020 665148 FETOPEN-RIA-2014-2015 665148 QCUMbER H2020 665233 FETOPEN-RIA-2014-2015 665233 CHROMAVISION H2020 H2020-FETOPEN-2015-CSA 713171 FETOPEN-CSA-FETEXCHANGE-2015 713171 SENSE H2020 H2020-FETOPEN-4-2016-2017 754514 FETOPEN-04-2016-2017 754514 ROMA H2020 754515 FETOPEN-04-2016-2017 754515 HHH H2020 H2020-FETOPEN-2014-CSA 665564 FETOPEN-2-2014 665564 CIRCLE H2020 665134 FETOPEN-2-2014 665134 FET-Event H2020 665083 FETOPEN-2-2014 665083 FET_TRACES H2020 665136 FETOPEN-2-2014 665136 OBSERVE H2020 ";
+
+ @Before
+ public void setup() throws Exception {
+ final PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
+ iterator = Lists.transform(Arrays.asList(resolver.getResources(PATTERN)), new Function() {
+ @Override
+ public String apply(final Resource resource) {
+ try {
+ return IOUtils.toString(resource.getInputStream());
+ } catch (IOException e) {
+ return null;
+ }
+ }
+ }).iterator();
+ fetBuilder = new BuildH2020FETTaxonomy();
+ }
+
+ @Test
+ public void parseFirstNFET() throws Exception {
+ fetBuilder.setIterator(iterator);
+ fetBuilder.parseFETProject(10);
+ String fetContext = fetBuilder.getTaxonomy();
+ Assert.assertNotNull(fetContext);
+ Assert.assertNotSame("", fetContext);
+ System.out.println(fetContext);
+ //Assert.assertTrue(fetContext.contains(outputCheck));
+ }
+
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/vre/VREPostJobNodeTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/vre/VREPostJobNodeTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/vre/VREPostJobNodeTest.java (revision 58540)
@@ -0,0 +1,24 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.vre;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Created by Alessia Bardi on 2019-06-20.
+ *
+ * @author Alessia Bardi
+ */
+public class VREPostJobNodeTest {
+
+ private String textWithNewLine = "Hello\\n World\\n!";
+ private String expected = "Hello\n World\n!";
+
+ @Test
+ public void testNewLine(){
+ String cleaned = textWithNewLine.replace("\\n", "\n");
+ Assert.assertEquals(expected, cleaned);
+
+ }
+
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/CleanActionSetsProfileJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/CleanActionSetsProfileJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/CleanActionSetsProfileJobNode.java (revision 58540)
@@ -0,0 +1,42 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.actions;
+
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+public class CleanActionSetsProfileJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(CleanActionSetsProfileJobNode.class);
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ private String xupdate;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ log.info("updating Action Sets profiles: " + getXupdate());
+ serviceLocator.getService(ISRegistryService.class).executeXUpdate(getXupdate());
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getXupdate() {
+ return xupdate;
+ }
+
+ @Required
+ public void setXupdate(final String xupdate) {
+ this.xupdate = xupdate;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/utils/CleaningXsltFunctionTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/utils/CleaningXsltFunctionTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/utils/CleaningXsltFunctionTest.java (revision 58540)
@@ -0,0 +1,21 @@
+package eu.dnetlib.msro.openaireplus.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Created by sandro on 11/28/16.
+ */
+public class CleaningXsltFunctionTest {
+
+ @Test
+ public void testCyrillic() {
+
+ final String inputString = "Издательский Дом \"Медицина и Просв";
+ final String cleanedString = CleaningXsltFunctions.clean(inputString);
+ System.out.println(cleanedString);
+
+ Assert.assertNotNull(cleanedString);
+
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/GarbageSetsHDFSJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/GarbageSetsHDFSJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/GarbageSetsHDFSJobNode.java (revision 58540)
@@ -0,0 +1,31 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.actions;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.actionmanager.rmi.ActionManagerService;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Created by claudio on 15/04/16.
+ */
+public class GarbageSetsHDFSJobNode extends BlackboardJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(GarbageSetsJobNode.class);
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ return getServiceLocator().getServiceId(ActionManagerService.class);
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ log.info("preparing garbage collection for Action sets Job");
+ job.setAction("GARBAGE_HDFS");
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimRelsJobNodeTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimRelsJobNodeTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimRelsJobNodeTest.java (revision 58540)
@@ -0,0 +1,43 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.claims;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import eu.dnetlib.data.proto.OafProtos.Oaf;
+import eu.dnetlib.msro.rmi.MSROException;
+import org.junit.Test;
+import org.postgresql.util.Base64;
+
+/**
+ * Created by Alessia Bardi on 26/06/2017.
+ *
+ * @author Alessia Bardi
+ */
+public class ApplyClaimRelsJobNodeTest {
+
+ private ApplyClaimRelsJobNode applyClaims = new ApplyClaimRelsJobNode();
+
+ @Test
+ public void testGetValue() throws MSROException, InvalidProtocolBufferException {
+
+ String sourceId = "40|corda_______::9f752db0b5ec9ca23673ca7f4cb0808e";
+ String semantics = "resultProject_outcome_produces";
+ String targetId = "50|userclaim___::6a8f649d968e734a6733c23a351c1859";
+ System.out.println(getValue(sourceId, semantics, targetId));
+ }
+
+ @Test(expected = MSROException.class)
+ public void testGetValueErrSem() throws MSROException, InvalidProtocolBufferException {
+
+ String sourceId = "40|corda_______::9f752db0b5ec9ca23673ca7f4cb0808e";
+ String semantics = "produces";
+ String targetId = "50|userclaim___::6a8f649d968e734a6733c23a351c1859";
+ getValue(sourceId, semantics, targetId);
+ }
+
+ private String getValue(final String rowKey, final String semantics, final String targetId) throws MSROException, InvalidProtocolBufferException {
+ long time = System.currentTimeMillis();
+ //final String sourceId, final String semantics, final String targetId, final long timestamp
+ String res = applyClaims.getValue(rowKey, semantics, targetId, time);
+ Oaf.Builder builder = Oaf.newBuilder().mergeFrom(Base64.decode(res));
+ return builder.build().toString();
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IncrementalTransformationJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IncrementalTransformationJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IncrementalTransformationJobNode.java (revision 58540)
@@ -0,0 +1,71 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.List;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class IncrementalTransformationJobNode extends IncrementalOperationJobNode {
+
+ private static final Log log = LogFactory.getLog(IncrementalTransformationJobNode.class);
+
+ @Autowired
+ private UniqueServiceLocator locator;
+
+ private static final DateUtils dateUtils = new DateUtils();
+
+ @Override
+ protected boolean forceRefresh(final NodeToken nodeToken, final Long lastSuccessStartDate, final String currentWfProfileId) throws Exception {
+ if (lastSuccessStartDate < 0) {
+ nodeToken.getFullEnv().setAttribute("OperationTypeInfo", "Last success start date < 0, transformation forced to REFRESH");
+ nodeToken.getFullEnv().setAttribute("operationType", "REFRESH");
+ return true;
+ }
+ String trRuleId = getTransformationId(currentWfProfileId);
+ final long lastUpdateDate = getLastTransformationRuleUpdate(trRuleId);
+ log.info(String.format("Last update date of the transformation rule with id %s is %s", trRuleId, DateUtils.calculate_ISO8601(lastUpdateDate)));
+
+ if (lastUpdateDate > lastSuccessStartDate) {
+ nodeToken.getFullEnv().setAttribute("OperationTypeInfo", "Transformation Rule has been updated, transformation forced to REFRESH");
+ nodeToken.getFullEnv().setAttribute("operationType", "REFRESH");
+ return true;
+ }
+ return false;
+ }
+
+
+
+
+
+ private String getTransformationId(final String workflowId) throws ISLookUpException {
+
+ final String query="for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType') where $x//RESOURCE_IDENTIFIER/@value='%s' " +
+ "return $x//PARAM[./@category='TRANSFORMATION_RULE_ID']/text()";
+ final ISLookUpService lookUpService = locator.getService(ISLookUpService.class);
+ final String queryInstance = String.format(query, workflowId);
+ log.debug("Query to find the Transformation Rule");
+ List transformationId = lookUpService.quickSearchProfile(queryInstance);
+ if(transformationId== null || transformationId.isEmpty())
+ throw new RuntimeException("Error unable to find the Transformation rule ID on workflow profile "+workflowId);
+ return transformationId.get(0);
+ }
+
+ private Long getLastTransformationRuleUpdate(final String trId) throws ISLookUpException {
+ final String query = "for $x in collection('/db/DRIVER/TransformationRuleDSResources/TransformationRuleDSResourceType') where $x//RESOURCE_IDENTIFIER/@value='%s' return $x//DATE_OF_CREATION/@value/string()";
+ log.debug("retrieve creation date from transformation ID "+trId);
+ final ISLookUpService lookUpService = locator.getService(ISLookUpService.class);
+ final String queryInstance = String.format(query, trId);
+ log.debug("Query to find the Transformation Rule");
+ List currentDate = lookUpService.quickSearchProfile(queryInstance);
+ if(currentDate== null || currentDate.isEmpty())
+ throw new RuntimeException("Error unable to find the creation date of the Transformation rule "+trId);
+ return dateUtils.parse(currentDate.get(0)).getTime();
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FilterManagedDatasourcesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FilterManagedDatasourcesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FilterManagedDatasourcesJobNode.java (revision 58540)
@@ -0,0 +1,87 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.io.StringReader;
+import java.util.Set;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;
+import eu.dnetlib.enabling.resultset.MappedResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Element;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class FilterManagedDatasourcesJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(FilterManagedDatasourcesJobNode.class);
+
+ private String inputEprParam;
+ private String outputEprParam;
+
+ @Autowired
+ private LocalOpenaireDatasourceManager dsManager;
+
+ @Autowired
+ private MappedResultSetFactory mappedResultSetFactory;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(getInputEprParam()));
+
+ final Set managedDatasources = dsManager.listManagedDatasourceIds();
+
+ log.info(String.format("found %s managed datasources", managedDatasources.size()));
+
+ final W3CEndpointReference outputEpr = mappedResultSetFactory.createMappedResultSet(inputEpr, s -> filterManaged(s, managedDatasources));
+
+ token.getEnv().setAttribute(getOutputEprParam(), outputEpr.toString());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ /**
+ * Extracts the datasource id from the input record and checks its existence in the given set.
+ * @param data
+ * @param filter
+ * @return The
+ * @throws IllegalStateException
+ */
+ private String filterManaged(final String data, Set filter) throws IllegalStateException {
+ try {
+ final Document doc = new SAXReader().read(new StringReader(data));
+
+ final String dsId = doc.valueOf("/record/metadata/ROWS/ROW[@table = 'dsm_datasources']/FIELD[@name = 'id']/text()");
+ if (filter.contains(dsId)) {
+ doc.selectSingleNode("/record/metadata/ROWS").detach();
+ ((Element) doc.selectSingleNode("/record/metadata")).addElement("ROWS");
+ return doc.asXML();
+ } else return data;
+ } catch (DocumentException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ public String getInputEprParam() {
+ return inputEprParam;
+ }
+
+ public void setInputEprParam(final String inputEprParam) {
+ this.inputEprParam = inputEprParam;
+ }
+
+ public String getOutputEprParam() {
+ return outputEprParam;
+ }
+
+ public void setOutputEprParam(final String outputEprParam) {
+ this.outputEprParam = outputEprParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input.xml (revision 58540)
@@ -0,0 +1,248 @@
+
+
+
+ r39633d1e8c4::00bbb9e987a2363ea886dc6004899be2
+ 10.1594/PANGAEA.807041
+ 2014-12-01T13:29:20.571Z
+
+ r39633d1e8c4
+ 1f978947-5dec-4e5a-a005-02b2035fdbb2_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+
+
+
+ info:eu-repo/grantAgreement/EC/FP7/265103
+
+
+
+
+
+ Kádár
+ Enikö
+ enik@pml.ac.uk
+
+
+ Fisher
+ Andrew
+
+
+ Stolpe
+ Björn
+
+
+ Harrison
+ Roy M
+
+
+ Parello
+ Francesco
+ parello@unipa.it
+
+
+ Lead
+ Jamie
+
+ 2013
+ Image analysis from TEM and AFM micrograph, Table S1, S2, S3, Figure S1, Figure S2
+ doi:10.1594/PANGAEA.807041
+ 2013-02-10T21:46:50
+ We report on metal enrichment along a natural pH gradient owing to increased CO2 degassing at cold, shal- low seeps of Vulcano Island in the Mediterranean Sea, off Sicily. We assessed composition of unfiltered and filtered seawater (b100 nm) along acidic zones ranging between ambient and pH 5, and showed that most seep derived elements are present as nanoclusters which then aggregate into larger colloids while mixing with ambient seawater along a pH gradient. Size and elemental composition of such naturally occurring nanoparticles assessed by modern characterisation methods were in good agreement with the results from conventional analytical methods.
+ We provide analytical evidence for the presence in the water column of a large fraction of seep derived ele- ments (e.g. approximately 50% of iron, over 80% of Mn, 100% of Cr, S and Zn) in the form of nano sized par- ticles (e.g. b100 nm) even at typical open ocean pHs. We launch in situ sampling protocols and sample preparation procedures for multi-method suitable to obtain accurate measurements on nanoparticles from environmental samples. Based on our results a first insight to the formation of natural nanoparticles at cold CO2 seeps is presented and the persistence of such nano-clusters in the surrounding seawater is stipulated.
+
+
+
+ Kádár
+ Enikö
+ enik@pml.ac.uk
+
+
+ Fisher
+ Andrew
+
+
+ Stolpe
+ Björn
+
+
+ Harrison
+ Roy M
+
+
+ Parello
+ Francesco
+ parello@unipa.it
+
+
+ Lead
+ Jamie
+
+ 2012
+ Metallic nanoparticle enrichment at low temperature, shallow CO2 seeps in Southern Italy
+ Marine Chemistry
+ doi:10.1016/j.marchem.2012.07.001
+ 140
+ 24-32
+
+
+
+
+ Capaccioni
+ B
+
+
+ Tassi
+ F
+ 0000-0002-3319-4257
+
+
+ Vaselli
+ O
+
+ 2001
+ Organic and inorganic geochemistry of low temperature gas discharges at the Baia di Levante beach, Vulcano Island, Italy
+ Journal of Volcanology and Geothermal Research
+ doi:10.1016/S0377-0273(00)00284-5
+ 108
+ 173-185
+
+
+
+ Kádár
+ Enikö
+ enik@pml.ac.uk
+
+
+ Costa
+ Valentina
+
+
+ Martins
+ Ines
+ 0000-0002-5362-9801
+
+
+ Santos
+ Ricado Serrao
+
+
+ Powell
+ Jonathan J
+
+ 2005
+ Enrichment in trace metals (Al, Mn, Co, Cu, Mo, Cd, Fe, Zn, Pb and Hg) of macro-invertebrate habitats at hydrothermal vents along the Mid-Atlantic Ridge
+ Hydrobiologia
+ doi:10.1007/s10750-005-4758-1
+ 548(1)
+ 191-205
+
+
+
+ Yücel
+ Mustafa
+
+
+ Gartman
+ Amy
+
+
+ Chan
+ Clara S
+
+
+ Luther
+ Georg W
+
+ 2011
+ Hydrothermal vents as a kinetically stable source of iron-sulphide-bearing nanoparticles to the ocean
+ Nature Geoscience
+ doi:10.1038/ngeo1148
+ 4
+ 367-371
+
+
+ Supplementary information - Image analysis from TEM and AFM micrographs
+ hdl:10013/epic.40911.d001
+
+
+
+ 14.9
+ 14.9
+ 38.53
+ 38.53
+ 14.9
+ 38.53
+
+ not specified
+
+
+ MedSeA
+ Mediterranean Sea Acidification in a Changing Climate
+ http://medsea-project.eu/
+
+
+ Vulcano_Island
+ 38.53
+ 14.9
+ Aeolian Islands, Mediterranean Sea, off Sicily
+
+ 2291.0
+
+ CC-BY
+ Creative Commons Attribution 3.0 Unported
+ http://creativecommons.org/licenses/by/3.0/
+
+
+ supplement
+ citable
+ author39176
+ author44941
+ author44942
+ author44943
+ author44944
+ author44946
+ author44947
+ author44948
+ author44949
+ author44950
+ author44951
+ author44952
+ author44953
+ author44954
+ author44955
+ author44956
+ author44957
+ dataset807041
+ event2678800
+ journal10213
+ journal10808
+ journal11670
+ journal6662
+ license1
+ location19077
+ project4142
+ ref39182
+ ref39183
+ ref39184
+ ref39185
+ ref54021
+ topotype1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/CollectDatasetsByJournalJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/CollectDatasetsByJournalJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/CollectDatasetsByJournalJobNode.java (revision 58540)
@@ -0,0 +1,180 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.datacite;
+
+import java.io.StringReader;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import javax.annotation.Resource;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.collector.plugins.datasets.DatasetsByJournalIterator;
+import eu.dnetlib.data.collector.plugins.datasets.PangaeaJournalInfo;
+import eu.dnetlib.enabling.resultset.IterableResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.io.SAXReader;
+
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class CollectDatasetsByJournalJobNode.
+ */
+public class CollectDatasetsByJournalJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(CollectDatasetsByJournalJobNode.class);
+
+ public static PangaeaJournalInfo END_QUEUE = new PangaeaJournalInfo();
+
+ /**
+ * The datasource id.
+ */
+ private String datasourceId;
+
+ /**
+ * The output epr param.
+ */
+ private String outputEprParam;
+
+ /**
+ * The input epr param.
+ */
+ private String inputEprParam;
+
+ /**
+ * The result set client factory.
+ */
+ private ResultSetClientFactory resultSetClientFactory;
+
+ /**
+ * The result set factory.
+ */
+ @Resource(name = "iterableResultSetFactory")
+ private IterableResultSetFactory resultSetFactory;
+
+ /**
+ * The executor.
+ */
+ private Executor executor = Executors.newSingleThreadExecutor();
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(inputEprParam));
+ final Iterable input = resultSetClientFactory.getClient(inputEpr);
+ final BlockingQueue publicationsQueue = new ArrayBlockingQueue(50);
+
+ executor.execute(new Runnable() {
+
+ @Override
+ public void run() {
+ final SAXReader reader = new SAXReader();
+
+ for (String inputString : input) {
+ try {
+ Document doc = reader.read(new StringReader(inputString));
+ final String jId = doc.valueOf("//FIELD[@name='id']");
+ final String dsId = doc.valueOf("//FIELD[@name='datasource']");
+ final String dsName = doc.valueOf("//FIELD[@name='name']");
+ final String jISSN = doc.valueOf("//FIELD[@name='issn']");
+ PangaeaJournalInfo info = new PangaeaJournalInfo();
+ info.setDatasourceId(dsId);
+ info.setJournalId(jId);
+ info.setJournalName(dsName);
+ info.setJournalISSN(jISSN);
+ publicationsQueue.put(info);
+ } catch (Exception e) {
+ log.error(e);
+ }
+ }
+ try {
+ publicationsQueue.put(END_QUEUE);
+ } catch (InterruptedException e) {
+ log.error(e);
+ }
+
+ }
+ });
+
+ IteratorOnQueue itOnQueue = new IteratorOnQueue(publicationsQueue, CollectDatasetsByJournalJobNode.END_QUEUE);
+
+ W3CEndpointReference eprOutput = resultSetFactory.createIterableResultSet(new DatasetsByJournalIterator(itOnQueue));
+ token.getEnv().setAttribute(getOutputEprParam(), eprOutput.toString());
+ return Arc.DEFAULT_ARC;
+ }
+
+ /**
+ * Gets the datasource id.
+ *
+ * @return the datasourceId
+ */
+ public String getDatasourceId() {
+ return datasourceId;
+ }
+
+ /**
+ * Sets the datasource id.
+ *
+ * @param datasourceId the datasourceId to set
+ */
+ public void setDatasourceId(final String datasourceId) {
+ this.datasourceId = datasourceId;
+ }
+
+ /**
+ * @return the outputEprParam
+ */
+ public String getOutputEprParam() {
+ return outputEprParam;
+ }
+
+ /**
+ * @param outputEprParam the outputEprParam to set
+ */
+ public void setOutputEprParam(final String outputEprParam) {
+ this.outputEprParam = outputEprParam;
+ }
+
+ /**
+ * @return the inputEprParam
+ */
+ public String getInputEprParam() {
+ return inputEprParam;
+ }
+
+ /**
+ * @param inputEprParam the inputEprParam to set
+ */
+ public void setInputEprParam(final String inputEprParam) {
+ this.inputEprParam = inputEprParam;
+ }
+
+ /**
+ * Gets the result set client factory.
+ *
+ * @return the resultSetClientFactory
+ */
+ public ResultSetClientFactory getResultSetClientFactory() {
+ return resultSetClientFactory;
+ }
+
+ /**
+ * Sets the result set client factory.
+ *
+ * @param resultSetClientFactory the resultSetClientFactory to set
+ */
+ public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+ this.resultSetClientFactory = resultSetClientFactory;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p4.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p4.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p4.xml (revision 58540)
@@ -0,0 +1,97 @@
+
+
+
+ corda__h2020::072c9a0f7177104264b881afedd4e0ab
+ 754514
+ 2017-03-01T11:26:37.65+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::754514
+ corda__h2020::754514
+ 754514
+ ROMA
+ Resource AuctiOning Engine for the Mobile Digital MArket
+ 01-JUN-2017
+ 30-NOV-2018
+ H2020-FETOPEN-4-2016-2017
+ true
+ No
+ openaire____::corda_h2020
+ CSA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-04-2016-2017
+ FET Innovation Launchpad
+
+
+ corda__h2020::754514::Francesco_DE PELLEGRINI
+ corda__h2020::754514::Francesco_DE PELLEGRINI
+ Francesco
+ DE PELLEGRINI
+ fdepellegrini@fbk.eu
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999625450
+ corda__h2020::999625450
+ CREATE-NET
+ FONDAZIONE BRUNO KESSLER
+ http://www.fbk.eu
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::754514@@corda__h2020::999625450
+ 1
+ corda__h2020::754514
+ corda__h2020::999625450
+ corda__h2020::754514::Francesco_DE PELLEGRINI
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-04-2016-2017
+ corda__h2020::FETOPEN-04-2016-2017
+ FET Innovation Launchpad
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::754514@@corda__h2020::FETOPEN-04-2016-2017
+ corda__h2020::754514
+ corda__h2020::FETOPEN-04-2016-2017
+
+
+ ec__________::EC::H2020::CSA@@corda__h2020::754514
+ ec__________::EC::H2020::CSA
+ corda__h2020::754514
+ 01-JUN-2017
+ 30-NOV-2018
+ CSA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/LoadEntityLinksJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/LoadEntityLinksJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/LoadEntityLinksJobNode.java (revision 58540)
@@ -0,0 +1,39 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.openaireplus.workflows.nodes.index.EntityGrouperConfigurationLoader;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class LoadEntityLinksJobNode extends SimpleJobNode {
+
+ private static final String INDEX_ENTITY_LINKS = "index.entity.links";
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(LoadEntityLinksJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Autowired
+ private EntityGrouperConfigurationLoader grouperConfLoader;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ log.info("loading entity link table");
+
+ String links = grouperConfLoader.load();
+
+ if (StringUtils.isBlank(links)) throw new MSROException("empty links table");
+
+ token.getEnv().setAttribute(INDEX_ENTITY_LINKS, links);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input3.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input3.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input3.xml (revision 58540)
@@ -0,0 +1,95 @@
+
+
+
+ datacite____::000000aaf19e1fdd1069b68c85cf5eda
+ 10.18421/tem53-12
+ 2018-10-28T00:39:04.337Z
+ datacite____
+ 10.18421/tem53-12
+ TIB.TEM
+
+
+
+ 10.18421/tem53-12
+
+
+ Žileska – Pančovska, Valentina
+
+
+ Blaževska-Stoilkovska, Biljana
+
+
+ Hanák, Tomáš
+
+
+
+ Management of Human Resources and Quality in Materials Supply Process in Construction Projects
+
+ UIKTEN
+ 2016
+
+ construction project
+ Human resources management
+ Quality management
+ materials supply process
+ satisfaction
+
+
+ 2016
+
+ eng
+ Article
+
+ https://doi.org/10.18421/tem53-12
+ http://dx.doi.org/10.18421/tem53-12
+
+
+ PDF
+
+ None
+
+ TEM Journal; Vol 5, No 3, 2016. ISSN 2217-8309
+ Construction projects participants were surveyed to assess how coordination and communication among stakeholders and quality control in building materials supply process were managed, as well as to evaluate their satisfaction with the structures quality. It was found that the participants’ role and working experience had a statistically significant interaction effect on satisfaction with the structures quality, but their effect on perceived coordination and communication among stakeholders and materials’ quality control wasn’t statistically significant. The results were discussed in light of their importance for effective construction project management from the aspect of human resources, building materials quality and satisfaction with the quality of structures.
+
+
+
+ 0038
+ 2016-01-01
+ UNKNOWN
+ eng
+
+
+
+
+
+
+ http://ip-90-147-167-25.ct1.garrservices.it:5000
+ 10.18421/tem53-12
+
+
+
+
+
+ false
+ false
+ 0.9
+
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dataFlowMonitoring/TriggerIndexMonitoringScenarioJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dataFlowMonitoring/TriggerIndexMonitoringScenarioJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dataFlowMonitoring/TriggerIndexMonitoringScenarioJobNode.java (revision 58540)
@@ -0,0 +1,51 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.dataFlowMonitoring;
+
+import java.io.IOException;
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.monitoring.core.sensors.SensorCreationException;
+import eu.dnetlib.monitoring.core.sensors.SensorFactory;
+import eu.dnetlib.monitoring.core.sensors.collection.index.solr.SolrSensor;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class TriggerIndexMonitoringScenarioJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(TriggerIndexMonitoringScenarioJobNode.class);
+ private static final String SOLR_COLLECTION_POSTFIX = "-index-openaire";
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+ private String mdFormat;
+ private SolrSensor solrSensor;
+
+ @Override
+ protected String execute(final NodeToken token) throws IOException, InstantiationException, IllegalAccessException, InterruptedException,
+ ISLookUpException, SensorCreationException {
+
+ /* Configuring SOLR sensor */
+ String actualCollection = mdFormat + SOLR_COLLECTION_POSTFIX;
+ log.info("Executing on " + actualCollection + " SOLR collection");
+ solrSensor = (SolrSensor) SensorFactory.createSensorWithConfiguration(SolrSensor.class, "index", "solr2");
+ solrSensor.getConfiguration().setParam("collection", actualCollection);
+ solrSensor.reinitialize();
+ log.info("Measuring...");
+ solrSensor.measure();
+ log.info("DONE!");
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getMdFormat() {
+ return mdFormat;
+ }
+
+ public void setMdFormat(final String mdFormat) {
+ this.mdFormat = mdFormat;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p2.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p2.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p2.xml (revision 58540)
@@ -0,0 +1,97 @@
+
+
+
+ corda__h2020::072c9a0f7177104264b881afedd4e0ab
+ 754514
+ 2017-03-01T11:26:37.65+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::754514
+ corda__h2020::754514
+ 754514
+ ROMA
+ Resource AuctiOning Engine for the Mobile Digital MArket
+ 01-JUN-2017
+ 30-NOV-2018
+ H2020-FETOPEN-4-2016-2017
+ true
+ No
+ openaire____::corda_h2020
+ CSA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-04-2016-2017
+ FET Innovation Launchpad
+
+
+ corda__h2020::754514::Francesco_DE PELLEGRINI
+ corda__h2020::754514::Francesco_DE PELLEGRINI
+ Francesco
+ DE PELLEGRINI
+ fdepellegrini@fbk.eu
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999625450
+ corda__h2020::999625450
+ CREATE-NET
+ FONDAZIONE BRUNO KESSLER
+ http://www.fbk.eu
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::754514@@corda__h2020::999625450
+ 1
+ corda__h2020::754514
+ corda__h2020::999625450
+ corda__h2020::754514::Francesco_DE PELLEGRINI
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-04-2016-2017
+ corda__h2020::FETOPEN-04-2016-2017
+ FET Innovation Launchpad
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::754514@@corda__h2020::FETOPEN-04-2016-2017
+ corda__h2020::754514
+ corda__h2020::FETOPEN-04-2016-2017
+
+
+ ec__________::EC::H2020::CSA@@corda__h2020::754514
+ ec__________::EC::H2020::CSA
+ corda__h2020::754514
+ 01-JUN-2017
+ 30-NOV-2018
+ CSA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/DownloadFromMetadataOpenaire.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/DownloadFromMetadataOpenaire.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/DownloadFromMetadataOpenaire.java (revision 58540)
@@ -0,0 +1,83 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.objectStore;
+
+import java.util.Map;
+
+import com.googlecode.sarasvati.Engine;
+import com.googlecode.sarasvati.NodeToken;
+import com.googlecode.sarasvati.env.Env;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.blackboard.BlackboardWorkflowJobListener;
+import eu.dnetlib.msro.workflows.nodes.download.DownloadFromMetadataJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class DownloadFromMetadataOpenaire extends DownloadFromMetadataJobNode {
+
+ private static final Log log = LogFactory.getLog(DownloadFromMetadataOpenaire.class);
+
+ private String basepath;
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ if (this.getBasepath() != null && this.getBasepath().length() > 0) {
+ job.getParameters().put("basePath", basepath);
+ }
+
+ int size = serviceLocator.getService(ObjectStoreService.class).getSize(getObjectStoreID());
+ log.debug(String.format("objectStore %s has size %s", getObjectStoreID(), size));
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "oldTotal", size);
+
+ super.prepareJob(job, token);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.msro.workflows.nodes.BlackboardJobNode#generateBlackboardListener(com.googlecode.sarasvati.Engine,
+ * com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ protected BlackboardWorkflowJobListener generateBlackboardListener(final Engine engine, final NodeToken token) {
+ return new BlackboardWorkflowJobListener(engine, token) {
+
+ @Override
+ protected void populateEnv(final Env env, final Map responseParams) {
+ env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", responseParams.get("total"));
+
+ final String base64 = responseParams.get("report");
+ if (StringUtils.isNotBlank(base64) && Base64.isBase64(base64.getBytes())) {
+ final String report = new String(Base64.decodeBase64(base64.getBytes()));
+ log.info("found download report");
+ log.debug(report);
+ env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "report", report);
+ } else {
+ log.warn("unable to find or decode download report");
+ }
+ }
+ };
+ }
+
+ /**
+ * @return the basepath
+ */
+ public String getBasepath() {
+ return basepath;
+ }
+
+ /**
+ * @param basepath the basepath to set
+ */
+ public void setBasepath(final String basepath) {
+ this.basepath = basepath;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p8.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p8.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p8.xml (revision 58540)
@@ -0,0 +1,217 @@
+
+
+
+ corda__h2020::19643e6a0e10790318ff84b3b7b01571
+ 686330
+ 2017-03-01T11:01:29.854+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::686330
+ corda__h2020::686330
+ 686330
+ FutureAgriculture
+ Transforming the future of agriculture through synthetic photorespiration
+ 01-JAN-2016
+ 31-DEC-2020
+ H2020-FETOPEN-2014-2015-RIA
+ true
+ Yes
+ openaire____::corda_h2020
+ RIA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-RIA-2014-2015
+ FET-Open research projects
+
+
+ corda__h2020::686330::Arren_BAR-EVEN
+ corda__h2020::686330::Arren_BAR-EVEN
+ Arren
+ BAR-EVEN
+ arren.be@gmail.com
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::925188814
+ corda__h2020::925188814
+ Evogene Ltd.
+ EVOGENE LTD
+ http://www.evogene.com/
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ IL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686330@@corda__h2020::925188814
+ 5
+ corda__h2020::686330
+ corda__h2020::925188814
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::983078414
+ corda__h2020::983078414
+ IN Srl
+ IN SRL
+ http://www.insrl.eu
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686330@@corda__h2020::983078414
+ 2
+ corda__h2020::686330
+ corda__h2020::983078414
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999979306
+ corda__h2020::999979306
+ WEIZMANN
+ WEIZMANN INSTITUTE OF SCIENCE
+ http://www.weizmann.ac.il
+ No
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ IL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686330@@corda__h2020::999979306
+ 4
+ corda__h2020::686330
+ corda__h2020::999979306
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999990267
+ corda__h2020::999990267
+ MPG
+ MAX-PLANCK-GESELLSCHAFT ZUR FORDERUNG DER WISSENSCHAFTEN EV
+ http://www.mpg.de
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686330@@corda__h2020::999990267
+ 1
+ corda__h2020::686330
+ corda__h2020::999990267
+ corda__h2020::686330::Arren_BAR-EVEN
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999993468
+ corda__h2020::999993468
+ Imperial
+ IMPERIAL COLLEGE OF SCIENCE TECHNOLOGY AND MEDICINE
+ http://www.imperial.ac.uk
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686330@@corda__h2020::999993468
+ 3
+ corda__h2020::686330
+ corda__h2020::999993468
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::FETOPEN-RIA-2014-2015
+ FET-Open research projects
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::686330@@corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::686330
+ corda__h2020::FETOPEN-RIA-2014-2015
+
+
+ ec__________::EC::H2020::RIA@@corda__h2020::686330
+ ec__________::EC::H2020::RIA
+ corda__h2020::686330
+ 01-JAN-2016
+ 31-DEC-2020
+ RIA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/PUMAIteratorRepository.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/PUMAIteratorRepository.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/PUMAIteratorRepository.java (revision 58540)
@@ -0,0 +1,36 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.objectStore;
+
+import java.io.File;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+
+public class PUMAIteratorRepository extends IteratorRepository {
+
+ public PUMAIteratorRepository(final Iterable input, final String repositoryPrefix, final String oaiPrefix) {
+ super(input, repositoryPrefix, oaiPrefix);
+ // TODO Auto-generated constructor stub
+ }
+
+ @Override
+ String generateNextElement() {
+ try {
+ String inputname = input.next().getCanonicalPath();
+ String[] values = inputname.split("/");
+ if (values.length > 1) {
+ ObjectStoreFile info = new ObjectStoreFile();
+ String fileName = values[values.length - 1].replace(".pdf", "");
+ info.setAccessProtocol(Protocols.None);
+ info.setObjectID(fileName);
+ info.setDownloadedURL(" ");
+ info.setMimeType("pdf");
+ info.setURI(inputname);
+ return info.toJSON();
+ }
+ } catch (Exception e) {
+ return null;
+ }
+ return null;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p20.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p20.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p20.xml (revision 58540)
@@ -0,0 +1,98 @@
+
+
+
+ corda__h2020::0059df285b7bff23913630f024172b7d
+ 658375
+ 2017-03-01T10:47:51.648+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::658375
+ corda__h2020::658375
+ 658375
+ EFTA DEVELOPMENT
+ The ‘other’ Europe: the formation and development of the European Free Trade Association (EFTA), 1958–92
+ 01-JAN-2016
+ 31-DEC-2017
+ H2020-MSCA-IF-2014
+ true
+ No
+ openaire____::corda_h2020
+ MSCA-IF-EF-ST
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ MSCA-IF-2014-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+
+
+ corda__h2020::658375::Henri_VOGT
+ corda__h2020::658375::Henri_VOGT
+ Henri
+ VOGT
+ henri.vogt@utu.fi
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999903064
+ corda__h2020::999903064
+ UNIVERSITY OF TURKU
+ TURUN YLIOPISTO
+ http://www.utu.fi
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ FI
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::658375@@corda__h2020::999903064
+ 1
+ corda__h2020::658375
+ corda__h2020::999903064
+ corda__h2020::658375::Henri_VOGT
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::MSCA-IF-2014-EF
+ corda__h2020::MSCA-IF-2014-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::658375@@corda__h2020::MSCA-IF-2014-EF
+ corda__h2020::658375
+ corda__h2020::MSCA-IF-2014-EF
+
+
+ ec__________::EC::H2020::MSCA-IF-EF-ST@@corda__h2020::658375
+ ec__________::EC::H2020::MSCA-IF-EF-ST
+ corda__h2020::658375
+ 01-JAN-2016
+ 31-DEC-2017
+ MSCA-IF-EF-ST
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p6.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p6.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p6.xml (revision 58540)
@@ -0,0 +1,248 @@
+
+
+
+ corda__h2020::0fa87a85cbbfb14f5cc2d4330c2b1cad
+ 736876
+ 2017-03-01T11:24:05.589+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::736876
+ corda__h2020::736876
+ 736876
+ VISORSURF
+ VisorSurf - A Hardware Platform for Software-driven Functional Metasurfaces
+ 01-JAN-2017
+ 30-JUN-2020
+ H2020-FETOPEN-1-2016-2017
+ true
+ No
+ openaire____::corda_h2020
+ RIA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-01-2016-2017
+ FET-Open research and innovation actions
+
+
+ corda__h2020::736876::Costas_SOUKOULIS
+ corda__h2020::736876::Costas_SOUKOULIS
+ Costas
+ SOUKOULIS
+ soukouli@iesl.forth.gr
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::991256096
+ corda__h2020::991256096
+ AALTO
+ AALTO-KORKEAKOULUSAATIO
+ http://www.aalto.fi
+ No
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ FI
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::991256096
+ 6
+ corda__h2020::736876
+ corda__h2020::991256096
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999542903
+ corda__h2020::999542903
+ SG
+ SIGNALGENERIX LTD
+ http://www.signalgenerix.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ CY
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999542903
+ 4
+ corda__h2020::736876
+ corda__h2020::999542903
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999835843
+ corda__h2020::999835843
+ UCY
+ UNIVERSITY OF CYPRUS
+ http://www.ucy.ac.cy
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ CY
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999835843
+ 2
+ corda__h2020::736876
+ corda__h2020::999835843
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999976202
+ corda__h2020::999976202
+ UPC
+ UNIVERSITAT POLITECNICA DE CATALUNYA
+ http://www.upc.edu
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ ES
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999976202
+ 5
+ corda__h2020::736876
+ corda__h2020::999976202
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999984059
+ corda__h2020::999984059
+ Fraunhofer
+ FRAUNHOFER GESELLSCHAFT ZUR FOERDERUNG DER ANGEWANDTEN FORSCHUNG E.V.
+ http://www.fraunhofer.de
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999984059
+ 3
+ corda__h2020::736876
+ corda__h2020::999984059
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999995893
+ corda__h2020::999995893
+ FORTH
+ FOUNDATION FOR RESEARCH AND TECHNOLOGY HELLAS
+ http://www.forth.gr
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ GR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999995893
+ 1
+ corda__h2020::736876
+ corda__h2020::999995893
+ corda__h2020::736876::Costas_SOUKOULIS
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-01-2016-2017
+ corda__h2020::FETOPEN-01-2016-2017
+ FET-Open research and innovation actions
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::736876@@corda__h2020::FETOPEN-01-2016-2017
+ corda__h2020::736876
+ corda__h2020::FETOPEN-01-2016-2017
+
+
+ ec__________::EC::H2020::RIA@@corda__h2020::736876
+ ec__________::EC::H2020::RIA
+ corda__h2020::736876
+ 01-JAN-2017
+ 30-JUN-2020
+ RIA
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISMainParamsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISMainParamsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISMainParamsJobNode.java (revision 58540)
@@ -0,0 +1,52 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.List;
+import java.util.Map;
+
+import com.google.gson.Gson;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import org.springframework.beans.factory.annotation.Required;
+
+@Deprecated
+public class PrepareIISMainParamsJobNode extends PrepareIISParams {
+
+ private String xqueryObjectStores;
+
+ private String objectStoreParam = "import_content_objectstores_csv";
+
+ // Enable/Disable inference modules
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ super.prepare(token);
+
+ token.getEnv().setAttribute(getObjectStoreParam(), getFilteredObjectStoreCSV(getXqueryObjectStores()));
+
+ @SuppressWarnings("unchecked") final List> sets = new Gson().fromJson(token.getEnv().getAttribute("sets"), List.class);
+ for (Map set : sets) {
+ token.getEnv().setAttribute(set.get("enablingProperty"), set.get("enabled"));
+ }
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getXqueryObjectStores() {
+ return xqueryObjectStores;
+ }
+
+ @Required
+ public void setXqueryObjectStores(final String xqueryObjectStores) {
+ this.xqueryObjectStores = xqueryObjectStores;
+ }
+
+ public String getObjectStoreParam() {
+ return objectStoreParam;
+ }
+
+ public void setObjectStoreParam(final String objectStoreParam) {
+ this.objectStoreParam = objectStoreParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p10.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p10.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p10.xml (revision 58540)
@@ -0,0 +1,249 @@
+
+
+
+ corda__h2020::232fe4024d869d852767d07f5e99f2fe
+ 713140
+ 2017-03-01T11:13:57.906+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::713140
+ corda__h2020::713140
+ 713140
+ MESO_BRAIN
+ Custom architecturally defined 3D stem cell derived functional human neural networks for
+ transformative progress in neuroscience and medicine
+
+ 01-SEP-2016
+ 31-AUG-2019
+ H2020-FETOPEN-2014-2015-RIA
+ true
+ No
+ openaire____::corda_h2020
+ RIA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-RIA-2014-2015
+ FET-Open research projects
+
+
+ corda__h2020::713140::Edik_RAFAILOV
+ corda__h2020::713140::Edik_RAFAILOV
+ Edik
+ RAFAILOV
+ e.rafailov@aston.ac.uk
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::936325772
+ corda__h2020::936325772
+ Axol
+ AXOL BIOSCIENCE LTD
+ http://www.axolbio.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Missing
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713140@@corda__h2020::936325772
+ 5
+ corda__h2020::713140
+ corda__h2020::936325772
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::995206033
+ corda__h2020::995206033
+ Kite Innovation
+ Kite Innovation (Europe) Limited
+ http://www.kiteinnovation.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713140@@corda__h2020::995206033
+ 6
+ corda__h2020::713140
+ corda__h2020::995206033
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999588590
+ corda__h2020::999588590
+ LZH
+ LZH LASER ZENTRUM HANNOVER E.V.
+
+ Yes
+ Yes
+ Yes
+ Yes
+ N/A
+ N/A
+ N/A
+ Missing
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713140@@corda__h2020::999588590
+ 2
+ corda__h2020::713140
+ corda__h2020::999588590
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999619436
+ corda__h2020::999619436
+ ICFO
+ FUNDACIO INSTITUT DE CIENCIES FOTONIQUES
+ http://www.icfo.eu
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ ES
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713140@@corda__h2020::999619436
+ 3
+ corda__h2020::713140
+ corda__h2020::999619436
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999896953
+ corda__h2020::999896953
+ ASTON U
+ ASTON UNIVERSITY
+
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ Missing
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713140@@corda__h2020::999896953
+ 1
+ corda__h2020::713140
+ corda__h2020::999896953
+ corda__h2020::713140::Edik_RAFAILOV
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999986387
+ corda__h2020::999986387
+ UB
+ UNIVERSITAT DE BARCELONA
+ http://www.ub.edu
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ ES
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713140@@corda__h2020::999986387
+ 4
+ corda__h2020::713140
+ corda__h2020::999986387
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::FETOPEN-RIA-2014-2015
+ FET-Open research projects
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::713140@@corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::713140
+ corda__h2020::FETOPEN-RIA-2014-2015
+
+
+ ec__________::EC::H2020::RIA@@corda__h2020::713140
+ ec__________::EC::H2020::RIA
+ corda__h2020::713140
+ 01-SEP-2016
+ 31-AUG-2019
+ RIA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/AbstractStatsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/AbstractStatsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/AbstractStatsJobNode.java (revision 58540)
@@ -0,0 +1,72 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import java.util.List;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Abstract BBJob node for sending BB messages to the StatsManagerService. Subclasses must implement the prepareJob method.
+ *
+ * @author alessia
+ */
+public abstract class AbstractStatsJobNode extends BlackboardJobNode {
+
+ private String xqueryForServiceIdParam;
+ /**
+ * BB parameter defining which is the portal affected by the action. Overrides the value in the env parameter below.
+ */
+ private String portal;
+ /**
+ * Name of the env parameter where to find the target portal value. It is used only if the portal parameter above is not explicitly set
+ * in the workflow.
+ */
+ private String portalParam;
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ final String xquery = token.getEnv().getAttribute(getXqueryForServiceIdParam());
+ List statsServiceIds;
+ try {
+ statsServiceIds = getServiceLocator().getService(ISLookUpService.class).quickSearchProfile(xquery);
+ if (statsServiceIds.size() > 1) throw new RuntimeException("Too many StatsManagerService ids found using query: " + xquery);
+ if (statsServiceIds.size() < 1) throw new RuntimeException("StatsManagerService id not found using query: " + xquery);
+ return statsServiceIds.get(0);
+ } catch (ISLookUpException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getPortalName(final NodeToken token) {
+ if (StringUtils.isNotBlank(portal)) return portal;
+ else return token.getEnv().getAttribute(portalParam);
+ }
+
+ public String getXqueryForServiceIdParam() {
+ return xqueryForServiceIdParam;
+ }
+
+ public void setXqueryForServiceIdParam(final String xqueryForServiceIdParam) {
+ this.xqueryForServiceIdParam = xqueryForServiceIdParam;
+ }
+
+ public String getPortal() {
+ return portal;
+ }
+
+ public void setPortal(final String portal) {
+ this.portal = portal;
+ }
+
+ public String getPortalParam() {
+ return portalParam;
+ }
+
+ public void setPortalParam(final String portalParam) {
+ this.portalParam = portalParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p13.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p13.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p13.xml (revision 58540)
@@ -0,0 +1,188 @@
+
+
+
+ corda__h2020::2fa7637c88313dd239c64ea52b04e9ad
+ 713171
+ 2017-03-01T11:13:57.907+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::713171
+ corda__h2020::713171
+ 713171
+ SENSE
+ SENSE - a roadmap for the ideal low light level sensor development
+ 01-SEP-2016
+ 31-AUG-2019
+ H2020-FETOPEN-2015-CSA
+ true
+ Yes
+ openaire____::corda_h2020
+ CSA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-CSA-FETEXCHANGE-2015
+ FET Exchange Coordination and Support Activities 2015
+
+
+ corda__h2020::713171::Katharina_HENJES-KUNST
+ corda__h2020::713171::Katharina_HENJES-KUNST
+ Katharina
+ HENJES-KUNST
+ katharina.henjes-kunst@desy.de
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::990797674
+ corda__h2020::990797674
+ KIT
+ KARLSRUHER INSTITUT FUER TECHNOLOGIE
+ http://www.kit.edu
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713171@@corda__h2020::990797674
+ 4
+ corda__h2020::713171
+ corda__h2020::990797674
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999974650
+ corda__h2020::999974650
+ UNIGE
+ UNIVERSITE DE GENEVE
+ http://www.unige.ch
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ CH
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713171@@corda__h2020::999974650
+ 2
+ corda__h2020::713171
+ corda__h2020::999974650
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999986969
+ corda__h2020::999986969
+ DESY
+ STIFTUNG DEUTSCHES ELEKTRONEN-SYNCHROTRON DESY
+ http://www.desy.de
+ No
+ Yes
+ Yes
+ Yes
+ N/A
+ N/A
+ N/A
+ Missing
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713171@@corda__h2020::999986969
+ 1
+ corda__h2020::713171
+ corda__h2020::999986969
+ corda__h2020::713171::Katharina_HENJES-KUNST
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999990267
+ corda__h2020::999990267
+ MPG
+ MAX-PLANCK-GESELLSCHAFT ZUR FORDERUNG DER WISSENSCHAFTEN EV
+ http://www.mpg.de
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::713171@@corda__h2020::999990267
+ 3
+ corda__h2020::713171
+ corda__h2020::999990267
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-CSA-FETEXCHANGE-2015
+ corda__h2020::FETOPEN-CSA-FETEXCHANGE-2015
+ FET Exchange Coordination and Support Activities 2015
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::713171@@corda__h2020::FETOPEN-CSA-FETEXCHANGE-2015
+ corda__h2020::713171
+ corda__h2020::FETOPEN-CSA-FETEXCHANGE-2015
+
+
+ ec__________::EC::H2020::CSA@@corda__h2020::713171
+ ec__________::EC::H2020::CSA
+ corda__h2020::713171
+ 01-SEP-2016
+ 31-AUG-2019
+ CSA
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/PromoteShadowStatsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/PromoteShadowStatsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/PromoteShadowStatsJobNode.java (revision 58540)
@@ -0,0 +1,40 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to promote the shadow stats db and cache to public for the portal specified by the additional BB parameter
+ * StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName()
.
+ *
+ *
+ * According to the StatsManagerService specifications in https://issue.openaire.research-infrastructures.eu/issues/914#note-49, this BB
+ * action will:
+ *
+ * promote the shadow cache to public
+ * make a backup of the current stats db schema under the name 'backup'
+ * replace it with the new one
+ *
+ *
+ *
+ * @author alessia
+ */
+public class PromoteShadowStatsJobNode extends AbstractStatsJobNode {
+
+ private static final Log log = LogFactory.getLog(PromoteShadowStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.action());
+ String portal = getPortalName(token);
+ if (StringUtils.isNotBlank(portal)) {
+ job.getParameters().put(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName(), portal);
+ } else {
+ log.warn(StatsManagerServiceBBAction.PROMOTE_SHADOW_STATS.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+ }
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p24.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p24.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p24.xml (revision 58540)
@@ -0,0 +1,368 @@
+
+
+
+ corda__h2020::3548e7151decbb665ee1ca31b2bc3a65
+ 665564
+ 2017-03-01T10:52:00.56+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::665564
+ corda__h2020::665564
+ 665564
+ CIRCLE
+ Coordinating European Research on Molecular Communications
+ 01-JUN-2015
+ 31-MAY-2017
+ H2020-FETOPEN-2014-CSA
+ true
+ Yes
+ openaire____::corda_h2020
+ CSA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-2-2014
+ Coordination and Support Activities
+
+
+ corda__h2020::665564::Alan_DAVY
+ corda__h2020::665564::Alan_DAVY
+ Alan
+ DAVY
+ adavy@tssg.org
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::990343811
+ corda__h2020::990343811
+ TUT
+ TTY-SAATIO
+ http://www.tut.fi
+ No
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ Missing
+ FI
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::990343811
+ 4
+ corda__h2020::665564
+ corda__h2020::990343811
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999608184
+ corda__h2020::999608184
+ IMINDS
+ IMINDS
+ http://www.iminds.be
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ BE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999608184
+ 8
+ corda__h2020::665564
+ corda__h2020::999608184
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999618660
+ corda__h2020::999618660
+ WIT
+ WATERFORD INSTITUTE OF TECHNOLOGY
+
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ IE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999618660
+ 1
+ corda__h2020::665564
+ corda__h2020::999618660
+ corda__h2020::665564::Alan_DAVY
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999649603
+ corda__h2020::999649603
+ CNIT
+ CONSORZIO NAZIONALE INTERUNIVERSITARIO PER LE TELECOMUNICAZIONI
+ http://www.cnit.it
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999649603
+ 9
+ corda__h2020::665564
+ corda__h2020::999649603
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999846319
+ corda__h2020::999846319
+ UNIPG
+ UNIVERSITA DEGLI STUDI DI PERUGIA
+
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ Missing
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999846319
+ 2
+ corda__h2020::665564
+ corda__h2020::999846319
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999901512
+ corda__h2020::999901512
+ KU
+ KOC UNIVERSITY
+ http://www.ku.edu.tr
+ Yes
+ Yes
+ Yes
+ No
+ Yes
+ No
+ No
+ No
+ TR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999901512
+ 5
+ corda__h2020::665564
+ corda__h2020::999901512
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999915189
+ corda__h2020::999915189
+ UMC
+ UNIVERSITAIR MEDISCH CENTRUM UTRECHT
+ http://www.umcutrecht.nl
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ NL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999915189
+ 6
+ corda__h2020::665564
+ corda__h2020::999915189
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999976202
+ corda__h2020::999976202
+ UPC
+ UNIVERSITAT POLITECNICA DE CATALUNYA
+ http://www.upc.edu
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ ES
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999976202
+ 3
+ corda__h2020::665564
+ corda__h2020::999976202
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999977172
+ corda__h2020::999977172
+ THE CHANCELLOR, MASTERS AND SCHOLARS OF THE UNIVERSITY OF CAMBRIDGE
+ THE CHANCELLOR, MASTERS AND SCHOLARS OF THE UNIVERSITY OF CAMBRIDGE
+ http://www.cam.ac.uk
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ Missing
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999977172
+ 7
+ corda__h2020::665564
+ corda__h2020::999977172
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999981149
+ corda__h2020::999981149
+ IMINDS
+ INTERUNIVERSITAIR MICRO-ELECTRONICA CENTRUM
+ http://www.imec.be
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ BE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665564@@corda__h2020::999981149
+ 8
+ corda__h2020::665564
+ corda__h2020::999981149
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-2-2014
+ corda__h2020::FETOPEN-2-2014
+ Coordination and Support Activities
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::665564@@corda__h2020::FETOPEN-2-2014
+ corda__h2020::665564
+ corda__h2020::FETOPEN-2-2014
+
+
+ ec__________::EC::H2020::CSA@@corda__h2020::665564
+ ec__________::EC::H2020::CSA
+ corda__h2020::665564
+ 01-JUN-2015
+ 31-MAY-2017
+ CSA
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p21.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p21.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p21.xml (revision 58540)
@@ -0,0 +1,98 @@
+
+
+
+ corda__h2020::005bd6e527663f5bb6063c16c9f72edb
+ 659064
+ 2017-03-01T10:48:23.062+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::659064
+ corda__h2020::659064
+ 659064
+ DISHY
+ Synergistic effects of DISpersant, oil and HYpoxia in a teleost fish: Investigating the impact of oil contamination in hypoxic areas and the use of dispersant as a response technique
+ 30-APR-2015
+ 29-APR-2017
+ H2020-MSCA-IF-2014
+ true
+ No
+ openaire____::corda_h2020
+ MSCA-IF-EF-ST
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ MSCA-IF-2014-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+
+
+ corda__h2020::659064::paolo_DOMENICI
+ corda__h2020::659064::paolo_DOMENICI
+ paolo
+ DOMENICI
+ paolo.domenici@cnr.it
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999979500
+ corda__h2020::999979500
+ CNR
+ CONSIGLIO NAZIONALE DELLE RICERCHE
+ http://www.cnr.it
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::659064@@corda__h2020::999979500
+ 1
+ corda__h2020::659064
+ corda__h2020::999979500
+ corda__h2020::659064::paolo_DOMENICI
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::MSCA-IF-2014-EF
+ corda__h2020::MSCA-IF-2014-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::659064@@corda__h2020::MSCA-IF-2014-EF
+ corda__h2020::659064
+ corda__h2020::MSCA-IF-2014-EF
+
+
+ ec__________::EC::H2020::MSCA-IF-EF-ST@@corda__h2020::659064
+ ec__________::EC::H2020::MSCA-IF-EF-ST
+ corda__h2020::659064
+ 30-APR-2015
+ 29-APR-2017
+ MSCA-IF-EF-ST
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISMainParamsV2JobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISMainParamsV2JobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISMainParamsV2JobNode.java (revision 58540)
@@ -0,0 +1,71 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.List;
+import java.util.Map;
+
+import com.google.gson.Gson;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import org.springframework.beans.factory.annotation.Required;
+
+public class PrepareIISMainParamsV2JobNode extends PrepareIISParamsV2 {
+
+ private String xqueryObjectStores;
+
+ private String objectStoreParam = "import_content_objectstores_csv";
+
+ private String importHbaseDumpLocation;
+
+ private String importHbaseDumpLocationParam = "import_hbase_dump_location";
+
+ // Enable/Disable inference modules
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ super.prepare(token);
+
+ token.getEnv().setAttribute(getObjectStoreParam(), getFilteredObjectStoreCSV(getXqueryObjectStores()));
+ token.getEnv().setAttribute(getImportHbaseDumpLocationParam(), getImportHbaseDumpLocation());
+
+ @SuppressWarnings("unchecked") final List> sets = new Gson().fromJson(token.getEnv().getAttribute("sets"), List.class);
+ for (Map set : sets) {
+ token.getEnv().setAttribute(set.get("enablingProperty"), set.get("enabled"));
+ }
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getXqueryObjectStores() {
+ return xqueryObjectStores;
+ }
+
+ @Required
+ public void setXqueryObjectStores(final String xqueryObjectStores) {
+ this.xqueryObjectStores = xqueryObjectStores;
+ }
+
+ public String getObjectStoreParam() {
+ return objectStoreParam;
+ }
+
+ public void setObjectStoreParam(final String objectStoreParam) {
+ this.objectStoreParam = objectStoreParam;
+ }
+
+ public String getImportHbaseDumpLocation() {
+ return importHbaseDumpLocation;
+ }
+
+ public void setImportHbaseDumpLocation(final String importHbaseDumpLocation) {
+ this.importHbaseDumpLocation = importHbaseDumpLocation;
+ }
+
+ public String getImportHbaseDumpLocationParam() {
+ return importHbaseDumpLocationParam;
+ }
+
+ public void setImportHbaseDumpLocationParam(final String importHbaseDumpLocationParam) {
+ this.importHbaseDumpLocationParam = importHbaseDumpLocationParam;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p14.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p14.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p14.xml (revision 58540)
@@ -0,0 +1,98 @@
+
+
+
+ corda__h2020::002f8a425bcd433f4bb17b12b0c1df17
+ 714692
+ 2017-03-01T11:14:49.222+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::714692
+ corda__h2020::714692
+ 714692
+ UNIGLASS
+ The Enigmatic Universality of Glass
+ 01-SEP-2017
+ 31-AUG-2022
+ ERC-2016-STG
+ true
+ No
+ openaire____::corda_h2020
+ ERC-STG
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ ERC-2016-STG
+ ERC Starting Grant
+
+
+ corda__h2020::714692::Andrew_FEFFERMAN
+ corda__h2020::714692::Andrew_FEFFERMAN
+ Andrew
+ FEFFERMAN
+ andrew.fefferman@neel.cnrs.fr
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999997930
+ corda__h2020::999997930
+ CNRS
+ CENTRE NATIONAL DE LA RECHERCHE SCIENTIFIQUE CNRS
+ http://www.cnrs.fr
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::714692@@corda__h2020::999997930
+ 1
+ corda__h2020::714692
+ corda__h2020::999997930
+ corda__h2020::714692::Andrew_FEFFERMAN
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::ERC-2016-STG
+ corda__h2020::ERC-2016-STG
+ ERC Starting Grant
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::714692@@corda__h2020::ERC-2016-STG
+ corda__h2020::714692
+ corda__h2020::ERC-2016-STG
+
+
+ ec__________::EC::H2020::ERC::ERC-STG@@corda__h2020::714692
+ ec__________::EC::H2020::ERC::ERC-STG
+ corda__h2020::714692
+ 01-SEP-2017
+ 31-AUG-2022
+ ERC-STG
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISPreprocessingParamsV2JobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISPreprocessingParamsV2JobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareIISPreprocessingParamsV2JobNode.java (revision 58540)
@@ -0,0 +1,106 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import org.springframework.beans.factory.annotation.Required;
+
+public class PrepareIISPreprocessingParamsV2JobNode extends PrepareIISParamsV2 {
+
+ private String xqueryWosMDStore;
+
+ private String mdStoreWosParam = "import_wos_mdstore_id";
+
+ private String xqueryDatabaseService;
+
+ private String databaseServiceLocationParam = "import_database_service_location";
+
+ private String xqueryDataciteObjectStore;
+
+ private String dataciteObjectStoreParam = "import_content_datacite_objectstores_csv";
+
+ private String xqueryWosObjectStore;
+
+ private String wosObjectStorePlaintextParam = "import_content_wos_plaintext_objectstores_csv";
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ super.prepare(token);
+
+ token.getEnv().setAttribute(getMdStoreWosParam(), getProfileId(getXqueryWosMDStore()));
+ token.getEnv().setAttribute(getDatabaseServiceLocationParam(), getServiceEndpoint(getXqueryDatabaseService()));
+
+ token.getEnv().setAttribute(getDataciteObjectStoreParam(), getFilteredObjectStoreCSV(getXqueryDataciteObjectStore()));
+ token.getEnv().setAttribute(getWosObjectStorePlaintextParam(), getProfileId(getXqueryWosObjectStore()));
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getXqueryWosMDStore() {
+ return xqueryWosMDStore;
+ }
+
+ @Required
+ public void setXqueryWosMDStore(final String xqueryWosMDStore) {
+ this.xqueryWosMDStore = xqueryWosMDStore;
+ }
+
+ public String getMdStoreWosParam() {
+ return mdStoreWosParam;
+ }
+
+ public void setMdStoreWosParam(final String mdStoreWosParam) {
+ this.mdStoreWosParam = mdStoreWosParam;
+ }
+
+ public String getXqueryDatabaseService() {
+ return xqueryDatabaseService;
+ }
+
+ @Required
+ public void setXqueryDatabaseService(final String xqueryDatabaseService) {
+ this.xqueryDatabaseService = xqueryDatabaseService;
+ }
+
+ public String getDatabaseServiceLocationParam() {
+ return databaseServiceLocationParam;
+ }
+
+ public void setDatabaseServiceLocationParam(final String databaseServiceLocationParam) {
+ this.databaseServiceLocationParam = databaseServiceLocationParam;
+ }
+
+ public String getWosObjectStorePlaintextParam() {
+ return wosObjectStorePlaintextParam;
+ }
+
+ public void setWosObjectStorePlaintextParam(final String wosObjectStorePlaintextParam) {
+ this.wosObjectStorePlaintextParam = wosObjectStorePlaintextParam;
+ }
+
+ public String getXqueryWosObjectStore() {
+ return xqueryWosObjectStore;
+ }
+
+ @Required
+ public void setXqueryWosObjectStore(final String xqueryWosObjectStore) {
+ this.xqueryWosObjectStore = xqueryWosObjectStore;
+ }
+
+ public String getXqueryDataciteObjectStore() {
+ return xqueryDataciteObjectStore;
+ }
+
+ @Required
+ public void setXqueryDataciteObjectStore(final String xqueryDataciteObjectStore) {
+ this.xqueryDataciteObjectStore = xqueryDataciteObjectStore;
+ }
+
+ public String getDataciteObjectStoreParam() {
+ return dataciteObjectStoreParam;
+ }
+
+ public void setDataciteObjectStoreParam(final String dataciteObjectStoreParam) {
+ this.dataciteObjectStoreParam = dataciteObjectStoreParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/propagation/communitytoresult/LoadCommunityIdsListJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/propagation/communitytoresult/LoadCommunityIdsListJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/propagation/communitytoresult/LoadCommunityIdsListJobNode.java (revision 58540)
@@ -0,0 +1,68 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.propagation.communitytoresult;
+
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.lang.reflect.Type;
+import java.util.List;
+
+public class LoadCommunityIdsListJobNode extends SimpleJobNode {
+
+ private static final Log log = LogFactory.getLog(LoadCommunityIdsListJobNode.class);
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ private String xquery;
+
+ private String communityIdParam;
+
+ public String getCommunityIdParam() {
+ return communityIdParam;
+ }
+
+ public void setCommunityIdParam(String communityIdParam) {
+ this.communityIdParam = communityIdParam;
+ }
+
+ public UniqueServiceLocator getServiceLocator() {
+ return serviceLocator;
+ }
+
+ public void setServiceLocator(UniqueServiceLocator serviceLocator) {
+ this.serviceLocator = serviceLocator;
+ }
+
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final List res = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(getXquery());
+ Type listType = new TypeToken>() {}.getType();
+
+ token.getEnv().setAttribute(getCommunityIdParam(),new Gson().toJson(res,listType));
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getXquery() {
+ log.debug(xquery);
+ return xquery;
+ }
+
+ public void setXquery(final String xquery) {
+ this.xquery = xquery;
+
+ }
+
+
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimRelsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimRelsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimRelsJobNode.java (revision 58540)
@@ -0,0 +1,132 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.claims;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.hadoop.rmi.HadoopService;
+import eu.dnetlib.data.proto.FieldTypeProtos.DataInfo;
+import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;
+import eu.dnetlib.data.proto.KindProtos.Kind;
+import eu.dnetlib.data.proto.OafProtos.Oaf;
+import eu.dnetlib.data.proto.OafProtos.OafRel;
+import eu.dnetlib.data.proto.RelMetadataProtos.RelMetadata;
+import eu.dnetlib.data.proto.RelTypeProtos.RelType;
+import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;
+import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject;
+import eu.dnetlib.data.proto.ResultProjectProtos.ResultProject.Outcome;
+import eu.dnetlib.data.proto.ResultResultProtos.ResultResult;
+import eu.dnetlib.data.proto.ResultResultProtos.ResultResult.PublicationDataset;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.utils.ontologies.OntologyLoader;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Created by alessia on 23/10/15.
+ */
+public class ApplyClaimRelsJobNode extends AbstractClaimsToHBASE {
+
+ private static final Log log = LogFactory.getLog(ApplyClaimRelsJobNode.class);
+
+ private final String SEPARATOR = "_";
+
+ @Override
+ protected String execute(NodeToken token) throws Exception {
+ //TODO: use claim.claim_date from the claim db
+ long timestamp = System.currentTimeMillis();
+ setTotal(getClaimDatabaseUtils().count(getCountQuery()));
+
+ final AtomicInteger totalWrites = new AtomicInteger(0);
+ final AtomicInteger discardedClaims = new AtomicInteger(0);
+
+ final HadoopService hadoopService = getServiceLocator().getService(HadoopService.class);
+
+ getClaimDatabaseUtils().query(getSql()).forEach(claim -> {
+ log.debug(claim);
+ try {
+ String sourceId = getFullId(getOpenAIREType(claim.getSourceType()), claim.getSource());
+ String targetId = getFullId(getOpenAIREType(claim.getTargetType()), claim.getTarget());
+ String value = getValue(sourceId, claim.getSemantics(), targetId, timestamp);
+ // adding relationship
+ hadoopService.addHBaseColumn(getClusterName(), getTableName(), sourceId, claim.getSemantics(), targetId, value);
+ totalWrites.incrementAndGet();
+
+ String inverseSemantics = OntologyLoader.fetchInverse(claim.getSemantics());
+ String inverseValue = getValue(targetId, inverseSemantics, sourceId, timestamp);
+ //adding inverse relationship
+ hadoopService.addHBaseColumn(getClusterName(), getTableName(), targetId, inverseSemantics, sourceId, inverseValue);
+ totalWrites.incrementAndGet();
+ incrementProcessed();
+ } catch (Exception e) {
+ log.error("Discarding claim " + claim + ". Cause: " + e.getMessage());
+ discardedClaims.incrementAndGet();
+ }
+ });
+ log.info("totalClaimRels: " + getTotal());
+ token.getEnv().setAttribute("claimRelsSize", getTotal());
+ log.info("claim rels writeOps: " + totalWrites.intValue());
+ token.getEnv().setAttribute("claimRelsWriteOps", totalWrites.intValue());
+ log.info("discardedClaimRels: " + discardedClaims.intValue());
+ token.getEnv().setAttribute("discardedClaimRels", discardedClaims.intValue());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ protected String getValue(final String sourceId, final String semantics, final String targetId, final long timestamp) throws MSROException {
+ log.debug(StringUtils.format("%s -- %s -- %s", sourceId, semantics, targetId));
+ String[] relInfo = semantics.split(SEPARATOR);
+ if (relInfo.length != 3) {
+ throw new MSROException("Semantics " + semantics + " not supported: must be splittable in 3 by '_'");
+ }
+ Qualifier.Builder semanticsBuilder = Qualifier.newBuilder().setClassid(relInfo[2]).setClassname(relInfo[2]);
+
+ Oaf.Builder builder = Oaf.newBuilder().setKind(Kind.relation).setLastupdatetimestamp(timestamp);
+ builder.setDataInfo(DataInfo.newBuilder().setTrust("0.91").setInferred(false)
+ .setProvenanceaction(
+ Qualifier.newBuilder()
+ .setClassid("user:claim")
+ .setClassname("user:claim")
+ .setSchemeid("dnet:provenanceActions")
+ .setSchemename("dnet:provenanceActions")
+ ));
+
+ final SubRelType subRelType = SubRelType.valueOf(relInfo[1]);
+ final OafRel.Builder relBuilder = OafRel.newBuilder()
+ .setSubRelType(subRelType)
+ .setRelClass(relInfo[2])
+ .setRelType(RelType.valueOf(relInfo[0]))
+ .setSource(sourceId).setTarget(targetId).setChild(false);
+
+ switch (relInfo[0]) {
+ case "resultProject":
+
+ relBuilder.setResultProject(ResultProject.newBuilder()
+ .setOutcome(Outcome.newBuilder().setRelMetadata(
+ RelMetadata.newBuilder().setSemantics(
+ semanticsBuilder
+ .setSchemeid("dnet:result_project_relations")
+ .setSchemename("dnet:result_project_relations")
+ .build()
+ ))));
+ break;
+ case "resultResult":
+ relBuilder.setResultResult(ResultResult.newBuilder()
+ .setPublicationDataset(PublicationDataset.newBuilder().setRelMetadata(
+ RelMetadata.newBuilder().setSemantics(
+ semanticsBuilder
+ .setSchemeid("dnet:result_result_relations")
+ .setSchemename("dnet:result_result_relations")
+ .build()
+ ))));
+ break;
+ default:
+ throw new MSROException("Semantics " + relInfo[0] + " not supported");
+ }
+
+ builder.setRel(relBuilder);
+ return Base64.encodeBase64String(builder.build().toByteArray());
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p17.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p17.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p17.xml (revision 58540)
@@ -0,0 +1,128 @@
+
+
+
+ corda__h2020::003d8b7877f8d86860ce30b7c24534d5
+ 734075
+ 2017-03-01T11:22:46.663+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::734075
+ corda__h2020::734075
+ 734075
+ CoDiS
+ Compact, high-power, frequency-converted diode laser systems (CoDiS)
+ 01-OCT-2016
+ 31-MAR-2019
+ H2020-SMEINST-2-2016-2017
+ true
+ No
+ openaire____::corda_h2020
+ SME-2
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ SMEInst-01-2016-2017
+ Open Disruptive Innovation Scheme
+
+
+ corda__h2020::734075::Oliver_HVIDT
+ corda__h2020::734075::Oliver_HVIDT
+ Oliver
+ HVIDT
+ olhv@norlase.com
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::938309131
+ corda__h2020::938309131
+ NORLASE
+ NORLASE APS
+ http://www.norlase.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ DK
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734075@@corda__h2020::938309131
+ 1
+ corda__h2020::734075
+ corda__h2020::938309131
+ corda__h2020::734075::Oliver_HVIDT
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::940765074
+ corda__h2020::940765074
+ eyP
+ EAGLEYARD PHOTONICS GMBH
+ http://www.eagleyard.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734075@@corda__h2020::940765074
+ 2
+ corda__h2020::734075
+ corda__h2020::940765074
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::SMEInst-01-2016-2017
+ corda__h2020::SMEInst-01-2016-2017
+ Open Disruptive Innovation Scheme
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::734075@@corda__h2020::SMEInst-01-2016-2017
+ corda__h2020::734075
+ corda__h2020::SMEInst-01-2016-2017
+
+
+ ec__________::EC::H2020::SME-2@@corda__h2020::734075
+ ec__________::EC::H2020::SME-2
+ corda__h2020::734075
+ 01-OCT-2016
+ 31-MAR-2019
+ SME-2
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/Claim.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/Claim.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/Claim.java (revision 58540)
@@ -0,0 +1,67 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.claims;
+
+/**
+ * Created by Alessia Bardi on 23/06/2017.
+ *
+ * @author Alessia Bardi
+ */
+public class Claim {
+
+ private String source, sourceType, target, targetType, semantics;
+
+ public String getSource() {
+ return source;
+ }
+
+ public Claim setSource(final String source) {
+ this.source = source;
+ return this;
+ }
+
+ public String getSourceType() {
+ return sourceType;
+ }
+
+ public Claim setSourceType(final String sourceType) {
+ this.sourceType = sourceType;
+ return this;
+ }
+
+ public String getTarget() {
+ return target;
+ }
+
+ public Claim setTarget(final String target) {
+ this.target = target;
+ return this;
+ }
+
+ public String getTargetType() {
+ return targetType;
+ }
+
+ public Claim setTargetType(final String targetType) {
+ this.targetType = targetType;
+ return this;
+ }
+
+ public String getSemantics() {
+ return semantics;
+ }
+
+ public Claim setSemantics(final String semantics) {
+ this.semantics = semantics;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "Claim{" +
+ "source='" + source + '\'' +
+ ", sourceType='" + sourceType + '\'' +
+ ", target='" + target + '\'' +
+ ", targetType='" + targetType + '\'' +
+ ", semantics='" + semantics + '\'' +
+ '}';
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p25.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p25.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p25.xml (revision 58540)
@@ -0,0 +1,128 @@
+
+
+
+ corda__h2020::818cb43dac3aeb307a264705549be581
+ 665134
+ 2017-03-01T10:51:52.49+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::665134
+ corda__h2020::665134
+ 665134
+ FET-Event
+ Ensuring the success of the next FET Conference and exhibition at a key period for FET
+ 01-SEP-2015
+ 30-NOV-2016
+ H2020-FETOPEN-2014-CSA
+ true
+ Yes
+ openaire____::corda_h2020
+ CSA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-2-2014
+ Coordination and Support Activities
+
+
+ corda__h2020::665134::Olga_KIEFFER
+ corda__h2020::665134::Olga_KIEFFER
+ Olga
+ KIEFFER
+ olga.kieffer@sigma-orionis.com
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999593828
+ corda__h2020::999593828
+ ERCIM
+ GEIE ERCIM
+ http://www.ercim.org
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665134@@corda__h2020::999593828
+ 2
+ corda__h2020::665134
+ corda__h2020::999593828
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999792096
+ corda__h2020::999792096
+ SIGMA
+ SIGMA ORIONIS SA
+ http://www.sigma-orionis.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665134@@corda__h2020::999792096
+ 1
+ corda__h2020::665134
+ corda__h2020::999792096
+ corda__h2020::665134::Olga_KIEFFER
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-2-2014
+ corda__h2020::FETOPEN-2-2014
+ Coordination and Support Activities
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::665134@@corda__h2020::FETOPEN-2-2014
+ corda__h2020::665134
+ corda__h2020::FETOPEN-2-2014
+
+
+ ec__________::EC::H2020::CSA@@corda__h2020::665134
+ ec__________::EC::H2020::CSA
+ corda__h2020::665134
+ 01-SEP-2015
+ 30-NOV-2016
+ CSA
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/utils/CleaningXsltFunctions.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/utils/CleaningXsltFunctions.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/utils/CleaningXsltFunctions.java (revision 58540)
@@ -0,0 +1,14 @@
+package eu.dnetlib.msro.openaireplus.utils;
+
+import java.text.Normalizer;
+
+public class CleaningXsltFunctions {
+
+ public static String clean(final String s) {
+ return Normalizer.normalize(s, Normalizer.Form.NFD)
+ .replaceAll("\\(.+\\)", "")
+ .replaceAll("(\\W|\\p{InCombiningDiacriticalMarks}|\\p{Punct}|\\n|\\s)+", "")
+ .toLowerCase()
+ .trim();
+ }
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p18.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p18.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p18.xml (revision 58540)
@@ -0,0 +1,98 @@
+
+
+
+ corda__h2020::003f2e50d3aaeb4c23eecf98126d33b2
+ 701859
+ 2017-03-01T11:08:24.367+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::701859
+ corda__h2020::701859
+ 701859
+ GSAORI
+ Guide Stars Adaptive Optics for Retinal Imaging
+ 01-APR-2016
+ 31-MAR-2018
+ H2020-MSCA-IF-2015
+ true
+ No
+ openaire____::corda_h2020
+ MSCA-IF-EF-ST
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ MSCA-IF-2015-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+
+
+ corda__h2020::701859::Michael_PIRCHER
+ corda__h2020::701859::Michael_PIRCHER
+ Michael
+ PIRCHER
+ michael.pircher@meduniwien.ac.at
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999989976
+ corda__h2020::999989976
+ MUW
+ MEDIZINISCHE UNIVERSITAET WIEN
+ http://www.meduniwien.ac.at
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ AT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::701859@@corda__h2020::999989976
+ 1
+ corda__h2020::701859
+ corda__h2020::999989976
+ corda__h2020::701859::Michael_PIRCHER
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::MSCA-IF-2015-EF
+ corda__h2020::MSCA-IF-2015-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::701859@@corda__h2020::MSCA-IF-2015-EF
+ corda__h2020::701859
+ corda__h2020::MSCA-IF-2015-EF
+
+
+ ec__________::EC::H2020::MSCA-IF-EF-ST@@corda__h2020::701859
+ ec__________::EC::H2020::MSCA-IF-EF-ST
+ corda__h2020::701859
+ 01-APR-2016
+ 31-MAR-2018
+ MSCA-IF-EF-ST
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextUtilsTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextUtilsTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/ContextUtilsTest.java (revision 58540)
@@ -0,0 +1,181 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.contexts;
+
+import java.io.StringReader;
+import java.util.List;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Created by claudio on 01/03/16.
+ */
+public class ContextUtilsTest {
+
+ private static final Log log = LogFactory.getLog(ContextUtilsTest.class);
+ private String params = "{param1:value1,param2:value2}";
+
+
+ @Test
+ public void testWrongInput() throws DocumentException {
+ Iterable it = Lists.newArrayList("irb_hr______::HRZZ HRZZ Croatian Science Foundation (CSF) Hrvatska zaklada za znanost HR ");
+ final ContextDesc context = ContextUtils.getContext(it, "HRZZ", "HRZZ", "funding", params, "all");
+ System.out.println(context.asDomElement().asXML());
+ }
+
+ @Test
+ public void testProfile() throws DocumentException {
+ Iterable it = Lists.newArrayList("irb_hr______::HRZZ HRZZ Croatian Science Foundation (CSF) Hrvatska zaklada za znanost HR ");
+ final ContextDesc contextDesc = ContextUtils.getContext(it, "HRZZ", "HRZZ", "funding", params, "all");
+ final String profile = "\n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " valueX\n"
+ + " \n"
+ + " Special Research initiative (Direct stimulation of the visual cortex: a flexible strategy for restoring high-acuity pattern vision)\n"
+ + " arc_________::ARC::Special Research initiative (Direct stimulation of the visual cortex: a flexible strategy for restoring high-acuity pattern vision)\n"
+ + " arc:fundingStream\n"
+ + " "
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " \n"
+ + " ";
+ final SAXReader reader = new SAXReader();
+ final Document doc = reader.read(new StringReader(profile));
+ final String profId = doc.valueOf("//HEADER/RESOURCE_IDENTIFIER/@value");
+ final Element ctxElem = (Element) doc.selectSingleNode("//CONFIGURATION/context[@id='" + contextDesc.getId() + "' and @type='" + contextDesc.getType() + "']");
+ updateContextParams(ctxElem, contextDesc);
+
+ for (ContextPart cat : contextDesc.getCategories().values()) {
+ final Node catElem = ctxElem.selectSingleNode("./category[@id='" + cat.getId() + "']");
+ if (catElem != null) {
+ catElem.detach();
+ }
+ ctxElem.add(cat.asDomElement("category"));
+ }
+ log.info("registering profile context " + contextDesc.getId());
+ System.out.println(doc.asXML());
+ }
+
+ private void updateContextParams(final Element ctxElement, ContextDesc desc) {
+ //removing old PARAMs
+ List oldParams = ctxElement.selectNodes("param");
+ for (Node n : oldParams) {
+ n.detach();
+ }
+ //adding new params
+ for (Element param : desc.getParamsAsElements()) {
+ ctxElement.add(param);
+ }
+ }
+
+ @Test
+ public void testContextFactory() throws DocumentException {
+
+ Iterable it = Lists.newArrayList(
+ "\t\t\n"
+ + "\t\t\t\n"
+ + "\t\t\t\t\n"
+ + "\t\t\t\t\tarc_________::ARC \n"
+ + "\t\t\t\t\tARC \n"
+ + "\t\t\t\t\tAustralian Research Council (ARC) \n"
+ + "\t\t\t\t\tAU \n"
+ + "\t\t\t\t \n"
+ + "\t\t\t \n"
+ + "\t\t ");
+
+ final ContextDesc context = ContextUtils.getContext(it, "ARC", "Australian Research Council (ARC)", "funding", params, "hidden");
+
+ final Iterable iter = Iterables.transform(context.getDbEntries().entrySet(), ContextUtils.getContextRowTransformer());
+
+ final SAXReader r = new SAXReader();
+
+ for (String f : iter) {
+
+ final Document fd = r.read(new StringReader(f));
+
+ assertTrue(fd != null);
+ assertTrue("AU".equals(fd.valueOf("//FIELD[@name='jurisdiction']")));
+ assertTrue("arc_________::ARC".equals(fd.valueOf("//FIELD[@name='funder']")));
+ }
+ System.out.println(context.asDomElement().asXML());
+ }
+
+ @Test
+ public void testEmptyFunding() throws DocumentException {
+
+ Iterable it = Lists.newArrayList(
+ "\t\t\n"
+ + "\t\t\t \n"
+ + "\t\t ");
+
+ final ContextDesc context = ContextUtils.getContext(it, "HRZZ", "HRZZ", "funding", params, "hidden");
+
+ final Iterable iter = Iterables.transform(context.getDbEntries().entrySet(), ContextUtils.getContextRowTransformer());
+
+ final SAXReader r = new SAXReader();
+
+ for (String f : iter) {
+ System.out.println(f);
+
+ /*final Document fd = r.read(new StringReader(f));
+
+ assertTrue(fd != null);
+ assertTrue("AU".equals(fd.valueOf("//FIELD[@name='jurisdiction']")));
+ assertTrue("arc_________::ARC".equals(fd.valueOf("//FIELD[@name='funder']")));
+ */
+ }
+ }
+
+ @Test
+ public void testCountry() throws DocumentException {
+
+ Iterable it = Lists.newArrayList(
+ "\t\t\n"
+ + "\t\t\t\n"
+ + "\t\t\t\t\n"
+ + "\t\t\t\t\tarc_________::ARC \n"
+ + "\t\t\t\t\tARC \n"
+ + "\t\t\t\t\tAustralian Research Council (ARC) \n"
+ + "\t\t\t\t\tAU \n"
+ + "\t\t\t\t \n"
+ + "\t\t\t \n"
+ + "\t\t ");
+
+ final ContextDesc context = ContextUtils.getContext(it, "ARC", "ARC", "funding", params, "hidden");
+
+ final Iterable iter = Iterables.transform(context.getDbEntries().entrySet(), ContextUtils.getContextRowTransformer());
+
+ final SAXReader r = new SAXReader();
+
+ for (String f : iter) {
+ System.out.println(f);
+
+ final Document fd = r.read(new StringReader(f));
+
+ assertTrue(fd != null);
+ assertTrue("AU".equals(fd.valueOf("//FIELD[@name='jurisdiction']")));
+ assertTrue("arc_________::ARC".equals(fd.valueOf("//FIELD[@name='funder']")));
+ }
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/index/oafRecord.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/index/oafRecord.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/index/oafRecord.xml (revision 58540)
@@ -0,0 +1,73 @@
+
+
+
+ lareferencia::29c95f5ef62c8bad9db1f4a6fdfd0a3a
+ 2018-03-09T15:22:59.538Z
+ 2018-09-19T09:51:37.481Z
+
+
+
+
+
+ Organizações & Sociedade v.20 n.67 2013
+ reponame:Escola de Administração da Universidade Federal da Bahia
+ instname:UFBA
+ text/html
+ Silva,Mônica de Aguiar Mac-Allister da
+ Andrade,José Célio Silveira
+ Editorial
+ 2013-12-01
+
+ Escola de Administração da Universidade Federal da Bahia
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ oai:periodicos.ibict.br.OrganizacoesESociedade:oai:scielo:S1984-92302013000400001
+
+
+
+
+ false
+ false
+ 0.9
+
+
+
+
+
+
+
+
+ 2013-12-01
+
+
+
+ http://www.scielo.br/scielo.php?script=sci_arttext&pid=S1984-92302013000400001
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_ariadne.json
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_ariadne.json (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_ariadne.json (revision 58540)
@@ -0,0 +1,25 @@
+{
+ "openaireId":"od_______278::d5b39693ac6c2197e8cf48008a982950",
+ "originalId": "oai:rudar.ruc.dk:1800/24690",
+ "type": "publication",
+ "title": "Autofiktion",
+ "authors": [
+ "Sloan, Patrick Alexander Raaby",
+ "Strange, Gustav Valdemar",
+ "Freund-Jensen, Sofie",
+ "Canvin, Emilie Meistrup",
+ "Faaborg, Ida Selvejer",
+ "Kruse, Mikkel"
+ ],
+ "publisher": "",
+ "description": "The following paper is a project on the autofictional book Færdig med Eddy Bellegueule by Édouard Louis (2015). The main focus is to ascertain how the novel is an expression of the author’s effort to connect social life and literature, and what part autofiction as a genre plays in this process. A textual analysis will describe the narrator’s position, and the aesthetic grip of the novel. Furthermore, we will analyze the central themes in Louis’ novel with theories by sociologist Pierre Bourdieu, gender theorist Judith Butler and postcolonial theorist Gayatri Spivak. This includes symbolic violence, gender roles, language and suppression. With use of classic literary theory and the more recent and specific theory on autofiction, the paper concludes, that Færdig med Eddy Bellegueule is based on non-fictional events, but presents these events through a fictionalization that raise the story into a literary work of art. The genre autofiction encircles a current tendency to blend non-fictional events and characters into literature, and Færdig med Eddy Bellegueule writes itself perfectly into this twilight zone between fact and fiction.",
+ "language": "eng",
+ "pids": [],
+ "licenseCode": "OPEN",
+ "resourceType": "0006",
+ "url": "http://rudar.ruc.dk/handle/1800/24690",
+ "collectedFromId": "opendoar____::278",
+ "hostedById": "opendoar____::278",
+ "linksToProjects": [],
+ "contexts": []
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidRepoProfilesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidRepoProfilesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidRepoProfilesJobNode.java (revision 58540)
@@ -0,0 +1,142 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+
+import eu.dnetlib.enabling.datasources.DatasourceManagerClients;
+import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.ProgressProvider;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+
+public class FindInvalidRepoProfilesJobNode extends SimpleJobNode implements ProgressJobNode {
+
+ @Autowired
+ private LocalOpenaireDatasourceManager dsManager;
+
+ @Autowired
+ private DatasourceManagerClients dsManagerClient;
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ private int current = 0;
+ private int total = 0;
+
+ private boolean deleteInvalidProfiles = false;
+ private boolean registerMissingProfiles = false;
+
+ private static final Log log = LogFactory.getLog(FindInvalidRepoProfilesJobNode.class);
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ final Map validIds = dsManagerClient.searchSQL("SELECT id FROM dsm_datasources", new HashMap<>())
+ .stream()
+ .map(m -> (String) m.get("id"))
+ .collect(Collectors.toMap(s -> s, s -> new AtomicInteger(0)));
+
+ final List list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(
+ "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') return concat($x//DATASOURCE_ORIGINAL_ID, ' @@@ ', $x//RESOURCE_IDENTIFIER/@value)");
+
+ this.current = 0;
+ this.total = list.size() + validIds.size();
+ int errorInvalidProfiles = 0;
+ int errorUnregisteredProfiles = 0;
+ int errorTooManyProfiles = 0;
+
+ for (final String s : list) {
+ current++;
+ final String oid = StringUtils.substringBefore(s, "@@@").trim();
+ final String profId = StringUtils.substringAfter(s, "@@@").trim();
+ log.info("Evaluating ds: " + oid);
+ if (validIds.containsKey(oid)) {
+ validIds.get(oid).incrementAndGet();
+ } else {
+ log.warn("Invalid profile " + profId + ", openaireId " + oid + " not registered properly");
+ errorInvalidProfiles++;
+ // token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalidProfile:" + oid, profId);
+ if (deleteInvalidProfiles) {
+ log.warn(" - Deleting profile " + profId);
+ serviceLocator.getService(ISRegistryService.class).deleteProfile(profId);
+ }
+ }
+ }
+
+ for (final Map.Entry e : validIds.entrySet()) {
+ current++;
+ final String dsId = e.getKey();
+ final int n = e.getValue().get();
+ if (n == 0) {
+ errorUnregisteredProfiles++;
+ // token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "unregisterdProfile:" + dsId, dsId);
+ log.warn("Missing profile for ds " + dsId);
+ if (registerMissingProfiles) {
+ dsManager.setManaged(dsId, dsManager.isManaged(dsId)); // This command should regenerate the repo profile
+ }
+ } else if (n > 1) {
+ errorTooManyProfiles++;
+ log.error("Too many profiles registerd " + n + " for ds " + dsId);
+ // token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "tooManyProfiles:" + dsId, n);
+ }
+ }
+
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "errorInvalidProfiles", errorInvalidProfiles);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "errorUnregisteredProfiles", errorUnregisteredProfiles);
+ token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "errorTooManyProfiles", errorTooManyProfiles);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ @Override
+ public ProgressProvider getProgressProvider() {
+ return new ProgressProvider() {
+
+ @Override
+ public int getTotalValue() {
+ return total;
+ }
+
+ @Override
+ public int getCurrentValue() {
+ return current;
+ }
+
+ @Override
+ public boolean isInaccurate() {
+ return false;
+ }
+ };
+ }
+
+ public boolean isDeleteInvalidProfiles() {
+ return deleteInvalidProfiles;
+ }
+
+ public void setDeleteInvalidProfiles(final boolean deleteInvalidProfiles) {
+ this.deleteInvalidProfiles = deleteInvalidProfiles;
+ }
+
+ public boolean isRegisterMissingProfiles() {
+ return registerMissingProfiles;
+ }
+
+ public void setRegisterMissingProfiles(final boolean registerMissingProfiles) {
+ this.registerMissingProfiles = registerMissingProfiles;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_zenodo_software.json
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_zenodo_software.json (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_zenodo_software.json (revision 58540)
@@ -0,0 +1,29 @@
+{
+ "authors":[
+ "Alouges, Fran\u00e7ois",
+ "Di Fratta, Giovanni"
+ ],
+ "collectedFromId":"opendoar____::2659",
+ "description":"This is a software",
+ "hostedById":"opendoar____::2659",
+ "licenseCode":"OPEN",
+ "originalId":"oai:zenodo.org:12345",
+ "pids":[
+ {
+ "type":"oai",
+ "value":"oai:zenodo.org:12345"
+ },
+ {
+ "type":"doi",
+ "value":"10.5281/zenodo.12345"
+ }
+ ],
+ "publisher":"Zenodo",
+ "resourceType":"0029",
+ "title":"Parking 3-sphere swimmer. I. Energy minimizing strokes",
+ "type":"software",
+ "url":"https://zenodo.org/record/12345",
+ "contexts": [
+ "mes"
+ ]
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/UpdateSetsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/UpdateSetsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/UpdateSetsJobNode.java (revision 58540)
@@ -0,0 +1,85 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.actions;
+
+import java.io.StringReader;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Resource;
+
+import com.google.gson.Gson;
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Attribute;
+import org.dom4j.Document;
+import org.dom4j.Element;
+import org.dom4j.io.SAXReader;
+
+public class UpdateSetsJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(UpdateSetsJobNode.class);
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ @SuppressWarnings("unchecked") final List> sets = new Gson().fromJson(token.getEnv().getAttribute("sets"), List.class);
+
+ final String lastUpdate = DateUtils.now_ISO8601();
+ for (Map set : sets) {
+
+ // update only the enabled sets.
+ if (isEnabled(set)) {
+ log.info("updating set: " + set.toString());
+ addLatestRawSet(set, lastUpdate);
+ } else {
+ log.info("skip set update: " + set.toString());
+ }
+ }
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ private boolean isEnabled(final Map set) {
+ return set.containsKey("enabled") && set.get("enabled").equals("true");
+ }
+
+ public void addLatestRawSet(final Map set, final String lastUpdate) throws MSROException {
+ final String q = "for $x in collection('/db/DRIVER/ActionManagerSetDSResources/ActionManagerSetDSResourceType') where $x//SET/@id = '" + set.get("set")
+ + "' return $x";
+ try {
+ final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(q);
+ final Document doc = new SAXReader().read(new StringReader(profile));
+ final String profId = doc.valueOf("//RESOURCE_IDENTIFIER/@value");
+ final Element latest = (Element) doc.selectSingleNode("//RAW_SETS/LATEST");
+ final Element expired = ((Element) doc.selectSingleNode("//RAW_SETS")).addElement("EXPIRED");
+
+ for (Object o : latest.attributes()) {
+ final Attribute a = (Attribute) o;
+ expired.addAttribute(a.getName(), a.getValue());
+ }
+
+ latest.addAttribute("id", set.get("rawset"));
+ latest.addAttribute("creationDate", set.get("creationDate"));
+ latest.addAttribute("lastUpdate", lastUpdate);
+
+ serviceLocator.getService(ISRegistryService.class).updateProfile(profId, doc.asXML(), "ActionManagerSetDSResourceType");
+ } catch (Exception e) {
+ String msg = "Error updating profile of set: " + set;
+ log.error(msg);
+ throw new MSROException(msg, e);
+ }
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_openaireId.json
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_openaireId.json (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_openaireId.json (revision 58540)
@@ -0,0 +1,38 @@
+{
+ "openaireId":"dedup_wf_001::ab42e8116f4ae3c4021fb7c643a8167a",
+ "originalId": "ORIG_ID_TEST",
+ "type": "publication",
+ "title": "TEST TITLE",
+ "authors": [
+ "Michele Artini",
+ "Claudio Atzori",
+ "Alessia Bardi"
+ ],
+ "publisher": "Test publisher",
+ "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",
+ "language": "ita",
+ "pids": [
+ {
+ "type": "doi",
+ "value": "10.000/xyz-123"
+ },
+ {
+ "type": "oai",
+ "value": "oai:1234"
+ }
+ ],
+ "licenseCode": "EMBARGO",
+ "embargoEndDate": "2018-02-02",
+ "resourceType": "0001",
+ "url": "http://test.it/xyz",
+ "collectedFromId": "opendoar____::2659",
+ "hostedById": "opendoar____::2659",
+ "linksToProjects": [
+ "info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",
+ "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"
+ ],
+ "contexts": [
+ "egi::classification::natsc::math::pure",
+ "egi::classification::natsc::math::stats"
+ ]
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/GarbageSetsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/GarbageSetsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/actions/GarbageSetsJobNode.java (revision 58540)
@@ -0,0 +1,28 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.actions;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.actionmanager.rmi.ActionManagerService;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class GarbageSetsJobNode extends BlackboardJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(GarbageSetsJobNode.class);
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ return getServiceLocator().getServiceId(ActionManagerService.class);
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ log.info("preparing garbage collection for Action sets Job");
+ job.setAction("GARBAGE");
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record.json
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record.json (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record.json (revision 58540)
@@ -0,0 +1,36 @@
+{
+ "originalId": "ORIG_ID_12345",
+ "title": "TEST TITLE",
+ "authors": [
+ "Michele Artini",
+ "Claudio Atzori",
+ "Alessia Bardi"
+ ],
+ "publisher": "Test publisher",
+ "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",
+ "language": "ita",
+ "pids": [
+ {
+ "type": "doi",
+ "value": "10.000/xyz-123"
+ },
+ {
+ "type": "oai",
+ "value": "oai:1234"
+ }
+ ],
+ "licenseCode": "OPEN",
+ "resourceType": "0001",
+ "url": "http://test.it/xyz",
+ "collectedFromId": "opendoar____::2659",
+ "hostedById": "opendoar____::2367",
+ "linksToProjects": [
+ "info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",
+ "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"
+ ],
+ "contexts": [
+ "egi::classification::natsc::math::pure",
+ "egi::classification::natsc::math::stats"
+ ]
+}
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repohi/RetrieveInterfaceInfoJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repohi/RetrieveInterfaceInfoJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repohi/RetrieveInterfaceInfoJobNode.java (revision 58540)
@@ -0,0 +1,41 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.repohi;
+
+import java.io.StringReader;
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+import org.dom4j.Document;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+
+public class RetrieveInterfaceInfoJobNode extends SimpleJobNode {
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ String datasourceId = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ID);
+ String interfaceID = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE);
+ final String profile = serviceLocator.getService(ISLookUpService.class).getResourceProfile(datasourceId);
+ final Document doc = new SAXReader().read(new StringReader(profile));
+ final Node ifcNode = doc.selectSingleNode("//INTERFACE[@id='" + interfaceID + "']");
+
+ String contentDescription = ifcNode.valueOf("./@contentDescription");
+ token.getEnv().setAttribute("objectStoreContentDescription", contentDescription);
+
+ final Node acProtNode = doc.selectSingleNode("//INTERFACE[@id='" + interfaceID + "']/ACCESS_PROTOCOL");
+
+ String basePath = acProtNode.valueOf("./@basePath");
+ token.getEnv().setAttribute("objectStoreBasePath", basePath);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/deploy.info
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/deploy.info (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/deploy.info (revision 58540)
@@ -0,0 +1 @@
+{"type_source": "SVN", "goal": "package -U source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaireplus-workflows/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-openaireplus-workflows"}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimUpdatesJobNodeTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimUpdatesJobNodeTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/claims/ApplyClaimUpdatesJobNodeTest.java (revision 58540)
@@ -0,0 +1,39 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.claims;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import eu.dnetlib.data.proto.OafProtos.Oaf;
+import eu.dnetlib.msro.rmi.MSROException;
+import org.junit.Test;
+import org.postgresql.util.Base64;
+
+/**
+ * Created by Alessia Bardi on 26/06/2017.
+ *
+ * @author Alessia Bardi
+ */
+public class ApplyClaimUpdatesJobNodeTest {
+
+ private ApplyClaimUpdatesJobNode applyClaims = new ApplyClaimUpdatesJobNode();
+
+ @Test
+ public void testGetValueEGI() throws MSROException, InvalidProtocolBufferException {
+ String context = "egi";
+ String rowkey = "50|userclaim___::6a8f649d968e734a6733c23a351c1859";
+ System.out.println(getValue(context, rowkey));
+ }
+
+ @Test
+ public void testGetValueEGILongId() throws MSROException, InvalidProtocolBufferException {
+ String context = "egi::classification::natsc::earths::other";
+ String rowkey = "50|userclaim___::6a8f649d968e734a6733c23a351c1859";
+ System.out.println(getValue(context, rowkey));
+ }
+
+ private String getValue(final String context, final String rowkey) throws MSROException, InvalidProtocolBufferException {
+ long time = System.currentTimeMillis();
+ //final String sourceId, final String semantics, final String targetId, final long timestamp
+ String res = applyClaims.getValue(rowkey, context, time);
+ Oaf.Builder builder = Oaf.newBuilder().mergeFrom(Base64.decode(res));
+ return builder.build().toString();
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_with_greek_chars.json
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_with_greek_chars.json (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_with_greek_chars.json (revision 58540)
@@ -0,0 +1,29 @@
+{
+ "originalId": "ORIG_ID_12345",
+ "title": "TEST TITLE WITH Greek Characters",
+ "authors": [
+ "αβγδεζηθικλμ νξοπρσςτυφχψω",
+ "ΑΒΓΔΕΖΗΘΙΚΛ ΜΝΞΟΠΡΣΤΥΦΧΨΩ"
+ ],
+ "publisher": "âââââââ",
+ "description": "Αν περιμένατε να βρίσκεται εδώ μια σελίδα και δεν υπάρχει, η σελίδα μπορεί να μην εμφανίζεται λόγω καθυστέρησης στην ανανέωση της βάσης δεδομένων, ή μπορεί να έχει διαγραφεί. (Δείτε την γρήγορη διαγραφή σελίδων για πιθανούς λόγους). Μπορείτε να δοκιμάστε την λειτουργία εκκαθάρισης, και να ελέγξετε το αρχείο διαγραφών.",
+ "language": "ell",
+ "pids": [
+ {
+ "type": "doi",
+ "value": "10.000/xyz-123-gr"
+ }
+ ],
+ "licenseCode": "EMBARGO",
+ "embargoEndDate": "2018-02-02",
+ "resourceType": "0001",
+ "url": "http://test.it/xyz",
+ "collectedFromId": "opendoar____::2659",
+ "hostedById": "opendoar____::2367",
+ "linksToProjects": [
+ "info:eu-repo/grantAgreement/EC/FP7/123456/EU//Test"
+ ],
+ "contexts": [
+ "egi::classification::natsc::math::stats"
+ ]
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/SplitterDatasetsIterator.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/SplitterDatasetsIterator.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/datacite/SplitterDatasetsIterator.java (revision 58540)
@@ -0,0 +1,145 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.datacite;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathFactory;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.w3c.dom.*;
+
+/**
+ * The Class SplitterDatasetsIterator.
+ */
+public class SplitterDatasetsIterator {
+
+ /**
+ * The Constant log.
+ */
+ private static final Log log = LogFactory.getLog(SplitterDatasetsIterator.class);
+
+ /**
+ * The end queue.
+ */
+ public static String END_QUEUE = "END_QUEUE";
+
+ /**
+ * The publications.
+ */
+ private Queue publications;
+
+ /**
+ * The input epr.
+ */
+ private Iterable inputEPR;
+
+ /**
+ * The root name.
+ */
+ private String rootName;
+
+ /**
+ * Instantiates a new splitter datasets iterator.
+ *
+ * @param publicationsQueue the publications queue
+ * @param inputEPR the input epr
+ * @param rootName the root name
+ */
+ public SplitterDatasetsIterator(final Queue publicationsQueue, final Iterable inputEPR, final String rootName) {
+ this.publications = publicationsQueue;
+ this.inputEPR = inputEPR;
+ this.rootName = rootName;
+ }
+
+ /**
+ * Populate queues.
+ */
+ public void populateQueues() {
+
+ if (this.inputEPR == null) return;
+ for (String inputXML : inputEPR) {
+
+ final ByteArrayInputStream bais = new ByteArrayInputStream(inputXML.getBytes(StandardCharsets.UTF_8));
+ final List publication_extracted = extractByTag(bais, "publication");
+
+ if (publication_extracted != null) {
+ publications.addAll(publication_extracted);
+
+ }
+ }
+ publications.add(END_QUEUE);
+ }
+
+ /**
+ * Extract by tag.
+ *
+ * @param inputXML the input xml
+ * @param tag the tag
+ * @return the list
+ */
+ private List extractByTag(final InputStream inputXML, final String tag) {
+ try {
+
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ Document doc = dbf.newDocumentBuilder().parse(inputXML);
+
+ XPath xpath = XPathFactory.newInstance().newXPath();
+
+ Node rootNode = (Node) xpath.evaluate("//*[local-name()='" + this.rootName + "']", doc, XPathConstants.NODE);
+
+ NamedNodeMap attributes = rootNode.getAttributes();
+
+ Map nameSpaces = Maps.newHashMap();
+
+ for (int i = 0; i < attributes.getLength(); i++) {
+ Node node = attributes.item(i);
+ String name = node.getNodeName();
+ if (name.startsWith("xmlns:")) {
+ nameSpaces.put(StringUtils.substringAfter(name, "xmlns:"), node.getNodeValue());
+ }
+
+ }
+ xpath = XPathFactory.newInstance().newXPath();
+ NodeList nodes = (NodeList) xpath.evaluate("//*[local-name()='" + tag + "']/*[local-name()='record']", doc, XPathConstants.NODESET);
+
+ if ((nodes != null) && (nodes.getLength() > 0)) {
+ List result = new ArrayList<>();
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Document currentDoc = dbf.newDocumentBuilder().newDocument();
+ Node imported = currentDoc.importNode(nodes.item(i), true);
+ for (String key : nameSpaces.keySet()) {
+ Element element = (Element) imported;
+ element.setAttributeNS(XMLConstants.XMLNS_ATTRIBUTE_NS_URI, "xmlns:" + key, nameSpaces.get(key));
+ }
+ Transformer transformer = TransformerFactory.newInstance().newTransformer();
+ DOMSource mydoc = new DOMSource(imported);
+ StringWriter writer = new StringWriter();
+ transformer.transform(mydoc, new StreamResult(writer));
+ String record = writer.toString();
+ result.add(record);
+ }
+ return result;
+ }
+ } catch (Exception e) {
+ log.error("Error on extracting " + tag, e);
+ return null;
+ }
+ return null;
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/SetHdfsFileJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/SetHdfsFileJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/SetHdfsFileJobNode.java (revision 58540)
@@ -0,0 +1,60 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class SetHdfsFileJobNode.
+ */
+public class SetHdfsFileJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(SetHdfsFileJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ /**
+ * The input records path.
+ */
+ private String hdfsPath;
+
+ /**
+ * The input records path param.
+ */
+ private String hdfsPathParam;
+
+ /**
+ * {@inheritDoc}
+ *
+ * @see com.googlecode.sarasvati.mem.MemNode#execute(com.googlecode.sarasvati.Engine, com.googlecode.sarasvati.NodeToken)
+ */
+ @Override
+ public String execute(final NodeToken token) {
+
+ log.info("hdfs path: " + getHdfsPath());
+
+ token.getEnv().setAttribute(getHdfsPathParam(), getHdfsPath());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ public String getHdfsPath() {
+ return hdfsPath;
+ }
+
+ public void setHdfsPath(final String hdfsPath) {
+ this.hdfsPath = hdfsPath;
+ }
+
+ public String getHdfsPathParam() {
+ return hdfsPathParam;
+ }
+
+ public void setHdfsPathParam(final String hdfsPathParam) {
+ this.hdfsPathParam = hdfsPathParam;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareLodParamJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareLodParamJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareLodParamJobNode.java (revision 58540)
@@ -0,0 +1,84 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import com.mongodb.DBObject;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Sorts;
+import eu.dnetlib.common.logging.DnetLogger;
+import eu.dnetlib.common.logging.dao.DnetLoggerMongoDao;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+public class PrepareLodParamJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(PrepareLodParamJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+ private static final String SYSTEM_PROFILE_ID = "system:profileId";
+ private static final String LOG_DATE = "log:date";
+ private static final String IS_COMPLETED_SUCCESSFULLY = "system:isCompletedSuccessfully";
+
+ @Value("${dnet.openaire.lod.jsonrels}")
+ private String lodJsonRels;
+
+ @Value("${dnet.openaire.lod.wfId}")
+ private String lodWfId;
+
+ @Value("${dnet.openaire.lod.default.date}")
+ private String defaultLodDate;
+
+ @Autowired
+ private DnetLogger dnetLogger;
+
+ @Autowired
+ private DnetLoggerMongoDao mongoDao;
+
+ private final SimpleDateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ log.info("prepare LOD update job");
+
+ token.getEnv().setAttribute("lod_jsonRels", lodJsonRels);
+ token.getEnv().setAttribute("lod_lastExecutionDate", getFormattedDate());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ /**
+ * Queries the MSRO logger to find the last time the LOD update workflow was successfully executed.
+ *
+ * @return the formatted date
+ */
+ private String getFormattedDate() {
+ final DBObject dbo = getLoggerDao()
+ .find(Filters.and(Filters.eq(SYSTEM_PROFILE_ID, lodWfId), Filters.eq(IS_COMPLETED_SUCCESSFULLY, "true")))
+ .sort(Sorts.descending(LOG_DATE))
+ .limit(1)
+ .first();
+
+ if (dbo == null || !dbo.containsField(LOG_DATE)) {
+ log.warn(String.format("Cannot find log entry for LOD update workflow, using default date: '%s'", defaultLodDate));
+ return defaultLodDate;
+ }
+
+ final String lastExecutionDate = dateFormat.format(new Date((Long) dbo.get(LOG_DATE)));
+ log.debug(String.format("Last LOD execution date: '%s'", lastExecutionDate));
+ return lastExecutionDate;
+ }
+
+ private MongoCollection getLoggerDao() {
+ return mongoDao.getDb().getCollection(dnetLogger.getName(), DBObject.class);
+ }
+
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareStatsParamJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareStatsParamJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareStatsParamJobNode.java (revision 58540)
@@ -0,0 +1,54 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.data.hadoop.config.ClusterName;
+import eu.dnetlib.data.hadoop.config.ConfigurationEnumerator;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+public class PrepareStatsParamJobNode extends SimpleJobNode {
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(PrepareStatsParamJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ private static final String xqueryIsLookupEndpoint =
+ "//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IS_LookUpServiceResourceType']//PROTOCOL[./@name='SOAP']/@address/string()";
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Resource
+ protected ConfigurationEnumerator configurationEnumerator;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ log.info("prepare stats update job");
+
+ Configuration conf = configurationEnumerator.get(ClusterName.DM);
+ String nameNode = conf.get("fs.defaultFS");
+
+ token.getEnv().setAttribute("nameNode", nameNode);
+ token.getEnv().setAttribute("jobTracker", conf.get("mapred.job.tracker"));
+
+ token.getEnv().setAttribute("isLookupEndpoint", getIsLookupEndpoint());
+
+ return Arc.DEFAULT_ARC;
+ }
+
+ private String getIsLookupEndpoint() throws ISLookUpDocumentNotFoundException, ISLookUpException {
+ return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xqueryIsLookupEndpoint);
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/utils/ontologies/OntologyLoaderTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/utils/ontologies/OntologyLoaderTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/utils/ontologies/OntologyLoaderTest.java (revision 58540)
@@ -0,0 +1,78 @@
+package eu.dnetlib.utils.ontologies;
+
+import com.google.gson.Gson;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.junit.Assert.*;
+
+/**
+ * Created by claudio on 12/12/2016.
+ */
+public class OntologyLoaderTest {
+
+ private String basePath = "/eu/dnetlib/test/profiles/OntologyDSResources/OntologyDSResourceType/";
+
+ @Test
+ public void testLoadOntologyFromCp() {
+
+ final InputStream i = getClass().getResourceAsStream(basePath + "result_result_relations.xml");
+
+ Ontology o = OntologyLoader.loadOntologyFromCp(i);
+ checkOntology(o);
+
+ String supplement = o.inverseOf("isSupplementedBy");
+ assertEquals(supplement, "isSupplementTo");
+
+ String part = o.inverseOf("isPartOf");
+ assertEquals(part, "hasPart");
+ }
+
+ @Test
+ public void testLoadOntologiesFromCp() throws IOException {
+
+ OntologyLoader.loadOntologiesFromCp().values().forEach(o -> checkOntology(o));
+ }
+
+ @Test
+ public void testLoadOntologiesSerialization() throws IOException {
+
+ final Ontologies o = OntologyLoader.loadOntologiesFromCp();
+ assertNotNull(o);
+
+ final String json = o.toJson(true);
+
+ assertFalse(StringUtils.isBlank(json));
+ assertFalse("{}".equals(json.trim()));
+
+ //System.out.println(json);
+
+ assertTrue(StringUtils.isNoneBlank(json));
+
+ final Ontologies o1 = new Gson().fromJson(json, Ontologies.class);
+
+ assertNotNull(o1);
+
+ o1.entrySet().forEach(e -> checkOntology(e.getValue()));
+ }
+
+ private void checkOntology(Ontology o) {
+ Assert.assertNotNull(o);
+ Assert.assertTrue(StringUtils.isNotBlank(o.getCode()));
+ Assert.assertTrue(StringUtils.isNotBlank(o.getDescription()));
+ Assert.assertNotNull(o.getTerms().values());
+
+ o.getTerms().values().forEach(it -> {
+ Assert.assertTrue(StringUtils.isNotBlank(it.getCode()));
+ Assert.assertTrue(StringUtils.isNotBlank(it.getEncoding()));
+ Assert.assertTrue(StringUtils.isNotBlank(it.getEnglishName()));
+ Assert.assertTrue(StringUtils.isNotBlank(it.getNativeName()));
+ Assert.assertTrue(StringUtils.isNotBlank(it.getInverseCode()));
+ Assert.assertNotNull(o.getTerms().get(it.getInverseCode()));
+ });
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/ArxivIteratorRepository.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/ArxivIteratorRepository.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/objectStore/ArxivIteratorRepository.java (revision 58540)
@@ -0,0 +1,51 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.objectStore;
+
+import java.io.File;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
+
+public class ArxivIteratorRepository extends IteratorRepository {
+
+ private Pattern pattern;
+
+ public ArxivIteratorRepository(final Iterable input, final String repositoryPrefix, final String oaiPrefix) {
+ super(input, repositoryPrefix, oaiPrefix);
+ pattern = Pattern.compile("[0-9]");
+
+ }
+
+ @Override
+ String generateNextElement() {
+ try {
+ String inputname = input.next().getCanonicalPath();
+ String[] values = inputname.split("/");
+ if (values.length > 1) {
+ ObjectStoreFile info = new ObjectStoreFile();
+ String fileName = values[values.length - 1].replace(".pdf", "");
+ Matcher matcher = pattern.matcher(fileName);
+ if (matcher.find()) {
+ if (matcher.start() != 0) {
+ fileName = fileName.substring(0, matcher.start()) + "/" + fileName.substring(matcher.start());
+ }
+ }
+ info.setDownloadedURL("http://arxiv.org/abs/" + fileName);
+ info.setAccessProtocol(Protocols.None);
+ // String value = "oai:arXiv.org:" + fileName;
+ String value = oaiPrefix + fileName;
+ String resultID = repositoryPrefix + "::" + DnetXsltFunctions.md5(value);
+ info.setObjectID(resultID + "::" + DnetXsltFunctions.md5(info.getDownloadedURL()));
+ info.setMimeType("pdf");
+ info.setURI(inputname);
+ return info.toJSON();
+ }
+ } catch (Exception e) {
+ return null;
+ }
+ return null;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/LoadContextsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/LoadContextsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/LoadContextsJobNode.java (revision 58540)
@@ -0,0 +1,39 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.msro.openaireplus.workflows.nodes.index.ContextLoader;
+import eu.dnetlib.msro.rmi.MSROException;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+public class LoadContextsJobNode extends SimpleJobNode {
+
+ public static final String CONTEXTMAP = "contextmap";
+
+ /**
+ * logger.
+ */
+ private static final Log log = LogFactory.getLog(LoadContextsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Autowired
+ private ContextLoader contextLoader;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+
+ log.info("start preparing job");
+
+ String contexts = contextLoader.load();
+
+ if (StringUtils.isBlank(contexts)) throw new MSROException("empty contexts");
+
+ token.getEnv().setAttribute(CONTEXTMAP, contexts);
+
+ return Arc.DEFAULT_ARC;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/ValidateShadowStatsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/ValidateShadowStatsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/ValidateShadowStatsJobNode.java (revision 58540)
@@ -0,0 +1,35 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to perform the content validation between the shadow stats db and the preproduction index (using the
+ * prepublic search service) and creates a report, which is serialized in the blackboard job parameters.
+ *
+ * The report is relative to the shadow stats db related to the portal specified by the additional BB parameter
+ * StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName()
+ *
+ *
+ * @author alessia
+ */
+public class ValidateShadowStatsJobNode extends AbstractStatsJobNode {
+
+ private static final Log log = LogFactory.getLog(ValidateShadowStatsJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.action());
+ String portal = getPortalName(token);
+ if (StringUtils.isNotBlank(portal)) {
+ job.getParameters().put(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName(), portal);
+ } else {
+ log.warn(StatsManagerServiceBBAction.VALIDATE_SHADOW_STATS.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+ }
+
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p1.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p1.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p1.xml (revision 58540)
@@ -0,0 +1,576 @@
+
+
+
+ corda__h2020::636a2309a19eed30ce928d92f058b268
+ 768977
+ 2017-09-29T10:03:47.191Z
+ abba4d10-e30b-4f60-b9ad-689bdfe298c9_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::768977
+ corda__h2020::768977
+ 768977
+ CHIST-ERA III
+ European coordinated research on long-term ICT and ICT-based scientific challenges
+ 01-DEC-2017
+ 30-NOV-2022
+ FETPROACT-2017
+ true
+ Yes
+ openaire____::corda_h2020
+ ERA-NET-Cofund
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETPROACT-02-2017
+ FET ERANET Cofund
+
+
+ corda__h2020::768977::Edouard_GEOFFROIS
+ corda__h2020::768977::Edouard_GEOFFROIS
+ Edouard
+ GEOFFROIS
+ edouard.geoffrois@agencerecherche.fr
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::927033172
+ corda__h2020::927033172
+ TA CR
+ TECHNOLOGICKA AGENTURA CESKE REPUBLIKY
+ http://www.tacr.cz/index.php/en/
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ Yes
+ CZ
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::927033172
+ 6
+ corda__h2020::768977
+ corda__h2020::927033172
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::954020706
+ corda__h2020::954020706
+ MINECO
+ MINISTERIO DE ECONOMIA, INDUSTRIA Y COMPETITIVIDAD
+ http://www.mineco.gob.es
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ Missing
+ ES
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::954020706
+ 8
+ corda__h2020::768977
+ corda__h2020::954020706
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::960264402
+ corda__h2020::960264402
+ NCN
+ NARODOWE CENTRUM NAUKI
+ http://www.ncn.gov.pl
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ PL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::960264402
+ 13
+ corda__h2020::768977
+ corda__h2020::960264402
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::972130024
+ corda__h2020::972130024
+ UEFISCDI
+ Unitatea Executiva pentru Finantarea Invatamantului Superior, a Cercetarii, Dezvoltarii si Inovarii
+ http://www.uefiscdi.gov.ro
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ RO
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::972130024
+ 14
+ corda__h2020::768977
+ corda__h2020::972130024
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::984222335
+ corda__h2020::984222335
+ BNSF
+ BULGARIAN NATIONAL SCIENCE FUND
+ http://www.fni.bg
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ BG
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::984222335
+ 4
+ corda__h2020::768977
+ corda__h2020::984222335
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::997808446
+ corda__h2020::997808446
+ LMT
+ Lietuvos mokslo taryba
+ http://www.lmt.lt/
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ Missing
+ LT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::997808446
+ 12
+ corda__h2020::768977
+ corda__h2020::997808446
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::998483760
+ corda__h2020::998483760
+ ETAg
+ SIHTASUTUS EESTI TEADUSAGENTUUR
+ http://www.etag.ee
+ No
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ EE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::998483760
+ 7
+ corda__h2020::768977
+ corda__h2020::998483760
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::998711613
+ corda__h2020::998711613
+ ANR
+ AGENCE NATIONALE DE LA RECHERCHE
+ http://www.agence-nationale-recherche.fr
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ Missing
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::998711613
+ 1
+ corda__h2020::768977
+ corda__h2020::998711613
+ corda__h2020::768977::Edouard_GEOFFROIS
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::998735960
+ corda__h2020::998735960
+ FWF
+ FONDS ZUR FÖRDERUNG DER WISSENSCHAFTLICHEN FORSCHUNG
+ http://www.fwf.ac.at
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ AT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::998735960
+ 2
+ corda__h2020::768977
+ corda__h2020::998735960
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999517586
+ corda__h2020::999517586
+ AKA
+ SUOMEN AKATEMIA
+ http://www.aka.fi
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ FI
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::999517586
+ 9
+ corda__h2020::768977
+ corda__h2020::999517586
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999530390
+ corda__h2020::999530390
+ SAV
+ SLOVENSKA AKADEMIA VIED
+ http://www.sav.sk
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ SK
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::999530390
+ 15
+ corda__h2020::768977
+ corda__h2020::999530390
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999540381
+ corda__h2020::999540381
+ FNRS
+ FONDS NATIONAL DE LA RECHERCHE SCIENTIFIQUE
+
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ Missing
+ BE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::999540381
+ 3
+ corda__h2020::768977
+ corda__h2020::999540381
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999540575
+ corda__h2020::999540575
+ SNSF
+ SCHWEIZERISCHER NATIONALFONDS ZUR FORDERUNG DER WISSENSCHAFTLICHEN FORSCHUNG
+ http://www.snf.ch
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ CH
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::999540575
+ 5
+ corda__h2020::768977
+ corda__h2020::999540575
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999563370
+ corda__h2020::999563370
+ HEA
+ AN TUDARAS UM ARD OIDEACHAS
+ http://www.hea.ie
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ Missing
+ IE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::999563370
+ 10
+ corda__h2020::768977
+ corda__h2020::999563370
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999587135
+ corda__h2020::999587135
+ TUBITAK
+ TURKIYE BILIMSEL VE TEKNOLOJIK ARASTIRMA KURUMU
+ http://www.tubitak.gov.tr
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ TR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::999587135
+ 16
+ corda__h2020::768977
+ corda__h2020::999587135
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999637963
+ corda__h2020::999637963
+ EPSRC
+ THE ENGINEERING AND PHYSICAL SCIENCES RESEARCH COUNCIL
+ http://www.epsrc.ac.uk
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ Missing
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::999637963
+ 17
+ corda__h2020::768977
+ corda__h2020::999637963
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999641358
+ corda__h2020::999641358
+ MIUR
+ MINISTERO DELL'ISTRUZIONE, DELL'UNIVERSITA' E DELLA RICERCA
+ http://www.istruzione.it
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::768977@@corda__h2020::999641358
+ 11
+ corda__h2020::768977
+ corda__h2020::999641358
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETPROACT-02-2017
+ corda__h2020::FETPROACT-02-2017
+ FET ERANET Cofund
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::768977@@corda__h2020::FETPROACT-02-2017
+ corda__h2020::768977
+ corda__h2020::FETPROACT-02-2017
+
+
+ ec__________::EC::H2020::ERA-NET-Cofund@@corda__h2020::768977
+ ec__________::EC::H2020::ERA-NET-Cofund
+ corda__h2020::768977
+ 01-DEC-2017
+ 30-NOV-2022
+ ERA-NET-Cofund
+ ec:h2020toas
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/BackupStatsCacheJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/BackupStatsCacheJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/BackupStatsCacheJobNode.java (revision 58540)
@@ -0,0 +1,31 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Ask the StatsManagerService to backup the stats cache of the portal specified by the additional BB parameter
+ * StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName()
. The cache can be restored with the last generated backup by
+ * sending a "restore" BB message.
+ *
+ * @author alessia
+ * @see RestoreStatsCacheJobNode
+ */
+public class BackupStatsCacheJobNode extends AbstractStatsJobNode {
+
+ private static final Log log = LogFactory.getLog(BackupStatsCacheJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+ job.setAction(StatsManagerServiceBBAction.BACKUP_CACHE.action());
+ String portal = getPortalName(token);
+ if (StringUtils.isNotBlank(portal)) {
+ job.getParameters().put(StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName(), portal);
+ } else {
+ log.warn(StatsManagerServiceBBAction.BACKUP_CACHE.getTargetPortalParamName() + " not set. The StatsManagerService will use its default.");
+ }
+ }
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/StatsManagerServiceBBAction.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/StatsManagerServiceBBAction.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/stats/StatsManagerServiceBBAction.java (revision 58540)
@@ -0,0 +1,68 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.stats;
+
+public enum StatsManagerServiceBBAction {
+ PROMOTE_SHADOW_STATS {
+ @Override
+ public String action() {
+ return "promoteShadow";
+ }
+ },
+ PROMOTE_SHADOW_CACHE {
+ @Override
+ public String action() {
+ return "promoteCache";
+ }
+ },
+ REFRESH_SHADOW_CACHE {
+ @Override
+ public String action() {
+ return "refreshCache";
+ }
+
+ },
+ VALIDATE_SHADOW_STATS {
+ @Override
+ public String action() {
+ return "validate";
+ }
+
+ },
+ BACKUP_CACHE {
+ @Override
+ public String action() {
+ return "backup";
+ }
+
+ },
+ RESTORE_CACHE {
+ @Override
+ public String action() {
+ return "restore";
+ }
+
+ },
+ MIGRATE_CACHE {
+ @Override
+ public String action() {
+ return "migrate";
+ }
+
+ @Override
+ public String getTargetPortalParamName() {
+ return "targetCache";
+ }
+
+ };
+
+ public abstract String action();
+
+ public String getSourcePortalParamName() {
+ return DEFAULT_PORTAL_PARAM_NAME;
+ }
+
+ public String getTargetPortalParamName() {
+ return DEFAULT_PORTAL_PARAM_NAME;
+ }
+
+ private static String DEFAULT_PORTAL_PARAM_NAME = "cache";
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p5.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p5.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p5.xml (revision 58540)
@@ -0,0 +1,248 @@
+
+
+
+ corda__h2020::0fa87a85cbbfb14f5cc2d4330c2b1cad
+ 736876
+ 2017-03-01T11:24:05.589+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::736876
+ corda__h2020::736876
+ 736876
+ VISORSURF
+ VisorSurf - A Hardware Platform for Software-driven Functional Metasurfaces
+ 01-JAN-2017
+ 30-JUN-2020
+ H2020-FETOPEN-1-2016-2017
+ true
+ No
+ openaire____::corda_h2020
+ RIA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-01-2016-2017
+ FET-Open research and innovation actions
+
+
+ corda__h2020::736876::Costas_SOUKOULIS
+ corda__h2020::736876::Costas_SOUKOULIS
+ Costas
+ SOUKOULIS
+ soukouli@iesl.forth.gr
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::991256096
+ corda__h2020::991256096
+ AALTO
+ AALTO-KORKEAKOULUSAATIO
+ http://www.aalto.fi
+ No
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ FI
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::991256096
+ 6
+ corda__h2020::736876
+ corda__h2020::991256096
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999542903
+ corda__h2020::999542903
+ SG
+ SIGNALGENERIX LTD
+ http://www.signalgenerix.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ CY
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999542903
+ 4
+ corda__h2020::736876
+ corda__h2020::999542903
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999835843
+ corda__h2020::999835843
+ UCY
+ UNIVERSITY OF CYPRUS
+ http://www.ucy.ac.cy
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ CY
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999835843
+ 2
+ corda__h2020::736876
+ corda__h2020::999835843
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999976202
+ corda__h2020::999976202
+ UPC
+ UNIVERSITAT POLITECNICA DE CATALUNYA
+ http://www.upc.edu
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ ES
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999976202
+ 5
+ corda__h2020::736876
+ corda__h2020::999976202
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999984059
+ corda__h2020::999984059
+ Fraunhofer
+ FRAUNHOFER GESELLSCHAFT ZUR FOERDERUNG DER ANGEWANDTEN FORSCHUNG E.V.
+ http://www.fraunhofer.de
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999984059
+ 3
+ corda__h2020::736876
+ corda__h2020::999984059
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999995893
+ corda__h2020::999995893
+ FORTH
+ FOUNDATION FOR RESEARCH AND TECHNOLOGY HELLAS
+ http://www.forth.gr
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ GR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::736876@@corda__h2020::999995893
+ 1
+ corda__h2020::736876
+ corda__h2020::999995893
+ corda__h2020::736876::Costas_SOUKOULIS
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-01-2016-2017
+ corda__h2020::FETOPEN-01-2016-2017
+ FET-Open research and innovation actions
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::736876@@corda__h2020::FETOPEN-01-2016-2017
+ corda__h2020::736876
+ corda__h2020::FETOPEN-01-2016-2017
+
+
+ ec__________::EC::H2020::RIA@@corda__h2020::736876
+ ec__________::EC::H2020::RIA
+ corda__h2020::736876
+ 01-JAN-2017
+ 30-JUN-2020
+ RIA
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/UpdateVocabularyEntriesJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/UpdateVocabularyEntriesJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/UpdateVocabularyEntriesJobNode.java (revision 58540)
@@ -0,0 +1,79 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import java.util.stream.Collectors;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.database.rmi.DatabaseService;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.IterableResultSetFactory;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
+import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
+import org.apache.commons.io.IOUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Required;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+public class UpdateVocabularyEntriesJobNode extends BlackboardJobNode {
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Autowired
+ private IterableResultSetFactory resultSetFactory;
+
+ private Resource xquery;
+
+ private String xsltClasspath;
+
+ private String db;
+
+ @Override
+ protected String obtainServiceId(final NodeToken token) {
+ return getServiceLocator().getServiceId(DatabaseService.class);
+ }
+
+ @Override
+ protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
+
+ final ApplyXslt t = new ApplyXslt(new ClassPathResource(xsltClasspath));
+
+ final W3CEndpointReference epr =
+ resultSetFactory.createIterableResultSet(
+ serviceLocator.getService(ISLookUpService.class).quickSearchProfile(IOUtils.toString(getXquery().getInputStream()))
+ .stream().map(t::evaluate).collect(Collectors.toList()));
+
+ job.setAction("IMPORT");
+ job.getParameters().put("db", db);
+ job.getParameters().put("epr", epr.toString());
+ }
+
+ public Resource getXquery() {
+ return xquery;
+ }
+
+ @Required
+ public void setXquery(final Resource xquery) {
+ this.xquery = xquery;
+ }
+
+ public String getXsltClasspath() {
+ return xsltClasspath;
+ }
+
+ public void setXsltClasspath(final String xsltClasspath) {
+ this.xsltClasspath = xsltClasspath;
+ }
+
+ public String getDb() {
+ return db;
+ }
+
+ public void setDb(final String db) {
+ this.db = db;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p9.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p9.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p9.xml (revision 58540)
@@ -0,0 +1,337 @@
+
+
+
+ corda__h2020::219eead6ff0077cb500106ded3ff7fcb
+ 686731
+ 2017-03-01T11:01:35.286+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::686731
+ corda__h2020::686731
+ 686731
+ SUPERTWIN
+ All Solid-State Super-Twinning Photon Microscope
+ 01-MAR-2016
+ 28-FEB-2019
+ H2020-FETOPEN-2014-2015-RIA
+ true
+ Yes
+ openaire____::corda_h2020
+ RIA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-RIA-2014-2015
+ FET-Open research projects
+
+
+ corda__h2020::686731::David_STOPPA
+ corda__h2020::686731::David_STOPPA
+ David
+ STOPPA
+ stoppa@fbk.eu
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::952944491
+ corda__h2020::952944491
+ Single Quantum
+ SINGLE QUANTUM BV
+ http://www.singlequantum.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ NL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::952944491
+ 5
+ corda__h2020::686731
+ corda__h2020::952944491
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::965904273
+ corda__h2020::965904273
+ LFOUNDRY SRL
+ LFOUNDRY SRL
+
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::965904273
+ 9
+ corda__h2020::686731
+ corda__h2020::965904273
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::993674015
+ corda__h2020::993674015
+ IPNASB
+ B.I. Stepanov Institute of Physics of the National Academy of Sciences of Belarus
+
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ Missing
+ BY
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::993674015
+ 8
+ corda__h2020::686731
+ corda__h2020::993674015
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::998354362
+ corda__h2020::998354362
+ APE
+ A.P.E. RESEARCH SRL
+ http://www.aperesearch.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::998354362
+ 2
+ corda__h2020::686731
+ corda__h2020::998354362
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999625450
+ corda__h2020::999625450
+ FBK
+ FONDAZIONE BRUNO KESSLER
+ http://www.fbk.eu
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::999625450
+ 1
+ corda__h2020::686731
+ corda__h2020::999625450
+ corda__h2020::686731::David_STOPPA
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999958839
+ corda__h2020::999958839
+ CSEM
+ CSEM CENTRE SUISSE D'ELECTRONIQUE ET DE MICROTECHNIQUE SA - RECHERCHE ET DEVELOPPEMENT
+ http://www.csem.ch
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ CH
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::999958839
+ 3
+ corda__h2020::686731
+ corda__h2020::999958839
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999971546
+ corda__h2020::999971546
+ III V LAB GIE
+ III-V LAB
+ http://www.3-5lab.fr
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::999971546
+ 4
+ corda__h2020::686731
+ corda__h2020::999971546
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999973971
+ corda__h2020::999973971
+ EPFL
+ ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE
+ http://www.epfl.ch
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ CH
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::999973971
+ 7
+ corda__h2020::686731
+ corda__h2020::999973971
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999976493
+ corda__h2020::999976493
+ UBERN
+ UNIVERSITAET BERN
+ http://www.unibe.ch
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ Missing
+ CH
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::686731@@corda__h2020::999976493
+ 6
+ corda__h2020::686731
+ corda__h2020::999976493
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::FETOPEN-RIA-2014-2015
+ FET-Open research projects
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::686731@@corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::686731
+ corda__h2020::FETOPEN-RIA-2014-2015
+
+
+ ec__________::EC::H2020::RIA@@corda__h2020::686731
+ ec__________::EC::H2020::RIA
+ corda__h2020::686731
+ 01-MAR-2016
+ 28-FEB-2019
+ RIA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/ObtainOpenaireDataSourceParamsJobNode.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/ObtainOpenaireDataSourceParamsJobNode.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/ObtainOpenaireDataSourceParamsJobNode.java (revision 58540)
@@ -0,0 +1,57 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes;
+
+import javax.annotation.Resource;
+
+import com.googlecode.sarasvati.Arc;
+import com.googlecode.sarasvati.NodeToken;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
+import eu.dnetlib.msro.workflows.util.WorkflowsConstants;
+
+public class ObtainOpenaireDataSourceParamsJobNode extends SimpleJobNode {
+
+ private String providerId;
+
+ @Resource
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected String execute(final NodeToken token) throws Exception {
+ final String query = "let $x := /*[.//RESOURCE_IDENTIFIER/@value='" + providerId + "']//EXTRA_FIELDS\n"
+ + "return concat($x/FIELD[./key='OpenAireDataSourceId']/value, ' @@@ ', $x/FIELD[./key='NamespacePrefix']/value)";
+
+ final String[] arr = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query).split("@@@");
+
+ final String origId = arr[0].trim();
+ final String nsPrefix = arr[1].trim();
+ //this is needed by the mdbuilder
+ //TODO: update mdbuilder to use the env attributes below, whose names are defined in WorkflowConstants
+ token.getEnv().setAttribute("parentDatasourceId", origId);
+ token.getEnv().setAttribute("namespacePrefix", nsPrefix);
+ token.getEnv().setAttribute("dateOfCollection", DateUtils.now_ISO8601());
+
+ //these are needed for validation and fill hostedby
+ token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID, origId);
+ token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_NAMESPACE_PREFIX, nsPrefix);
+
+ token.getFullEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID, origId);
+ token.getFullEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_NAMESPACE_PREFIX, nsPrefix);
+
+ token.getProcess().getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID, origId);
+ token.getProcess().getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_NAMESPACE_PREFIX, nsPrefix);
+
+ return Arc.DEFAULT_ARC;
+
+ }
+
+ public String getProviderId() {
+ return providerId;
+ }
+
+ public void setProviderId(final String providerId) {
+ this.providerId = providerId;
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p12.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p12.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p12.xml (revision 58540)
@@ -0,0 +1,217 @@
+
+
+
+ corda__h2020::29d6d8c73672fc18badf81efc66fda4f
+ 665233
+ 2017-03-01T10:51:55.199+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::665233
+ corda__h2020::665233
+ 665233
+ CHROMAVISION
+ Super-resolution visualisation and manipulation of metaphase chromosomes
+ 01-JUN-2015
+ 31-MAY-2019
+ H2020-FETOPEN-2014-2015-RIA
+ true
+ Yes
+ openaire____::corda_h2020
+ RIA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-RIA-2014-2015
+ FET-Open research projects
+
+
+ corda__h2020::665233::gijs_WUITE
+ corda__h2020::665233::gijs_WUITE
+ gijs
+ WUITE
+ gwuite@gmail.com
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::941077220
+ corda__h2020::941077220
+ LUMICKS
+ LUMICKS B.V.
+ http://www.lumicks.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Missing
+ NL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665233@@corda__h2020::941077220
+ 5
+ corda__h2020::665233
+ corda__h2020::941077220
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::954530344
+ corda__h2020::954530344
+ VU/VUmc
+ STICHTING VU
+ http://www.vu.nl
+ No
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ NL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665233@@corda__h2020::954530344
+ 1
+ corda__h2020::665233
+ corda__h2020::954530344
+ corda__h2020::665233::gijs_WUITE
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999975620
+ corda__h2020::999975620
+ UNIVERSITY COLLEGE LONDON
+ UNIVERSITY COLLEGE LONDON
+ http://www.ucl.ac.uk
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ N/A
+ N/A
+ No
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665233@@corda__h2020::999975620
+ 4
+ corda__h2020::665233
+ corda__h2020::999975620
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999990655
+ corda__h2020::999990655
+ DTU
+ DANMARKS TEKNISKE UNIVERSITET
+ http://www.dtu.dk
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ DK
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665233@@corda__h2020::999990655
+ 2
+ corda__h2020::665233
+ corda__h2020::999990655
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999991043
+ corda__h2020::999991043
+ UCPH
+ KOBENHAVNS UNIVERSITET
+ http://www.ku.dk
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ DK
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665233@@corda__h2020::999991043
+ 3
+ corda__h2020::665233
+ corda__h2020::999991043
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::FETOPEN-RIA-2014-2015
+ FET-Open research projects
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::665233@@corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::665233
+ corda__h2020::FETOPEN-RIA-2014-2015
+
+
+ ec__________::EC::H2020::RIA@@corda__h2020::665233
+ ec__________::EC::H2020::RIA
+ corda__h2020::665233
+ 01-JUN-2015
+ 31-MAY-2019
+ RIA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/utils/ontologies/OntologyLoader.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/utils/ontologies/OntologyLoader.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/main/java/eu/dnetlib/utils/ontologies/OntologyLoader.java (revision 58540)
@@ -0,0 +1,147 @@
+package eu.dnetlib.utils.ontologies;
+
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Created by claudio on 12/12/2016.
+ */
+@Component
+public class OntologyLoader {
+
+ private static final Log log = LogFactory.getLog(OntologyLoader.class);
+
+ private static final String PATTERN = "classpath*:/eu/dnetlib/test/profiles/OntologyDSResources/OntologyDSResourceType/*.xml";
+
+ private static UniqueServiceLocator staticServiceLocator;
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+
+ //
+ public static String fetchInverse(final String relType) throws ISLookUpException {
+ final String xquery = "let $x:= /RESOURCE_PROFILE["
+ + " .//RESOURCE_TYPE/@value = 'OntologyDSResourceType' and "
+ + " .//TERM/@encoding='" + relType + "']"
+ + "let $y:= $x//TERM[./@encoding='" + relType + "']//RELATION[./@type='inverseOf']/@code/string() "
+ + "return $x//TERM[./@code = $y]/@encoding/string()";
+ return staticServiceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
+ }
+
+ @PostConstruct
+ public void init() {
+ OntologyLoader.staticServiceLocator = serviceLocator;
+ }
+
+ public static Ontology loadOntologyFromCp(final InputStream profile) {
+ return loadOntologyProfilefromPath(profile);
+ }
+
+ public static Ontologies loadOntologiesFromCp() throws IOException {
+ final PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
+ final List files = Arrays.asList(resolver.getResources(PATTERN)).stream().map(r -> {
+ try {
+ return r.getInputStream();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }).collect(Collectors.toList());
+
+ return createOntologies(files);
+ }
+
+ public static Ontologies createOntologies(List files) {
+ final Ontologies ontologies = new Ontologies();
+ files.stream().map(
+ s -> loadOntologyProfilefromPath(s))
+ .collect(Collectors.toMap(Ontology::getCode, Function.identity()))
+ .entrySet()
+ .forEach(it -> ontologies.put(it.getKey(), it.getValue()));
+ return ontologies;
+ }
+
+ public static Ontologies loadOntologiesFromIS() throws ISLookUpException {
+ final ISLookUpService lookUpService = staticServiceLocator.getService(ISLookUpService.class);
+ List files = lookUpService.quickSearchProfile("for $x in collection('/db/DRIVER/OntologyDSResources/OntologyDSResourceType') return $x");
+ final Ontologies ontologies = new Ontologies();
+ files.stream().map
+ (OntologyLoader::getOntologyFromProfile)
+ .collect(Collectors.toMap(Ontology::getCode, Function.identity()))
+ .entrySet()
+ .forEach(it -> ontologies.put(it.getKey(), it.getValue()));
+ return ontologies;
+ }
+
+ private static Ontology loadOntologyProfilefromPath(final InputStream profile) {
+ final String profileXML;
+ try {
+ profileXML = IOUtils.toString(profile);
+ } catch (IOException e) {
+ log.error(e);
+ return null;
+ }
+ return getOntologyFromProfile(profileXML);
+ }
+
+ private static Ontology getOntologyFromProfile(final String profile) {
+ SAXReader saxReader = new SAXReader();
+ Document doc = null;
+ StringReader reader = new StringReader(profile);
+
+ try {
+ doc = saxReader.read(reader);
+ } catch (DocumentException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ return new Ontology()
+ .setCode(doc.valueOf("//ONTOLOGY_NAME/@code"))
+ .setDescription(doc.valueOf("//ONTOLOGY_DESCRIPTION/text()"))
+ .setTerms(asTerms(doc.selectNodes("//TERMS/TERM")));
+ }
+
+ private static Map asTerms(final List list) {
+ final Map terms = Maps.newHashMap();
+ if (list != null) {
+ list.forEach(o -> {
+ final Element e = (Element) o;
+ final OntologyTerm ot = OntologyTerm.newInstance()
+ .setCode(e.attributeValue("code"))
+ .setEncoding(e.attributeValue("encoding"))
+ .setEnglishName(e.attributeValue("english_name"))
+ .setNativeName(e.attributeValue("native_name"))
+ .setInverseCode(((Node) o).valueOf("./RELATIONS/RELATION[@type = 'inverseOf']/@code"));
+
+ terms.put(ot.getCode(), ot);
+ });
+ }
+ return terms;
+ }
+
+
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedByTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedByTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedByTest.java (revision 58540)
@@ -0,0 +1,47 @@
+package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby;
+
+import java.io.IOException;
+import java.lang.annotation.Documented;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+
+import com.google.common.collect.Maps;
+import junit.framework.Assert;
+import org.apache.commons.io.IOUtils;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.io.SAXReader;
+import org.junit.Test;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+public class PatchHostedByTest {
+ private PatchHostedBy p = new PatchHostedBy(new HashMap<>(),"", new HostedByCounters());
+ private Resource xmlInput = new ClassPathResource("/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input.xml");
+ private Resource xmlInput2 = new ClassPathResource("/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input2.xml");
+ private Resource xmlInput3 = new ClassPathResource("/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input3.xml");
+
+ private final SAXReader reader = new SAXReader();
+
+ private final String xpath = "substring-after(//*[local-name()='supplementTo']/*[local-name()='source']/@id,'.')";
+
+ @Test
+ public void testEvaluate() throws DocumentException, IOException {
+ final Document doc = reader.read(xmlInput.getInputStream());
+ for (Object o : doc.selectNodes(this.xpath)) {
+ Assert.assertEquals("journal10808", o);
+ }
+ }
+
+
+ @Test
+ public void testEvaluate3() throws Exception{
+ HashMap map = Maps.newHashMap();
+ map.put("TIB.TEM", new HostedByEntry("123", "test repo"));
+ PatchHostedBy phb = new PatchHostedBy(map,"//*[local-name() = 'setSpec']", new HostedByCounters());
+ String res = phb.evaluate(IOUtils.toString(xmlInput3.getInputStream(), StandardCharsets.UTF_8));
+ Assert.assertTrue(res.contains("123"));
+
+ }
+
+}
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p23.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p23.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p23.xml (revision 58540)
@@ -0,0 +1,97 @@
+
+
+
+ corda__h2020::006a3cdb2edabcf0d82d914bb4b2ac7f
+ 678282
+ 2017-03-01T10:57:33.372+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::678282
+ corda__h2020::678282
+ 678282
+ LENA
+ non-LinEar sigNal processing for solving data challenges in Astrophysics
+ 01-SEP-2016
+ 31-AUG-2021
+ ERC-2015-STG
+ true
+ No
+ openaire____::corda_h2020
+ ERC-STG
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ ERC-StG-2015
+ ERC Starting Grant
+
+
+ corda__h2020::678282::Jerome_BOBIN
+ corda__h2020::678282::Jerome_BOBIN
+ Jerome
+ BOBIN
+ jbobin@cea.fr
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999992401
+ corda__h2020::999992401
+ CEA
+ COMMISSARIAT A L ENERGIE ATOMIQUE ET AUX ENERGIES ALTERNATIVES
+ http://www.cea.fr
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::678282@@corda__h2020::999992401
+ 1
+ corda__h2020::678282
+ corda__h2020::999992401
+ corda__h2020::678282::Jerome_BOBIN
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::ERC-StG-2015
+ corda__h2020::ERC-StG-2015
+ ERC Starting Grant
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::678282@@corda__h2020::ERC-StG-2015
+ corda__h2020::678282
+ corda__h2020::ERC-StG-2015
+
+
+ ec__________::EC::H2020::ERC::ERC-STG@@corda__h2020::678282
+ ec__________::EC::H2020::ERC::ERC-STG
+ corda__h2020::678282
+ 01-SEP-2016
+ 31-AUG-2021
+ ERC-STG
+ ec:h2020toas
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/index/OpenaireLayoutToRecordStylesheetTest.java
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/index/OpenaireLayoutToRecordStylesheetTest.java (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/java/eu/dnetlib/msro/openaireplus/workflows/index/OpenaireLayoutToRecordStylesheetTest.java (revision 58540)
@@ -0,0 +1,127 @@
+package eu.dnetlib.msro.openaireplus.workflows.index;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.common.collect.Maps;
+import eu.dnetlib.functionality.index.model.document.IndexDocument;
+import eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory;
+import eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
+import eu.dnetlib.miscutils.functional.xml.IndentXmlString;
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.common.SolrInputDocument;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+import org.junit.Test;
+import org.springframework.core.io.ClassPathResource;
+
+import javax.xml.stream.XMLStreamException;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.text.SimpleDateFormat;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+public class OpenaireLayoutToRecordStylesheetTest {
+
+ private static final String OPENAIRE_LAYOUT_TO_RECORD_STYLESHEET_XSL = "/eu/dnetlib/msro/openaireplus/workflows/index/openaireLayoutToRecordStylesheet.xsl";
+
+ private static final String MDFORMAT_FIELDS_PROFILE = "/eu/dnetlib/test/profiles/MDFormatDSResources/MDFormatDSResourceType/2-8b9503d9-8a86-4330-93ef-7e0cd9bc87c2.xml";
+
+ private static final String OAF_RECORD = "/eu/dnetlib/msro/openaireplus/workflows/index/oafRecord.xml";
+
+ private final static String[] dateFormats = { "yyyy-MM-dd'T'hh:mm:ss", "yyyy-MM-dd", "dd-MM-yyyy", "dd/MM/yyyy", "yyyy" };
+
+ @Test
+ public void testConvertRecord() throws IOException, TransformerException, XMLStreamException, DocumentException {
+ String xsl = prepareXslt("DMF");
+ assertNotNull(xsl);
+ assertFalse(xsl.isEmpty());
+
+ System.out.println(xsl);
+
+ ApplyXslt xslt = new ApplyXslt(xsl);
+
+ String indexRecord = xslt.evaluate(readFromClasspath(OAF_RECORD));
+
+ assertNotNull(indexRecord);
+ assertFalse(indexRecord.isEmpty());
+
+ System.out.println(IndentXmlString.apply(indexRecord));
+
+ StreamingInputDocumentFactory factory = new StreamingInputDocumentFactory();
+
+ SolrInputDocument doc = factory.parseDocument(DateUtils.now_ISO8601(), indexRecord, "dsId", "dnetResult");
+ assertNotNull(doc);
+ assertFalse(doc.isEmpty());
+
+ System.out.println("SolrDocument");
+
+ System.out.println(doc.values());
+
+ System.out.println("SolrDocument fields");
+
+ Map fields = Maps.newHashMap();
+
+ doc.entrySet().stream()
+ .forEach(e -> {
+ Object value = e.getValue().getValueCount() == 1 ? e.getValue().getFirstValue() : e.getValue().getValues();
+ fields.put(e.getKey(), value);
+ });
+
+ //new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(fields);
+
+ final ObjectWriter objectWriter = new ObjectMapper()
+ .setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"))
+ .writerWithDefaultPrettyPrinter();
+
+ System.out.println(objectWriter.writeValueAsString(fields));
+ }
+
+ protected String prepareXslt(final String format) throws IOException, TransformerException, DocumentException {
+
+ final Transformer layoutTransformer = TransformerFactory.newInstance().newTransformer(streamSource(OPENAIRE_LAYOUT_TO_RECORD_STYLESHEET_XSL));
+
+ final StreamResult result = new StreamResult(new StringWriter());
+
+ layoutTransformer.setParameter("format", format);
+
+ final StreamSource fields = indexFieldsFromProfile(MDFORMAT_FIELDS_PROFILE);
+
+ layoutTransformer.transform(fields , result);
+
+ return result.getWriter().toString();
+ }
+
+ private StreamSource streamSource(final String s) throws IOException {
+ return new StreamSource(new StringReader(readFromClasspath(s)));
+ }
+
+ private String readFromClasspath(final String s) throws IOException {
+ ClassPathResource resource = new ClassPathResource(s);
+ InputStream inputStream = resource.getInputStream();
+ return IOUtils.toString(inputStream);
+ }
+
+ private StreamSource indexFieldsFromProfile(final String path) throws IOException, DocumentException {
+ final Document doc = new SAXReader().read(new StringReader(readFromClasspath(path)));
+ final Node layout = doc.selectSingleNode("//LAYOUT[@name='index']");
+ return new StreamSource(new StringReader(layout.asXML()));
+ }
+
+}
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p16.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p16.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p16.xml (revision 58540)
@@ -0,0 +1,428 @@
+
+
+
+ corda__h2020::003a34b5811de10e17597e29e32214b4
+ 721577
+ 2017-03-01T11:17:28.987+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::721577
+ corda__h2020::721577
+ 721577
+ Keep Control
+ Industrial Academic Initial Training Network towards specific diagnosis and treatment of age-related gait and balance deficits
+ 01-MAR-2017
+ 28-FEB-2021
+ H2020-MSCA-ITN-2016
+ true
+ No
+ openaire____::corda_h2020
+ MSCA-ITN-ETN
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ MSCA-ITN-2016
+ Innovative Training Networks
+
+
+ corda__h2020::721577::Walter_MAETZLER
+ corda__h2020::721577::Walter_MAETZLER
+ Walter
+ MAETZLER
+ w.maetzler@neurologie.uni-kiel.de
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::916516238
+ corda__h2020::916516238
+ Patients Know Best
+ PATIENTS KNOW BEST LIMITED
+ http://www.patientsknowbest.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::916516238
+ 12
+ corda__h2020::721577
+ corda__h2020::916516238
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::932602718
+ corda__h2020::932602718
+ Gait Up
+ GAIT UP SA
+ http://www.gaitup.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ CH
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::932602718
+ 10
+ corda__h2020::721577
+ corda__h2020::932602718
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::954530344
+ corda__h2020::954530344
+ VU/VUmc
+ STICHTING VU
+ http://www.vu.nl
+ No
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ NL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::954530344
+ 6
+ corda__h2020::721577
+ corda__h2020::954530344
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::969048334
+ corda__h2020::969048334
+ Motek Medical
+ MOTEK MEDICAL B.V.
+ http://www.motekmedical.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ NL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::969048334
+ 9
+ corda__h2020::721577
+ corda__h2020::969048334
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::969946457
+ corda__h2020::969946457
+ HASOMED GmbH
+ HASOMED HARD-UND SOFTWARE FUER MEDIZIN GESELLSCHAFT MBH
+ http://www.hasomed.de
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Yes
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::969946457
+ 11
+ corda__h2020::721577
+ corda__h2020::969946457
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999587232
+ corda__h2020::999587232
+ CHU LILLE
+ CENTRE HOSPITALIER REGIONAL ET UNIVERSITAIRE DE LILLE
+
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ Missing
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::999587232
+ 8
+ corda__h2020::721577
+ corda__h2020::999587232
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999839529
+ corda__h2020::999839529
+ CAU
+ CHRISTIAN-ALBRECHTS-UNIVERSITAET ZU KIEL
+ http://www.uni-kiel.de
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ Missing
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::999839529
+ 1
+ corda__h2020::721577
+ corda__h2020::999839529
+ corda__h2020::721577::Walter_MAETZLER
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999894916
+ corda__h2020::999894916
+ UPORTO
+ UNIVERSIDADE DO PORTO
+ http://www.up.pt
+ No
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ PT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::999894916
+ 5
+ corda__h2020::721577
+ corda__h2020::999894916
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999914801
+ corda__h2020::999914801
+ UMCG
+ ACADEMISCH ZIEKENHUIS GRONINGEN
+ http://www.umcg.nl
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ NL
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::999914801
+ 7
+ corda__h2020::721577
+ corda__h2020::999914801
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999978433
+ corda__h2020::999978433
+ LMU
+ LUDWIG-MAXIMILIANS-UNIVERSITAET MUENCHEN
+ http://www.uni-muenchen.de
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ N/A
+ N/A
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::999978433
+ 4
+ corda__h2020::721577
+ corda__h2020::999978433
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999985417
+ corda__h2020::999985417
+ UNEW
+ UNIVERSITY OF NEWCASTLE UPON TYNE
+ http://www.newcastle.ac.uk
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::999985417
+ 3
+ corda__h2020::721577
+ corda__h2020::999985417
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999991334
+ corda__h2020::999991334
+ KU Leuven
+ KATHOLIEKE UNIVERSITEIT LEUVEN
+ http://www.kuleuven.be
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ BE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::721577@@corda__h2020::999991334
+ 2
+ corda__h2020::721577
+ corda__h2020::999991334
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::MSCA-ITN-2016
+ corda__h2020::MSCA-ITN-2016
+ Innovative Training Networks
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::721577@@corda__h2020::MSCA-ITN-2016
+ corda__h2020::721577
+ corda__h2020::MSCA-ITN-2016
+
+
+ ec__________::EC::H2020::MSCA-ITN-ETN@@corda__h2020::721577
+ ec__________::EC::H2020::MSCA-ITN-ETN
+ corda__h2020::721577
+ 01-MAR-2017
+ 28-FEB-2021
+ MSCA-ITN-ETN
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input2.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input2.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/input2.xml (revision 58540)
@@ -0,0 +1,336 @@
+
+
+
+
+ Expert Review of Proteomics
+ Expert Review of Proteomics
+ 1478-9450
+ 1744-8387
+
+
+
+ 11
+ 11
+ 2016
+
+
+ 12
+ 2016
+
+
+ 13
+
+ 12
+
+
+
+ Urinary biomarkers for renal tract malformations
+
+
+
+ Pedro
+ Magalhães
+
+
+ Joost P.
+ Schanstra
+
+
+ Emma
+ Carrick
+
+
+ Harald
+ Mischak
+
+
+ Petra
+ Zürbig
+
+
+
+ 11
+ 15
+ 2016
+
+
+ 12
+ 2016
+
+
+ 1121
+ 1129
+
+
+ 7
+ 10.1080/14789450.2016.1254555
+
+
+ 1
+ 10.1080/tandf_crossmark_01
+
+
+ www.tandfonline.com
+
+
+ true
+
+ The publishing and review policy for this title is described in its Aims & Scope.
+ http://www.tandfonline.com/action/journalInformation?show=aimsScope&journalCode=ieru20
+
+
+
+ Horizon 2020
+ http://dx.doi.org/10.13039/100010661
+
+ 642937-RENALTRACT
+
+
+
+ Seventh Framework Programme
+ http://dx.doi.org/10.13039/501100004963
+
+ 305608-EURenOmics
+ 608332-iMODECKD
+
+
+
+
+
+ 10.1080/14789450.2016.1254555
+ https://www.tandfonline.com/doi/full/10.1080/14789450.2016.1254555
+
+ -
+
http://www.tandfonline.com/doi/pdf/10.1080/14789450.2016.1254555
+
+
+
+
+
+ 10.1034/j.1399-3046.2003.00029.x
+
+
+ 10.2215/CJN.03310412
+
+
+ 10.1074/mcp.E800010-MCP200
+
+
+ 10.1126/scitranslmed.3001249
+
+
+ 10.1586/14789450.2014.872564
+
+
+ 10.1111/j.1523-1755.2004.00352.x
+
+
+ 10.1002/pmic.200401012
+
+
+ 10.1002/prca.200780082
+
+
+ 10.1053/j.ackd.2010.09.004
+
+
+ 10.1074/mcp.R800001-MCP200
+
+
+ 10.1016/j.kint.2015.10.010
+
+
+ 10.1111/j.1523-1755.2004.00659.x
+
+
+ 10.1002/pmic.200600230
+
+
+ 10.1038/nrneph.2014.247
+
+
+ 10.1681/ASN.2006090956
+
+
+ 10.1042/CS20050006
+
+
+ 10.1038/nrc2902
+
+
+ 10.1586/14789450.4.3.421
+
+
+ 10.1093/ndt/gfv091
+
+
+ 10.1002/prca.201400113
+
+
+ 10.1681/ASN.2014050423
+
+
+ 10.1021/pr070529w
+
+
+ 10.1016/j.bbapap.2013.06.016
+
+
+ 10.1074/mcp.M110.001917
+
+
+ 10.1093/eurheartj/ehs185
+
+
+ 10.1074/mcp.M700394-MCP200
+
+
+ 10.4161/cbt.8.12.8634
+
+
+ 10.1016/j.molonc.2014.03.006
+
+
+ 10.1038/msb.2008.61
+
+
+ 10.1586/14737159.2015.1093937
+
+
+ 10.1002/pmic.201100436
+
+
+ 10.1007/s00467-015-3291-3
+
+
+ 10.1007/s00467-014-2790-y
+
+
+ 10.1007/s00467-009-1251-5
+
+
+ Rays
+ Manzoni C
+ 131
+ 27
+ 2
+ 2002
+
+
+ 10.1177/000992280204100503
+
+
+ 10.1542/peds.2010-3467
+
+
+ 10.1111/j.1464-410X.2006.06641.x
+
+
+ 10.5114/aoms.2013.36912
+
+
+ 10.1590/S1677-55382009000300008
+
+
+ 10.4103/2277-9175.133196
+
+
+ 10.1046/j.1523-1755.2000.00153.x
+
+
+ 10.1016/j.jpedsurg.2010.07.057
+
+
+ 10.1007/s00467-012-2243-4
+
+
+ 10.1016/j.jpurol.2012.11.011
+
+
+ 10.1111/apa.12324
+
+
+ 10.1074/mcp.M116.059386
+
+
+ 10.1007/s00467-011-1773-5
+
+
+ 10.1007/s00467-012-2217-6
+
+
+ 10.1038/nm1384
+
+
+ 10.1007/s00467-010-1455-8
+
+
+ 10.1002/pmic.200800560
+
+
+ 10.1016/j.juro.2011.10.169
+
+
+ 10.1074/mcp.M114.040121
+
+
+ 10.1016/j.juro.2010.03.061
+
+
+ 10.1016/j.urology.2013.06.009
+
+
+ 10.1016/j.kint.2016.02.020
+
+
+ 10.1093/jjco/hys200
+
+
+ 10.1016/S0022-5347(05)65444-4
+
+
+ 10.1111/j.1464-410X.1992.tb15704.x
+
+
+ 10.1126/scitranslmed.3005807
+
+
+ 10.2215/CJN.09640911
+
+
+ 10.1007/s00467-013-2506-8
+
+
+ 10.1681/ASN.2007020245
+
+
+ Dan Med J
+ Madsen MG
+ B4582
+ 60
+ 2013
+
+
+ Mol Med Rep
+ Zhao Q
+ 681
+ 14
+ 1
+ 2016
+
+
+ Int J Clin Exp Pathol
+ Zhao Q
+ 2615
+ 8
+ 3
+ 2015
+
+
+ 10.1152/ajprenal.00244.2013
+
+
+
+
+
+ info:eu-repo/grantAgreement/EC/H2020/642937 10.1080/14789450.2016.1254555
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p27.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p27.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p27.xml (revision 58540)
@@ -0,0 +1,97 @@
+
+
+
+ corda__h2020::e12e28f98e22f9a9c85b086adfcf34d5
+ 665136
+ 2017-03-01T10:51:55.188+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::665136
+ corda__h2020::665136
+ 665136
+ OBSERVE
+ Observing Emergence
+ 01-JUN-2015
+ 31-MAY-2017
+ H2020-FETOPEN-2014-CSA
+ true
+ Yes
+ openaire____::corda_h2020
+ CSA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-2-2014
+ Coordination and Support Activities
+
+
+ corda__h2020::665136::Philine_WARNKE
+ corda__h2020::665136::Philine_WARNKE
+ Philine
+ WARNKE
+ philine.warnke@isi.fraunhofer.de
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999984059
+ corda__h2020::999984059
+ Fraunhofer
+ FRAUNHOFER GESELLSCHAFT ZUR FOERDERUNG DER ANGEWANDTEN FORSCHUNG E.V.
+ http://www.fraunhofer.de
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665136@@corda__h2020::999984059
+ 1
+ corda__h2020::665136
+ corda__h2020::999984059
+ corda__h2020::665136::Philine_WARNKE
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-2-2014
+ corda__h2020::FETOPEN-2-2014
+ Coordination and Support Activities
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::665136@@corda__h2020::FETOPEN-2-2014
+ corda__h2020::665136
+ corda__h2020::FETOPEN-2-2014
+
+
+ ec__________::EC::H2020::CSA@@corda__h2020::665136
+ ec__________::EC::H2020::CSA
+ corda__h2020::665136
+ 01-JUN-2015
+ 31-MAY-2017
+ CSA
+ ec:h2020toas
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p3.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p3.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p3.xml (revision 58540)
@@ -0,0 +1,97 @@
+
+
+
+ corda__h2020::072c9a0f7177104224b881afedd4e0ab
+ 754515
+ 2017-03-01T11:26:37.65+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::754515
+ corda__h2020::754515
+ 754515
+ HHH
+ Hello
+ 01-JUN-2345
+ 30-APR-2018
+ H2020-FETOPEN-4-2016-2017
+ true
+ No
+ openaire____::corda_h2020
+ CSA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-04-2016-2017
+ FET Innovation Launchpad
+
+
+ corda__h2020::754515::Francesco_DE PELLEGRINI
+ corda__h2020::754515::Francesco_DE PELLEGRINI
+ Francesco
+ DE PELLEGRINI
+ fdepellegrini@fbk.eu
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999625450
+ corda__h2020::999625450
+ CREATE-NET
+ FONDAZIONE BRUNO KESSLER
+ http://www.fbk.eu
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::754515@@corda__h2020::999625450
+ 1
+ corda__h2020::754515
+ corda__h2020::999625450
+ corda__h2020::754515::Francesco_DE PELLEGRINI
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-04-2016-2017
+ corda__h2020::FETOPEN-04-2016-2017
+ FET Innovation Launchpad
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::754515@@corda__h2020::FETOPEN-04-2016-2017
+ corda__h2020::754515
+ corda__h2020::FETOPEN-04-2016-2017
+
+
+ ec__________::EC::H2020::CSA@@corda__h2020::754515
+ ec__________::EC::H2020::CSA
+ corda__h2020::754515
+ 01-JUN-2017
+ 30-NOV-2018
+ CSA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p7.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p7.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p7.xml (revision 58540)
@@ -0,0 +1,247 @@
+
+
+
+ corda__h2020::1455cc535553ceef0b94e58da9abfd52
+ 737164
+ 2017-03-01T11:24:08.344+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::737164
+ corda__h2020::737164
+ 737164
+ SUMCASTEC
+ Semiconductor-based Ultrawideband Micromanipulation of CAncer STEm Cells
+ 01-JAN-2017
+ 30-JUN-2020
+ H2020-FETOPEN-1-2016-2017
+ true
+ Yes
+ openaire____::corda_h2020
+ RIA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-01-2016-2017
+ FET-Open research and innovation actions
+
+
+ corda__h2020::737164::Arnaud_POTHIER
+ corda__h2020::737164::Arnaud_POTHIER
+ Arnaud
+ POTHIER
+ arnaud.pothier@xlim.fr
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::939948916
+ corda__h2020::939948916
+ CREO
+ CREO MEDICAL LIMITED
+ http://www.creomedical.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Missing
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::737164@@corda__h2020::939948916
+ 6
+ corda__h2020::737164
+ corda__h2020::939948916
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::998267838
+ corda__h2020::998267838
+ UNILIM
+ UNIVERSITE DE LIMOGES
+ http://www.unilim.fr
+ Yes
+ Yes
+ Yes
+ No
+ Yes
+ No
+ No
+ No
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::737164@@corda__h2020::998267838
+ 1
+ corda__h2020::737164
+ corda__h2020::998267838
+ corda__h2020::737164::Arnaud_POTHIER
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999606438
+ corda__h2020::999606438
+ IHP
+ IHP GMBH - INNOVATIONS FOR HIGH PERFORMANCE MICROELECTRONICS/LEIBNIZ-INSTITUT FUER INNOVATIVE MIKROELEKTRONIK
+
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::737164@@corda__h2020::999606438
+ 3
+ corda__h2020::737164
+ corda__h2020::999606438
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999823815
+ corda__h2020::999823815
+ BANGOR
+ BANGOR UNIVERSITY
+ http://www.bangor.ac.uk
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ N/A
+ N/A
+ No
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::737164@@corda__h2020::999823815
+ 2
+ corda__h2020::737164
+ corda__h2020::999823815
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999988521
+ corda__h2020::999988521
+ ENEA
+ AGENZIA NAZIONALE PER LE NUOVE TECNOLOGIE, L'ENERGIA E LO SVILUPPO ECONOMICO SOSTENIBILE
+ http://www.enea.it
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::737164@@corda__h2020::999988521
+ 4
+ corda__h2020::737164
+ corda__h2020::999988521
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999995602
+ corda__h2020::999995602
+ UNIPD
+ UNIVERSITA DEGLI STUDI DI PADOVA
+ http://www.unipd.it
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::737164@@corda__h2020::999995602
+ 5
+ corda__h2020::737164
+ corda__h2020::999995602
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-01-2016-2017
+ corda__h2020::FETOPEN-01-2016-2017
+ FET-Open research and innovation actions
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::737164@@corda__h2020::FETOPEN-01-2016-2017
+ corda__h2020::737164
+ corda__h2020::FETOPEN-01-2016-2017
+
+
+ ec__________::EC::H2020::RIA@@corda__h2020::737164
+ ec__________::EC::H2020::RIA
+ corda__h2020::737164
+ 01-JAN-2017
+ 30-JUN-2020
+ RIA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/index/fields.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/index/fields.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/index/fields.xml (revision 58540)
@@ -0,0 +1,172 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_wrong_openaireId.json
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_wrong_openaireId.json (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_wrong_openaireId.json (revision 58540)
@@ -0,0 +1,38 @@
+{
+ "openaireId":"dedup_wf_001::ab42e811",
+ "originalId": "ORIG_ID_TEST",
+ "type": "publication",
+ "title": "TEST TITLE",
+ "authors": [
+ "Michele Artini",
+ "Claudio Atzori",
+ "Alessia Bardi"
+ ],
+ "publisher": "Test publisher",
+ "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",
+ "language": "ita",
+ "pids": [
+ {
+ "type": "doi",
+ "value": "10.000/xyz-123"
+ },
+ {
+ "type": "oai",
+ "value": "oai:1234"
+ }
+ ],
+ "licenseCode": "EMBARGO",
+ "embargoEndDate": "2018-02-02",
+ "resourceType": "0001",
+ "url": "http://test.it/xyz",
+ "collectedFromId": "opendoar____::2659",
+ "hostedById": "opendoar____::2659",
+ "linksToProjects": [
+ "info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",
+ "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"
+ ],
+ "contexts": [
+ "egi::classification::natsc::math::pure",
+ "egi::classification::natsc::math::stats"
+ ]
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p11.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p11.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p11.xml (revision 58540)
@@ -0,0 +1,247 @@
+
+
+
+ corda__h2020::236e667820ae0bcf403cb3a8d604e9f4
+ 665148
+ 2017-03-01T10:51:55.189+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::665148
+ corda__h2020::665148
+ 665148
+ QCUMbER
+ Quantum Controlled Ultrafast Multimode Entanglement and Measurement
+ 01-SEP-2015
+ 31-AUG-2018
+ H2020-FETOPEN-2014-2015-RIA
+ true
+ No
+ openaire____::corda_h2020
+ RIA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-RIA-2014-2015
+ FET-Open research projects
+
+
+ corda__h2020::665148::Brian_SMITH
+ corda__h2020::665148::Brian_SMITH
+ Brian
+ SMITH
+ b.smith1@physics.ox.ac.uk
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999837492
+ corda__h2020::999837492
+ UPB
+ UNIVERSITAET PADERBORN
+ http://www.uni-paderborn.de
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665148@@corda__h2020::999837492
+ 3
+ corda__h2020::665148
+ corda__h2020::999837492
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999852430
+ corda__h2020::999852430
+ UROS
+ UNIVERSITAET ROSTOCK
+ http://www.uni-rostock.de
+ Yes
+ Yes
+ Yes
+ No
+ Yes
+ No
+ No
+ Missing
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665148@@corda__h2020::999852430
+ 5
+ corda__h2020::665148
+ corda__h2020::999852430
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999866107
+ corda__h2020::999866107
+ UNIROMA3
+ UNIVERSITA DEGLI STUDI ROMA TRE
+ http://www.uniroma3.it
+ Yes
+ Yes
+ Yes
+ No
+ Yes
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665148@@corda__h2020::999866107
+ 6
+ corda__h2020::665148
+ corda__h2020::999866107
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999899378
+ corda__h2020::999899378
+ USTL
+ UNIVERSITE DES SCIENCES ET TECHNOLOGIES DE LILLE - LILLE I
+
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665148@@corda__h2020::999899378
+ 4
+ corda__h2020::665148
+ corda__h2020::999899378
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999984350
+ corda__h2020::999984350
+ UOXF
+ THE CHANCELLOR, MASTERS AND SCHOLARS OF THE UNIVERSITY OF OXFORD
+ http://www.ox.ac.uk
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665148@@corda__h2020::999984350
+ 1
+ corda__h2020::665148
+ corda__h2020::999984350
+ corda__h2020::665148::Brian_SMITH
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999986193
+ corda__h2020::999986193
+ UPMC
+ UNIVERSITE PIERRE ET MARIE CURIE - PARIS 6
+ http://www.upmc.fr
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665148@@corda__h2020::999986193
+ 2
+ corda__h2020::665148
+ corda__h2020::999986193
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::FETOPEN-RIA-2014-2015
+ FET-Open research projects
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::665148@@corda__h2020::FETOPEN-RIA-2014-2015
+ corda__h2020::665148
+ corda__h2020::FETOPEN-RIA-2014-2015
+
+
+ ec__________::EC::H2020::RIA@@corda__h2020::665148
+ ec__________::EC::H2020::RIA
+ corda__h2020::665148
+ 01-SEP-2015
+ 31-AUG-2018
+ RIA
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/submit_test_record.sh
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/submit_test_record.sh (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/submit_test_record.sh (revision 58540)
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+curl -H "Content-Type: application/json" --data @test_record.json "http://localhost:8280/app/mvc/api/publications/feedObject"
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p22.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p22.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p22.xml (revision 58540)
@@ -0,0 +1,98 @@
+
+
+
+ corda__h2020::005c3dcb8b95f0ead48dcef46d7a934b
+ 703374
+ 2017-03-01T11:09:28.795+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::703374
+ corda__h2020::703374
+ 703374
+ NMR4Nanos
+ Development and application of NMR-based tools to inorganic nanocarriers for effective vaccine delivery
+ .--
+ .--
+ H2020-MSCA-IF-2015
+ true
+ No
+ openaire____::corda_h2020
+ MSCA-IF-EF-ST
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ MSCA-IF-2015-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+
+
+ corda__h2020::703374::Jose_MARTINS
+ corda__h2020::703374::Jose_MARTINS
+ Jose
+ MARTINS
+ jose.martins@ugent.be
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999986096
+ corda__h2020::999986096
+ UGENT
+ UNIVERSITEIT GENT
+ http://www.ugent.be
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ BE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::703374@@corda__h2020::999986096
+ 1
+ corda__h2020::703374
+ corda__h2020::999986096
+ corda__h2020::703374::Jose_MARTINS
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::MSCA-IF-2015-EF
+ corda__h2020::MSCA-IF-2015-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::703374@@corda__h2020::MSCA-IF-2015-EF
+ corda__h2020::703374
+ corda__h2020::MSCA-IF-2015-EF
+
+
+ ec__________::EC::H2020::MSCA-IF-EF-ST@@corda__h2020::703374
+ ec__________::EC::H2020::MSCA-IF-EF-ST
+ corda__h2020::703374
+ .--
+ .--
+ MSCA-IF-EF-ST
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p15.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p15.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p15.xml (revision 58540)
@@ -0,0 +1,97 @@
+
+
+
+ corda__h2020::0036ca7301de596bcb73e3a5d78fdb62
+ 701676
+ 2017-03-01T11:08:16.309+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::701676
+ corda__h2020::701676
+ 701676
+ SF-QFT
+ Fundamental physics with intense laser fields
+ 01-OCT-2016
+ 30-SEP-2018
+ H2020-MSCA-IF-2015
+ true
+ No
+ openaire____::corda_h2020
+ MSCA-IF-EF-ST
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ MSCA-IF-2015-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+
+
+ corda__h2020::701676::Thomas_HEINZL
+ corda__h2020::701676::Thomas_HEINZL
+ Thomas
+ HEINZL
+ thomas.heinzl@plymouth.ac.uk
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999854661
+ corda__h2020::999854661
+ UOP
+ UNIVERSITY OF PLYMOUTH
+
+ Yes
+ Yes
+ Yes
+ Yes
+ Yes
+ N/A
+ N/A
+ Missing
+ GB
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::701676@@corda__h2020::999854661
+ 1
+ corda__h2020::701676
+ corda__h2020::999854661
+ corda__h2020::701676::Thomas_HEINZL
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::MSCA-IF-2015-EF
+ corda__h2020::MSCA-IF-2015-EF
+ Marie Skłodowska-Curie Individual Fellowships (IF-EF)
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::701676@@corda__h2020::MSCA-IF-2015-EF
+ corda__h2020::701676
+ corda__h2020::MSCA-IF-2015-EF
+
+
+ ec__________::EC::H2020::MSCA-IF-EF-ST@@corda__h2020::701676
+ ec__________::EC::H2020::MSCA-IF-EF-ST
+ corda__h2020::701676
+ 01-OCT-2016
+ 30-SEP-2018
+ MSCA-IF-EF-ST
+ ec:h2020toas
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p26.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p26.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p26.xml (revision 58540)
@@ -0,0 +1,128 @@
+
+
+
+ corda__h2020::afbcda36b1a26e7e606daae82ad48762
+ 665083
+ 2017-03-01T10:51:52.482+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::665083
+ corda__h2020::665083
+ 665083
+ FET_TRACES
+ Tracing impacts of the FET programme
+ 15-JUL-2015
+ 14-NOV-2017
+ H2020-FETOPEN-2014-CSA
+ true
+ No
+ openaire____::corda_h2020
+ CSA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ FETOPEN-2-2014
+ Coordination and Support Activities
+
+
+ corda__h2020::665083::Bernd_BECKERT
+ corda__h2020::665083::Bernd_BECKERT
+ Bernd
+ BECKERT
+ bernd.beckert@isi.fraunhofer.de
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::999584128
+ corda__h2020::999584128
+ AIT Austrian Institute of Technology GmbH
+ AIT AUSTRIAN INSTITUTE OF TECHNOLOGY GMBH
+ http://www.ait.ac.at/
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ AT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665083@@corda__h2020::999584128
+ 2
+ corda__h2020::665083
+ corda__h2020::999584128
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999984059
+ corda__h2020::999984059
+ Fraunhofer Gesellschaft
+ FRAUNHOFER GESELLSCHAFT ZUR FOERDERUNG DER ANGEWANDTEN FORSCHUNG E.V.
+ http://www.fraunhofer.de
+ No
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::665083@@corda__h2020::999984059
+ 1
+ corda__h2020::665083
+ corda__h2020::999984059
+ corda__h2020::665083::Bernd_BECKERT
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::FETOPEN-2-2014
+ corda__h2020::FETOPEN-2-2014
+ Coordination and Support Activities
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::665083@@corda__h2020::FETOPEN-2-2014
+ corda__h2020::665083
+ corda__h2020::FETOPEN-2-2014
+
+
+ ec__________::EC::H2020::CSA@@corda__h2020::665083
+ ec__________::EC::H2020::CSA
+ corda__h2020::665083
+ 15-JUL-2015
+ 14-NOV-2017
+ CSA
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p19.xml
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p19.xml (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/workflows/nodes/contexts/fetprojects/p19.xml (revision 58540)
@@ -0,0 +1,458 @@
+
+
+
+ corda__h2020::0043a658bfc1be5e0b2997128cec0b11
+ 734140
+ 2017-03-01T11:22:49.39+01:00
+ f70dfee1-c5bb-4872-ba0f-0b029f066747_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+ corda__h2020
+
+
+
+
+ corda__h2020::734140
+ corda__h2020::734140
+ 734140
+ PJ27 IOPVLD
+ Flight Object Interoperability VLD Demonstration
+ 01-NOV-2016
+ 31-DEC-2019
+ H2020-SESAR-2015-2
+ true
+ No
+ openaire____::corda_h2020
+ SESAR-IA
+ ec:h2020toas
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+ SESAR.IR-VLD.Wave1-28-2015
+ Flight Information Exchange
+
+
+ corda__h2020::734140::Peter_GLOMBIOWSKI
+ corda__h2020::734140::Peter_GLOMBIOWSKI
+ Peter
+ GLOMBIOWSKI
+ peter.glombiowski@dfs.de
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+
+
+ corda__h2020::923671249
+ corda__h2020::923671249
+ Naviair/COOPANS
+ NAVIAIR
+ http://www.naviair.dk
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Missing
+ DK
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::923671249
+ 13
+ corda__h2020::734140
+ corda__h2020::923671249
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::924128216
+ corda__h2020::924128216
+ CCL/COOPANS
+ CROATIA CONTROL, CROATIAN AIR NAVIGATION SERVICES LTD
+ http://www.crocontrol.hr
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Missing
+ HR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::924128216
+ 10
+ corda__h2020::734140
+ corda__h2020::924128216
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::927009019
+ corda__h2020::927009019
+ IAA/COOPANS
+ UDARAS EITLIOCHTA NA HEIREANN THE IRISH AVIATION AUTHORITY
+ http://www.iaa.ie
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Missing
+ IE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::927009019
+ 11
+ corda__h2020::734140
+ corda__h2020::927009019
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::928673636
+ corda__h2020::928673636
+ DSNA
+ DIRECTION DES SERVICES DE LA NAVIGATION AERIENNE
+ http://www.dgac.fr
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::928673636
+ 2
+ corda__h2020::734140
+ corda__h2020::928673636
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::942346077
+ corda__h2020::942346077
+ LFV/COOPANS
+ LUFTFARTSVERKET
+ http://www.lfv.se
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ SE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::942346077
+ 12
+ corda__h2020::734140
+ corda__h2020::942346077
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::997701843
+ corda__h2020::997701843
+ ENAIRE
+ ENTIDAD PUBLICA EMPRESARIAL ENAIRE
+ http://www.enaire.es
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ No
+ No
+ ES
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::997701843
+ 8
+ corda__h2020::734140
+ corda__h2020::997701843
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::998197513
+ corda__h2020::998197513
+ ENAV
+ ENAV SPA
+ http://www.enav.it
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ No
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::998197513
+ 3
+ corda__h2020::734140
+ corda__h2020::998197513
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::998627417
+ corda__h2020::998627417
+ FINMECCANICA
+ LEONARDO - FINMECCANICA SPA
+ http://www.finmeccanica.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Missing
+ IT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::998627417
+ 6
+ corda__h2020::734140
+ corda__h2020::998627417
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::998956635
+ corda__h2020::998956635
+ ACG/COOPANS
+ AUSTRO CONTROL OSTERREICHISCHE GESELLSCHAFT FUR ZIVILLUFTFAHRT MBH
+ http://www.austrocontrol.at
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ No
+ AT
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::998956635
+ 9
+ corda__h2020::734140
+ corda__h2020::998956635
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999483733
+ corda__h2020::999483733
+ EUROCONTROL
+ EUROCONTROL - EUROPEAN ORGANISATION FOR THE SAFETY OF AIR NAVIGATION
+ http://www.eurocontrol.int
+ Yes
+ Yes
+ Yes
+ Yes
+ No
+ No
+ No
+ No
+ BE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::999483733
+ 4
+ corda__h2020::734140
+ corda__h2020::999483733
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999936820
+ corda__h2020::999936820
+ DFS
+ DFS DEUTSCHE FLUGSICHERUNG GMBH
+ http://www.dfs.de
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ Missing
+ DE
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::999936820
+ 1
+ corda__h2020::734140
+ corda__h2020::999936820
+ corda__h2020::734140::Peter_GLOMBIOWSKI
+ coordinator
+ dnet:project_organization_relations
+
+
+ corda__h2020::999959130
+ corda__h2020::999959130
+ INDRA
+ INDRA SISTEMAS SA
+ http://www.indracompany.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ No
+ ES
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::999959130
+ 5
+ corda__h2020::734140
+ corda__h2020::999959130
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::999961555
+ corda__h2020::999961555
+ THALES AIR SYS
+ THALES AIR SYSTEMS SAS
+ http://www.thalesgroup.com
+ No
+ Yes
+ No
+ No
+ No
+ No
+ No
+ No
+ FR
+ dnet:countries
+ openaire____::corda_h2020
+ sysimport:crosswalk:entityregistry
+ dnet:provenanceActions
+ 0.92
+
+
+ corda__h2020::734140@@corda__h2020::999961555
+ 7
+ corda__h2020::734140
+ corda__h2020::999961555
+
+ participant
+ dnet:project_organization_relations
+
+
+ corda__h2020::SESAR.IR-VLD.Wave1-28-2015
+ corda__h2020::SESAR.IR-VLD.Wave1-28-2015
+ Flight Information Exchange
+ ec:h2020topics
+ dnet:subject_classification_typologies
+
+
+ corda__h2020::734140@@corda__h2020::SESAR.IR-VLD.Wave1-28-2015
+ corda__h2020::734140
+ corda__h2020::SESAR.IR-VLD.Wave1-28-2015
+
+
+ ec__________::EC::H2020::SESAR-IA@@corda__h2020::734140
+ ec__________::EC::H2020::SESAR-IA
+ corda__h2020::734140
+ 01-NOV-2016
+ 31-DEC-2019
+ SESAR-IA
+ ec:h2020toas
+
+
+
+
+
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_zenodo.json
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_zenodo.json (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_zenodo.json (revision 58540)
@@ -0,0 +1,26 @@
+{
+ "authors":[
+ "Alouges, Fran\u00e7ois",
+ "Di Fratta, Giovanni"
+ ],
+ "collectedFromId":"opendoar____::2659",
+ "description":"The paper is about the parking 3-sphere swimmer ($\\text{sPr}_3$). This is a low-Reynolds number model swimmer composed of three balls of equal radii. The three balls can move along three horizontal axes (supported in the same plane) that mutually meet at the center of $\\text{sPr}_3$ with angles of $120^{\u2218}$ . The governing dynamical system is introduced and the implications of its geometric symmetries revealed. It is then shown that, in the first order range of small strokes, optimal periodic strokes are ellipses embedded in 3d space, i.e. closed curves of the form $t\ud835\udfc4 [0,2\u03c0] \u21a6 (\\cos t)u + (\\sin t)v$ for suitable orthogonal vectors $u$ and $v$ of $\u211d^3$. A simple analytic expression for the vectors $u$ and $v$ is derived. The results of the paper are used in a second article where the real physical dynamics of $\\text{sPr}_3$ is analyzed in the asymptotic range of very long arms. ; Comment: 17 pages, 4 figures",
+ "hostedById":"opendoar____::2659",
+ "licenseCode":"OPEN",
+ "originalId":"oai:zenodo.org:996201",
+ "pids":[
+ {
+ "type":"oai",
+ "value":"oai:zenodo.org:996201"
+ },
+ {
+ "type":"doi",
+ "value":"10.5281/zenodo.996201"
+ }
+ ],
+ "publisher":"Zenodo",
+ "resourceType":"0020",
+ "title":"Parking 3-sphere swimmer. I. Energy minimizing strokes",
+ "type":"publication",
+ "url":"https://zenodo.org/record/996201"
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_complete.json
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_complete.json (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/eu/dnetlib/msro/openaireplus/api/objects/test_record_complete.json (revision 58540)
@@ -0,0 +1,37 @@
+{
+ "originalId": "ORIG_ID_TEST",
+ "type": "publication",
+ "title": "TEST TITLE",
+ "authors": [
+ "Michele Artini",
+ "Claudio Atzori",
+ "Alessia Bardi"
+ ],
+ "publisher": "Test publisher",
+ "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",
+ "language": "ita",
+ "pids": [
+ {
+ "type": "doi",
+ "value": "10.000/xyz-123"
+ },
+ {
+ "type": "oai",
+ "value": "oai:1234"
+ }
+ ],
+ "licenseCode": "EMBARGO",
+ "embargoEndDate": "2018-02-02",
+ "resourceType": "0001",
+ "url": "http://test.it/xyz",
+ "collectedFromId": "opendoar____::2659",
+ "hostedById": "opendoar____::2659",
+ "linksToProjects": [
+ "info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",
+ "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"
+ ],
+ "contexts": [
+ "egi::classification::natsc::math::pure",
+ "egi::classification::natsc::math::stats"
+ ]
+}
\ No newline at end of file
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/log4j.properties
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/log4j.properties (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6/src/test/resources/log4j.properties (revision 58540)
@@ -0,0 +1,10 @@
+### Root Level ###
+log4j.rootLogger=WARN, CONSOLE
+### Configuration for the CONSOLE appender ###
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c - %m%n
+org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger
+### Application Level ###
+log4j.logger.eu.dnetlib=INFO
+log4j.logger.eu.dnetlib.msro.openaireplus.workflows.nodes.contexts=DEBUG
Index: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6
===================================================================
--- modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6 (nonexistent)
+++ modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6 (revision 58540)
Property changes on: modules/dnet-openaireplus-workflows/tags/dnet-openaireplus-workflows-7.0.6
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,9 ##
+.classpath
+.settings
+target
+velocity.log
+.project
+.springBeans
+*.iml
+.idea
+.factorypath