D-Net

package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.concurrent.atomic.AtomicInteger;

import java.util.stream.Collectors;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.datasources.DatasourceManagerClients;

import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

import eu.dnetlib.msro.workflows.util.ProgressProvider;

import eu.dnetlib.msro.workflows.util.WorkflowsConstants;

public class FindInvalidRepoProfilesJobNode extends SimpleJobNode implements ProgressJobNode {

	@Autowired

	private LocalOpenaireDatasourceManager dsManager;

	@Autowired

	private DatasourceManagerClients dsManagerClient;

	@Autowired

	private UniqueServiceLocator serviceLocator;

	private int current = 0;

	private int total = 0;

	private boolean deleteInvalidProfiles = false;

	private boolean registerMissingProfiles = false;

	private static final Log log = LogFactory.getLog(FindInvalidRepoProfilesJobNode.class);

	@Override

	protected String execute(final NodeToken token) throws Exception {

		final Map<String, AtomicInteger> validIds = dsManagerClient.searchSQL("SELECT id FROM dsm_datasources", new HashMap<>())

				.stream()

				.map(m -> (String) m.get("id"))

				.collect(Collectors.toMap(s -> s, s -> new AtomicInteger(0)));

		final List<String> list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(

				"for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') return concat($x//DATASOURCE_ORIGINAL_ID, ' @@@ ', $x//RESOURCE_IDENTIFIER/@value)");

		this.current = 0;

		this.total = list.size() + validIds.size();

		int errorInvalidProfiles = 0;

		int errorUnregisteredProfiles = 0;

		int errorTooManyProfiles = 0;

		for (final String s : list) {

			current++;

			final String oid = StringUtils.substringBefore(s, "@@@").trim();

			final String profId = StringUtils.substringAfter(s, "@@@").trim();

			log.info("Evaluating ds: " + oid);

			if (validIds.containsKey(oid)) {

				validIds.get(oid).incrementAndGet();

			} else {

				log.warn("Invalid profile " + profId + ", openaireId " + oid + " not registered properly");

				errorInvalidProfiles++;

				// token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalidProfile:" + oid, profId);

				if (deleteInvalidProfiles) {

					log.warn("  - Deleting profile " + profId);

					serviceLocator.getService(ISRegistryService.class).deleteProfile(profId);

				}

			}

		}

		for (final Map.Entry<String, AtomicInteger> e : validIds.entrySet()) {

			current++;

			final String dsId = e.getKey();

			final int n = e.getValue().get();

			if (n == 0) {

				errorUnregisteredProfiles++;

				// token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "unregisterdProfile:" + dsId, dsId);

				log.warn("Missing profile for ds " + dsId);

				if (registerMissingProfiles) {

					dsManager.setManaged(dsId, dsManager.isManaged(dsId)); // This command should regenerate the repo profile

				}

			} else if (n > 1) {

				errorTooManyProfiles++;

				log.error("Too many profiles registerd " + n + " for ds " + dsId);

				// token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "tooManyProfiles:" + dsId, n);

			}

		}

		token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "errorInvalidProfiles", errorInvalidProfiles);

		token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "errorUnregisteredProfiles", errorUnregisteredProfiles);

		token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "errorTooManyProfiles", errorTooManyProfiles);

		return Arc.DEFAULT_ARC;

	}

	@Override

	public ProgressProvider getProgressProvider() {

		return new ProgressProvider() {

			@Override

			public int getTotalValue() {

				return total;

			}

			@Override

			public int getCurrentValue() {

				return current;

			}

			@Override

			public boolean isInaccurate() {

				return false;

			}

		};

	}

	public boolean isDeleteInvalidProfiles() {

		return deleteInvalidProfiles;

	}

	public void setDeleteInvalidProfiles(final boolean deleteInvalidProfiles) {

		this.deleteInvalidProfiles = deleteInvalidProfiles;

	}

	public boolean isRegisterMissingProfiles() {

		return registerMissingProfiles;

	}

	public void setRegisterMissingProfiles(final boolean registerMissingProfiles) {

		this.registerMissingProfiles = registerMissingProfiles;

	}

}

ALTER TABLE datasources

	ADD COLUMN issn CHARACTER VARYING(20);

ALTER TABLE datasources

	ADD COLUMN eissn CHARACTER VARYING(20);

ALTER TABLE datasources

	ADD COLUMN lissn CHARACTER VARYING(20);

ALTER TABLE datasources

	ALTER COLUMN databaseaccesstype TYPE VARCHAR(32);

ALTER TABLE datasources

	ALTER COLUMN datauploadtype TYPE VARCHAR(32);

ALTER TABLE datasources

	ALTER COLUMN databaseaccessrestriction TYPE VARCHAR(32);

ALTER TABLE datasources

	ALTER COLUMN datauploadrestriction TYPE VARCHAR(32);

ALTER TABLE datasources

	ALTER COLUMN qualitymanagementkind TYPE VARCHAR(32);

ALTER TABLE datasources

	ADD COLUMN registeredby CHARACTER VARYING(255);

DELETE FROM apicollections

WHERE api NOT LIKE 'api_________::%';

DELETE FROM api

WHERE id NOT LIKE 'api_________::%';

ALTER TABLE apicollections

	DROP COLUMN name;

ALTER TABLE apicollections

	RENAME COLUMN accessparam TO param;

ALTER TABLE apicollections

	RENAME COLUMN accessvalue TO original;

ALTER TABLE apicollections

	ADD COLUMN edited VARCHAR(255) DEFAULT NULL;

ALTER TABLE apicollections

	ADD COLUMN accessparam BOOLEAN DEFAULT TRUE;

ALTER TABLE apicollections

	DROP CONSTRAINT apicollection_pkey;

ALTER TABLE apicollections

	ALTER COLUMN original SET DEFAULT '';

UPDATE apicollections

SET param = 'set'

WHERE param = 'SET';

UPDATE apicollections

SET param = 'format'

WHERE param = 'FORMAT';

UPDATE apicollections

SET param = 'baseUrl'

WHERE param = 'BASEURL';

-- FIX A PROBLEM WITH MULTIPLE SET

DELETE FROM apicollections

WHERE api = 'api_________::opendoar____::1560::1' AND param = 'set';

INSERT INTO apicollections (param, original, api) VALUES

	('set', 'hdl_10251_3829, hdl_10251_321, hdl_10251_11256, hdl_10251_11206, hdl_10251_8702, hdl_10251_3026, hdl_10251_11076',

	 'api_________::opendoar____::1560::1');

DELETE FROM apicollections

WHERE api = 'api_________::driver______::d0b68287-678d-4efc-b354-c47cbacaed3c::0' AND param = 'set';

INSERT INTO apicollections (param, original, api) VALUES

	('set', 'pub-type:article, pub-type:conf-proceeding, pub-type:lecture, pub-type:paper, pub-type:report',

	 'api_________::driver______::d0b68287-678d-4efc-b354-c47cbacaed3c::0');

DELETE FROM apicollections

WHERE api = 'api_________::driver______::79476713-c428-4f23-bf97-8f27500f754a::0' AND param = 'set';

INSERT INTO apicollections (param, original, api)

VALUES ('set', 'dad:SIQG, jhs,sal:CNF+PA', 'api_________::driver______::79476713-c428-4f23-bf97-8f27500f754a::0');

DELETE FROM apicollections

WHERE api = 'api_________::opendoar____::202::0' AND param = 'set';

INSERT INTO apicollections (param, original, api) VALUES ('set', 'hdl_1887_4540, hdl_1887_4951', 'api_________::opendoar____::202::0');

DELETE FROM apicollections

WHERE api = 'api_________::driver______::c75b9f23-25a8-4e45-aadd-ba449211e2a8::0' AND param = 'set';

INSERT INTO apicollections (param, original, api) VALUES ('set', 'ijdc:ART, ijdc:PAP', 'api_________::driver______::c75b9f23-25a8-4e45-aadd-ba449211e2a8::0');

-- AND FIX

ALTER TABLE apicollections

	ADD CONSTRAINT apicollection_pkey PRIMARY KEY (api, param);

INSERT INTO apicollections (param, original, edited, api) SELECT

	                                                          'baseUrl',

	                                                          COALESCE(original, ''),

	                                                          COALESCE(edited, ''),

	                                                          id

                                                          FROM api

                                                          WHERE originalprotocolclass = editedprotocolclass;

INSERT INTO apicollections (param, original, api) SELECT

	                                                  'baseUrl',

	                                                  COALESCE(original, ''),

	                                                  id

                                                  FROM api

                                                  WHERE originalprotocolclass != editedprotocolclass OR editedprotocolclass IS NULL;

UPDATE apicollections

SET edited = NULL

WHERE edited = '';

UPDATE datasources

SET datasourceclass = 'websource'

WHERE id = 'openaire____::webcrawl';

INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'repository', 'api_________::opendoar::0');

INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'ROW', 'api_________::re3data::0');

INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'Record', 'api_________::wellcometrust::0');

INSERT INTO apicollections (param, original, api) VALUES ('splitOnElement', 'ROW', 'api_________::corda::0');

UPDATE apicollections

SET _dnet_resource_identifier_ = api || '@@' || param;

INSERT INTO scheme (_dnet_resource_identifier_, code, name)

VALUES ('dnet:content_description_typologies', 'dnet:content_description_typologies', 'D-Net Content Description Typologies');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('UNKNOWN', 'UNKNOWN', 'UNKNOWN');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('native', 'native', 'native');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('native@@dnet:compatibilityLevel', 'native', 'dnet:compatibilityLevel');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('files', 'files', 'files');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('files@@dnet:compatibilityLevel', 'files', 'dnet:compatibilityLevel');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('metadata', 'metadata', 'metadata');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('file::WoS', 'file::WoS', 'file::WoS');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('file::PDF', 'file::PDF', 'file::PDF');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('file::EuropePMC', 'file::EuropePMC', 'file::EuropePMC');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme)

VALUES ('metadata@@dnet:content_description_typologies', 'metadata', 'dnet:content_description_typologies');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme)

VALUES ('file::WoS@@dnet:content_description_typologies', 'file::WoS', 'dnet:content_description_typologies');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme)

VALUES ('file::PDF@@dnet:content_description_typologies', 'file::PDF', 'dnet:content_description_typologies');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme)

VALUES ('file::EuropePMC@@dnet:content_description_typologies', 'file::EuropePMC', 'dnet:content_description_typologies');

ALTER TABLE api

	ADD COLUMN contentdescriptionclass VARCHAR(255) DEFAULT 'metadata';

ALTER TABLE api

	ADD COLUMN contentdescriptionscheme VARCHAR(255) DEFAULT 'dnet:content_description_typologies';

ALTER TABLE api

	DROP COLUMN editedprotocolclass;

ALTER TABLE api

	DROP COLUMN editedprotocolscheme;

ALTER TABLE api

	DROP COLUMN original;

ALTER TABLE api

	DROP COLUMN edited;

ALTER TABLE api

	DROP COLUMN useedited;

ALTER TABLE api

	RENAME COLUMN originalprotocolclass TO protocolclass;

ALTER TABLE api

	RENAME COLUMN originalprotocolscheme TO protocolscheme;

ALTER TABLE api

	ADD COLUMN active BOOLEAN DEFAULT FALSE;

ALTER TABLE api

	ADD COLUMN removable BOOLEAN DEFAULT FALSE;

ALTER TABLE api

	ADD COLUMN typologyclass VARCHAR(255) DEFAULT 'UNKNOWN';

ALTER TABLE api

	ADD COLUMN typologyscheme VARCHAR(255) DEFAULT 'dnet:datasource_typologies';

ALTER TABLE api

	ADD CONSTRAINT api_typologyclass_fkey FOREIGN KEY (typologyclass) REFERENCES class (code);

ALTER TABLE api

	ADD CONSTRAINT api_typologyscheme_fkey FOREIGN KEY (typologyscheme) REFERENCES scheme (code);

ALTER TABLE api

	ADD COLUMN compatibilityclass VARCHAR(255) DEFAULT 'UNKNOWN';

ALTER TABLE api

	ADD COLUMN compatibilityscheme VARCHAR(255) DEFAULT 'dnet:compatibilityLevel';

ALTER TABLE api

	ADD CONSTRAINT api_compatibilityclass_fkey FOREIGN KEY (compatibilityclass) REFERENCES class (code);

ALTER TABLE api

	ADD CONSTRAINT api_compatibilityscheme_fkey FOREIGN KEY (compatibilityscheme) REFERENCES scheme (code);

ALTER TABLE api

	ADD CONSTRAINT api_contentdescriptionclass_fkey FOREIGN KEY (contentdescriptionclass) REFERENCES class (code);

ALTER TABLE api

	ADD CONSTRAINT api_contentdescriptionscheme_fkey FOREIGN KEY (contentdescriptionscheme) REFERENCES scheme (code);

UPDATE api

SET typologyclass = d.datasourceclass FROM datasources d

WHERE datasource = d.id;

--UPDATE api SET compatibilityclass = 'driver' where typology = 'driver';

--UPDATE api SET compatibilityclass = 'openaire2.0' FROM apicollections ac WHERE typology = 'openaire' AND id = ac.api AND ac.param = 'set' AND ac.original like '%ec_fundedresources%';

--UPDATE api SET compatibilityclass = 'openaire3.0' FROM apicollections ac WHERE typology = 'openaire' AND id = ac.api AND ac.param = 'set' AND ac.original like '%openaire%';

-- Jochen version

UPDATE api

SET compatibilityclass = ds.openairecompatibilityclass FROM datasources ds

WHERE ds.id = api.datasource;

UPDATE api

SET compatibilityclass = 'driver'

WHERE compatibilityclass = 'driver-openaire2.0' AND typology = 'driver';

UPDATE api

SET compatibilityclass = 'openaire2.0'

WHERE compatibilityclass = 'driver-openaire2.0' AND typology = 'openaire';

UPDATE api

SET compatibilityclass = 'UNKNOWN'

WHERE compatibilityclass = 'openaire2.0-openaire3.0'

-- END Jochen

UPDATE api

SET (typologyclass, compatibilityclass) = ('entityregistry', 'native')

WHERE id = 'api_________::opendoar::0';

UPDATE api

SET (typologyclass, compatibilityclass) = ('entityregistry', 'native')

WHERE id = 'api_________::re3data::0';

UPDATE api

SET (typologyclass, compatibilityclass) = ('entityregistry', 'native')

WHERE id = 'api_________::wellcometrust::0';

UPDATE api

SET (typologyclass, compatibilityclass) = ('entityregistry', 'native')

WHERE id = 'api_________::corda::0';

ALTER TABLE api

	DROP COLUMN typology;

ALTER TABLE datasources

	DROP COLUMN openairecompatibilityclass;

ALTER TABLE datasources

	DROP COLUMN openairecompatibilityscheme;

DELETE FROM class_scheme

WHERE scheme = 'dnet:compatibilityLevel' AND class = 'driver-openaire2.0';

DELETE FROM class_scheme

WHERE scheme = 'dnet:compatibilityLevel' AND class = 'driver-openaire2.0-openaire3.0';

DELETE FROM class_scheme

WHERE scheme = 'dnet:compatibilityLevel' AND class = 'driver-openaire3.0';

DELETE FROM class_scheme

WHERE scheme = 'dnet:compatibilityLevel' AND class = 'openaire2.0-openaire3.0';

DELETE FROM class

WHERE code = 'driver-openaire2.0';

DELETE FROM class

WHERE code = 'driver-openaire2.0-openaire3.0';

DELETE FROM class

WHERE code = 'driver-openaire3.0';

DELETE FROM class

WHERE code = 'openaire2.0-openaire3.0';

DELETE FROM apicollections

WHERE api = 'api_________::openaire____::webcrawl::0';

DELETE FROM api

WHERE id = 'api_________::openaire____::webcrawl::0';

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('fileCSV', 'fileCSV', 'fileCSV');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('fileCSV@@dnet:protocols', 'fileCSV', 'dnet:protocols');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('files_from_mdstore', 'files_from_mdstore', 'files_from_mdstore');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('files_from_mdstore@@dnet:protocols', 'files_from_mdstore', 'dnet:protocols');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('files_from_metadata', 'files_from_metadata', 'files_from_metadata');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('files_from_metadata@@dnet:protocols', 'files_from_metadata', 'dnet:protocols');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('files_by_rpc', 'files_by_rpc', 'files_by_rpc');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('files_by_rpc@@dnet:protocols', 'files_by_rpc', 'dnet:protocols');

INSERT INTO class (_dnet_resource_identifier_, code, name) VALUES ('filesystem', 'filesystem', 'filesystem');

INSERT INTO class_scheme (_dnet_resource_identifier_, class, scheme) VALUES ('filesystem@@dnet:protocols', 'filesystem', 'dnet:protocols');

INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass)

VALUES ('api_________::openaire____::webcrawl::csv', 'api_________::openaire____::webcrawl::csv', 'fileCSV', 'openaire____::webcrawl', 'websource', 'native');

INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES

	('api_________::openaire____::webcrawl::files', 'api_________::openaire____::webcrawl::files', 'files_from_mdstore', 'openaire____::webcrawl', 'websource',

	 'files', 'file::WoS');

INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES

	('api_________::opendoar____::18::files', 'api_________::opendoar____::18::files', 'files_from_metadata', 'opendoar____::18', 'pubsrepository::unknown',

	 'files', 'file::PDF');

INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES

	('api_________::opendoar____::908::files', 'api_________::opendoar____::908::files', 'files_from_metadata', 'opendoar____::908', 'pubsrepository::unknown',

	 'files', 'file::PDF');

INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES

	('api_________::opendoar____::2367::files', 'api_________::opendoar____::2367::files', 'files_from_metadata', 'opendoar____::2367', 'pubsrepository::unknown',

	 'files', 'file::PDF');

INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES

	('api_________::opendoar____::165::files', 'api_________::opendoar____::165::files', 'files_from_metadata', 'opendoar____::165', 'pubsrepository::unknown',

	 'files', 'file::PDF');

INSERT INTO api (_dnet_resource_identifier_, id, protocolclass, datasource, typologyclass, compatibilityclass, contentdescriptionclass) VALUES

	('api_________::infrastruct_::openaire::jdbc', 'api_________::infrastruct_::openaire::jdbc', 'jdbc', 'infrastruct_::openaire', 'scholarcomminfra',

	 'openaire2.0', 'metadata');

INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'jdbc:postgresql://localhost:5432', 'api_________::infrastruct_::openaire::jdbc');

INSERT INTO apicollections (param, original, api) VALUES ('dbName', 'dnet_openaireplus', 'api_________::infrastruct_::openaire::jdbc');

INSERT INTO apicollections (param, original, api) VALUES

	('sql', 'SELECT regexp_replace(xml, ''&(?!amp;)'', ''&'', ''g'') as xml FROM claims WHERE type = ''dmf2actions'' and set = ''userclaim_dmf''',

	 'api_________::infrastruct_::openaire::jdbc');

INSERT INTO apicollections (param, original, api) VALUES ('format', 'OAF', 'api_________::infrastruct_::openaire::jdbc');

INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', 'file:///var/lib/dnet/wos/metadata.tsv', 'api_________::openaire____::webcrawl::csv');

INSERT INTO apicollections (param, original, api) VALUES ('separator', E'\\t', 'api_________::openaire____::webcrawl::csv');

INSERT INTO apicollections (param, original, api) VALUES ('identifier', '56', 'api_________::openaire____::webcrawl::csv');

INSERT INTO apicollections (param, original, api) VALUES ('header', 'true', 'api_________::openaire____::webcrawl::csv');

INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::openaire____::webcrawl::files');

INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::openaire____::webcrawl::files');

INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//column[./@isID=''true'']', 'api_________::openaire____::webcrawl::files');

INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::opendoar____::18::files');

INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::opendoar____::18::files');

INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//dc:identifier', 'api_________::opendoar____::18::files');

INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::opendoar____::908::files');

INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::opendoar____::908::files');

INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//dc:identifier', 'api_________::opendoar____::908::files');

INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::opendoar____::2367::files');

INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::opendoar____::2367::files');

INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//dc:identifier', 'api_________::opendoar____::2367::files');

INSERT INTO apicollections (param, original, api) VALUES ('baseUrl', '', 'api_________::opendoar____::165::files');

INSERT INTO apicollections (param, original, api) VALUES ('mdstoreId', '', 'api_________::opendoar____::165::files');

INSERT INTO apicollections (param, original, api) VALUES ('xpath', '//dc:identifier', 'api_________::opendoar____::165::files');

INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '//repository/@rID', 'api_________::opendoar::0', FALSE);

INSERT INTO apicollections (param, original, api, accessparam)

VALUES ('metadata_identifier_path', '//*[local-name()=''repository'']/*[local-name()=''identifier'']', 'api_________::re3data::0', FALSE);

INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '//Position', 'api_________::wellcometrust::0', FALSE);

INSERT INTO apicollections (param, original, api, accessparam) VALUES ('metadata_identifier_path', '//ProjectId', 'api_________::corda::0', FALSE);

DELETE FROM apicollections

WHERE param = 'splitOnElement' AND api = 'api_________::corda::0';

INSERT INTO apicollections (param, original, api) VALUES ('filter', '*.xml', 'api_________::corda::0');

INSERT INTO apicollections (param, original, api) VALUES ('username', '***', 'api_________::corda::0');

INSERT INTO apicollections (param, original, api) VALUES ('password', '***', 'api_________::corda::0');

INSERT INTO apicollections (param, original, api) VALUES ('recursive', 'false', 'api_________::corda::0');

UPDATE apicollections

SET original = 'ftp://fts.ec.europa.eu'

WHERE api = 'api_________::corda::0' AND param = 'baseUrl';

UPDATE api

SET protocolclass = 'ftp'

WHERE id = 'api_________::corda::0';

UPDATE api

SET protocolclass = 'filesystem'

WHERE id = 'api_________::re3data::0';

UPDATE apicollections

SET original = '/var/lib/dnet/re3data'

WHERE api = 'api_________::re3data::0' AND param = 'baseUrl';

UPDATE api

SET compatibilityclass = 'native'

WHERE id = 'api_________::datacite::0';

-- metadata_identifier_path for Datacite and WoS

INSERT INTO apicollections (param, original, api, accessparam)

VALUES ('metadata_identifier_path', '//*[local-name()=''header'']/*[local-name()=''identifier'']', 'api_________::datacite::0', FALSE);

INSERT INTO apicollections (param, original, api, accessparam)

VALUES ('metadata_identifier_path', '//*[local-name()=''column'' and @isID=''true'']', 'api_________::openaire____::webcrawl::csv', FALSE);

UPDATE apicollections

SET _dnet_resource_identifier_ = api || '@@' || param;

-- get the orgs whose legalshortname is in upper case (probably good sort name, i.e. acronym)

select id, legalshortname from dsm_organizations where id like 're3data%' and legalshortname ~ '^[^a-z]*$';

-- removed legalshortnames of re3data orgs when the legalshortname is not all uppercase

update dsm_organizations set legalshortname = '' where id like 're3data%' and legalshortname !~ '^[^a-z]*$';

-- noinspection SqlNoDataSourceInspectionForFile

-- cleanup the unnecessary functions: the following SQL produces the deletion statement list

SELECT 'DROP FUNCTION ' || ns.nspname || '.' || proname || '(' || oidvectortypes(proargtypes) || ') CASCADE;'

FROM pg_proc

	INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid)

WHERE ns.nspname = 'public'

ORDER BY proname;

-- cleanup the unnecessary triggers: the following SQL produces the deletion statement list

SELECT 'DROP TRIGGER ' || trg.tgname || ' ON ' || tbl.relname || ';'

FROM pg_trigger trg

 JOIN pg_class tbl on trg.tgrelid = tbl.oid

 JOIN pg_namespace ns ON ns.oid = tbl.relnamespace

WHERE trg.tgname like 'notify_%';

-- export the new dnet_dsm schema using

--    pg_dump --schema-only -h localhost -U dnet dnet_dsm | gzip -c > /tmp/dnet_dsm.sql.gz

-- rename tables and index names in order to include prefix "dsm_"

-- import the dsm table schema to the existing openaire database:

--    psql -h localhost -U dnet dnet_openaireplus -f dnet_dsm.sql

ALTER TABLE dsm_datasources ALTER COLUMN officialname TYPE VARCHAR(512);

ALTER TABLE dsm_datasources ALTER COLUMN englishname TYPE VARCHAR(512);

ALTER TABLE dsm_datasources ALTER COLUMN missionstatementurl TYPE VARCHAR(512);

ALTER TABLE dsm_datasources ALTER COLUMN citationguidelineurl TYPE VARCHAR(512);

ALTER TABLE dsm_datasources ALTER COLUMN languages TYPE text;

ALTER TABLE dsm_datasources ALTER COLUMN od_contenttypes TYPE text;

ALTER TABLE dsm_datasources ALTER COLUMN subjects TYPE text;

ALTER TABLE dsm_datasources ALTER COLUMN description TYPE text;

ALTER TABLE dsm_datasources ALTER COLUMN certificates TYPE text;

ALTER TABLE dsm_datasources ALTER COLUMN pidsystems TYPE text;

-- insert into the new dsm_* tables picking the information from the old tables

insert into dsm_datasources (id, officialname, englishname, websiteurl, logourl, contactemail, latitude, longitude, timezone, namespaceprefix, languages, od_contenttypes, collectedfrom, dateofvalidation,

                             typology, provenanceaction, dateofcollection, platform, activationid, description,releasestartdate, releaseenddate, missionstatementurl, dataprovider, serviceprovider,

                             databaseaccesstype, datauploadtype,databaseaccessrestriction, datauploadrestriction, versioning, citationguidelineurl, qualitymanagementkind,pidsystems, certificates,

                             aggregator, issn, eissn, lissn, registeredby, subjects)

select d.id, 

	d.officialname, 

	d.englishname, 

	d.websiteurl, 

	d.logourl, 

	d.contactemail, 

	d.latitude, 

	d.longitude, 

	d.timezone, 

	d.namespaceprefix as namespaceprefix,

	d.od_languages as languages,

	d.od_contenttypes,

	d.collectedfrom,

	d.dateofvalidation,

	d.datasourceclass as typology,

	d.provenanceactionclass as provenanceaction,

	d.dateofcollection,	

	d.typology as platform,

	d.activationid,

	d.description,

	d.releasestartdate,

	d.releaseenddate,

	d.missionstatementurl,

	d.dataprovider as dataprovider,

	d.serviceprovider as serviceprovider,

	d.databaseaccesstype,

	d.datauploadtype,

	d.databaseaccessrestriction,

	d.datauploadrestriction,

	d.versioning,

	d.citationguidelineurl,

	d.qualitymanagementkind,

	d.pidsystems,

	d.certificates,

	d.aggregator,

	d.issn,

	d.eissn,

	d.lissn,

	d.registeredby,

	ARRAY_AGG(s.name) as subjects

from datasources d

	left outer join datasource_subject sd on (d.id = sd.datasource)

	left outer join subjects s on (sd.subject = s.id)

group by

	d.id;

insert into dsm_api (id, protocol, datasource, contentdescription, active, removable, typology, compatibility)

select a.id,

	a.protocolclass as protocol,

	a.datasource,

	a.contentdescriptionclass as contentdescription,

	a.active,

	a.removable,

	a.typologyclass as typology,

	a.compatibilityclass as compatibility

from api a;

update dsm_api da set metadata_identifier_path = ac.original from apicollections ac where da.id = ac.api and ac.param = 'metadata_identifier_path' ;

update dsm_api da set last_collection_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_collection_total' ;

update dsm_api da set last_collection_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_collection_date' ;

update dsm_api da set last_collection_mdid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_collection_mdId' ;

update dsm_api da set last_aggregation_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_total' ;

update dsm_api da set last_aggregation_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_date' ;

update dsm_api da set last_aggregation_mdid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_aggregation_mdId' ;

update dsm_api da set last_download_total = ac.original::integer from apicollections ac where da.id = ac.api and ac.param = 'last_download_total' ;

update dsm_api da set last_download_date = ac.original::date from apicollections ac where da.id = ac.api and ac.param = 'last_download_date' ;

update dsm_api da set last_download_objid = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_download_objId' ;

update dsm_api da set last_validation_job = ac.original from apicollections ac where da.id = ac.api and ac.param = 'last_validation_job' ;

update dsm_api da set baseurl = CASE WHEN (ac.edited is not null and ac.edited != '') THEN ac.edited ELSE ac.original END from apicollections ac where da.id = ac.api and ac.param = 'baseUrl' ;

insert into dsm_apiparams (param,value, api)

select

	a.param,

	CASE

		WHEN (a.edited is not null and a.edited != '') THEN a.edited ELSE a.original

	END AS value,

	a.api

from apicollections a

where	param != 'baseUrl' AND accessparam is true;

INSERT INTO dsm_organizations (id, legalshortname,legalname,websiteurl, logourl,ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests,

                               ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, country, collectedfrom, dateofcollection, provenanceaction)

	SELECT

		o.id,

		o.legalshortname,

		o.legalname,

		o.websiteurl,

		o.logourl,

		o.ec_legalbody,

		o.ec_legalperson,

		o.ec_nonprofit,

		o.ec_researchorganization,

		o.ec_highereducation,

		o.ec_internationalorganizationeurinterests,

		o.ec_internationalorganization,

		o.ec_enterprise,

		o.ec_smevalidated,

		o.ec_nutscode,

		o.countryclass          AS country,

		o.collectedfrom,

		o.dateofcollection,

		o.provenanceactionclass AS provenanceaction

	FROM organizations o;

INSERT INTO dsm_datasource_organization (datasource, organization)

	SELECT

		dorg.datasource,

		dorg.organization

	FROM datasource_organization dorg;

INSERT INTO dsm_identities (pid, issuertype)

	SELECT

		i.pid,

		i.issuertypeclass AS issuertype

	FROM identities i;

INSERT INTO dsm_datasourcepids (datasource, pid)

	SELECT

		dp.datasource,

		dp.pid

	FROM datasourcepids dp;

-- is this table needed? --

INSERT INTO dsm_organizationpids (organization, pid)

	SELECT

		op.organization,

		op.pid

	FROM organizationpids op;

-- Migrate the managed status

update dsm_datasources d set managed = true where d.id in (

	SELECT d.id

	FROM dsm_datasources d

	LEFT OUTER JOIN dsm_api a ON (d.id = a.datasource)

	LEFT OUTER JOIN apicollections ap ON (ap.api = a.id)

	WHERE a.active = TRUE

	  OR a.removable = TRUE

	  OR ap.edited IS NOT NULL

	GROUP BY d.id );

-- !!!!! data insert complete !!!!!

delete from organizations where id like 'opendoar%' ;

delete from organizations where id like 're3data%' ;

delete from organizations where collectedfrom is null ;

delete from dsm_organizations where collectedfrom = 'openaire____::corda'  ;

delete from dsm_organizations where collectedfrom = 'openaire____::snsf'  ;

delete from dsm_organizations where collectedfrom = 'openaire____::conicytf'  ;

delete from dsm_organizations where collectedfrom = 'openaire____::irb_hr' ;

delete from dsm_organizations where collectedfrom = 'openaire____::fwf'  ;

delete from dsm_organizations where collectedfrom = 'openaire____::sfi'  ;

delete from dsm_organizations where collectedfrom = 'openaire____::wellcometrust'  ;

delete from dsm_organizations where collectedfrom = 'openaire____::nsf'  ;

delete from dsm_organizations where collectedfrom = 'openaire____::corda_h2020'  ;

ALTER TABLE dsm_datasources ALTER COLUMN databaseaccesstype TYPE varchar(64);

ALTER TABLE dsm_datasources ALTER COLUMN datauploadtype TYPE varchar(64);

ALTER TABLE dsm_datasources ALTER COLUMN databaseaccessrestriction TYPE varchar(64);

ALTER TABLE dsm_datasources ALTER COLUMN datauploadrestriction TYPE varchar(64);

ALTER TABLE dsm_datasources ALTER COLUMN qualitymanagementkind TYPE varchar(64);

ALTER TABLE dsm_datasources ALTER COLUMN missionstatementurl TYPE text;

	<version>7.1.19-SNAPSHOT</version>

			<version>[3.0.0,4.0.0)</version>

			<version>[1.0.0-SNAPSHOT,2.0.0)</version>

import com.googlecode.sarasvati.Arc;

import com.googlecode.sarasvati.NodeToken;

import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;

import eu.dnetlib.enabling.resultset.MappedResultSetFactory;

import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;

import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;

	private String filterManaged(final String data, Set<String> filter) throws IllegalStateException {

			final String dsId = doc.valueOf("/record/metadata/ROWS/ROW[@table = 'dsm_datasources']/FIELD[@name = 'id']/text()");

			} else return data;

		} catch (DocumentException e) {

import javax.annotation.Resource;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

	@Resource

	private UniqueServiceLocator serviceLocator;

		final String query = "let $x := /*[.//RESOURCE_IDENTIFIER/@value='" + providerId + "']//EXTRA_FIELDS\n"

				+ "return concat($x/FIELD[./key='OpenAireDataSourceId']/value, ' @@@ ', $x/FIELD[./key='NamespacePrefix']/value)";

		final String[] arr = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query).split("@@@");

		final String origId = arr[0].trim();

		final String nsPrefix = arr[1].trim();

		//this is needed by the mdbuilder

		//TODO: update mdbuilder to use the env attributes below, whose names are defined in WorkflowConstants

		token.getEnv().setAttribute("parentDatasourceId", origId);

		//these are needed for validation and fill hostedby

		token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID, origId);

		token.getFullEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID, origId);

		token.getProcess().getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID, origId);

import javax.annotation.Resource;

import org.antlr.stringtemplate.StringTemplate;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import java.io.UnsupportedEncodingException;

import java.net.URLEncoder;

	@Resource

		if ((inputEpr == null) || inputEpr.isEmpty()) throw new MSROException("InputEprParam (" + inputEprParam + ") not found in ENV");

			st.setAttribute("xpath", getMetadataIdentifierXPath());

			st.setAttribute("baseurl", URLEncoder.encode(getBaseUrl(), "UTF-8"));

			st.setAttribute("metadatanamespace", getMetadataNamespace());

		} catch (ISLookUpException e) {

		} catch (UnsupportedEncodingException e) {

	private String getMetadataIdentifierXPath() throws ISLookUpException {

		String xQuery =

				String.format("for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') "

						+ "return $x//INTERFACE[@id='%s']/INTERFACE_EXTRA_FIELD[@name='metadata_identifier_path']/string()", getDatasourceInterface());

		return hasOverridingMetadataIdentifierXPath() ? getOverridingMetadataIdentifierXPath() : serviceLocator.getService(ISLookUpService.class)

				.getResourceProfileByQuery(xQuery);

	}

	private String getBaseUrl() throws ISLookUpException {

		String xQuery = "/*[.//RESOURCE_IDENTIFIER/@value='{datasourceId}']//INTERFACE[@id='{interfaceId}']//BASE_URL/string()";

		xQuery = xQuery.replace("{interfaceId}", datasourceInterface).replace("{datasourceId}", datasourceId);

		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xQuery);

	}

	private String getMetadataNamespace() {

					"let $x := /*[.//RESOURCE_IDENTIFIER/@value='{datasourceId}']//INTERFACE[@id='{interfaceId}']/ACCESS_PROTOCOL/@format/string() "

							+ "return /*[.//RESOURCE_TYPE/@value='MetadataFormatDSResourceType']//METADATAFORMAT[@Prefix=$x]/@NameSpace/string()";

			xQuery = xQuery.replace("{interfaceId}", datasourceInterface).replace("{datasourceId}", datasourceId);

		} catch (ISLookUpException e) {

					+ "cannot managed here and it will be leaved empty (for the time being)");

	private boolean hasOverridingMetadataIdentifierXPath() {

		return !getOverridingMetadataIdentifierXPath().isEmpty();

	}

	private LocalDatasourceManager<Datasource<?, ?>, Api<?>> dsManager;

		final String q1 = "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')\n" +

				"let $id := $x//RESOURCE_IDENTIFIER/@value\n" +

				"for $api in $x//INTERFACE/@id\n" +

				"return concat($id,'|',$api)";

		final String q2 = "for $x in collection(' /db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType') \n" +

				"where $x//DATAPROVIDER/@id and $x//DATAPROVIDER/@interface\n" +

				"return concat($x//RESOURCE_IDENTIFIER/@value,'@@@',$x//DATAPROVIDER/@id,'|',$x//DATAPROVIDER/@interface)";

		final Set<String> valids = new HashSet<>(lookup.quickSearchProfile(q1));

		for (final String s : lookup.quickSearchProfile(q2)) {

import java.util.HashMap;

import java.util.Map;

import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;

import eu.dnetlib.enabling.datasources.common.ApiParam;

	private LocalOpenaireDatasourceManager dsManager;

	@Autowired

	private UniqueServiceLocator serviceLocator;

	private ISRegistryService registry;

	private final Map<String, String> openaireIds = new HashMap<>();

	private boolean alwaysUpdate = false;

		try {

			openaireIds.putAll(lookup.quickSearchProfile(

					"for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') return concat($x//DATASOURCE_ORIGINAL_ID, ' @@@ ', $x//RESOURCE_IDENTIFIER/@value)")

					.stream()

					.collect(Collectors.toMap(

							s -> StringUtils.substringBefore(s, "@@@").trim(),

							s -> StringUtils.substringAfter(s, "@@@").trim())));

		} catch (final ISLookUpException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

				for (final Api<ApiParam> api : dsManager.getApis(dsId)) {

	private void verifyApi(final String dsId, final Api<ApiParam> api)

			throws DsmException, ISRegistryException, IOException, ISLookUpException, MDStoreServiceException, ObjectStoreServiceException {

		st.setAttribute("dsId", openaireIds.get(dsId));

				.stream()

				.map(s -> {

					try {

						return reader.read(new StringReader(s));

					} catch (final DocumentException e) {

						return null;

					}

				})

				.filter(Objects::nonNull)

				.collect(Collectors.toList());

import java.net.URI;

import java.util.HashMap;

import java.util.Map;

import com.google.gson.Gson;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.rmi.MSROException;

import org.apache.commons.lang3.StringUtils;

import org.springframework.web.util.UriComponentsBuilder;

import javax.annotation.Resource;

    private static final Log log = LogFactory.getLog(CreateMDStoreHadoopJobNode.class);

    public static final String MDSTORE_CREATE_URL_AND_PARAMS = "/new/{format}/{layout}/{interpretation}?dsName={dsName}&dsId={dsid}&apiId={apiId}";

    /* Workflow params */

    private String format;

    private String layout;

    private String interpretation;

    private String outputPrefix = "mdstore";

    /* Spring managed params */

    private String mdStoreManagerUrl;

    @Resource

    private UniqueServiceLocator serviceLocator;

    @Override

    protected String execute(final NodeToken token) throws Exception {

        final String repositoryId = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ID);

        final String xquery = String.format(

                "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') \n" +

                "where .//RESOURCE_IDENTIFIER[./@value='%s']\n" +

                "return $x//EXTRA_FIELDS/FIELD[./key/text() = 'OpenAireDataSourceId']/value/text()",

                repositoryId);

        log.info("resolving openaAireDatasourceId: " + xquery);

        final String openAireDataSourceId = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);

        if (StringUtils.isBlank(openAireDataSourceId)) {

            throw new MSROException("unable to find openAireDataSourceId in repository profile " + repositoryId);

        }

        log.info(String.format("got OpenAIRE datasource id '%s' for repository profile '%s'", openAireDataSourceId, repositoryId));

        final String urlTemplate = getMdStoreManagerUrl() + MDSTORE_CREATE_URL_AND_PARAMS;

        final RestTemplate restTemplate = new RestTemplate();

        final MDStoreWithInfo result = restTemplate.getForObject(

                urlTemplate,

                MDStoreWithInfo.class,

                getFormat(),

                getLayout(),

                getInterpretation(),

                token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_NAME),

                openAireDataSourceId,

                token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE));

        log.info("created mdstore: " + new Gson().toJson(result));

        token.getEnv().setAttribute(getOutputPrefix() + "format", format);

        token.getEnv().setAttribute(getOutputPrefix() + "layout", layout);

        token.getEnv().setAttribute(getOutputPrefix() + "interpretation", interpretation);

        token.getEnv().setAttribute(getOutputPrefix() + "id", result.getId());

        return Arc.DEFAULT_ARC;

    }

    public String getFormat() {

        return format;

    }

    public void setFormat(final String format) {

        this.format = format;

    }

    public String getLayout() {

        return layout;

    }

    public void setLayout(final String layout) {

        this.layout = layout;

    }

    public String getInterpretation() {

        return interpretation;

    }

    public void setInterpretation(final String interpretation) {

        this.interpretation = interpretation;

    }

    public String getOutputPrefix() {

        return outputPrefix;

    }

    public void setOutputPrefix(final String outputPrefix) {

        this.outputPrefix = outputPrefix;

    }

    public String getMdStoreManagerUrl() {

        return mdStoreManagerUrl;

    }

    @Required

    public void setMdStoreManagerUrl(final String mdStoreManagerUrl) {

        this.mdStoreManagerUrl = mdStoreManagerUrl;

    }

import eu.dnetlib.common.logging.DnetLogger;

import eu.dnetlib.dhp.collection.ApiDescriptor;

import eu.dnetlib.dhp.model.mdstore.Provenance;

import eu.dnetlib.enabling.datasources.common.Datasource;

import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;

import eu.dnetlib.msro.rmi.MSROException;

import org.springframework.beans.factory.annotation.Value;

import javax.annotation.Resource;

import static eu.dnetlib.dhp.common.Constants.*;

    private static final Log log = LogFactory.getLog(PrepareEnvCollectHadoopJobNode.class);

    public static final String METADATA_IDENTIFIER_PATH = "metadata_identifier_path";

    public static final String DATE_FORMAT = "yyyy-MM-dd";

    private long ONE_DAY = 1000 * 60 * 60 * 24;

    @Resource(name = "msroWorkflowLogger")

    private DnetLogger dnetLogger;

    @Autowired

    private LocalDatasourceManager<?, ?> dsManager;