D-Net

#dnet.data.path 								=	/tmp/dnet

dnet.bootstrap.schemas                          =   classpath*:/eu/dnetlib/bootstrap/schemas/**/*.xsd

dnet.bootstrap.profiles                         =   classpath*:/eu/dnetlib/bootstrap/profiles/**/*.xml

msro.wf.logger.name					            =	wf_logs_dli

services.datasourceManager.core                 =   simpleDnetDataSourceManagerCore

dnet.datasourcemanager.db.name                  =   dnet_dli

msro.wf.mail.smtp.host                          =   smtp.isti.cnr.it

msro.wf.mail.smtp.user                          =   smtp-dnet

msro.wf.mail.smtp.password                      =   hhr*7932

msro.wf.mail.cc                                 =   michele.artini@isti.cnr.it

services.pid.resolver.header                    =   dli_resolver::

service.index.solr.rank.enable                  =   true

services.publisher.oai.host                     =   localhost

services.publisher.oai.port                     =   27017

services.objectstore.dao                        =   gridFSObjectstoreDao

services.transformation.blacklist_api           = http://localhost:8280/validator-service/worfklows?request=GetBlacklistedRecords&datasourceId=

import org.junit.Ignore;

    @Ignore

<record xmlns="http://www.scholix.org" xmlns:oaf="http://namespace.dnet.eu/oaf"

        xmlns:dc="http://purl.org/dc/elements/1.1/"

        xmlns:dri="http://www.driver-repository.eu/namespace/dri">

    <oai:header xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

                xmlns:oai="http://www.openarchives.org/OAI/2.0/">

        <dri:objIdentifier>crossref____::6b85fd4ba09dfc3cd0417e2f758755e5</dri:objIdentifier>

        <dri:resolvedDate>2018-12-13T12:29:43.770</dri:resolvedDate>

        <dri:resolvedDate>2018-12-13T11:13:45.043</dri:resolvedDate>

        <dri:recordIdentifier>10.1577/1548-8675(1998)018<0569:phbitw>2.0.co;2::doi</dri:recordIdentifier>

        <dri:dateOfCollection>2018-12-11T12:27:26.758+01:00</dri:dateOfCollection>

        <dri:repositoryId>0cf05336-894e-4793-a1f0-58d9705c4e6f_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId>

        <dri:datasourceprefix>crossref____</dri:datasourceprefix>

    <metadata>

        <oaf:pid type ="doi">10.1577/1548-8675(1998)018&lt;0569:PHBITW&gt;2.0.CO;2</oaf:pid>

        <dc:identifier>http://dx.doi.org/10.1577/1548-8675(1998)018&lt;0569:PHBITW&gt;2.0.CO;2</dc:identifier>

        <dc:date></dc:date>

        <dc:description></dc:description>

        <dc:type>publication</dc:type>

        <oaf:relatedIdentifier relatedIdentifierType="dnet" relationType="References" inverseRelationType="" entityType="publication">dli_resolver::60f9cf6d317e38bd7081f7733d92d80a</oaf:relatedIdentifier>

    <oaf:about>

        <oaf:datainfo>

            <oaf:completionStatus>incomplete</oaf:completionStatus>

            <oaf:collectedFrom id="dli_________::crossref" name="Crossref" completionStatus="incomplete"/>

        </oaf:datainfo>

    </oaf:about>

</record>

import eu.dnetlib.msro.workflows.util.ProgressProvider;

public class MergeDLIRecord extends BlackboardJobNode implements ProcessAware, ProgressProvider {

    private String numExecutor;

            job.getParameters().put("numExecutor", getNumExecutor());

    public String getNumExecutor() {

        return numExecutor;

    }

    public void setNumExecutor(String numExecutor) {

        this.numExecutor = numExecutor;

    }

    @Override

    public String getProgressDescription() {

        return null;

    }

			log.error("Input record: " + record);

                    VtdUtilityParser.getTextValuesWithAttributes(ap, vn, "//*[local-name()='resource']//*[local-name()='relatedIdentifier']", Arrays.asList("relatedIdentifierType", "relationType", "inverseRelationType", "entityType"));

                    final String relatedEntityType = relationMap.getAttributes().get("entityType");

                    currentRelation.setTargetType(ObjectType.valueOf(relatedEntityType));

        final String number_of_core= params.get("numExecutor");

            final String [] command= {sparkPath+"bin/spark-submit",sparkJobPath ,host, transactionManager.getDb().getName(), mdStoreCollection, nsPrefix, number_of_core, sparkApplicationName };

    @Override

    public String getStatus() {

        return "30/100";

    }

import java.util.function.Function;

        return resolveRelations(inputRecord, inputObject, shouldUpdate,s -> {

			try {

				return DLIUtils.getInverse(s);

			} catch (Exception e) {

				log.error("Error on getting Inverse relation from "+s, e);

				return "";

			}

		});

        dli.setDatasourceProvenance(

                DnetStreamSupport.generateStreamFromIterator(root.getAsJsonArray("datasourceProvenance").iterator())

                        .map(JsonElement::getAsJsonObject)

                        .map(it -> {

                            final DLIObjectProvenance provenance =new DLIObjectProvenance();

                            provenance.setDatasourceId(it.get("datasourceId").getAsString());

                            provenance.setCompletionStatus(it.get("completionStatus").getAsString());

                            provenance.setProvisionMode(it.get("provisionMode").getAsString());

                            provenance.setDatasource(it.get("datasource").getAsString());

                            provenance.setPublisher(getStringValue(it, "publisher"));

                            return provenance;

                        })

                        .collect(Collectors.toList())

        );

		<identifier identifierType="$object.pidType$">$object.escapedXMLPid$</identifier>

	void run(MDStoreDao dao, Map<String, String> params, DoneCallback doneCallback) throws MDStoreServiceException;

	default String getStatus() {

		return "";

	}

import java.util.concurrent.Executors;

class StatusThread extends Thread {

	boolean shouldUpdate = false;

	final BlackboardServerHandler handler;

	final BlackboardJob job;

	final long timeout;

	final MDStorePlugin currentPlugin;

	public StatusThread(final BlackboardServerHandler handler, final BlackboardJob job, final long timeout, final  MDStorePlugin currentPlugin) {

		this.job = job;

		this.handler = handler;

		this.timeout = timeout;

		this.currentPlugin = currentPlugin;

	}

	@Override

	public void run() {

		while (shouldUpdate) {

			job.getParameters().put("ongoingPercentage",currentPlugin.getStatus());

			handler.ongoing(job);

			try {

				sleep(timeout);

			} catch (InterruptedException e) {

			}

		}

	}

	public boolean isShouldUpdate() {

		return shouldUpdate;

	}

	public void setShouldUpdate(boolean shouldUpdate) {

		this.shouldUpdate = shouldUpdate;

	}

}

		final StatusThread st = new StatusThread(handler, job,20000, plugins.get(name));

		st.setShouldUpdate(true);

		st.start();

			st.setShouldUpdate(false);

import java.util.function.Function;

    protected String resolveRelations(String inputRecord, ResolvedObject inputObject, boolean shouldUpdate, Function<String,String> getInverseRelation) {

                                rel.setInverseRelation(getInverseRelation.apply(rel.getRelationSemantics()));

import com.google.common.collect.Lists;

import com.mongodb.client.model.*;

		final UpdateOptions f = new UpdateOptions().upsert(true);

		final List<WriteModel<DBObject>> upsertList = new ArrayList<>();

		final BulkWriteOptions writeOptions = new BulkWriteOptions().ordered(false);

		final int bulkSize = 1000;

		long validOpCounter  =0;

		for (DBObject object : dbObjects) {

			if (StringUtils.isNotBlank(object.get("id").toString())) {

				final DBObject replacedObj = BasicDBObjectBuilder.start()

						.add("body", object.get("body").toString())

						.add("resolved_ts", object.get("resolved_ts"))

						.get();

				upsertList.add(new ReplaceOneModel(new BasicDBObject("id",  object.get("id").toString()), replacedObj, f));

				validOpCounter++;

				if (((validOpCounter % bulkSize) == 0) && (validOpCounter != 0)) {

					currentMdStoreCollection.bulkWrite(upsertList, writeOptions);

					upsertList.clear();

					log.info("Transaction commit: Upserting: "+validOpCounter);

				}

			}

//		if (upsertList.size()>0){

//			mdstore.bulkWrite(upsertList, writeOptions);

//		}

//

//

//

//        for (DBObject object : dbObjects) {

//			Bson query = Filters.eq("id", object.get("id").toString());

//			final DBObject replacedObj = BasicDBObjectBuilder.start()

//					.add("body", object.get("body").toString())

//                    .add("resolved_ts", object.get("resolved_ts"))

//                    .get();

//			Bson newDocument = new Document("$set", replacedObj);

//			currentMdStoreCollection.findOneAndUpdate(query, newDocument);

//            i++;

//

//		}

import org.apache.commons.lang3.StringEscapeUtils;

        if (args == null || args.length != 6) {

        final String core_number = args[4];

        final String applicationName = args[5];

                .master("local["+core_number+"]")

            final String newBody = getSerializerInstance().serializeReplacingXML(resultObject.getBody(), parse, objIdentifier, StringEscapeUtils.escapeXml11(originalId));

            <scope>provided</scope>

            <scope>provided</scope>