D-Net

<?xml version="1.0" ?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-directindex-api</artifactId>

	<packaging>jar</packaging>

	<version>4.0.1</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-directindex-api/tags/dnet-directindex-api-4.0.1</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaireplus-mapping-utils</artifactId>

			<version>[7.0.0,8.0.0)</version>

			<exclusions>

				<exclusion>

					<groupId>eu.dnetlib</groupId>

					<artifactId>dnet-hadoop-commons</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-client</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>io.springfox</groupId>

			<artifactId>springfox-swagger2</artifactId>

			<version>${springfox-version}</version>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.9.5</version>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity</artifactId>

			<version>1.7</version>

			<exclusions>

				<exclusion>

					<artifactId>antlr</artifactId>

					<groupId>antlr</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity-tools</artifactId>

			<version>2.0</version>

			<exclusions>

				<exclusion>

					<artifactId>antlr</artifactId>

					<groupId>antlr</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-context-support</artifactId>

			<version>${spring.version}</version>

		</dependency>

		<dependency>

			<groupId>com.fasterxml.jackson.core</groupId>

			<artifactId>jackson-annotations</artifactId>

			<version>2.9.4</version>

		</dependency>

    </dependencies>

	<properties>

		<springfox-version>2.5.0</springfox-version>

	</properties>

</project>

package eu.dnetlib.openaire.directindex.api;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import com.mongodb.*;

import com.mongodb.client.model.IndexOptions;

import eu.dnetlib.miscutils.datetime.DateUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

/**

 * Created by michele on 11/11/15.

 */

public class RecentResultsQueue implements Iterable<String> {

	private static final Log log = LogFactory.getLog(RecentResultsQueue.class);

	private DB db;

	private String collection;

	public void init() {

		if (!db.collectionExists(collection)) {

			log.info(String.format("creating collection %s", collection));

			db.createCollection(collection, new BasicDBObject());

		}

		db.getCollection(collection).createIndex(new BasicDBObject("id", 1), new BasicDBObject("background", true));

	}

	@Override

	public Iterator<String> iterator() {

		final DBCursor cursor = db.getCollection(collection).find();

		return new Iterator<String>() {

			@Override

			public boolean hasNext() {

				return cursor.hasNext();

			}

			@Override

			public String next() {

				final DBObject obj = cursor.next();

				return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : "";

			}

			@Override

			public void remove() {

				throw new RuntimeException("NOT IMPLEMENTED");

			}

		};

	}

	synchronized public void add(final String oaf) throws Exception {

		final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']");

		log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection);

		final DBCollection coll = db.getCollection(collection);

		final DBObject obj = BasicDBObjectBuilder.start()

				.append("id", id)

				.append("record", oaf)

				.append("date", DateUtils.now())

				.get();

		coll.update(new BasicDBObject("id", id), obj, true, false);

	}

	public void remove(final List<String> list) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : list) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public void remove(final String... ids) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : ids) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public DB getDb() {

		return db;

	}

	@Required

	public void setDb(final DB db) {

		this.db = db;

	}

	public String getCollection() {

		return collection;

	}

	@Required

	public void setCollection(final String collection) {

		this.collection = collection;

	}

}

{

  "originalId": "oai:zenodo.org:8282705",

  "title": "স্বাধীনতার পঁচাত্তর",

  "authors": [

    "মহম্মদ সেলিম"

  ],

  "publisher": "Zenodo",

  "description": "\u003cp\u003e\u0026nbsp;স্বাধীনতার পঁচাত্তর\u003c/p\u003e",

  "language": "ben",

  "pids": [

    {

      "type": "oai",

      "value": "oai:zenodo.org:8282705"

    },

    {

      "type": "doi",

      "value": "10.5281/zenodo.8282705"

    }

  ],

  "licenseCode": "OPEN",

  "type": "publication",

  "resourceType": "0001",

  "url": "https://zenodo.org/record/8282705",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "contexts": [],

  "linksToProjects": []

}

package eu.dnetlib.openaire.directindex.objects;

import java.io.Serializable;

public class DsmDatasourceInfo implements Serializable {

	private static final long serialVersionUID = -593392381920400974L;

	private String id;

	private String officialname;

	private String namespaceprefix;

	public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public String getOfficialname() {

		return officialname;

	}

	public void setOfficialname(final String officialname) {

		this.officialname = officialname;

	}

	public String getNamespaceprefix() {

		return namespaceprefix;

	}

	public void setNamespaceprefix(final String namespaceprefix) {

		this.namespaceprefix = namespaceprefix;

	}

}

{

  "linksToProjects": [

    "info:eu-repo/grantAgreement/EC/H2020/101004870//A Competitive Intelligence Cloud%2FHPC Platform for AI-based STI Policy Making/IntelComp"

  ],

  "description": "<p>IntelComp sets out to build an innovative Cloud Platform that...",

  "publisher": "Zenodo",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "originalId": "oai:zenodo.org:5109933",

  "resourceType": "0045",

  "version": "1",

  "licenseCode": "OPEN",

  "authors": [

    "Elli Papadopoulou"

  ],

  "url": "https://zenodo.org/record/5109933",

  "title": "Data Management Plan for IntelComp. D8.2.",

  "pids": [

    {

      "type": "oai",

      "value": "oai:zenodo.org:5109933"

    },

    {

      "type": "doi",

      "value": "10.5281/zenodo.5109933"

    }

  ],

  "type": "publication"

}

{

  "openaireId":"od_______278::d5b39693ac6c2197e8cf48008a982951",

  "originalId": "oai:rudar.ruc.dk:1800/24691",

  "type": "software",

  "title": "Software test",

  "authors": [

	"Sloan, Patrick Alexander Raaby",

	"Strange, Gustav Valdemar"

  ],

  "publisher": "",

  "description": "This is for testing software",

  "language": "eng",

  "pids": [],

  "licenseCode": "OPEN SOURCE",

  "resourceType": "0000",

  "url": "http://rudar.ruc.dk/handle/1800/24691",

  "collectedFromId": "opendoar____::278",

  "hostedById": "opendoar____::278",

  "linksToProjects": [],

  "contexts": []

}

package eu.dnetlib.openaire.directindex.objects;

import java.io.Serializable;

import java.util.List;

public class DsmSearchResponse implements Serializable {

	private static final long serialVersionUID = 8944122902111813747L;

	private List<DsmDatasourceInfo> datasourceInfo;

	public List<DsmDatasourceInfo> getDatasourceInfo() {

		return datasourceInfo;

	}

	public void setDatasourceInfo(final List<DsmDatasourceInfo> datasourceInfo) {

		this.datasourceInfo = datasourceInfo;

	}

}

package eu.dnetlib.openaire.directindex.objects;

import java.util.ArrayList;

import java.util.List;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;

@JsonIgnoreProperties(ignoreUnknown = true)

public class ZenodoContextList {

        private String zenodoid;

        private List<String> openAirecommunitylist;

        public List<String> getOpenAirecommunitylist() {

            return openAirecommunitylist;

        }

        public void setOpenAirecommunitylist(List<String> openAirecommunitylist) {

            this.openAirecommunitylist = openAirecommunitylist;

        }

        public String getZenodoid() {

            return zenodoid;

        }

        public void setZenodoid(String zenodoid) {

            this.zenodoid = zenodoid;

        }

    }

package eu.dnetlib.openaire.directindex.objects;

import io.swagger.annotations.ApiModelProperty;

/**

 * Created by michele on 02/12/15.

 */

public class PidEntry {

	private String type;

	private String value;

	public PidEntry() {

	}

	public PidEntry(final String type, final String value) {

		this.type = type;

		this.value = value;

	}

	@ApiModelProperty(required = true, value = "E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types")

	public String getType() {

		return type;

	}

	public void setType(final String type) {

		this.type = type;

	}

	@ApiModelProperty(required = true)

	public String getValue() {

		return value;

	}

	public void setValue(final String value) {

		this.value = value;

	}

}

package eu.dnetlib.openaire.directindex.api;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import javax.annotation.Resource;

import java.util.concurrent.Executors;

import java.util.concurrent.ScheduledExecutorService;

import java.util.concurrent.TimeUnit;

public class ResultSubmitterService {

    private static final Log log = LogFactory.getLog(ResultSubmitterService.class);

    @Autowired

    private IndexDSRetriever indexDSRetriever;

    @Resource(name = "indexClientManager")

    private IndexClientManager clientManager;

    /**

     * Autocommit feature activation flag

     */

    @Value(value = "${openaire.api.directindex.autocommit.active}")

    private boolean autocommitactive;

    /**

     * Autocommit frequency (Seconds)

     */

    @Value(value = "${openaire.api.directindex.autocommit.frequency}")

    private long commitfrquency = 60;

    private ScheduledExecutorService executor;

    public ResultSubmitterService() {

        executor = Executors.newSingleThreadScheduledExecutor();

        updateCommitSchedule();

    }

    private void updateCommitSchedule() {

        log.info("updating commit schedule");

        executor.scheduleAtFixedRate(() -> {

            if (isAutocommitactive()) {

                try {

                    IndexDsInfo info = indexDSRetriever.calculateCurrentIndexDsInfo();

                    CloudSolrClient client = clientManager.getClient(info);

                    log.info("performing commit on " + info.getColl());

                    client.commit();

                } catch (Throwable e) {

                    log.error("unable to perform commit", e);

                }

            }

        }, 0, getCommitfrquency(), TimeUnit.SECONDS);

    }

    public boolean isAutocommitactive() {

        return autocommitactive;

    }

    public synchronized void setAutocommitactive(boolean autocommitactive) {

        this.autocommitactive = autocommitactive;

    }

    public long getCommitfrquency() {

        return commitfrquency;

    }

    public synchronized void setCommitfrquency(long commitfrquency) {

        this.commitfrquency = commitfrquency;

    }

}

package eu.dnetlib.openaire.directindex.utils;

import java.io.IOException;

import java.io.StringReader;

import java.io.StringWriter;

import javax.annotation.Resource;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import javax.xml.transform.stream.StreamSource;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.miscutils.functional.xml.ApplyXslt;

import org.apache.commons.io.IOUtils;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.ClassPathResource;

/**

 * Created by michele on 15/12/15.

 */

public class OafToIndexRecordFactory {

	private ClassPathResource layoutToRecord;

	@Resource

	private UniqueServiceLocator serviceLocator;

	public ApplyXslt newTransformer(final String format) throws ISLookUpException, IOException, TransformerException {

		final TransformerFactory factory = TransformerFactory.newInstance();

		final Transformer layoutTransformer = factory.newTransformer(readLayoutToRecord());

		final StreamResult layoutToXsltXslt = new StreamResult(new StringWriter());

		layoutTransformer.setParameter("format", format);

		layoutTransformer.transform(new StreamSource(new StringReader(getLayoutSource(format))), layoutToXsltXslt);

		return new ApplyXslt(layoutToXsltXslt.getWriter().toString());

	}

	private StreamSource readLayoutToRecord() throws IOException {

		return new StreamSource(new StringReader(IOUtils.toString(layoutToRecord.getInputStream())));

	}

	private String getLayoutSource(final String format) throws ISLookUpDocumentNotFoundException, ISLookUpException {

		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(

				"collection('/db/DRIVER/MDFormatDSResources/MDFormatDSResourceType')[.//NAME='" + format + "']//LAYOUT[@name='index']");

	}

	public ClassPathResource getLayoutToRecord() {

		return layoutToRecord;

	}

	@Required

	public void setLayoutToRecord(final ClassPathResource layoutToRecord) {

		this.layoutToRecord = layoutToRecord;

	}

}

package eu.dnetlib.openaire.directindex.api;

import java.io.StringWriter;

import java.util.ArrayList;

import java.util.Collection;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import java.util.Map;

import java.util.stream.Collectors;

import org.apache.commons.lang.StringUtils;

import org.springframework.web.client.RestClientException;

import org.springframework.web.client.RestTemplate;

import com.google.common.collect.Sets;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.openaire.directindex.objects.DatasourceEntry;

import eu.dnetlib.openaire.directindex.objects.DsmSearchRequest;

import eu.dnetlib.openaire.directindex.objects.DsmSearchResponse;

import eu.dnetlib.openaire.directindex.objects.ZenodoContextList;

/**

 * Created by michele on 15/01/16.

 */

public class OpenAIRESubmitterUtils {

	private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(OpenAIRESubmitterUtils.class);

	private static final String ZENODO_COMMUNITY = "zenodo.org/communities/";

	private final String community_api;

	private final String dsm_api;

	public OpenAIRESubmitterUtils(final String community_api, final String dsm_api) {

		this.community_api = community_api;

		this.dsm_api = dsm_api;

	}

	public DatasourceEntry findDatasource(final String dsId) {

		final RestTemplate rt = new RestTemplate();

		final String url = dsm_api + "/searchdetails/0/1?requestSortBy=id&order=ASCENDING";

		try {

			final DsmSearchResponse res = rt.postForObject(url, new DsmSearchRequest(dsId), DsmSearchResponse.class);

			return res.getDatasourceInfo()

				.stream()

				.map(ds -> new DatasourceEntry(ds.getId(), ds.getOfficialname(), ds.getNamespaceprefix()))

				.findFirst()

				.orElse(null);

		} catch (final RestClientException rce) {

			log.error("Unable to get object for " + url);

			return null;

		}

	}

	public Map<String, String> calculateProjectInfo(final String link) {

		final Map<String, String> info = new HashMap<>();

		final String[] arr = link.split("/");

		// info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble

		if (arr.length > 4) {

			final String acronym = arr.length > 7 ? arr[7] : "";

			final String title = arr.length > 6 ? StringUtils.isNotBlank(arr[6]) ? arr[6] : acronym : "";

			final String jurisdiction = arr.length > 5 ? arr[5] : "";

			final String funderId = calculateFunderId(arr[2], arr[3]);

			final String funderShortName = fixFunderShortName(arr[2]);

			final String fundingName = fixFundingName(funderShortName, arr[3]);

			info.put("id", calculateProjectId(arr[2], arr[3], arr[4]));

			info.put("funderShortName", fixFunderShortName(arr[2]));

			info.put("fundingName", fundingName);

			info.put("code", unescape(arr[4]));

			info.put("jurisdiction", jurisdiction);

			info.put("title", title);

			info.put("acronym", acronym);

			info.put("funderId", funderId);

			info.put("funderName", calculateFunderName(arr[2]));

			if (StringUtils.isNotBlank(arr[3])) {

				info.put("fundingId", funderId + "::" + fundingName);

			}

		}

		return info;

	}

	// TODO: remove me when Zenodo ingests the good UKRI projects

	protected String fixFunderShortName(final String funderShortName) {

		switch (funderShortName) {

		case "RCUK":

			return "UKRI";

		default:

			return funderShortName;

		}

	}

	protected String calculateFunderPrefix(final String funderShortName, final String funding) {

		switch (funderShortName.toLowerCase()) {

		case "chist-era":

			return "chistera____::";

		case "conicyt":

			return "conicytf____::";

		case "dfg":

			return "dfgf________::";

		case "ec":

			switch (funding.toLowerCase()) {

			case "fp7":

				return "corda_______::";

			case "h2020":

				return "corda__h2020::";

			default:

				return "corda_____he::";

			}

		case "eea":

			return "euenvagency_::";

		case "hrzz":

		case "mzos":

			return "irb_hr______::";

		case "tara":

			return "taraexp_____::";

		case "tubitak":

			return "tubitakf____::";

		case "rcuk":

			return "ukri________::";

		default:

			String prefix = funderShortName.toLowerCase();

			// ensure we have 12 chars

			while (prefix.length() < 12) {

				prefix += "_";

			}

			return prefix + "::";

		}

	}

	protected String calculateProjectId(final String funderShortName, final String funding, final String code) {

		final String suffix = Hashing.md5(unescape(code));

		final String funderPrefix = calculateFunderPrefix(funderShortName, funding);

		return funderPrefix + suffix;

	}

	private String unescape(final String code) {

		return code.replace("%2F", "/");

	}

	protected String calculateFunderId(final String funderShortName, final String funding) {

		switch (funderShortName.toLowerCase()) {

		case "ec":

			return "ec__________::EC";

		default:

			final String fixedFunderShortName = fixFunderShortName(funderShortName);

			final String prefix = calculateFunderPrefix(fixedFunderShortName, funding);

			return prefix + fixedFunderShortName.toUpperCase();

		}

	}

	protected String calculateFunderName(final String funderShortName) {

		switch (funderShortName.toLowerCase()) {

		case "aff":

		case "aka":

			return "Academy of Finland";

		case "anr":

			return "French National Research Agency (ANR)";

		case "arc":

			return "Australian Research Council (ARC)";

		case "asap":

			return "Aligning Science Across Parkinson's";

		case "chist-era":

			return "CHIST-ERA";

		case "cihr":

			return "Canadian Institutes of Health Research";

		case "conicyt":

			return "Comisión Nacional de Investigación Científica y Tecnológica";

		case "dfg":

			return "Deutsche Forschungsgemeinschaft";

		case "ec":

			return "European Commission";

		case "eea":

			return "European Environment Agency";

		case "fct":

			return "Fundação para a Ciência e a Tecnologia, I.P.";

		case "fwf":

			return "Austrian Science Fund (FWF)";

		case "gsrt":

			return "General Secretariat of Research and Technology (GSRT)";

		case "hrzz":

			return "Croatian Science Foundation (CSF)";

		case "innoviris":

			return "INNOVIRIS";

		case "mestd":

			return "Ministry of Education, Science and Technological Development of Republic of Serbia";

		case "miur":

			return "Ministero dell'Istruzione dell'Università e della Ricerca";

		case "mzos":

			return "Ministry of Science, Education and Sports of the Republic of Croatia (MSES)";

		case "nhmrc":

			return "National Health and Medical Research Council (NHMRC)";

		case "nih":

			return "National Institutes of Health";

		case "nsf":

			return "National Science Foundation";

		case "nserc":

			return "Natural Sciences and Engineering Research Council of Canada";

		case "nwo":

			return "Netherlands Organisation for Scientific Research (NWO)";

		case "rcuk":

		case "ukri":

			return "UK Research and Innovation";

		case "rif":

		case "rpf":

			return "Research and Innovation Foundation";

		case "rsf":

			return "Russian Science Foundation";

		case "sfi":

			return "Science Foundation Ireland";

		case "sgov":

			return "Gobierno de España";

		case "snsf":

			return "Swiss National Science Foundation";

		case "sshrc":

			return "Social Sciences and Humanities Research Council";

		case "tara":

			return "Tara Expeditions Foundation";

		case "tubitak":

			return "Türkiye Bilimsel ve Teknolojik Araştırma Kurumu";

		case "wt":

			return "Wellcome Trust";

		default:

			log.error("Funder short name '" + funderShortName + "' not managed");

			return "";

		}

	}

	protected String fixFundingName(final String funderShortName, final String fundingName) {

		switch (funderShortName) {

		case "EC":

			if (fundingName.toLowerCase().startsWith("horizon 2020")) { return "H2020"; }

			if (fundingName.toLowerCase().startsWith("horizon europe")) { return "HE"; }

		default:

			return fundingName;

		}

	}

	public Collection<String> translateZenodoCommunity(final String community) {

		if (community.contains(ZENODO_COMMUNITY)) {

			final String context = community.substring(community.lastIndexOf("/") + 1);

			final RestTemplate rt = new RestTemplate();

			try {

				return new HashSet<>(rt.getForObject(community_api + context + "/openairecommunities", ZenodoContextList.class)

					.getOpenAirecommunitylist());

			} catch (final RestClientException rce) {

				log.error("Unable to get object for " + community_api + context + "/openairecommunities");

				log.error(rce.getMessage());

				return new HashSet<>();

			}

		} else {

			return Sets.newHashSet(community);

		}

	}

	private ContextInfo createContextInfo(final String[] arr, final int pos, final Map<String, String> labelMap) {

		final StringWriter id = new StringWriter();

		id.write(arr[0]);

		for (int i = 0; i < pos; i++) {

			id.write("::");

			id.write(arr[i + 1]);

		}

		final String label = labelMap.get(id.toString());

		final String elem = pos == 0 ? "context" : pos == 1 ? "category" : "concept";

		final ContextInfo info = new ContextInfo(elem, id.toString(), label);

		if (pos + 1 < arr.length) {

			info.getChildren().add(createContextInfo(arr, pos + 1, labelMap));

		}

		return info;

	}

	public List<ContextInfo> processContexts(final List<String> zenodoCommunities, final Map<String, String> labelMap) {

		return zenodoCommunities.stream()

			.map(c -> translateZenodoCommunity(c))

			.flatMap(coll -> coll.stream())

			.map(ctx -> createContextInfo(ctx.split("::"), 0, labelMap))

			.filter(info -> StringUtils.isNotBlank(info.getLabel()))

			.collect(Collectors.toList());

	}

	public class ContextInfo {

		private String elem;

		private String id;

		private String label;

		private List<ContextInfo> children = new ArrayList<>();

		public ContextInfo(final String elem,

			final String id, final String label) {

			this.elem = elem;

			this.id = id;

			this.label = label;

		}

		public String getElem() {

			return elem;

		}

		public void setElem(final String elem) {

			this.elem = elem;

		}

		public String getId() {

			return id;

		}

		public void setId(final String id) {

			this.id = id;

		}

		public List<ContextInfo> getChildren() {

			return children;

		}

		public void setChildren(final List<ContextInfo> children) {

			this.children = children;

		}

		public boolean isRoot() {

			return elem.equals("context");

		}

		public String getLabel() {

			return label;

		}

		public void setLabel(final String label) {

			this.label = label;

		}

	}

}

package eu.dnetlib.openaire.directindex.api;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.cache.annotation.CacheEvict;

import org.springframework.cache.annotation.Cacheable;

import org.springframework.core.io.ClassPathResource;

import javax.annotation.Resource;

import java.io.IOException;

public class IndexDSRetriever {

	private static final Log log = LogFactory.getLog(IndexDSRetriever.class);

	@Value(value = "${openaire.api.directindex.findSolrIndexUrl.xquery}")

	private ClassPathResource findSolrIndexUrl;

	@Value(value = "${openaire.api.directindex.findIndexDsInfo.xquery}")

	private ClassPathResource findIndexDsInfo;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Cacheable("indexDsInfo")

	public IndexDsInfo calculateCurrentIndexDsInfo() throws IOException, ISLookUpException {

		log.info("Not using cache");

		final String queryUrl = IOUtils.toString(findSolrIndexUrl.getInputStream());

		final String queryDs = IOUtils.toString(findIndexDsInfo.getInputStream());

		final ISLookUpService lu = serviceLocator.getService(ISLookUpService.class);

		final String indexBaseUrl = lu.getResourceProfileByQuery(queryUrl);

		final String idxDs = lu.getResourceProfileByQuery(queryDs);

		if (idxDs.isEmpty()) {

			throw new IllegalStateException(queryDs + "\n\nreturned no results, check IS profiles");

		}

		final String[] arr = idxDs.split("@@@");

		return new IndexDsInfo(indexBaseUrl, arr[0].trim(), arr[1].trim(), arr[2].trim());

	}

	@CacheEvict(value="indexDsInfo", allEntries = true)

	public void evictCache(){

		log.info("Evicting indexDsInfo cache");

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

       xmlns:p="http://www.springframework.org/schema/p"

       xmlns="http://www.springframework.org/schema/beans"

       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">

	<bean id="recentResultsQueue" class="eu.dnetlib.openaire.directindex.api.RecentResultsQueue"

	      p:db-ref="msroApiMongoDB" p:collection="${openaire.api.directindex.mongo.collection}"

	      init-method="init"/>

	<bean id="apiMongoServer" class="com.mongodb.Mongo">

		<constructor-arg index="0" type="com.mongodb.ServerAddress">

			<bean class="com.mongodb.ServerAddress">

				<constructor-arg index="0"

				                 value="${openaire.api.directindex.mongo.host}"/>

				<constructor-arg index="1"

				                 value="${openaire.api.directindex.mongo.port}"/>

			</bean>

		</constructor-arg>

	</bean>

	<bean id="apiOafToIndexRecordFactory" class="eu.dnetlib.openaire.directindex.utils.OafToIndexRecordFactory"

	      p:layoutToRecord="${openaire.api.directindex.layoutToRecord.xslt}"/>

	<bean id="msroApiMongoDB" factory-bean="apiMongoServer"

	      factory-method="getDB">

		<constructor-arg index="0" value="${openaire.api.directindex.mongo.db}"/>

	</bean>

	<bean id="openaireplusApisVelocityEngine" class="org.springframework.ui.velocity.VelocityEngineFactoryBean">

		<property name="velocityProperties">

			<value>

				resource.loader=class

				class.resource.loader.class=org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader

				runtime.log.logsystem.class=org.apache.velocity.runtime.log.Log4JLogChute

			</value>

		</property>

	</bean>

	<bean id="indexClientManager" class="eu.dnetlib.openaire.directindex.api.IndexClientManager" />

	<bean id="resultSubmitterService" class="eu.dnetlib.openaire.directindex.api.ResultSubmitterService" />

	<bean id="indexDSRetriever" class="eu.dnetlib.openaire.directindex.api.IndexDSRetriever" />

	<bean id="directIndexCacheManager" class="org.springframework.cache.ehcache.EhCacheCacheManager"

	      p:cacheManager-ref="directIndexCache" />

	<bean id="directIndexCache"

	      class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"

	      p:cacheManagerName="directIndexCache"

	      p:configLocation="classpath:/eu/dnetlib/openaire/directindex/api/ehcache.xml" />

</beans>

package eu.dnetlib.openaire.directindex.api;

import eu.dnetlib.common.rmi.RMIException;

public class DirecIndexApiException extends RMIException {

	/**

	 *

	 */

	private static final long serialVersionUID = -3888037031334809448L;

	public DirecIndexApiException(final String string) {

		super(string);

	}

	public DirecIndexApiException(final String string, final Throwable exception) {

		super(string, exception);

	}

	public DirecIndexApiException(final Throwable exception) {

		super(exception);

	}

}

{"type_source": "SVN", "goal": "package -U source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-directindex-api/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-directindex-api"}

{

  "openaireId":"dedup_wf_001::ab42e811",

  "originalId": "ORIG_ID_TEST",

  "type": "publication",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "licenseCode": "EMBARGO",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

{

  "openaireId":"dedup_wf_001::ab42e8116f4ae3c4021fb7c643a8167a",

  "originalId": "ORIG_ID_TEST",

  "type": "publication",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "accessRightCode": "EMBARGO",

  "licenseCode": "OPEN",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

<ehcache>

	<!-- Sets the path to the directory where cache .data files are created. 

		If the path is a Java System Property it is replaced by its value in the 

		running VM. The following properties are translated: user.home - User's home 

		directory user.dir - User's current working directory java.io.tmpdir - Default 

		temp file path -->

	<diskStore path="java.io.tmpdir" />

	<!--Default Cache configuration. These will applied to caches programmatically 

		created through the CacheManager. The following attributes are required for 

		defaultCache: maxInMemory - Sets the maximum number of objects that will 

		be created in memory eternal - Sets whether elements are eternal. If eternal, 

		timeouts are ignored and the element is never expired. timeToIdleSeconds 

		- Sets the time to idle for an element before it expires. i.e. The maximum 

		amount of time between accesses before an element expires Is only used if 

		the element is not eternal. Optional attribute. A value of 0 means that an 

		Element can idle for infinity timeToLiveSeconds - Sets the time to live for 

		an element before it expires. i.e. The maximum time between creation time 

		and when an element expires. Is only used if the element is not eternal. 

		overflowToDisk - Sets whether elements can overflow to disk when the in-memory 

		cache has reached the maxInMemory limit. -->

	<!-- <defaultCache maxElementsInMemory="1000" overflowToDisk="true" -->

	<!-- eternal="true" diskPersistent="false" timeToIdleSeconds="0" -->

	<!-- timeToLiveSeconds="0" memoryStoreEvictionPolicy="LRU" -->

	<!-- diskExpiryThreadIntervalSeconds="120" diskSpoolBufferSizeMB="5" /> -->

<!-- Expires once in a day -->

	<cache name="indexDsInfo" maxElementsInMemory="5" eternal="false"

		timeToIdleSeconds="86400" timeToLiveSeconds="86400" overflowToDisk="false" />

<!-- 	<cache name="publisherfields" maxElementsInMemory="30" eternal="false" -->

<!-- 		timeToIdleSeconds="1800" timeToLiveSeconds="1800" overflowToDisk="false" /> -->

<!-- 	<cache name="publisheridscheme" maxElementsInMemory="1" eternal="false" -->

<!-- 		timeToIdleSeconds="1800" timeToLiveSeconds="1800" overflowToDisk="false" /> -->

<!-- 	<cache name="publisheridnamespace" maxElementsInMemory="1" -->

<!-- 		eternal="false" timeToIdleSeconds="1800" timeToLiveSeconds="1800" -->

<!-- 		overflowToDisk="false" /> -->

</ehcache>

{

  "originalId": "ORIG_ID_12345",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "accessRightCode": "OPEN",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2367",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

{

  "openaireId":"od_______278::d5b39693ac6c2197e8cf48008a982951",

  "originalId": "oai:rudar.ruc.dk:1800/24691",

  "type": "other",

  "title": "Other test record",

  "authors": [

    "Sloan, Patrick Alexander Raaby",

    "Strange, Gustav Valdemar"

  ],

  "publisher": "",

  "description": "This is for testing software",

  "language": "eng",

  "pids": [],

  "licenseCode": "OPEN SOURCE",

  "resourceType": "0000",

  "url": "http://rudar.ruc.dk/handle/1800/24691",

  "collectedFromId": "opendoar____::278",

  "hostedById": "opendoar____::278",

  "linksToProjects": [],

  "contexts": []

}

<?xml version="1.0" encoding="UTF-8"?>

#macro(context $ctx)

        <$!ctx.elem id="$!esc.evaluate($!ctx.id)" label="$!esc.evaluate($!ctx.label)" #if($!ctx.isRoot()) type="community"#end>

	#foreach($child in $!ctx.children)#context($child)#end

        </$!ctx.elem>

#end

<record xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oaf="http://namespace.openaire.eu/oaf">

	<result>

		<header>

			<dri:objIdentifier>$!esc.evaluate($!objIdentifier)</dri:objIdentifier>

			<dri:dateOfCollection>$!esc.evaluate($!dateOfCollection)</dri:dateOfCollection>

			<dri:status>under curation</dri:status>

			<counters/>

		</header>

		<metadata>

			<oaf:entity schemaLocation="http://namespace.openaire.eu/oaf $!oafSchemaLocation">

				<oaf:result>

					<title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">

						$!esc.evaluate($!pub.title)

					</title>

					#foreach($author in $!pub.authors)

                    	<creator rank="$velocityCount" name="" surname="">$!esc.evaluate($!author)</creator>

                    #end

					<dateofacceptance/>

					<resulttype classid="$!esc.evaluate($!pub.type)" classname="$!esc.evaluate($!resultTypes.get($!pub.type))" schemeid="dnet:result_typologies" schemename="dnet:result_typologies"/>

					<language classid="$!esc.evaluate($!pub.language)" classname="$!esc.evaluate($!languages.get($!pub.language))" schemeid="dnet:languages"

					          schemename="dnet:languages"/>

					<description>

						$!esc.evaluate($!pub.description)

					</description>

					<country classid="" classname="" schemeid="" schemename=""/>

					<subject classid="" classname="" schemeid="" schemename=""/>

					<relevantdate classid="" classname="" schemeid="" schemename=""/>

					<publisher>$!esc.evaluate($!pub.publisher)</publisher>

					<embargoenddate>$!esc.evaluate($!pub.embargoEndDate)</embargoenddate>

					<journal issn="" eissn="" lissn="" ep="" iss="" sp="" vol=""/>

					<source/>

					<fulltext/>

					<format/>

					<storagedate/>

					<resourcetype classid="" classname="" schemeid="" schemename=""/>

					<device/>

					<size/>

					<version/>

					<lastmetadataupdate/>

					<metadataversionnumber/>

					<documentationUrl/>

					<codeRepositoryUrl/>

					<programmingLanguage classid="" classname="" schemeid="" schemename="" />

					<contactperson />

					<contactgroup />

					<tool />

					<originalId>$!esc.evaluate($!pub.originalId)</originalId>

					<collectedfrom name="$!esc.evaluate($!collectedFrom.name)" id="$!esc.evaluate($!collectedFrom.calculateOpenaireId())"/>

					#foreach($pid in $!pub.pids)

						<pid classid="$!esc.evaluate($!pid.type)" classname="$!esc.evaluate($!pidTypes.get($!pid.type))" schemeid="dnet:pid_types"

						     schemename="dnet:pid_types">$!esc.evaluate($pid.value)</pid>

					#end

					#if($!pub.accessRightCode)

						<bestaccessright classid="$!esc.evaluate($!pub.accessRightCode)" classname="$!esc.evaluate($!rights.get($!pub.accessRightCode))"

						                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>

					#elseif($!pub.licenseCode)

							<bestaccessright classid="$!esc.evaluate($!pub.licenseCode)" classname="$!esc.evaluate($!rights.get($!pub.licenseCode))"

							                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>

					#else

						<bestaccessright classid="UNKNOWN" classname="not available" schemeid="dnet:access_modes" schemename="dnet:access_modes" />

					#end

					#foreach($ctx in $contextInfo)

                        #context($ctx)

                    #end

					<datainfo>

						<inferred>false</inferred>

						<deletedbyinference>false</deletedbyinference>

						<trust>0.9</trust>

						<inferenceprovenance/>

						<provenanceaction classid="user:insert" classname="user:insert" schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>

					</datainfo>

					<rels>

						#foreach($link in $!pub.linksToProjects)

							#set( $info = $!util.calculateProjectInfo($!link) )

							<rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="user:claim">

								<to class="isProducedBy" scheme="dnet:result_project_relations" type="project">$!esc.evaluate($!info.id)</to>

								<code>$!esc.evaluate($!info.code)</code>

								<acronym>$!esc.evaluate($!info.acronym)</acronym>

								<title>$!esc.evaluate($!info.title)</title>

								<contracttype classid="" classname="" schemeid="" schemename=""/>

								<funding>

									<funder id="$!esc.evaluate($!info.funderId)"

									        shortname="$!esc.evaluate($!info.funderShortName)"

									        name="$!esc.evaluate($!info.funderName)"

									        jurisdiction="$!esc.evaluate($!info.jurisdiction)"/>

									#if($!info.fundingId)

									<funding_level_0 name="$!esc.evaluate($!info.fundingName)">$!esc.evaluate($!info.fundingId)</funding_level_0>

									#end

								</funding>

								<websiteurl/>

							</rel>

						#end

					</rels>

					<children>

						<instance id="$!esc.evaluate($!objIdentifier)">

							<instancetype classid="$!esc.evaluate($!pub.resourceType)" classname="$!esc.evaluate($resourceTypes.get($!pub.resourceType))"

							              schemeid="dnet:publication_resource" schemename="dnet:publication_resource"/>

							<collectedfrom name="$!esc.evaluate($!collectedFrom.name)" id="$!esc.evaluate($!collectedFrom.calculateOpenaireId())"/>

							<hostedby name="$!esc.evaluate($!hostedBy.name)" id="$!esc.evaluate($!hostedBy.calculateOpenaireId())"/>

							#if($!pub.accessRightCode)

								<accessright classid="$!esc.evaluate($!pub.accessRightCode)" classname="$!esc.evaluate($!rights.get($!pub.accessRightCode))"

								                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>

							#elseif($!pub.licenseCode)

								<accessright classid="$!esc.evaluate($!pub.licenseCode)" classname="$!esc.evaluate($!rights.get($!pub.licenseCode))"

								                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>

							#else

								<accessright classid="UNKNOWN" classname="not available" schemeid="dnet:access_modes" schemename="dnet:access_modes" />

							#end

							<dateofacceptance/>

							<webresource>

								<url>$!esc.evaluate($!pub.url)</url>

							</webresource>

						</instance>

					</children>

				</oaf:result>

			</oaf:entity>

		</metadata>

	</result>

</record>

{

  "authors":[

	"Alouges, Fran\u00e7ois",

	"Di Fratta, Giovanni"

  ],

  "collectedFromId":"opendoar____::2659",

  "description":"The paper is about the parking 3-sphere swimmer ($\\text{sPr}_3$). This is a low-Reynolds number model swimmer composed of three balls of equal radii. The three balls can move along three horizontal axes (supported in the same plane) that mutually meet at the center of $\\text{sPr}_3$ with angles of $120^{\u2218}$ . The governing dynamical system is introduced and the implications of its geometric symmetries revealed. It is then shown that, in the first order range of small strokes, optimal periodic strokes are ellipses embedded in 3d space, i.e. closed curves of the form $t\ud835\udfc4 [0,2\u03c0] \u21a6 (\\cos t)u + (\\sin t)v$ for suitable orthogonal vectors $u$ and $v$ of $\u211d^3$. A simple analytic expression for the vectors $u$ and $v$ is derived. The results of the paper are used in a second article where the real physical dynamics of $\\text{sPr}_3$ is analyzed in the asymptotic range of very long arms. ; Comment: 17 pages, 4 figures",

  "hostedById":"opendoar____::2659",

  "licenseCode":"OPEN",

  "originalId":"oai:zenodo.org:996201",

  "pids":[

	{

	  "type":"oai",

	  "value":"oai:zenodo.org:996201"

	},

	{

	  "type":"doi",

	  "value":"10.5281/zenodo.996201"

	}

  ],

  "publisher":"Zenodo",

  "resourceType":"0020",

  "title":"Parking 3-sphere swimmer. I. Energy minimizing strokes",

  "type":"publication",

  "url":"https://zenodo.org/record/996201"

}

distinct-values(for $s in collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')//SERVICE_PROPERTIES[./PROPERTY[@key = 'infrastructure' and @value = 'public'] ]

let $format := $s/PROPERTY[@key = "mdformat"]/@value/string()

return(

for $x in collection('/db/DRIVER/IndexDSResources/IndexDSResourceType')

where

    $x//METADATA_FORMAT =$format and

            $x//METADATA_FORMAT_INTERPRETATION = 'openaire' and

            $x//METADATA_FORMAT_LAYOUT = 'index'

return

    concat($x//RESOURCE_IDENTIFIER/@value/string(), ' @@@ ', $format , ' @@@ ', $format , '-index-openaire')

))