D-Net

{

  "authors": [

	"Conradt, Tobias",

	"& members of the ISIMIP project (original data provision), cf. Hempel et al. 2013, https://doi.org/10.5194/esd-4-219-2013"

  ],

  "collectedFromId": "re3data_____::r3d100010468",

  "description": "<p>ISIMIP-2a climate data cutout provided for Sardinia in the framework of SIM4NEXUS & X<\/p>",

  "hostedById": "re3data_____::r3d100010468",

  "licenseCode": "OPEN",

  "linksToProjects": ["info:eu-repo/grantAgreement/EC/H2020/689150//Sustainable Integrated Management FOR the NEXUS of water-land-food-energy-climate for a resource-efficient Europe & beyond/SIM4NEXUS"],

  "originalId": "10.5281/zenodo.1460665",

  "pids": [

	{

	  "type": "oai",

	  "value": "oai:zenodo.org:1460665"

	},

	{

	  "type": "doi",

	  "value": "10.5281/zenodo.1460665"

	}

  ],

  "publisher": "Zenodo & Zenodo",

  "resourceType": "0021",

  "title": "sardinia_tasmax_MIROC-ESM-CHEM_rcp4p5_2006-2099_daily.nc4 & the title has && in it",

  "type": "dataset",

  "url": "https://zenodo.org/record/1460665"

}

#!/bin/bash

curl -H "Content-Type: application/json" --data @test_record.json "http://localhost:8280/app/mvc/api/publications/feedObject"

{

  "originalId": "ORIG_ID_TEST",

  "type": "publication",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "accessRightCode": "EMBARGO",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

<?xml version="1.0" ?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-directindex-api</artifactId>

	<packaging>jar</packaging>

	<version>4.0.0</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-directindex-api/tags/dnet-directindex-api-4.0.0</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaireplus-mapping-utils</artifactId>

			<version>[7.0.0,8.0.0)</version>

			<exclusions>

				<exclusion>

					<groupId>eu.dnetlib</groupId>

					<artifactId>dnet-hadoop-commons</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-client</artifactId>

			<version>[3.0.0,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>io.springfox</groupId>

			<artifactId>springfox-swagger2</artifactId>

			<version>${springfox-version}</version>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.9.5</version>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity</artifactId>

			<version>1.7</version>

			<exclusions>

				<exclusion>

					<artifactId>antlr</artifactId>

					<groupId>antlr</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity-tools</artifactId>

			<version>2.0</version>

			<exclusions>

				<exclusion>

					<artifactId>antlr</artifactId>

					<groupId>antlr</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-context-support</artifactId>

			<version>${spring.version}</version>

		</dependency>

		<dependency>

			<groupId>com.fasterxml.jackson.core</groupId>

			<artifactId>jackson-annotations</artifactId>

			<version>2.9.4</version>

		</dependency>

    </dependencies>

	<properties>

		<springfox-version>2.5.0</springfox-version>

	</properties>

</project>

package eu.dnetlib.openaire.directindex.api;

import static org.mockito.Mockito.CALLS_REAL_METHODS;

import static org.mockito.Mockito.doReturn;

import static org.mockito.Mockito.mock;

import static org.mockito.Mockito.withSettings;

import java.util.ArrayList;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.runners.MockitoJUnitRunner;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import com.google.gson.Gson;

import eu.dnetlib.openaire.directindex.objects.DatasourceEntry;

/**

 * Created by Alessia Bardi on 26/05/2017.

 *

 * @author Alessia Bardi

 */

@RunWith(MockitoJUnitRunner.class)

public class OpenAIRESubmitterUtilsTest {

	private OpenAIRESubmitterUtils utils;

	final String fullProject = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble";

	final String minimalProject = "info:eu-repo/grantAgreement/EC/FP7/244909///WorkAble";

	final String onlyId = "info:eu-repo/grantAgreement/EC/FP7/244909/";

	final String onlyTitle = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work";

	final String escapedId = "info:eu-repo/grantAgreement/RCUK//NE%2FL003066%2F1/";

	final String heProjectFromZenodo =

		"info:eu-repo/grantAgreement/EC/Horizon Europe Framework Programme - HORIZON-CSA\\HORIZON Action Grant Budget-Based/101058527/";

	final String chistera = "info:eu-repo/grantAgreement/CHIST-ERA//CHIST-ERA-19-XAI-002/";

	final String intelcomp = "info:eu-repo/grantAgreement/EC/H2020/101004870//A Competitive Intelligence Cloud%2FHPC Platform for AI-based STI Policy Making/IntelComp";

	@Before

	public void setup() {

		utils = mock(OpenAIRESubmitterUtils.class, withSettings().defaultAnswer(CALLS_REAL_METHODS));

		doReturn(Lists.newArrayList("dh-ch", "dariah", "bar", "x")).when(utils).translateZenodoCommunity("https://zenodo.org/communities/dimpo");

		doReturn(Lists.newArrayList()).when(utils).translateZenodoCommunity("https://zenodo.org/communities/dumbo");

	}

	@Test

	public void testCalculateProjectInfoCHISTERA() {

		final Map<String, String> project = utils.calculateProjectInfo(chistera);

		print(project);

		Assert.assertEquals("CHIST-ERA-19-XAI-002", project.get("code"));

		Assert.assertEquals("chistera____::289ec73ce0e9ece89a0cbdf74c445cd8", project.get("id"));

	}

	@Test

	public void testCalculateProjectInfoZenodo() {

		final Map<String, String> project = utils.calculateProjectInfo(onlyId);

		print(project);

		Assert.assertEquals("244909", project.get("code"));

	}

	@Test

	public void testCalculateProjectInfoZenodoHE() {

		final Map<String, String> project = utils.calculateProjectInfo(heProjectFromZenodo);

		print(project);

		Assert.assertEquals("101058527", project.get("code"));

		Assert.assertEquals("corda_____he::eab43774d3762e87f5e66030c6e01d2b", project.get("id"));

	}

	@Test

	public void testCalculateProjectInfoZenodoIntelComp() {

		final Map<String, String> project = utils.calculateProjectInfo(intelcomp);

		print(project);

		Assert.assertEquals("101004870", project.get("code"));

		Assert.assertEquals("corda__h2020::8264fd9d5f5d59b1b76fc618f3a8af24", project.get("id"));

		Assert.assertEquals("IntelComp", project.get("acronym"));

	}

	@Test

	public void testCalculateProjectInfoEscaped() {

		final Map<String, String> project = utils.calculateProjectInfo(escapedId);

		print(project);

		Assert.assertEquals("NE/L003066/1", project.get("code"));

	}

	@Test

	public void testCalculateProjectInfoFull() {

		final Map<String, String> project = utils.calculateProjectInfo(fullProject);

		print(project);

		Assert.assertEquals(10, project.size());

		for (final String k : project.keySet()) {

			Assert.assertNotNull(project.get(k));

			Assert.assertNotSame("", project.get(k));

		}

	}

	@Test

	public void testCalculateProjectInfoOnlyId() {

		final Map<String, String> project = utils.calculateProjectInfo(onlyId);

		print(project);

		Assert.assertEquals("244909", project.get("code"));

	}

	@Test

	public void testCalculateProjectInfoMinimalAcro() {

		final Map<String, String> project = utils.calculateProjectInfo(minimalProject);

		print(project);

		Assert.assertEquals("244909", project.get("code"));

		Assert.assertEquals("WorkAble", project.get("acronym"));

	}

	@Test

	public void testCalculateProjectInfoOnlyTitle() {

		final Map<String, String> project = utils.calculateProjectInfo(onlyTitle);

		print(project);

		Assert.assertEquals("244909", project.get("code"));

		Assert.assertEquals("Making Capabilities Work", project.get("title"));

	}

	@Test

	public void testJerusalem() {

		final String s =

			"info:eu-repo/grantAgreement/EC/FP7/337895/EU/Opening Jerusalem Archives: For a connected History of ‘Citadinité’ in the Holy City (1840-1940)/OPEN-JERUSALEM";

		final Map<String, String> project = utils.calculateProjectInfo(s);

		print(project);

		Assert.assertEquals("337895", project.get("code"));

		Assert.assertEquals("Opening Jerusalem Archives: For a connected History of ‘Citadinité’ in the Holy City (1840-1940)", project.get("title"));

		Assert.assertEquals("OPEN-JERUSALEM", project.get("acronym"));

	}

	private void print(final Map<String, String> map) {

		for (final Entry e : map.entrySet()) {

			System.out.println(e.getKey() + " = " + e.getValue());

		}

	}

	@Test

	public void testContext() {

		final List<String> zenodoCommunities = new ArrayList<>();

		zenodoCommunities.add("https://zenodo.org/communities/dimpo");

		zenodoCommunities.add("https://zenodo.org/communities/dumbo");

		final Map<String, String> labelMap = Maps.newHashMap();

		labelMap.put("dariah", "DARIAH");

		labelMap.put("dh-ch", "DH-CH");

		labelMap.put("foo", "FOO");

		labelMap.put("bar", "");

		final List<OpenAIRESubmitterUtils.ContextInfo> tmp = utils.processContexts(zenodoCommunities, labelMap);

		Assert.assertEquals(2, tmp.size());

		Assert.assertTrue(tmp.get(0).getId().equals("dh-ch") || tmp.get(1).getId().equalsIgnoreCase("dh-ch"));

		Assert.assertTrue(tmp.get(0).getId().equals("dariah") || tmp.get(1).getId().equalsIgnoreCase("dariah"));

	}

	@Test

	public void testWrongH2020() {

		final String link = "info:eu-repo/grantAgreement/EC/Horizon 2020 Framework Programme - Research and Innovation action/881825/";

		final Map<String, String> info = utils.calculateProjectInfo(link);

		System.out.println(info);

		Assert.assertEquals("H2020", info.get("fundingName"));

	}

	@Test

	@Ignore

	public void testFindDatasource_01() {

		final OpenAIRESubmitterUtils utils = new OpenAIRESubmitterUtils("", "https://services.openaire.eu/openaire/ds/");

		final DatasourceEntry res = utils.findDatasource("opendoar____::1112");

		System.err.println(new Gson().toJson(res));

	}

	@Test

	@Ignore

	public void testFindDatasourceCrossref() {

		final OpenAIRESubmitterUtils utils = new OpenAIRESubmitterUtils("", "https://services.openaire.eu/openaire/ds/");

		final DatasourceEntry res = utils.findDatasource("openaire____::crossref");

		System.err.println(new Gson().toJson(res));

	}

}

{

  "linksToProjects": [

    "info:eu-repo/grantAgreement/EC/H2020/101004870//A Competitive Intelligence Cloud%2FHPC Platform for AI-based STI Policy Making/IntelComp"

  ],

  "description": "<p>IntelComp sets out to build an innovative Cloud Platform that...",

  "publisher": "Zenodo",

  "contexts": [

    "https://zenodo.org/communities/argos",

    "https://zenodo.org/communities/intelcomp"

  ],

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "originalId": "oai:zenodo.org:5109933",

  "resourceType": "0045",

  "version": "1",

  "licenseCode": "OPEN",

  "authors": [

    "Elli Papadopoulou"

  ],

  "url": "https://zenodo.org/record/5109933",

  "title": "Data Management Plan for IntelComp. D8.2.",

  "pids": [

    {

      "type": "oai",

      "value": "oai:zenodo.org:5109933"

    },

    {

      "type": "doi",

      "value": "10.5281/zenodo.5109933"

    }

  ],

  "type": "publication"

}

{

  "openaireId":"dedup_wf_001::ab42e811",

  "originalId": "ORIG_ID_TEST",

  "type": "publication",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "licenseCode": "EMBARGO",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

{

  "originalId": "ORIG_ID_12345",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "accessRightCode": "OPEN",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2367",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

{

  "authors":[

	"Alouges, Fran\u00e7ois",

	"Di Fratta, Giovanni"

  ],

  "collectedFromId":"opendoar____::2659",

  "description":"The paper is about the parking 3-sphere swimmer ($\\text{sPr}_3$). This is a low-Reynolds number model swimmer composed of three balls of equal radii. The three balls can move along three horizontal axes (supported in the same plane) that mutually meet at the center of $\\text{sPr}_3$ with angles of $120^{\u2218}$ . The governing dynamical system is introduced and the implications of its geometric symmetries revealed. It is then shown that, in the first order range of small strokes, optimal periodic strokes are ellipses embedded in 3d space, i.e. closed curves of the form $t\ud835\udfc4 [0,2\u03c0] \u21a6 (\\cos t)u + (\\sin t)v$ for suitable orthogonal vectors $u$ and $v$ of $\u211d^3$. A simple analytic expression for the vectors $u$ and $v$ is derived. The results of the paper are used in a second article where the real physical dynamics of $\\text{sPr}_3$ is analyzed in the asymptotic range of very long arms. ; Comment: 17 pages, 4 figures",

  "hostedById":"opendoar____::2659",

  "licenseCode":"OPEN",

  "originalId":"oai:zenodo.org:996201",

  "pids":[

	{

	  "type":"oai",

	  "value":"oai:zenodo.org:996201"

	},

	{

	  "type":"doi",

	  "value":"10.5281/zenodo.996201"

	}

  ],

  "publisher":"Zenodo",

  "resourceType":"0020",

  "title":"Parking 3-sphere swimmer. I. Energy minimizing strokes",

  "type":"publication",

  "url":"https://zenodo.org/record/996201"

}

{

  "authors": ["Milanese, Alessio", "Mende, Daniel", "Zeller, Georg", "Sunagawa, Shinichi"],

  "collectedFromId": "re3data_____::r3d100010468",

  "description": "<p>We simulated ten human gut metagenomic samples to assess the taxonomic quantification accuracy of the mOTUs tool (<a href=\"http: \// motu-tool.org \/\">link</a>). In this directory you can find the metagenomic samples, the gold standard (used to produce them) and the profiles obtained with four metagenomic profiler tools.</p>\\n\\n<p>Check README.txt for more information.</p>",

  "hostedById": "re3data_____::r3d100010468",

  "licenseCode": "OPEN",

  "originalId": "10.5281/zenodo.1473645",

  "pids": [

    {"type": "oai", "value": "oai:zenodo.org:1473645"},

    {"type": "doi", "value": "10.5281/zenodo.1473645"}

  ],

  "publisher": "Zenodo",

  "resourceType": "0021",

  "title": "Simulated Human gut metagenomic samples to benchmark mOTUs v2",

  "type": "dataset",

  "url": "https://zenodo.org/record/1473645",

  "version": "2"

}

{"authors":["test"],

  "collectedFromId":"re3data_____::r3d100010468",

  "contexts":["https://zenodo.org/communities/oac-ni"],

  "description":"<p>test</p>",

  "hostedById":"re3data_____::r3d100010468",

  "licenseCode":"OPEN",

  "originalId":"oai:zenodo.org:315784",

  "pids":[{"type":"oai","value":"oai:zenodo.org:315784"},

    {"type":"doi","value":"10.5072/zenodo.315784"}],"resourceType":"0020","title":"test","type":"other","url":"https://zenodo.org/record/315784"}

<xsl:stylesheet version="1.0"

                xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:nxsl="http://www.w3.org/1999/XSL/TransformXX">

	<xsl:output omit-xml-declaration="yes" method="xml"/>

	<xsl:namespace-alias stylesheet-prefix="nxsl" result-prefix="xsl"/>

	<xsl:param name="format"/>

	<xsl:template match="/">

		<xsl:apply-templates select="//LAYOUT"/>

	</xsl:template>

	<xsl:template match="LAYOUT">

		<nxsl:stylesheet version="1.0"

		                 xmlns:dnet="eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions"

		                 xmlns:dri="http://www.driver-repository.eu/namespace/dri"

		                 exclude-result-prefixes="dnet">

			<nxsl:output version="1.0" omit-xml-declaration="yes"

			             method="xml"/>

			<nxsl:variable name="format">

				<xsl:value-of select="$format"/>

			</nxsl:variable>

			<nxsl:template match="/">

				<indexRecord>

					<indexRecordIdentifier>

						<nxsl:value-of select="//dri:objIdentifier"/>

					</indexRecordIdentifier>

					<targetFields>

						<nxsl:if test="count(//*[local-name()='metadata']/*) &gt; 0">

							<xsl:apply-templates select="FIELDS/FIELD[@indexable='true']"/>

						</nxsl:if>

					</targetFields>

					<dnetResult>

						<nxsl:copy-of select="/*[local-name()='record']/*[local-name()='result']/*[local-name()='header']"/>

						<metadata>

							<xsl:apply-templates select="FIELDS/FIELD"

							                     mode="result"/>

						</metadata>

					</dnetResult>

				</indexRecord>

			</nxsl:template>

		</nxsl:stylesheet>

	</xsl:template>

	<xsl:template match="FIELD[@indexable='true']">

		<xsl:choose>

			<xsl:when test="@constant">

				<xsl:element name="{@name}">

					<xsl:value-of select="@constant"/>

				</xsl:element>

			</xsl:when>

			<xsl:when test="@value and not(@xpath)">

				<xsl:choose>

					<xsl:when test="contains(@type, 'date')">

						<nxsl:variable name="{@name}" select="dnet:normalizeDate(normalize-space({normalize-space(@value)}), false())"/>

						<nxsl:if test="string-length(${@name}) &gt; 0">

							<nxsl:element name="{@name}">

								<nxsl:value-of select="${@name}"/>

							</nxsl:element>

						</nxsl:if>

					</xsl:when>

					<xsl:otherwise>

						<nxsl:element name="{@name}">

							<nxsl:value-of select="{@value}"/>

						</nxsl:element>

					</xsl:otherwise>

				</xsl:choose>

			</xsl:when>

			<xsl:otherwise>

				<xsl:variable name="value">

					<xsl:choose>

						<xsl:when test="@value">

							<xsl:value-of select="@value"/>

						</xsl:when>

						<xsl:otherwise>

							.

						</xsl:otherwise>

					</xsl:choose>

				</xsl:variable>

				<nxsl:for-each select="{@xpath}">

					<xsl:element name="{@name}">

						<xsl:choose>

							<xsl:when test="@tokenizable='false'">

								<nxsl:value-of select="normalize-space({normalize-space($value)})"/>

							</xsl:when>

							<xsl:otherwise>

								<nxsl:value-of select="{normalize-space($value)}"/>

							</xsl:otherwise>

						</xsl:choose>

					</xsl:element>

				</nxsl:for-each>

			</xsl:otherwise>

		</xsl:choose>

	</xsl:template>

	<xsl:template match="FIELD" mode="result">

		<xsl:if test="@result='true'">

			<nxsl:copy-of select="{@xpath}"/>

		</xsl:if>

	</xsl:template>

	<xsl:template match="FIELD" mode="header">

		<xsl:if test="@header='true'">

			<nxsl:copy-of select="{@xpath}"/>

		</xsl:if>

	</xsl:template>

</xsl:stylesheet>

package eu.dnetlib.openaire.directindex.objects;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.concurrent.TimeUnit;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.ui.velocity.VelocityEngineUtils;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.miscutils.functional.string.EscapeXml;

import eu.dnetlib.openaire.directindex.api.DirecIndexApiException;

import eu.dnetlib.openaire.directindex.api.OpenAIRESubmitterUtils;

public class ResultEntryToOaf {

	private static final Log log = LogFactory.getLog(ResultEntryToOaf.class);

	private static long last_cache_update = 0;

	private static final Map<String, Map<String, String>> cached_vocabularies = new HashMap<>();

	private static final Map<String, DatasourceEntry> cached_datasources = new HashMap<>();

	private static final Map<String, String> cached_contexts = new HashMap<>();

	private static final String ZENODO_ID="opendoar____::2659";

	private OpenAIRESubmitterUtils utils;

	public ResultEntryToOaf(final OpenAIRESubmitterUtils utils) {

		this.utils = utils;

	}

	public String asOafRecord(final ResultEntry entry,

		final VelocityEngine ve,

		final ISLookUpService lookupService,

		final String oafSchemaLocation) throws Exception {

		if (StringUtils.isBlank(entry.getOriginalId())

			&& StringUtils

				.isBlank(entry.getOpenaireId())) {

			throw new DirecIndexApiException("One of the following fields is required: originalId or openaireId");

		}

		if (StringUtils.isBlank(entry.getTitle())) { throw new DirecIndexApiException("A required field is missing: title"); }

		if (StringUtils.isBlank(entry.getUrl())) { throw new DirecIndexApiException("A required field is missing: url"); }

		if (StringUtils.isBlank(entry.getLicenseCode()) && StringUtils.isBlank(entry.getAccessRightCode())) {

			throw new DirecIndexApiException("A required field is missing: accessRightCode");

		}

		if (StringUtils.isBlank(entry.getResourceType())) { throw new DirecIndexApiException("A required field is missing: resourceType"); }

		if (StringUtils.isBlank(entry.getCollectedFromId())) { throw new DirecIndexApiException("A required field is missing: collectedFromId"); }

		if (StringUtils.isBlank(entry.getType())) { throw new DirecIndexApiException("A required field is missing: type"); }

		final DatasourceEntry collectedFromEntry = getDatasourceInfo(entry.getCollectedFromId(), utils);

		final DatasourceEntry hostedByEntry = getDatasourceInfo(entry.getHostedById(), utils);

		if (StringUtils.isBlank(entry.getOpenaireId())) {

			if(entry.getCollectedFromId().equals(ZENODO_ID)){

				String doi = entry.getPids().stream().filter(e -> e.getType().equalsIgnoreCase("doi")).findFirst().get().getValue();

				entry.setOpenaireId(calculateOpenaireId(doi, collectedFromEntry));

			}

			else entry.setOpenaireId(calculateOpenaireId(entry.getOriginalId(), collectedFromEntry));

		}

		if (!entry.getOpenaireId()

			.matches("^\\w{12}::\\w{32}$")) {

			throw new DirecIndexApiException(

				"Invalid openaireId: " + entry.getOpenaireId() + " - regex ^\\w{12}::\\w{32}$ not matched");

		}

		final List<OpenAIRESubmitterUtils.ContextInfo> contextInfos = utils.processContexts(entry.getContexts(), getContexts(lookupService));

		final Map<String, Object> model = new HashMap<>();

		model.put("esc", new EscapeXml());

		model.put("util", utils);

		model.put("pub", entry);

		model.put("objIdentifier", entry.getOpenaireId());

		model.put("oafSchemaLocation", oafSchemaLocation);

		model.put("resultTypes", getVocabulary("dnet:result_typologies", lookupService));

		model.put("rights", getVocabulary("dnet:access_modes", lookupService));

		model.put("resourceTypes", getVocabulary("dnet:publication_resource", lookupService));

		model.put("pidTypes", getVocabulary("dnet:pid_types", lookupService));

		model.put("languages", getVocabulary("dnet:languages", lookupService));

		model.put("contexts", getContexts(lookupService));

		model.put("contextInfo", contextInfos);

		model.put("dateOfCollection", new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'").format(new Date()));

		model.put("collectedFrom", collectedFromEntry);

		model.put("hostedBy", hostedByEntry);

		return VelocityEngineUtils.mergeTemplateIntoString(ve, "/eu/dnetlib/openaire/directindex/indexRecord.xml.vm", "UTF-8", model);

	}

	private synchronized static DatasourceEntry getDatasourceInfo(final String dsId, final OpenAIRESubmitterUtils utils) throws ISLookUpException {

		if (StringUtils

			.isBlank(dsId)) {

			return new DatasourceEntry("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "unknown_____");

		}

		if (!cached_datasources.containsKey(dsId)) {

			final DatasourceEntry ds = utils.findDatasource(dsId);

			if (ds == null || StringUtils.isBlank(ds.getName()) || StringUtils.isBlank(ds.getPrefix())) {

				log.error("Invalid datasource id: " + dsId);

				throw new ISLookUpException("Invalid datasource id: " + dsId);

			} else {

				cached_datasources.put(dsId, ds);

			}

		}

		return cached_datasources.get(dsId);

	}

	private synchronized static Map<String, String> getVocabulary(final String voc, final ISLookUpService lookupService) throws ISLookUpException {

		if (DateUtils.now() - last_cache_update < TimeUnit.MINUTES.toMillis(15) && cached_vocabularies.containsKey(voc)) {

			return cached_vocabularies.get(voc);

		} else {

			final String query = "collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code='" + voc

				+ "']//TERM/concat(@code, ' @@@ ', @english_name)";

			final Map<String, String> map = new HashMap<>();

			for (final String s : lookupService.quickSearchProfile(query)) {

				final String[] arr = s.split("@@@");

				map.put(arr[0].trim(), arr[1].trim());

			}

			cached_vocabularies.put(voc, map);

			last_cache_update = DateUtils.now();

			return map;

		}

	}

	private synchronized static Map<String, String> getContexts(final ISLookUpService lookupService) throws ISLookUpException {

		if (DateUtils.now() - last_cache_update > TimeUnit.MINUTES.toMillis(15) || cached_contexts.isEmpty()) {

			final String query =

				"collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community' or .//context/@type='ri']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)";

			cached_contexts.clear();

			for (final String s : lookupService.quickSearchProfile(query)) {

				final String[] arr = s.split("@@@");

				cached_contexts.put(arr[0].trim(), arr[1].trim());

			}

			last_cache_update = DateUtils.now();

		}

		return cached_contexts;

	}

	private static String calculateOpenaireId(final String originalId, final DatasourceEntry collectedFromEntry) {

		return collectedFromEntry.getPrefix() + "::" + Hashing.md5(originalId);

	}

	public static String calculateOpenaireId(final String originalId, final String collectedFromId, final OpenAIRESubmitterUtils utils)

		throws ISLookUpException {

		return calculateOpenaireId(originalId, getDatasourceInfo(collectedFromId, utils));

	}

	public OpenAIRESubmitterUtils getUtils() {

		return utils;

	}

	public void setUtils(final OpenAIRESubmitterUtils utils) {

		this.utils = utils;

	}

}

package eu.dnetlib.openaire.directindex.objects;

import java.io.Serializable;

public class DsmDatasourceInfo implements Serializable {

	private static final long serialVersionUID = -593392381920400974L;

	private String id;

	private String officialname;

	private String namespaceprefix;

	public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public String getOfficialname() {

		return officialname;

	}

	public void setOfficialname(final String officialname) {

		this.officialname = officialname;

	}

	public String getNamespaceprefix() {

		return namespaceprefix;

	}

	public void setNamespaceprefix(final String namespaceprefix) {

		this.namespaceprefix = namespaceprefix;

	}

}

package eu.dnetlib.openaire.directindex.objects;

import java.io.Serializable;

public class DsmSearchRequest implements Serializable {

	private static final long serialVersionUID = -2532361140043817319L;

	private String id;

	public DsmSearchRequest() {}

	public DsmSearchRequest(final String id) {

		this.id = id;

	}

	public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

}

package eu.dnetlib.openaire.directindex.objects;

import java.io.Serializable;

import java.util.List;

public class DsmSearchResponse implements Serializable {

	private static final long serialVersionUID = 8944122902111813747L;

	private List<DsmDatasourceInfo> datasourceInfo;

	public List<DsmDatasourceInfo> getDatasourceInfo() {

		return datasourceInfo;

	}

	public void setDatasourceInfo(final List<DsmDatasourceInfo> datasourceInfo) {

		this.datasourceInfo = datasourceInfo;

	}

}

package eu.dnetlib.openaire.directindex.objects;

import com.google.gson.Gson;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.miscutils.functional.string.EscapeXml;

import eu.dnetlib.openaire.directindex.api.DirecIndexApiException;

import eu.dnetlib.openaire.directindex.api.OpenAIRESubmitterUtils;

import io.swagger.annotations.ApiModelProperty;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.ui.velocity.VelocityEngineUtils;

import java.text.SimpleDateFormat;

import java.util.*;

import java.util.concurrent.TimeUnit;

/**

 * Created by michele on 02/12/15.

 */

public class ResultEntry {

	private String openaireId;

	private String originalId;

	private String title;

	private List<String> authors = new ArrayList<>();

	private String publisher;

	private String description;

	private String language;

	private List<PidEntry> pids = new ArrayList<>();

	/**

	 * @Deprecated: use accessRightCode

	 */

	@Deprecated

	private String licenseCode;

	private String accessRightCode;

	private String embargoEndDate;

	/**

	 * One of publication, dataset, software, other. Default value is publication.

	 */

	private String type = "publication";

	private String resourceType;

	private String url;

	private String collectedFromId;

	private String hostedById;

	// String according to the EGI context profile, example: egi::classification::natsc::math

	private List<String> contexts = new ArrayList<>();

	// String according to openaire guidelines:

	// info:eu-repo/grantAgreement/Funder/FundingProgram/ProjectID/[Jurisdiction]/[ProjectName]/[ProjectAcronym]

	private List<String> linksToProjects = new ArrayList<>();

	private static final Log log = LogFactory.getLog(ResultEntry.class);

	public ResultEntry() {}

	public String getOpenaireId() {

		return openaireId;

	}

	public void setOpenaireId(final String openaireId) {

		this.openaireId = openaireId;

	}

	public String getOriginalId() {

		return originalId;

	}

	public void setOriginalId(final String originalId) {

		this.originalId = originalId;

	}

	@ApiModelProperty(required = true)

	public String getTitle() {

		return title;

	}

	public void setTitle(final String title) {

		this.title = title;

	}

	public List<String> getAuthors() {

		return authors;

	}

	public void setAuthors(final List<String> authors) {

		this.authors = authors;

	}

	public String getPublisher() {

		return publisher;

	}

	public void setPublisher(final String publisher) {

		this.publisher = publisher;

	}

	public String getDescription() {

		return description;

	}

	public void setDescription(final String description) {

		this.description = description;

	}

	@ApiModelProperty(value = "ISO Alpha-3 code. E.g. 'eng', 'ita'")

	public String getLanguage() {

		return language;

	}

	public void setLanguage(final String language) {

		this.language = language;

	}

	public List<PidEntry> getPids() {

		return pids;

	}

	public void setPids(final List<PidEntry> pids) {

		this.pids = pids;

	}

	@Deprecated

	@ApiModelProperty(required = false, allowableValues = "OPEN, CLOSED, RESTRICTED, EMBARGO, UNKNOWN, OTHER, OPEN SOURCE")

	public String getLicenseCode() {

		return licenseCode;

	}

	@Deprecated

	public void setLicenseCode(final String licenseCode) {

		this.licenseCode = licenseCode;

	}

	/**

	 * Set required = true when the deprecated licenseCode is not used anymore by our client and it is removed

	 * @return access rights code

	 */

	@ApiModelProperty(required = false, allowableValues = "OPEN, CLOSED, RESTRICTED, EMBARGO, UNKNOWN, OTHER, OPEN SOURCE")

	public String getAccessRightCode() {

		return accessRightCode;

	}

	public void setAccessRightCode(final String accessRightCode) {

		this.accessRightCode = accessRightCode;

	}

	@ApiModelProperty(required = true, value = "Use 001 for articles, 021 for datasets, 0029 for software. See: http://api.openaire.eu/vocabularies/dnet:publication_resource for all the available resource types.")

	public String getResourceType() {

		return resourceType;

	}

	public void setResourceType(final String resourceType) {

		this.resourceType = resourceType;

	}

	@ApiModelProperty(required = true)

	public String getUrl() {

		return url;

	}

	public void setUrl(final String url) {

		this.url = url;

	}

	@ApiModelProperty(required = true, value = "Use opendoar___::2659 for Zenodo Publications; re3data_____::r3d100010468 for Zenodo datasets; infrastruct::openaire for OpenAIRE portal.")

	public String getCollectedFromId() {

		return collectedFromId;

	}

	public void setCollectedFromId(final String collectedFromId) {

		this.collectedFromId = collectedFromId;

	}

	public String getHostedById() {

		return hostedById;

	}

	public void setHostedById(final String hostedById) {

		this.hostedById = hostedById;

	}

	@ApiModelProperty(value = "E.g. fet, egi::classification::natsc::math::pure, egi::projects::EMI")

	public List<String> getContexts() {

		return contexts;

	}

	public void setContexts(final List<String> contexts) {

		this.contexts = contexts;

	}

	@ApiModelProperty(value = "E.g. info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus")

	public List<String> getLinksToProjects() {

		return linksToProjects;

	}

	public void setLinksToProjects(final List<String> linksToProjects) {

		this.linksToProjects = linksToProjects;

	}

	@ApiModelProperty(allowableValues = "publication, dataset, software, other")

	public String getType() {

		return type;

	}

	public void setType(final String type) {

		this.type = type;

	}

	public String getEmbargoEndDate() {

		return embargoEndDate;

	}

	public void setEmbargoEndDate(final String embargoEndDate) {

		this.embargoEndDate = embargoEndDate;

	}

	@Override

	public String toString() {

		return new Gson().toJson(this);

	}

	public String getAnyId() {

		return StringUtils.isNotBlank(openaireId) ? openaireId : originalId;

	}

}

package eu.dnetlib.openaire.directindex.objects;

import java.util.ArrayList;

import java.util.List;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;

@JsonIgnoreProperties(ignoreUnknown = true)

public class ZenodoContextList {

        private String zenodoid;

        private List<String> openAirecommunitylist;

        public List<String> getOpenAirecommunitylist() {

            return openAirecommunitylist;

        }

        public void setOpenAirecommunitylist(List<String> openAirecommunitylist) {

            this.openAirecommunitylist = openAirecommunitylist;

        }

        public String getZenodoid() {

            return zenodoid;

        }

        public void setZenodoid(String zenodoid) {

            this.zenodoid = zenodoid;

        }

    }

package eu.dnetlib.openaire.directindex.objects;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import org.apache.commons.lang.StringUtils;

/**

 * Created by michele on 02/12/15.

 */

public class DatasourceEntry {

	private String id;

	private String name;

	private String prefix;

	public DatasourceEntry() {

	}

	public DatasourceEntry(final String id, final String name, final String prefix) {

		this.id = id;

		this.name = name;

		this.prefix = prefix;

	}

	public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public String getName() {

		return name;

	}

	public void setName(final String name) {

		this.name = name;

	}

	public String getPrefix() {

		return prefix;

	}

	public void setPrefix(final String prefix) {

		this.prefix = prefix;

	}

	public String calculateOpenaireId() {

		if (StringUtils.isNotBlank(id)) {

			final String[] arr = id.split("::");

			if (arr.length == 2) {

				return String.format("%s::%s", arr[0], Hashing.md5(arr[1]));

			}

		}

		return "";

	}

}

package eu.dnetlib.openaire.directindex.objects;

import io.swagger.annotations.ApiModelProperty;

/**

 * Created by michele on 02/12/15.

 */

public class PidEntry {

	private String type;

	private String value;

	public PidEntry() {

	}

	public PidEntry(final String type, final String value) {

		this.type = type;

		this.value = value;

	}

	@ApiModelProperty(required = true, value = "E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types")

	public String getType() {

		return type;

	}

	public void setType(final String type) {

		this.type = type;

	}

	@ApiModelProperty(required = true)

	public String getValue() {

		return value;

	}

	public void setValue(final String value) {

		this.value = value;

	}

}

package eu.dnetlib.openaire.directindex.api;

import java.text.SimpleDateFormat;

import java.util.Date;

import javax.annotation.PostConstruct;

import javax.annotation.Resource;

import org.apache.commons.lang.exception.ExceptionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.common.SolrInputDocument;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.http.HttpStatus;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.ExceptionHandler;

import org.springframework.web.bind.annotation.PathVariable;

import org.springframework.web.bind.annotation.RequestBody;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestMethod;

import org.springframework.web.bind.annotation.RequestParam;

import org.springframework.web.bind.annotation.ResponseBody;

import org.springframework.web.bind.annotation.ResponseStatus;

import com.google.gson.Gson;

import eu.dnetlib.common.rmi.DNetRestDocumentation;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import eu.dnetlib.openaire.directindex.objects.ResultEntry;

import eu.dnetlib.openaire.directindex.objects.ResultEntryToOaf;

import eu.dnetlib.openaire.directindex.utils.OafToIndexRecordFactory;

/**

 * Created by michele on 11/11/15.

 */

@Controller

@DNetRestDocumentation

public class OpenaireResultSubmitter {

	private static final Log log = LogFactory.getLog(OpenaireResultSubmitter.class);

	@Value(value = "${openaire.api.community}")

	private String community_api;

	@Value(value = "${openaire.api.dsm}")

	private String dsm_api;

	@Value(value = "oaf.schema.location")

	private String oafSchemaLocation;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Resource

	private OafToIndexRecordFactory oafToIndexRecordFactory;

	@Resource

	private RecentResultsQueue recentResultsQueue;

	@Resource(name = "openaireplusApisVelocityEngine")

	private VelocityEngine velocityEngine;

	@Resource(name = "indexClientManager")

	private IndexClientManager clientManager;

	@Resource(name = "resultSubmitterService")

	private ResultSubmitterService submitterService;

	@Autowired

	private IndexDSRetriever indexDSRetriever;

	private OpenAIRESubmitterUtils utils;

	@PostConstruct

	public void init() {

		utils = new OpenAIRESubmitterUtils(community_api, dsm_api);

	}

	@RequestMapping(value = {

		"/api/admin/autocommit/active"

	}, method = RequestMethod.GET)

	public @ResponseBody Boolean getAutocommit() throws DirecIndexApiException {

		return submitterService.isAutocommitactive();

	}

	@RequestMapping(value = {

		"/api/admin/autocommit/active"

	}, method = RequestMethod.POST)

	public @ResponseBody Boolean setAutocommit(@RequestParam(value = "active", required = true) final Boolean active) throws DirecIndexApiException {

		submitterService.setAutocommitactive(active);

		log.info(String.format("automatic commit, active '%s', frequency '%s'", submitterService.isAutocommitactive(), submitterService.getCommitfrquency()));

		return submitterService.isAutocommitactive();

	}

	@RequestMapping(value = "/api/admin/evictCache", method = RequestMethod.GET)

	@ResponseStatus(value = HttpStatus.OK)

	public void evictCache() {

		indexDSRetriever.evictCache();

	}

	@Deprecated

	@RequestMapping(value = {

		"/api/publications/feedJson", "/api/results/feedJson"

	}, method = RequestMethod.POST)

	public @ResponseBody String feedObjectJson(@RequestParam(value = "json", required = true) final String json,

		@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirecIndexApiException {

		log.debug(json);

		final ResultEntry pub = new Gson().fromJson(json, ResultEntry.class);

		return feedObject(pub, commit);

	}

	@RequestMapping(value = {

		"/api/results/feedObject"

	}, method = RequestMethod.POST)

	public @ResponseBody String feedResult(@RequestBody final ResultEntry pub,

		@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)

		throws DirecIndexApiException {

		return feed(pub, commit);

	}

	@Deprecated

	@RequestMapping(value = {

		"/api/publications/feedObject"

	}, method = RequestMethod.POST)

	public @ResponseBody String feedObject(@RequestBody final ResultEntry pub,

		@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)

		throws DirecIndexApiException {

		return feed(pub, commit);

	}

	@RequestMapping(value = "/api/result/{openaireId}", method = RequestMethod.DELETE)

	public @ResponseBody boolean deleteResultWithOpenaireId(

		@PathVariable(value = "openaireId") final String openaireId,

		@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirecIndexApiException {

		return deleteResult(openaireId);

	}

	@RequestMapping(value = "/api/results", method = RequestMethod.DELETE)

	public @ResponseBody boolean deleteResultWithOriginalId(

		@RequestParam(value = "originalId", required = true) final String originalId,

		@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,

		@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws Exception {