D-Net

<?xml version="1.0" ?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-directindex-api</artifactId>

	<packaging>jar</packaging>

	<version>2.1.7-solr75</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-directindex-api/tags/dnet-directindex-api-2.1.7-solr75</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaireplus-mapping-utils</artifactId>

			<version>[6.2.21-solr75]</version>

			<exclusions>

				<exclusion>

					<groupId>eu.dnetlib</groupId>

					<artifactId>dnet-hadoop-commons</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-client</artifactId>

			<version>[2.3.2-solr75]</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>io.springfox</groupId>

			<artifactId>springfox-swagger2</artifactId>

			<version>${springfox-version}</version>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.9.5</version>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity</artifactId>

			<version>1.7</version>

			<exclusions>

				<exclusion>

					<artifactId>antlr</artifactId>

					<groupId>antlr</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity-tools</artifactId>

			<version>2.0</version>

			<exclusions>

				<exclusion>

					<artifactId>antlr</artifactId>

					<groupId>antlr</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-context-support</artifactId>

			<version>${spring.version}</version>

		</dependency>

	</dependencies>

	<properties>

		<springfox-version>2.5.0</springfox-version>

	</properties>

</project>

{

  "authors":[

	"Alouges, Fran\u00e7ois",

	"Di Fratta, Giovanni"

  ],

  "collectedFromId":"opendoar____::2659",

  "description":"The paper is about the parking 3-sphere swimmer ($\\text{sPr}_3$). This is a low-Reynolds number model swimmer composed of three balls of equal radii. The three balls can move along three horizontal axes (supported in the same plane) that mutually meet at the center of $\\text{sPr}_3$ with angles of $120^{\u2218}$ . The governing dynamical system is introduced and the implications of its geometric symmetries revealed. It is then shown that, in the first order range of small strokes, optimal periodic strokes are ellipses embedded in 3d space, i.e. closed curves of the form $t\ud835\udfc4 [0,2\u03c0] \u21a6 (\\cos t)u + (\\sin t)v$ for suitable orthogonal vectors $u$ and $v$ of $\u211d^3$. A simple analytic expression for the vectors $u$ and $v$ is derived. The results of the paper are used in a second article where the real physical dynamics of $\\text{sPr}_3$ is analyzed in the asymptotic range of very long arms. ; Comment: 17 pages, 4 figures",

  "hostedById":"opendoar____::2659",

  "licenseCode":"OPEN",

  "originalId":"oai:zenodo.org:996201",

  "pids":[

	{

	  "type":"oai",

	  "value":"oai:zenodo.org:996201"

	},

	{

	  "type":"doi",

	  "value":"10.5281/zenodo.996201"

	}

  ],

  "publisher":"Zenodo",

  "resourceType":"0020",

  "title":"Parking 3-sphere swimmer. I. Energy minimizing strokes",

  "type":"publication",

  "url":"https://zenodo.org/record/996201"

}

{

  "openaireId":"dedup_wf_001::ab42e811",

  "originalId": "ORIG_ID_TEST",

  "type": "publication",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "licenseCode": "EMBARGO",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

{

  "originalId": "ORIG_ID_12345",

  "title": "TEST TITLE WITH Greek Characters",

  "authors": [

	"αβγδεζηθικλμ νξοπρσςτυφχψω",

	"ΑΒΓΔΕΖΗΘΙΚΛ ΜΝΞΟΠΡΣΤΥΦΧΨΩ"

  ],

  "publisher": "âââââââ",

  "description": "Αν περιμένατε να βρίσκεται εδώ μια σελίδα και δεν υπάρχει, η σελίδα μπορεί να μην εμφανίζεται λόγω καθυστέρησης στην ανανέωση της βάσης δεδομένων, ή μπορεί να έχει διαγραφεί. (Δείτε την γρήγορη διαγραφή σελίδων για πιθανούς λόγους). Μπορείτε να δοκιμάστε την λειτουργία εκκαθάρισης, και να ελέγξετε το αρχείο διαγραφών.",

  "language": "ell",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123-gr"

	}

  ],

  "licenseCode": "EMBARGO",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2367",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/123456/EU//Test"

  ],

  "contexts": [

	"egi::classification::natsc::math::stats"

  ]

}

package eu.dnetlib.openaire.directindex.api;

import com.google.gson.Gson;

import eu.dnetlib.common.rmi.DNetRestDocumentation;

import eu.dnetlib.data.index.CloudIndexClient;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.openaire.directindex.objects.ResultEntry;

import eu.dnetlib.openaire.directindex.utils.OafToIndexRecordFactory;

import org.apache.commons.lang.exception.ExceptionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.http.HttpStatus;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.*;

import javax.annotation.Resource;

import java.util.Map;

/**

 * Created by michele on 11/11/15.

 */

@Controller

@DNetRestDocumentation

public class OpenaireResultSubmitter {

	private static final Log log = LogFactory.getLog(OpenaireResultSubmitter.class);

	@Value(value = "oaf.schema.location")

	private String oafSchemaLocation;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Resource

	private OafToIndexRecordFactory oafToIndexRecordFactory;

	@Resource

	private RecentResultsQueue recentResultsQueue;

	@Resource(name = "openaireplusApisVelocityEngine")

	private VelocityEngine velocityEngine;

	@Resource(name = "indexClientMap")

	private IndexClientMap clientMap;

	@Resource(name = "resultSubmitterService")

	private ResultSubmitterService submitterService;

	@RequestMapping(value = { "/api/admin/autocommit/active" }, method = RequestMethod.GET)

	public @ResponseBody Boolean getAutocommit() throws DirecIndexApiException {

		return submitterService.isAutocommitactive();

	}

	@RequestMapping(value = { "/api/admin/autocommit/active" }, method = RequestMethod.POST)

	public @ResponseBody Boolean setAutocommit(@RequestParam(value = "active", required = true) final Boolean active) throws DirecIndexApiException {

		submitterService.setAutocommitactive(active);

		log.info(String.format("automatic commit, active '%s', frequency '%s'", submitterService.isAutocommitactive(), submitterService.getCommitfrquency()));

		return submitterService.isAutocommitactive();

	}

	@Deprecated

	@RequestMapping(value = { "/api/publications/feedJson", "/api/results/feedJson" }, method = RequestMethod.POST)

	public @ResponseBody String feedObjectJson(@RequestParam(value = "json", required = true) final String json,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirecIndexApiException {

		final ResultEntry pub = new Gson().fromJson(json, ResultEntry.class);

		return feedObject(pub, commit);

	}

	@RequestMapping(value = { "/api/results/feedObject" }, method = RequestMethod.POST)

	public @ResponseBody String feedResult(@RequestBody final ResultEntry pub,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)

			throws DirecIndexApiException {

		return feed(pub, commit);

	}

	@Deprecated

	@RequestMapping(value = { "/api/publications/feedObject" }, method = RequestMethod.POST)

	public @ResponseBody String feedObject(@RequestBody final ResultEntry pub,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)

			throws DirecIndexApiException {

		return feed(pub, commit);

	}

	@RequestMapping(value = "/api/result/{openaireId}", method = RequestMethod.DELETE)

	public @ResponseBody boolean deleteResultWithOpenaireId(

			@PathVariable(value = "openaireId") final String openaireId,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirecIndexApiException {

		return deleteResult(openaireId, commit);

	}

	@RequestMapping(value = "/api/results", method = RequestMethod.DELETE)

	public @ResponseBody boolean deleteResultWithOriginalId(

			@RequestParam(value = "originalId", required = true) final String originalId,

			@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws Exception {

		final String openaireId = ResultEntry.calculateOpenaireId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class));

		return deleteResult(openaireId, commit);

	}

	@Deprecated

	@RequestMapping(value = { "/api/publications/deleteObject", "/api/results/delete" }, method = RequestMethod.POST)

	public @ResponseBody boolean deleteResultPost(

			@RequestParam(value = "originalId", required = true) final String originalId,

			@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws Exception {

		final String openaireId = ResultEntry.calculateOpenaireId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class));

		return deleteResult(openaireId, commit);

	}

	@Deprecated

	private String feed(final ResultEntry pub, final boolean commit) throws DirecIndexApiException {

		return feed(pub);

	}

	private String feed(final ResultEntry pub) throws DirecIndexApiException {

		try {

			final String oafRecord = pub.asOafRecord(velocityEngine, serviceLocator.getService(ISLookUpService.class), oafSchemaLocation);

			for(Map.Entry<IndexDsInfo, CloudIndexClient> e : clientMap.getClients().entrySet()) {

				final IndexDsInfo idx = e.getKey();

				final CloudIndexClient client = e.getValue();

				client.feed(oafRecord, idx.getIndexDsId(), oafToIndexRecordFactory.newTransformer(idx.getFormat()), false);

			}

			recentResultsQueue.add(oafRecord);

			return pub.getOpenaireId();

		} catch (final Throwable e) {

			log.error("Error saving record", e);

			log.debug(pub.toString());

			throw new DirecIndexApiException("Error adding publication: " + e.getMessage(), e);

		}

	}

	private boolean deleteResult(final String openaireId, final boolean commit) throws DirecIndexApiException {

		try {

			for(Map.Entry<IndexDsInfo, CloudIndexClient> e : clientMap.getClients().entrySet()) {

				final IndexDsInfo idx = e.getKey();

				final CloudIndexClient client = e.getValue();

				client.remove(openaireId, false);

				log.info("Deleted result with id: " + openaireId + " from: " + idx.getIndexBaseUrl());

			}

			recentResultsQueue.remove(openaireId);

			return true;

		} catch (Throwable e) {

			throw new DirecIndexApiException("Error deleting publication: " + e.getMessage(), e);

		}

	}

	@ExceptionHandler(Exception.class)

	@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)

	public @ResponseBody ErrorMessage handleException(final Exception e) {

		log.error("Error in direct index API", e);

		return new ErrorMessage(e);

	}

	public class ErrorMessage {

		private final String message;

		private final String stacktrace;

		public ErrorMessage(final Exception e) {

			this(e.getMessage(), ExceptionUtils.getStackTrace(e));

		}

		public ErrorMessage(final String message, final String stacktrace) {

			this.message = message;

			this.stacktrace = stacktrace;

		}

		public String getMessage() {

			return this.message;

		}

		public String getStacktrace() {

			return this.stacktrace;

		}

	}

}

{"type_source": "SVN", "goal": "package -U source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-directindex-api/branches/solr75", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-directindex-api_solr75"}

package eu.dnetlib.openaire.directindex.api;

import eu.dnetlib.data.index.CloudIndexClient;

import eu.dnetlib.data.index.CloudIndexClientException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Value;

import javax.annotation.Resource;

import java.util.Map;

import java.util.concurrent.Executors;

import java.util.concurrent.ScheduledExecutorService;

import java.util.concurrent.TimeUnit;

public class ResultSubmitterService {

    private static final Log log = LogFactory.getLog(ResultSubmitterService.class);

    @Resource(name = "indexClientMap")

    private IndexClientMap clientMap;

    /**

     * Autocommit feature activation flag

     */

    @Value(value = "${openaire.api.directindex.autocommit.active}")

    private boolean autocommitactive;

    /**

     * Autocommit frequency (Seconds)

     */

    @Value(value = "${openaire.api.directindex.autocommit.frequency}")

    private long commitfrquency = 60;

    private ScheduledExecutorService executor;

    public ResultSubmitterService() {

        executor = Executors.newSingleThreadScheduledExecutor();

        updateCommitSchedule();

    }

    private void updateCommitSchedule() {

        log.info("updating commit schedule");

        executor.scheduleAtFixedRate(() -> {

            if (isAutocommitactive()) {

                try {

                    for (Map.Entry<IndexDsInfo, CloudIndexClient> entry : clientMap.getClients().entrySet()) {

                        final IndexDsInfo i = entry.getKey();

                        final CloudIndexClient client = entry.getValue();

                        log.info("performing commit on " + i.getColl());

                        try {

                            client.commit();

                        } catch (CloudIndexClientException e) {

                            log.error("error performing commit on " + i.getColl(), e);

                        }

                    }

                } catch (DirecIndexApiException e) {

                    log.error(e);

                }

            }

        }, 0, getCommitfrquency(), TimeUnit.SECONDS);

    }

    public boolean isAutocommitactive() {

        return autocommitactive;

    }

    public synchronized void setAutocommitactive(boolean autocommitactive) {

        this.autocommitactive = autocommitactive;

    }

    public long getCommitfrquency() {

        return commitfrquency;

    }

    public synchronized void setCommitfrquency(long commitfrquency) {

        this.commitfrquency = commitfrquency;

    }

}

package eu.dnetlib.openaire.directindex.api;

public class IndexDsInfo {

    private final String indexBaseUrl;

    private final String indexDsId;

    private final String format;

    private final String coll;

    public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) {

        this.indexBaseUrl = indexBaseUrl;

        this.indexDsId = indexDsId;

        this.format = format;

        this.coll = coll;

    }

    public String getIndexBaseUrl() {

        return indexBaseUrl;

    }

    public String getIndexDsId() {

        return indexDsId;

    }

    public String getFormat() {

        return format;

    }

    public String getColl() {

        return coll;

    }

    @Override

    public int hashCode() {

        return getColl().hashCode();

    }

    @Override

    public boolean equals(Object other) {

        if (!(other instanceof IndexDsInfo)) return false;

        return getColl().equals(((IndexDsInfo) other).getColl());

    }

}

package eu.dnetlib.openaire.directindex.api;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.data.index.CloudIndexClient;

import eu.dnetlib.data.index.CloudIndexClientException;

import eu.dnetlib.data.index.CloudIndexClientFactory;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.core.io.ClassPathResource;

import javax.annotation.Resource;

import java.io.IOException;

import java.util.List;

import java.util.Map;

public class IndexClientMap {

    private static final Log log = LogFactory.getLog(IndexClientMap.class);

    @Value(value = "${openaire.api.directindex.findSolrIndexUrl.xquery}")

    private ClassPathResource findSolrIndexUrl;

    @Value(value = "${openaire.api.directindex.findIndexDsInfo.xquery}")

    private ClassPathResource findIndexDsInfo;

    @Resource

    private UniqueServiceLocator serviceLocator;

    private Map<IndexDsInfo, CloudIndexClient> clients = Maps.newHashMap();

    public Map<IndexDsInfo, CloudIndexClient> getClients() throws DirecIndexApiException {

        try {

            final List<IndexDsInfo> idxList = calculateCurrentIndexDsInfo();

            if (idxList == null || idxList.isEmpty()) {

                throw new DirecIndexApiException("cannot create index: no public Search Service found");

            }

            if (idxList.size() > 1) {

                log.warn("found more than 1 public search service");

            }

            for(IndexDsInfo i : idxList) {

                if (!clients.containsKey(i)) {

                    clients.put(i, CloudIndexClientFactory.newIndexClient(i.getIndexBaseUrl(), i.getColl(), false));

                }

            }

            return clients;

        } catch (IOException | ISLookUpException | CloudIndexClientException e) {

            throw new DirecIndexApiException(e);

        }

    }

    private List<IndexDsInfo> calculateCurrentIndexDsInfo() throws IOException, ISLookUpException {

        final List<IndexDsInfo> list = Lists.newArrayList();

        final String queryUrl = IOUtils.toString(findSolrIndexUrl.getInputStream());

        final String queryDs = IOUtils.toString(findIndexDsInfo.getInputStream());

        final ISLookUpService lu = serviceLocator.getService(ISLookUpService.class);

        final String indexBaseUrl = lu.getResourceProfileByQuery(queryUrl);

        final List<String> idxDs = lu.quickSearchProfile(queryDs);

        for (final String idx : idxDs) {

            final String[] arr = idx.split("@@@");

            list.add(new IndexDsInfo(indexBaseUrl, arr[0].trim(), arr[1].trim(), arr[2].trim()));

        }

        return list;

    }

}

package eu.dnetlib.openaire.directindex.api;

import java.io.StringWriter;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import org.apache.commons.lang.StringUtils;

/**

 * Created by michele on 15/01/16.

 */

public class OpenAIRESubmitterUtils {

	private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(OpenAIRESubmitterUtils.class);

	public Map<String, String> calculateProjectInfo(final String link) {

		final Map<String, String> info = new HashMap<String, String>();

		final String[] arr = link.split("/");

		// info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble

		if (arr.length > 4) {

			final String acronym = arr.length > 7 ? arr[7] : "";

			final String title = arr.length > 6 ? StringUtils.isNotBlank(arr[6]) ? arr[6] : acronym : "";

			final String jurisdiction = arr.length > 5 ? arr[5] : "";

			final String funderId = calculateFunderId(arr[2], arr[3]);

			info.put("id", calculateProjectId(arr[2], arr[3], arr[4]));

			info.put("funderShortName", arr[2]);

			info.put("fundingName", arr[3]);

			info.put("code", arr[4]);

			info.put("jurisdiction", jurisdiction);

			info.put("title", title);

			info.put("acronym", acronym);

			info.put("funderId", funderId);

			info.put("funderName", calculateFunderName(arr[2]));

			info.put("fundingId", funderId + "::" + arr[3]);

		}

		return info;

	}

	protected String calculateFunderPrefix(final String funderShortName, final String funding){

		switch(funderShortName.toLowerCase()){

		case "conicyt":

			return "conicytf____::";

		case "ec":

			if (funding.equalsIgnoreCase("fp7")) {

				return "corda_______::";

			} else {

				return "corda__h2020::";

			}

		case "hrzz":

		case "mzos":

			return "irb_hr______::";

		case "tara":

			return "taraexp_____::";

		case "tubitak":

			return "tubitakf____::";

		default:

			String prefix = funderShortName.toLowerCase();

			//ensure we have 12 chars

			while(prefix.length() < 12) prefix += "_";

			return prefix +"::";

		}

	}

	protected String calculateProjectId(final String funderShortName, final String funding, final String code) {

		final String suffix = Hashing.md5(code);

		final String funderPrefix = calculateFunderPrefix(funderShortName, funding);

		return funderPrefix + suffix;

	}

	protected String calculateFunderId(final String funderShortName, final String funding) {

		switch (funderShortName.toLowerCase()) {

		case "ec":

			return "ec__________::EC";

		default:

			String prefix = calculateFunderPrefix(funderShortName, funding);

			return prefix + funderShortName.toUpperCase();

		}

	}

	protected String calculateFunderName(final String funderShortName) {

		switch (funderShortName.toLowerCase()) {

		case "aff":

		case "aka":

			return "Academy of Finland";

		case "arc":

			return "Australian Research Council (ARC)";

		case "conicyt":

			return "Comisión Nacional de Investigación Científica y Tecnológica";

		case "ec":

			return "European Commission";

		case "fct":

			return "Fundação para a Ciência e a Tecnologia, I.P.";

		case "fwf":

			return "Austrian Science Fund (FWF)";

		case "hrzz":

			return "Croatian Science Foundation (CSF)";

		case "mestd":

			return "Ministry of Education, Science and Technological Development of Republic of Serbia";

		case "miur":

			return "Ministero dell'Istruzione dell'Università e della Ricerca";

		case "mzos":

			return "Ministry of Science, Education and Sports of the Republic of Croatia (MSES)";

		case "nhmrc":

			return "National Health and Medical Research Council (NHMRC)";

		case "nih":

			return "National Institutes of Health";

		case "nsf":

			return "National Science Foundation";

		case "nwo":

			return "Netherlands Organisation for Scientific Research (NWO)";

		case "rcuk":

			return "Research Council UK";

		case "sfi":

			return "Science Foundation Ireland";

		case "sgov":

			return "Gobierno de España";

		case "snsf":

			return "Swiss National Science Foundation";

		case "tara":

			return "Tara Expeditions Foundation";

		case "tubitak":

			return "Türkiye Bilimsel ve Teknolojik Araştırma Kurumu";

		case "wt":

			return "Wellcome Trust";

		default:

			log.error("Funder short name '"+funderShortName+"' not managed");

			return "";

		}

	}

	public List<ContextInfo> processContexts(final List<String> list) {

		return Lists.newArrayList(Lists.transform(list, new Function<String, ContextInfo>() {

			@Override

			public ContextInfo apply(final String s) {

				return createContextInfo(s.split("::"), 0);

			}

			private ContextInfo createContextInfo(final String[] arr, final int pos) {

				final StringWriter id = new StringWriter();

				id.write(arr[0]);

				for (int i = 0; i < pos; i++) {

					id.write("::");

					id.write(arr[i + 1]);

				}

				final String elem = (pos == 0) ? "context" : (pos == 1) ? "category" : "concept";

				final ContextInfo info = new ContextInfo(elem, id.toString());

				if ((pos + 1) < arr.length) {

					info.getChildren().add(createContextInfo(arr, pos + 1));

				}

				return info;

			}

		}));

	}

	public class ContextInfo {

		private String elem;

		private String id;

		private List<ContextInfo> children = new ArrayList<ContextInfo>();

		public ContextInfo(final String elem,

				final String id) {

			this.elem = elem;

			this.id = id;

		}

		public String getElem() {

			return elem;

		}

		public void setElem(final String elem) {

			this.elem = elem;

		}

		public String getId() {

			return id;

		}

		public void setId(final String id) {

			this.id = id;

		}

		public List<ContextInfo> getChildren() {

			return children;

		}

		public void setChildren(final List<ContextInfo> children) {

			this.children = children;

		}

		public boolean isRoot() {

			return elem.equals("context");

		}

	}

}

package eu.dnetlib.openaire.directindex.api;

import eu.dnetlib.common.rmi.RMIException;

public class DirecIndexApiException extends RMIException {

	/**

	 *

	 */

	private static final long serialVersionUID = -3888037031334809448L;

	public DirecIndexApiException(final String string) {

		super(string);

	}

	public DirecIndexApiException(final String string, final Throwable exception) {

		super(string, exception);

	}

	public DirecIndexApiException(final Throwable exception) {

		super(exception);

	}

}

package eu.dnetlib.openaire.directindex.api;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import com.mongodb.*;

import eu.dnetlib.miscutils.datetime.DateUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

/**

 * Created by michele on 11/11/15.

 */

public class RecentResultsQueue implements Iterable<String> {

	private static final Log log = LogFactory.getLog(RecentResultsQueue.class);

	private DB db;

	private String collection;

	public void init() {

		if (!db.collectionExists(collection)) {

			log.info(String.format("creating collection %s", collection));

			db.createCollection(collection, new BasicDBObject());

		}

	}

	@Override

	public Iterator<String> iterator() {

		final DBCursor cursor = db.getCollection(collection).find();

		return new Iterator<String>() {

			@Override

			public boolean hasNext() {

				return cursor.hasNext();

			}

			@Override

			public String next() {

				final DBObject obj = cursor.next();

				return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : "";

			}

			@Override

			public void remove() {

				throw new RuntimeException("NOT IMPLEMENTED");

			}

		};

	}

	synchronized public void add(final String oaf) throws Exception {

		final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']");

		log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection);

		final DBCollection coll = db.getCollection(collection);

		final DBObject obj = BasicDBObjectBuilder.start()

				.append("id", id)

				.append("record", oaf)

				.append("date", DateUtils.now())

				.get();

		coll.update(new BasicDBObject("id", id), obj, true, false);

	}

	public void remove(final List<String> list) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : list) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public void remove(final String... ids) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : ids) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public DB getDb() {

		return db;

	}

	@Required

	public void setDb(final DB db) {

		this.db = db;

	}

	public String getCollection() {

		return collection;

	}

	@Required

	public void setCollection(final String collection) {

		this.collection = collection;

	}

}

package eu.dnetlib.openaire.directindex.objects;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Date;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.concurrent.TimeUnit;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.ui.velocity.VelocityEngineUtils;

import com.google.gson.Gson;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.miscutils.functional.string.EscapeXml;

import eu.dnetlib.openaire.directindex.api.DirecIndexApiException;

import eu.dnetlib.openaire.directindex.api.OpenAIRESubmitterUtils;

import io.swagger.annotations.ApiModelProperty;

/**

 * Created by michele on 02/12/15.

 */

public class ResultEntry {

	private String openaireId;

	private String originalId;

	private String title;

	private List<String> authors = new ArrayList<>();

	private String publisher;

	private String description;

	private String language;

	private List<PidEntry> pids = new ArrayList<>();

	/**

	 * @Deprecated: use accessRightCode

	 */

	@Deprecated

	private String licenseCode;

	private String accessRightCode;

	private String embargoEndDate;

	/**

	 * One of publication, dataset, software, other. Default value is publication.

	 */

	private String type = "publication";

	private String resourceType;

	private String url;

	private String collectedFromId;

	private String hostedById;

	// String according to the EGI context profile, example: egi::classification::natsc::math

	private List<String> contexts = new ArrayList<>();

	// String according to openaire guidelines:

	// info:eu-repo/grantAgreement/Funder/FundingProgram/ProjectID/[Jurisdiction]/[ProjectName]/[ProjectAcronym]

	private List<String> linksToProjects = new ArrayList<>();

	private static long last_cache_update = 0;

	private static final Map<String, Map<String, String>> cached_vocabularies = new HashMap<>();

	private static final Map<String, DatasourceEntry> cached_datasources = new HashMap<>();

	private static final Map<String, String> cached_contexts = new HashMap<>();

	private static final Log log = LogFactory.getLog(ResultEntry.class);

	public ResultEntry() {}

	public String getOpenaireId() {

		return openaireId;

	}

	public void setOpenaireId(final String openaireId) {

		this.openaireId = openaireId;

	}

	public String getOriginalId() {

		return originalId;

	}

	public void setOriginalId(final String originalId) {

		this.originalId = originalId;

	}

	@ApiModelProperty(required = true)

	public String getTitle() {

		return title;

	}

	public void setTitle(final String title) {

		this.title = title;

	}

	public List<String> getAuthors() {

		return authors;

	}

	public void setAuthors(final List<String> authors) {

		this.authors = authors;

	}

	public String getPublisher() {

		return publisher;

	}

	public void setPublisher(final String publisher) {

		this.publisher = publisher;

	}

	public String getDescription() {

		return description;

	}

	public void setDescription(final String description) {

		this.description = description;

	}

	@ApiModelProperty(value = "ISO Alpha-3 code. E.g. 'eng', 'ita'")

	public String getLanguage() {

		return language;

	}

	public void setLanguage(final String language) {

		this.language = language;

	}

	public List<PidEntry> getPids() {

		return pids;

	}

	public void setPids(final List<PidEntry> pids) {

		this.pids = pids;

	}

	@Deprecated

	@ApiModelProperty(required = false, allowableValues = "OPEN, CLOSED, RESTRICTED, EMBARGO, UNKNOWN, OTHER, OPEN SOURCE")

	public String getLicenseCode() {

		return licenseCode;

	}

	@Deprecated

	public void setLicenseCode(final String licenseCode) {

		this.licenseCode = licenseCode;

	}

	/**

	 * Set required = true when the deprecated licenseCode is not used anymore by our client and it is removed

	 * @return

	 */

	@ApiModelProperty(required = false, allowableValues = "OPEN, CLOSED, RESTRICTED, EMBARGO, UNKNOWN, OTHER, OPEN SOURCE")

	public String getAccessRightCode() {

		return accessRightCode;

	}

	public void setAccessRightCode(final String accessRightCode) {

		this.accessRightCode = accessRightCode;

	}

	@ApiModelProperty(required = true, value = "Use 001 for articles, 021 for datasets. See: http://api.openaire.eu/vocabularies/dnet:publication_resource.")

	public String getResourceType() {

		return resourceType;

	}

	public void setResourceType(final String resourceType) {

		this.resourceType = resourceType;

	}

	@ApiModelProperty(required = true)

	public String getUrl() {

		return url;

	}

	public void setUrl(final String url) {

		this.url = url;

	}

	@ApiModelProperty(required = true, value = "Use opendoar___::2659 for Zenodo Publications; re3data_____::r3d100010468 for Zenodo datasets; infrastruct::openaire for OpenAIRE portal.")

	public String getCollectedFromId() {

		return collectedFromId;

	}

	public void setCollectedFromId(final String collectedFromId) {

		this.collectedFromId = collectedFromId;

	}

	public String getHostedById() {

		return hostedById;

	}

	public void setHostedById(final String hostedById) {

		this.hostedById = hostedById;

	}

	@ApiModelProperty(value = "E.g. fet, egi::classification::natsc::math::pure, egi::projects::EMI")

	public List<String> getContexts() {

		return contexts;

	}

	public void setContexts(final List<String> contexts) {

		this.contexts = contexts;

	}

	@ApiModelProperty(value = "E.g. info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus")

	public List<String> getLinksToProjects() {

		return linksToProjects;

	}

	public void setLinksToProjects(final List<String> linksToProjects) {

		this.linksToProjects = linksToProjects;

	}

	@ApiModelProperty(allowableValues = "publication, dataset")

	public String getType() {

		return type;

	}

	public void setType(final String type) {

		this.type = type;

	}

	public String getEmbargoEndDate() {

		return embargoEndDate;

	}

	public void setEmbargoEndDate(final String embargoEndDate) {

		this.embargoEndDate = embargoEndDate;

	}

	public String asOafRecord(final VelocityEngine ve,

			final ISLookUpService lookupService,

			final String oafSchemaLocation) throws Exception {

		if (StringUtils.isBlank(getOriginalId())

				&& StringUtils

						.isBlank(getOpenaireId())) { throw new DirecIndexApiException("One of the following fields is required: originalId or openaireId"); }

		if (StringUtils.isBlank(getTitle())) { throw new DirecIndexApiException("A required field is missing: title"); }

		if (StringUtils.isBlank(getUrl())) { throw new DirecIndexApiException("A required field is missing: url"); }

		if (StringUtils.isBlank(getLicenseCode()) && StringUtils.isBlank(getAccessRightCode())) { throw new DirecIndexApiException("A required field is missing: accessRightCode"); }

		if (StringUtils.isBlank(getResourceType())) { throw new DirecIndexApiException("A required field is missing: resourceType"); }

		if (StringUtils.isBlank(getCollectedFromId())) { throw new DirecIndexApiException("A required field is missing: collectedFromId"); }

		if (StringUtils.isBlank(getType())) { throw new DirecIndexApiException("A required field is missing: type"); }

		final DatasourceEntry collectedFromEntry = getDatasourceInfo(collectedFromId, lookupService);

		final DatasourceEntry hostedByEntry = getDatasourceInfo(hostedById, lookupService);

		if (StringUtils.isBlank(openaireId)) {

			setOpenaireId(calculateOpenaireId(originalId, collectedFromEntry));

		}

		if (!openaireId

				.matches("^\\w{12}::\\w{32}$")) { throw new DirecIndexApiException(

						"Invalid openaireId: " + openaireId + " - regex ^\\w{12}::\\w{32}$ not matched"); }

		final Map<String, Object> model = new HashMap<>();

		model.put("esc", new EscapeXml());

		model.put("util", new OpenAIRESubmitterUtils());

		model.put("pub", this);

		model.put("objIdentifier", getOpenaireId());

		model.put("oafSchemaLocation", oafSchemaLocation);

		model.put("resultTypes", getVocabulary("dnet:result_typologies", lookupService));

		model.put("rights", getVocabulary("dnet:access_modes", lookupService));

		model.put("resourceTypes", getVocabulary("dnet:publication_resource", lookupService));

		model.put("pidTypes", getVocabulary("dnet:pid_types", lookupService));

		model.put("languages", getVocabulary("dnet:languages", lookupService));

		model.put("contexts", getContexts(lookupService));

		model.put("dateOfCollection", new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'").format(new Date()));

		model.put("collectedFrom", collectedFromEntry);

		model.put("hostedBy", hostedByEntry);

		return VelocityEngineUtils.mergeTemplateIntoString(ve, "/eu/dnetlib/openaire/directindex/indexRecord.xml.vm", "UTF-8", model);

	}

	private static String calculateOpenaireId(final String originalId, final DatasourceEntry collectedFromEntry) {

		return collectedFromEntry.getPrefix() + "::" + Hashing.md5(originalId);

	}

	public static String calculateOpenaireId(final String originalId, final String collectedFromId, final ISLookUpService lookupService)

			throws ISLookUpException {

		return calculateOpenaireId(originalId, getDatasourceInfo(collectedFromId, lookupService));

	}

	private synchronized static DatasourceEntry getDatasourceInfo(final String dsId, final ISLookUpService lookupService) throws ISLookUpException {

		if (StringUtils

				.isBlank(dsId)) { return new DatasourceEntry("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "unknown_____"); }

		if (!cached_datasources.containsKey(dsId)) {

			final String query =

					"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//CONFIGURATION[./DATASOURCE_ORIGINAL_ID='" + dsId

							+ "']/concat(./OFFICIAL_NAME, ' @@@ ', .//FIELD/value[../key='NamespacePrefix'])";

			final String s = lookupService.getResourceProfileByQuery(query);

			final String[] arr = s.split("@@@");

			final DatasourceEntry ds = new DatasourceEntry(dsId, arr[0].trim(), arr[1].trim());

			if (StringUtils.isBlank(ds.getName()) || StringUtils.isBlank(ds.getPrefix())) {

				log.error("Invalid datasource id: " + dsId);

				throw new ISLookUpException("Invalid datasource id: " + dsId);

			} else {

				cached_datasources.put(dsId, ds);

			}

		}

		return cached_datasources.get(dsId);

	}

	private synchronized static Map<String, String> getVocabulary(final String voc, final ISLookUpService lookupService) throws ISLookUpException {

		if (DateUtils.now() - last_cache_update < TimeUnit.MINUTES.toMillis(15) && cached_vocabularies.containsKey(voc)) {

			return cached_vocabularies.get(voc);

		} else {

			final String query = "collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code='" + voc

					+ "']//TERM/concat(@code, ' @@@ ', @english_name)";

			final Map<String, String> map = new HashMap<>();

			for (final String s : lookupService.quickSearchProfile(query)) {

				final String[] arr = s.split("@@@");

				map.put(arr[0].trim(), arr[1].trim());

			}

			cached_vocabularies.put(voc, map);

			last_cache_update = DateUtils.now();

			return map;

		}

	}

	private synchronized static Map<String, String> getContexts(final ISLookUpService lookupService) throws ISLookUpException {

		if (DateUtils.now() - last_cache_update > TimeUnit.MINUTES.toMillis(15) || cached_contexts.isEmpty()) {

			final String query =

					"collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community' or .//context/@type='ri']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)";

			cached_contexts.clear();

			for (final String s : lookupService.quickSearchProfile(query)) {

				final String[] arr = s.split("@@@");

				cached_contexts.put(arr[0].trim(), arr[1].trim());

			}

			last_cache_update = DateUtils.now();

		}

		return cached_contexts;

	}

	@Override

	public String toString() {

		return new Gson().toJson(this);

	}

	public String getAnyId() {

		return StringUtils.isNotBlank(openaireId) ? openaireId : originalId;

	}

}

package eu.dnetlib.openaire.directindex.objects;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import org.apache.commons.lang.StringUtils;

/**

 * Created by michele on 02/12/15.

 */

public class DatasourceEntry {

	private String id;

	private String name;

	private String prefix;

	public DatasourceEntry() {

	}

	public DatasourceEntry(final String id, final String name, final String prefix) {

		this.id = id;

		this.name = name;

		this.prefix = prefix;

	}

	public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public String getName() {

		return name;

	}

	public void setName(final String name) {

		this.name = name;

	}

	public String getPrefix() {

		return prefix;

	}

	public void setPrefix(final String prefix) {

		this.prefix = prefix;

	}

	public String calculateOpenaireId() {

		if (StringUtils.isNotBlank(id)) {

			final String[] arr = id.split("::");

			if (arr.length == 2) {

				return String.format("%s::%s", arr[0], Hashing.md5(arr[1]));

			}

		}

		return "";

	}

}

package eu.dnetlib.openaire.directindex.objects;

import io.swagger.annotations.ApiModelProperty;

/**

 * Created by michele on 02/12/15.

 */

public class PidEntry {

	private String type;

	private String value;

	public PidEntry() {

	}

	public PidEntry(final String type, final String value) {

		this.type = type;

		this.value = value;

	}

	@ApiModelProperty(required = true, value = "E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types")

	public String getType() {

		return type;

	}

	public void setType(final String type) {

		this.type = type;

	}

	@ApiModelProperty(required = true)

	public String getValue() {

		return value;

	}

	public void setValue(final String value) {

		this.value = value;

	}

}

package eu.dnetlib.openaire.directindex.utils;

import java.io.IOException;

import java.io.StringReader;

import java.io.StringWriter;

import javax.annotation.Resource;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import javax.xml.transform.stream.StreamSource;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.miscutils.functional.xml.ApplyXslt;

import org.apache.commons.io.IOUtils;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.ClassPathResource;

/**

 * Created by michele on 15/12/15.

 */

public class OafToIndexRecordFactory {

	private ClassPathResource layoutToRecord;

	@Resource

	private UniqueServiceLocator serviceLocator;

	public ApplyXslt newTransformer(final String format) throws ISLookUpException, IOException, TransformerException {

		final TransformerFactory factory = TransformerFactory.newInstance();

		final Transformer layoutTransformer = factory.newTransformer(readLayoutToRecord());

		final StreamResult layoutToXsltXslt = new StreamResult(new StringWriter());

		layoutTransformer.setParameter("format", format);

		layoutTransformer.transform(new StreamSource(new StringReader(getLayoutSource(format))), layoutToXsltXslt);

		return new ApplyXslt(layoutToXsltXslt.getWriter().toString());

	}

	private StreamSource readLayoutToRecord() throws IOException {

		return new StreamSource(new StringReader(IOUtils.toString(layoutToRecord.getInputStream())));

	}

	private String getLayoutSource(final String format) throws ISLookUpDocumentNotFoundException, ISLookUpException {

		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(

				"collection('/db/DRIVER/MDFormatDSResources/MDFormatDSResourceType')[.//NAME='" + format + "']//LAYOUT[@name='index']");

	}

	public ClassPathResource getLayoutToRecord() {

		return layoutToRecord;

	}

	@Required

	public void setLayoutToRecord(final ClassPathResource layoutToRecord) {

		this.layoutToRecord = layoutToRecord;

	}

}