D-Net

package eu.dnetlib.msro.openaireplus.api;

import java.util.Map;

import java.util.Map.Entry;

import org.junit.Test;

/**

 * Created by Alessia Bardi on 26/05/2017.

 *

 * @author Alessia Bardi

 */

public class OpenAIRESubmitterUtilsTest {

	private OpenAIRESubmitterUtils utils = new OpenAIRESubmitterUtils();

	final String fullProject = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble";

	final String minimalProject = "info:eu-repo/grantAgreement/EC/FP7/244909///WorkAble";

	final String onlyTitle = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work";

	@Test

	public void testCalculateProjectInfoFull() {

		final Map<String, String> project = utils.calculateProjectInfo(fullProject);

		print(project);

	}

	@Test

	public void testCalculateProjectInfoMinimalAcro() {

		final Map<String, String> project = utils.calculateProjectInfo(minimalProject);

		print(project);

	}

	@Test

	public void testCalculateProjectInfoOnlyTitle() {

		final Map<String, String> project = utils.calculateProjectInfo(onlyTitle);

		print(project);

	}

	private void print(final Map<String, String> map) {

		for (final Entry e : map.entrySet()) {

			System.out.println(e.getKey() + " = " + e.getValue());

		}

	}

}

package eu.dnetlib.msro.openaireplus.api.objects;

import static org.mockito.Matchers.anyString;

import static org.mockito.Mockito.when;

import java.io.InputStreamReader;

import java.io.StringReader;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Properties;

import org.apache.velocity.app.VelocityEngine;

import org.dom4j.Document;

import org.dom4j.io.OutputFormat;

import org.dom4j.io.SAXReader;

import org.dom4j.io.XMLWriter;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.invocation.InvocationOnMock;

import org.mockito.runners.MockitoJUnitRunner;

import org.mockito.stubbing.Answer;

import com.google.gson.Gson;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

/**

 * Created by michele on 14/12/15.

 */

@RunWith(MockitoJUnitRunner.class)

public class ResultEntryTest {

	private VelocityEngine ve;

	@Mock

	private ISLookUpService lookUpService;

	@Before

	public void setUp() throws Exception {

		final Properties props = new Properties();

		props.setProperty("resource.loader", "class");

		props.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader");

		props.setProperty("runtime.log.logsystem.class", "org.apache.velocity.runtime.log.Log4JLogChute");

		ve = new VelocityEngine();

		ve.init(props);

		when(lookUpService.quickSearchProfile(anyString())).thenAnswer(new Answer<Object>() {

			@Override

			public Object answer(final InvocationOnMock invocation) throws Throwable {

				final String query = invocation.getArguments()[0].toString();

				if (query.contains("dnet:result_typologies")) {

					return Arrays.asList("publication @@@ publication", "dataset @@@ dataset");

				} else if (query.contains("dnet:access_modes")) {

					return Arrays.asList("OPEN @@@ Open Access");

				} else if (query.contains("dnet:publication_resource")) {

					return Arrays.asList("0001 @@@ Journal Article");

				} else if (query.contains("dnet:pid_types")) {

					return Arrays.asList("oai @@@ Open Archive Initiative", "doi @@@ Digital object identifier");

				} else if (query.contains("dnet:languages")) {

					return Arrays.asList("ita @@@ Italian");

				} else if (query.contains("ContextDSResourceType")) {

					return Arrays.asList(

							"egi::classification::natsc::math::stats @@@ Statistics and Probability",

							"egi::classification::natsc::math::pure @@@ Pure Mathematics",

							"egi::classification::natsc::math @@@ Mathematics",

							"egi::classification::natsc @@@ Natural Sciences",

							"egi::classification @@@ EGI classification scheme",

							"egi @@@ EGI");

				} else {

					return new ArrayList<String>();

				}

			}

		});

		when(lookUpService.getResourceProfileByQuery(anyString())).thenAnswer(new Answer<Object>() {

			@Override

			public Object answer(final InvocationOnMock invocation) throws Throwable {

				return "REPO NAME @@@ repo________";

			}

		});

	}

	@Test

	public void testAsIndexRecord() throws Exception {

		final ResultEntry pub = new ResultEntry();

		pub.setOriginalId("ORIG_ID_1234");

		pub.setTitle("TEST TITLE <test>");

		pub.getAuthors().add("Michele Artini");

		pub.getAuthors().add("Claudio Atzori");

		pub.getAuthors().add("Alessia Bardi");

		pub.setPublisher("Test publisher");

		pub.setDescription("DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION ");

		pub.setLanguage("ita");

		pub.setLicenseCode("OPEN");

		pub.setResourceType("0001");

		pub.setUrl("http://test.it/a=1&b=2");

		pub.getPids().add(new PidEntry("doi", "10.000/xyz-123"));

		pub.getPids().add(new PidEntry("oai", "oai:1234"));

		pub.setCollectedFromId("test________::zenodo");

		pub.setHostedById("test________::UNKNOWN");

		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus");

		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble");

		pub.getContexts().add("egi::classification::natsc::math::pure");

		pub.getContexts().add("egi::classification::natsc::math::stats");

		final String xml = pub.asOafRecord(ve, lookUpService, "http://oaf/oaf.xsd");

		final Document doc = (new SAXReader()).read(new StringReader(xml));

		final OutputFormat format = OutputFormat.createPrettyPrint();

		final XMLWriter writer = new XMLWriter(System.out, format);

		writer.write(doc);

		/* writer.close(); */

	}

	@Test

	public void testAsIndexRecord_json() throws Exception {

		final ResultEntry pub = (new Gson()).fromJson(new InputStreamReader(getClass().getResourceAsStream("test_record.json")), ResultEntry.class);

		final String xml = pub.asOafRecord(ve, lookUpService, "http://oaf/oaf.xsd");

		final Document doc = (new SAXReader()).read(new StringReader(xml));

		final OutputFormat format = OutputFormat.createPrettyPrint();

		final XMLWriter writer = new XMLWriter(System.out, format);

		writer.write(doc);

		/* writer.close(); */

	}

	@Test

	public void testAsIndexRecord_json_with_greek_chars() throws Exception {

		final ResultEntry pub =

				(new Gson()).fromJson(new InputStreamReader(getClass().getResourceAsStream("test_record_with_greek_chars.json")), ResultEntry.class);

		final String xml = pub.asOafRecord(ve, lookUpService, "http://oaf/oaf.xsd");

		final Document doc = (new SAXReader()).read(new StringReader(xml));

		final OutputFormat format = OutputFormat.createPrettyPrint();

		final XMLWriter writer = new XMLWriter(System.out, format);

		writer.write(doc);

		/* writer.close(); */

	}

}

{

  "originalId": "ORIG_ID_12345",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "licenseCode": "OPEN",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2367",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

#!/bin/bash

curl -H "Content-Type: application/json" --data @test_record.json "http://localhost:8280/app/mvc/api/publications/feedObject"

{

  "originalId": "ORIG_ID_TEST",

  "type":"publication",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "licenseCode": "EMBARGO",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

{

  "originalId": "ORIG_ID_12345",

  "title": "TEST TITLE WITH Greek Characters",

  "authors": [

	"αβγδεζηθικλμ νξοπρσςτυφχψω",

	"ΑΒΓΔΕΖΗΘΙΚΛ ΜΝΞΟΠΡΣΤΥΦΧΨΩ"

  ],

  "publisher": "âââââââ",

  "description": "Αν περιμένατε να βρίσκεται εδώ μια σελίδα και δεν υπάρχει, η σελίδα μπορεί να μην εμφανίζεται λόγω καθυστέρησης στην ανανέωση της βάσης δεδομένων, ή μπορεί να έχει διαγραφεί. (Δείτε την γρήγορη διαγραφή σελίδων για πιθανούς λόγους). Μπορείτε να δοκιμάστε την λειτουργία εκκαθάρισης, και να ελέγξετε το αρχείο διαγραφών.",

  "language": "ell",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123-gr"

	}

  ],

  "licenseCode": "EMBARGO",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2367",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/123456/EU//Test"

  ],

  "contexts": [

	"egi::classification::natsc::math::stats"

  ]

}

package eu.dnetlib.msro.openaireplus.api;

import java.io.IOException;

import java.util.List;

import javax.annotation.Resource;

import com.google.common.collect.Lists;

import com.google.gson.Gson;

import eu.dnetlib.common.rmi.DNetRestDocumentation;

import eu.dnetlib.data.index.CloudIndexClient;

import eu.dnetlib.data.index.CloudIndexClientFactory;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.msro.openaireplus.api.objects.ResultEntry;

import eu.dnetlib.msro.openaireplus.utils.OafToIndexRecordFactory;

import eu.dnetlib.msro.rmi.MSROException;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.lang.exception.ExceptionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.core.io.ClassPathResource;

import org.springframework.http.HttpStatus;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.*;

/**

 * Created by michele on 11/11/15.

 */

@Controller

@DNetRestDocumentation

public class OpenaireResultSubmitter {

	private static final Log log = LogFactory.getLog(OpenaireResultSubmitter.class);

	public class IndexDsInfo {

		private final String indexBaseUrl;

		private final String indexDsId;

		private final String format;

		private final String coll;

		public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) {

			this.indexBaseUrl = indexBaseUrl;

			this.indexDsId = indexDsId;

			this.format = format;

			this.coll = coll;

		}

		public String getIndexBaseUrl() {

			return indexBaseUrl;

		}

		public String getIndexDsId() {

			return indexDsId;

		}

		public String getFormat() {

			return format;

		}

		public String getColl() {

			return coll;

		}

	}

	@Value(value = "oaf.schema.location")

	private String oafSchemaLocation;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Resource

	private OafToIndexRecordFactory oafToIndexRecordFactory;

	@Resource

	private RecentResultsQueue recentResultsQueue;

	@Resource(name = "openaireplusApisVelocityEngine")

	private VelocityEngine velocityEngine;

	@Value(value = "${openaireplus.msro.api.findSolrIndexUrl.xquery}")

	private ClassPathResource findSolrIndexUrl;

	@Value(value = "${openaireplus.msro.api.findIndexDsInfo.xquery}")

	private ClassPathResource findIndexDsInfo;

	@Deprecated

	@RequestMapping(value = {"/api/publications/feedJson", "/api/results/feedJson"}, method = RequestMethod.POST)

	public

	@ResponseBody

	boolean feedObjectJson(@RequestParam(value = "json", required = true) final String json,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws MSROException {

		final ResultEntry pub = new Gson().fromJson(json, ResultEntry.class);

		return feedObject(pub, commit);

	}

	@RequestMapping(value = {"/api/results/feedObject"} , method = RequestMethod.POST)

	public

	@ResponseBody

	String feedResult(@RequestBody final ResultEntry pub,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)

			throws MSROException {

		try {

			if(feedObject(pub, commit)) {

				return ResultEntry.calculateObjId(pub.getOriginalId(), pub.getCollectedFromId(), serviceLocator.getService(ISLookUpService.class));

			}

			else return null;

		} catch (final Throwable e) {

			throw new MSROException("Error adding publication: " + e.getMessage(), e);

		}

	}

	@RequestMapping(value = {"/api/publications/feedObject"} , method = RequestMethod.POST)

	public

	@ResponseBody

	boolean feedObject(@RequestBody final ResultEntry pub,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)

			throws MSROException {

		checkParameters(pub);

		final List<IndexDsInfo> idxList;

		try {

			idxList = calculateCurrentIndexDsInfo();

			if (idxList == null || idxList.isEmpty()) {

				throw new MSROException("Cannot add result: " + pub.getOriginalId() + " : No public Search Service found");

			}

			if (idxList.size() > 1) log.warn("Found more than 1 public search service");

			final String oafRecord = pub.asOafRecord(velocityEngine, serviceLocator.getService(ISLookUpService.class), oafSchemaLocation);

			for (IndexDsInfo idx : idxList) {

				CloudIndexClient idxClient = null;

				try {

					idxClient = CloudIndexClientFactory.newIndexClient(idx.getIndexBaseUrl(), idx.getColl(), false);

					idxClient.feed(oafRecord, idx.getIndexDsId(), oafToIndexRecordFactory.newTransformer(idx.getFormat()), commit);

				} catch (final Throwable e) {

					throw new MSROException("Error adding publication: " + e.getMessage(), e);

				} finally {

					if (idxClient != null) {

						idxClient.close();

					}

				}

			}

			recentResultsQueue.add(oafRecord);

			return true;

		} catch (final Throwable e) {

			throw new MSROException("Error adding publication: " + e.getMessage(), e);

		}

	}

	@RequestMapping(value = "/api/results", method = RequestMethod.DELETE)

	public

	@ResponseBody

	boolean deleteResult(

			@RequestParam(value = "originalId", required = true) final String originalId,

			@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws MSROException {

		return deleteResultPost(originalId, collectedFromId, commit);

	}

	@RequestMapping(value = {"/api/publications/deleteObject", "/api/results/delete"}, method = RequestMethod.POST)

	public

	@ResponseBody

	boolean deleteResultPost(

			@RequestParam(value = "originalId", required = true) final String originalId,

			@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws MSROException {

		final List<IndexDsInfo> idxList;

		try {

			idxList = calculateCurrentIndexDsInfo();

			if (idxList == null || idxList.isEmpty()) {

				throw new MSROException("Cannot delete result: " + originalId + " : No public Search Service found");

			}

			if (idxList.size() > 1) log.warn("Found more than 1 public search service");

			final String objId = ResultEntry.calculateObjId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class));

			for (IndexDsInfo idx : idxList) {

				CloudIndexClient idxClient = null;

				try {

					idxClient = CloudIndexClientFactory.newIndexClient(idx.getIndexBaseUrl(), idx.getColl(), false);

					idxClient.remove(objId, commit);

					log.info("Deleted result with id: " + objId + " from: " + idx.getIndexBaseUrl());

				} catch (final Throwable e) {

					throw new MSROException("Error deleting publication: " + e.getMessage(), e);

				} finally {

					if (idxClient != null) {

						idxClient.close();

					}

				}

			}

			recentResultsQueue.remove(objId);

			return true;

		} catch (IOException | ISLookUpException e) {

			throw new MSROException("Error deleting publication: " + e.getMessage(), e);

		}

	}

	@ExceptionHandler(Exception.class)

	@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)

	public

	@ResponseBody

	ErrorMessage handleException(final Exception e) {

		log.error("Error in direct index API", e);

		return new ErrorMessage(e);

	}

	private void checkParameters(ResultEntry pub) throws MSROException{

		if (StringUtils.isBlank(pub.getOriginalId())) { throw new MSROException("A required field is missing: originalId"); }

		if (StringUtils.isBlank(pub.getTitle())) { throw new MSROException("A required field is missing: title"); }

		if (StringUtils.isBlank(pub.getUrl())) { throw new MSROException("A required field is missing: url"); }

		if (StringUtils.isBlank(pub.getLicenseCode())) { throw new MSROException("A required field is missing: licenseCode"); }

		if (StringUtils.isBlank(pub.getResourceType())) { throw new MSROException("A required field is missing: resourceType"); }

		if (StringUtils.isBlank(pub.getCollectedFromId())) { throw new MSROException("A required field is missing: collectedFromId"); }

		if (StringUtils.isBlank(pub.getType())) { throw new MSROException("A required field is missing: type"); }

	}

	private List<IndexDsInfo> calculateCurrentIndexDsInfo() throws IOException, ISLookUpException {

		List<IndexDsInfo> list = Lists.newArrayList();

		final String queryUrl = IOUtils.toString(findSolrIndexUrl.getInputStream());

		final String queryDs = IOUtils.toString(findIndexDsInfo.getInputStream());

		final ISLookUpService lu = serviceLocator.getService(ISLookUpService.class);

		final String indexBaseUrl = lu.getResourceProfileByQuery(queryUrl);

		final List<String> idxDs = lu.quickSearchProfile(queryDs);

		for (String idx : idxDs) {

			final String[] arr = idx.split("@@@");

			list.add(new IndexDsInfo(indexBaseUrl, arr[0].trim(), arr[1].trim(), arr[2].trim()));

		}

		return list;

	}

	public class ErrorMessage {

		private final String message;

		private final String stacktrace;

		public ErrorMessage(final Exception e) {

			this(e.getMessage(), ExceptionUtils.getStackTrace(e));

		}

		public ErrorMessage(final String message, final String stacktrace) {

			this.message = message;

			this.stacktrace = stacktrace;

		}

		public String getMessage() {

			return this.message;

		}

		public String getStacktrace() {

			return this.stacktrace;

		}

	}

}

package eu.dnetlib.msro.openaireplus.api;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import com.mongodb.BasicDBObject;

import com.mongodb.BasicDBObjectBuilder;

import com.mongodb.DB;

import com.mongodb.DBCollection;

import com.mongodb.DBCursor;

import com.mongodb.DBObject;

import eu.dnetlib.miscutils.datetime.DateUtils;

/**

 * Created by michele on 11/11/15.

 */

public class RecentPublicationsQueue implements Iterable<String> {

	private static final Log log = LogFactory.getLog(RecentPublicationsQueue.class);

	private DB db;

	private String collection;

	public void init() {

		if (!db.collectionExists(collection)) {

			log.info(String.format("creating collection %s", collection));

			db.createCollection(collection, new BasicDBObject());

		}

	}

	@Override

	public Iterator<String> iterator() {

		final DBCursor cursor = db.getCollection(collection).find();

		return new Iterator<String>() {

			@Override

			public boolean hasNext() {

				return cursor.hasNext();

			}

			@Override

			public String next() {

				final DBObject obj = cursor.next();

				return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : "";

			}

			@Override

			public void remove() {

				throw new RuntimeException("NOT IMPLEMENTED");

			}

		};

	}

	synchronized public void add(final String oaf) throws Exception {

		final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']");

		log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection);

		final DBCollection coll = db.getCollection(collection);

		final DBObject obj = BasicDBObjectBuilder.start()

				.append("id", id)

				.append("record", oaf)

				.append("date", DateUtils.now())

				.get();

		coll.update(new BasicDBObject("id", id), obj, true, false);

	}

	public void remove(final List<String> list) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : list) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public void remove(final String... ids) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : ids) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public DB getDb() {

		return db;

	}

	@Required

	public void setDb(final DB db) {

		this.db = db;

	}

	public String getCollection() {

		return collection;

	}

	@Required

	public void setCollection(final String collection) {

		this.collection = collection;

	}

}

package eu.dnetlib.msro.openaireplus.api;

import java.io.StringWriter;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import org.apache.commons.lang.StringUtils;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import eu.dnetlib.miscutils.functional.hash.Hashing;

/**

 * Created by michele on 15/01/16.

 */

public class OpenAIRESubmitterUtils {

	public Map<String, String> calculateProjectInfo(final String link) {

		final Map<String, String> info = new HashMap<String, String>();

		final String[] arr = link.split("/");

		// info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble

		if (arr.length > 4) {

			final String acronym = arr.length > 7 ? arr[7] : "";

			final String title = arr.length > 6 ? StringUtils.isNotBlank(arr[6]) ? arr[6] : acronym : "";

			final String jurisdiction = arr.length > 5 ? arr[5] : "";

			final String funderId = calculateFunderId(arr[2]);

			info.put("id", calculateProjectId(arr[2], arr[3], arr[4]));

			info.put("funderShortName", arr[2]);

			info.put("fundingName", arr[3]);

			info.put("code", arr[4]);

			info.put("jurisdiction", jurisdiction);

			info.put("title", title);

			info.put("acronym", acronym);

			info.put("funderId", funderId);

			info.put("funderName", calculateFunderName(arr[2]));

			info.put("fundingId", funderId + "::" + arr[3]);

		}

		return info;

	}

	private String calculateProjectId(final String funderShortName, final String funding, final String code) {

		final String suffix = Hashing.md5(code);

		if (funderShortName.equalsIgnoreCase("ec")) {

			if (funding.equalsIgnoreCase("fp7")) {

				return "corda_______::" + suffix;

			} else {

				return "corda__h2020::" + suffix;

			}

		} else if (funderShortName.equalsIgnoreCase("arc")) {

			return "arc_________::" + suffix;

		} else if (funderShortName.equalsIgnoreCase("fct")) {

			return "fct_________::" + suffix;

		} else if (funderShortName.equalsIgnoreCase("nhmrc")) {

			return "nhmrc_______::" + suffix;

		} else if (funderShortName.equalsIgnoreCase("wt")) {

			return "wt__________::" + suffix;

		} else {

			return "";

		}

	}

	private String calculateFunderId(final String funderShortName) {

		if (funderShortName.equalsIgnoreCase("ec")) {

			return "ec__________::EC";

		} else if (funderShortName.equalsIgnoreCase("arc")) {

			return "arc_________::ARC";

		} else if (funderShortName.equalsIgnoreCase("fct")) {

			return "fct_________::FCT";

		} else if (funderShortName.equalsIgnoreCase("nhmrc")) {

			return "nhmrc_______::NHMRC";

		} else if (funderShortName.equalsIgnoreCase("wt")) {

			return "wt__________::WT";

		} else {

			return "";

		}

	}

	private String calculateFunderName(final String funderShortName) {

		if (funderShortName.equalsIgnoreCase("ec")) {

			return "European Commission";

		} else if (funderShortName.equalsIgnoreCase("arc")) {

			return "Australian Research Council (ARC)";

		} else if (funderShortName.equalsIgnoreCase("fct")) {

			return "Fundação para a Ciência e a Tecnologia";

		} else if (funderShortName.equalsIgnoreCase("nhmrc")) {

			return "National Health and Medical Research Council (NHMRC)";

		} else if (funderShortName.equalsIgnoreCase("wt")) {

			return "Wellcome Trust";

		} else {

			return "";

		}

	}

	public List<ContextInfo> processContexts(final List<String> list) {

		return Lists.newArrayList(Lists.transform(list, new Function<String, ContextInfo>() {

			@Override

			public ContextInfo apply(final String s) {

				return createContextInfo(s.split("::"), 0);

			}

			private ContextInfo createContextInfo(final String[] arr, final int pos) {

				final StringWriter id = new StringWriter();

				id.write(arr[0]);

				for (int i = 0; i < pos; i++) {

					id.write("::");

					id.write(arr[i + 1]);

				}

				final String elem = (pos == 0) ? "context" : (pos == 1) ? "category" : "concept";

				final ContextInfo info = new ContextInfo(elem, id.toString());

				if ((pos + 1) < arr.length) {

					info.getChildren().add(createContextInfo(arr, pos + 1));

				}

				return info;

			}

		}));

	}

	public class ContextInfo {

		private String elem;

		private String id;

		private List<ContextInfo> children = new ArrayList<ContextInfo>();

		public ContextInfo(final String elem,

				final String id) {

			this.elem = elem;

			this.id = id;

		}

		public String getElem() {

			return elem;

		}

		public void setElem(final String elem) {

			this.elem = elem;

		}

		public String getId() {

			return id;

		}

		public void setId(final String id) {

			this.id = id;

		}

		public List<ContextInfo> getChildren() {

			return children;

		}

		public void setChildren(final List<ContextInfo> children) {

			this.children = children;

		}

		public boolean isRoot() {

			return elem.equals("context");

		}

	}

}

package eu.dnetlib.msro.openaireplus.api;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import com.mongodb.BasicDBObject;

import com.mongodb.BasicDBObjectBuilder;

import com.mongodb.DB;

import com.mongodb.DBCollection;

import com.mongodb.DBCursor;

import com.mongodb.DBObject;

import eu.dnetlib.miscutils.datetime.DateUtils;

/**

 * Created by michele on 11/11/15.

 */

public class RecentResultsQueue implements Iterable<String> {

	private static final Log log = LogFactory.getLog(RecentResultsQueue.class);

	private DB db;

	private String collection;

	public void init() {

		if (!db.collectionExists(collection)) {

			log.info(String.format("creating collection %s", collection));

			db.createCollection(collection, new BasicDBObject());

		}

	}

	@Override

	public Iterator<String> iterator() {

		final DBCursor cursor = db.getCollection(collection).find();

		return new Iterator<String>() {

			@Override

			public boolean hasNext() {

				return cursor.hasNext();

			}

			@Override

			public String next() {

				final DBObject obj = cursor.next();

				return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : "";

			}

			@Override

			public void remove() {

				throw new RuntimeException("NOT IMPLEMENTED");

			}

		};

	}

	synchronized public void add(final String oaf) throws Exception {

		final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']");

		log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection);

		final DBCollection coll = db.getCollection(collection);

		final DBObject obj = BasicDBObjectBuilder.start()

				.append("id", id)

				.append("record", oaf)

				.append("date", DateUtils.now())

				.get();

		coll.update(new BasicDBObject("id", id), obj, true, false);

	}

	public void remove(final List<String> list) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : list) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public void remove(final String... ids) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : ids) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public DB getDb() {

		return db;

	}

	@Required

	public void setDb(final DB db) {

		this.db = db;

	}

	public String getCollection() {

		return collection;

	}

	@Required

	public void setCollection(final String collection) {

		this.collection = collection;

	}

}

package eu.dnetlib.msro.openaireplus.api.objects;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Date;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.concurrent.TimeUnit;

//import io.swagger.annotations.ApiModelProperty;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.velocity.app.VelocityEngine;

import org.apache.velocity.tools.generic.EscapeTool;

import org.springframework.ui.velocity.VelocityEngineUtils;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.msro.openaireplus.api.OpenAIRESubmitterUtils;

/**

 * Created by michele on 02/12/15.

 */

public class ResultEntry {

	private String originalId;

	private String title;

	private List<String> authors = new ArrayList<String>();

	private String publisher;

	private String description;

	private String language;

	private List<PidEntry> pids = new ArrayList<PidEntry>();

	private String licenseCode;

	private String embargoEndDate;

	private String type = "publication";

	private String resourceType;

	private String url;

	private String collectedFromId;

	private String hostedById;

	// String according to openaire guidelines:

	// info:eu-repo/grantAgreement/Funder/FundingProgram/ProjectID/[Jurisdiction]/[ProjectName]/[ProjectAcronym]

	private List<String> contexts = new ArrayList<String>();

	// String according to the EGI context profile, example: egi::classification::natsc::math

	private List<String> linksToProjects = new ArrayList<String>();

	private static long last_cache_update = 0;

	private static final Map<String, Map<String, String>> cached_vocabularies = new HashMap<String, Map<String, String>>();

	private static final Map<String, DatasourceEntry> cached_datasources = new HashMap<String, DatasourceEntry>();

	private static final Map<String, String> cached_contexts = new HashMap<String, String>();

	private static final Log log = LogFactory.getLog(ResultEntry.class);

	public ResultEntry() {}

//	@ApiModelProperty(required = true)

	public String getOriginalId() {

		return originalId;

	}

	public void setOriginalId(final String originalId) {

		this.originalId = originalId;

	}

//	@ApiModelProperty(required = true)

	public String getTitle() {

		return title;

	}

	public void setTitle(final String title) {

		this.title = title;

	}

	public List<String> getAuthors() {

		return authors;

	}

	public void setAuthors(final List<String> authors) {

		this.authors = authors;

	}

	public String getPublisher() {

		return publisher;

	}

	public void setPublisher(final String publisher) {

		this.publisher = publisher;

	}

	public String getDescription() {

		return description;

	}

	public void setDescription(final String description) {

		this.description = description;

	}

//	@ApiModelProperty(value="ISO Alpha-3 code. E.g. 'eng', 'ita'")

	public String getLanguage() {

		return language;

	}

	public void setLanguage(final String language) {

		this.language = language;

	}

	public List<PidEntry> getPids() {

		return pids;

	}

	public void setPids(final List<PidEntry> pids) {

		this.pids = pids;

	}

//	@ApiModelProperty(required = true, allowableValues = "OPEN, CLOSED, RESTRICTED, EMBARGO, UNKNOWN, OTHER")

	public String getLicenseCode() {

		return licenseCode;

	}

	public void setLicenseCode(final String licenseCode) {

		this.licenseCode = licenseCode;

	}

//	@ApiModelProperty(required = true, value="Use 001 for articles, 021 for datasets. See: http://api.openaire.eu/vocabularies/dnet:publication_resource.")

	public String getResourceType() {

		return resourceType;

	}

	public void setResourceType(final String resourceType) {

		this.resourceType = resourceType;

	}

//	@ApiModelProperty(required = true)

	public String getUrl() {

		return url;

	}

	public void setUrl(final String url) {

		this.url = url;

	}

//	@ApiModelProperty(required = true, value="Use opendoar___::2659 for Zenodo Publications; re3data_____::r3d100010468 for Zenodo datasets; infrastruct::openaire for OpenAIRE portal.")

	public String getCollectedFromId() {

		return collectedFromId;

	}

	public void setCollectedFromId(final String collectedFromId) {

		this.collectedFromId = collectedFromId;

	}

	public String getHostedById() {

		return hostedById;

	}

	public void setHostedById(final String hostedById) {

		this.hostedById = hostedById;

	}

//	@ApiModelProperty(value="E.g. fet, egi::classification::natsc::math::pure, egi::projects::EMI")

	public List<String> getContexts() {

		return contexts;

	}

	public void setContexts(final List<String> contexts) {

		this.contexts = contexts;

	}

//	@ApiModelProperty(value="E.g. info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus")

	public List<String> getLinksToProjects() {

		return linksToProjects;

	}

	public void setLinksToProjects(final List<String> linksToProjects) {

		this.linksToProjects = linksToProjects;

	}

//	@ApiModelProperty(allowableValues = "publication, dataset")

	public String getType() {

		return type;

	}

	public void setType(final String type) {

		this.type = type;

	}

	public String getEmbargoEndDate() {

		return embargoEndDate;

	}

	public void setEmbargoEndDate(final String embargoEndDate) {

		this.embargoEndDate = embargoEndDate;

	}

	public String asOafRecord(final VelocityEngine ve,

			final ISLookUpService lookupService,

			final String oafSchemaLocation) throws Exception {

		final DatasourceEntry collectedFromEntry = getDatasourceInfo(collectedFromId, lookupService);

		final DatasourceEntry hostedByEntry = getDatasourceInfo(hostedById, lookupService);

		final String objId = calculateObjId(originalId, collectedFromEntry);

		final Map<String, Object> model = new HashMap<String, Object>();

		model.put("esc", new EscapeTool());

		model.put("util", new OpenAIRESubmitterUtils());

		model.put("pub", this);

		model.put("objIdentifier", objId);

		model.put("oafSchemaLocation", oafSchemaLocation);

		model.put("resultTypes", getVocabulary("dnet:result_typologies", lookupService));

		model.put("licenses", getVocabulary("dnet:access_modes", lookupService));

		model.put("resourceTypes", getVocabulary("dnet:publication_resource", lookupService));

		model.put("pidTypes", getVocabulary("dnet:pid_types", lookupService));

		model.put("languages", getVocabulary("dnet:languages", lookupService));

		model.put("contexts", getContexts(lookupService));

		model.put("dateOfCollection", (new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'")).format(new Date()));

		model.put("collectedFrom", collectedFromEntry);

		model.put("hostedBy", hostedByEntry);

		return VelocityEngineUtils.mergeTemplateIntoString(ve, "/eu/dnetlib/msro/openaireplus/api/indexRecord.xml.vm", "UTF-8", model);

	}

	public static String calculateObjId(final String originalId, final DatasourceEntry collectedFromEntry) {

		return collectedFromEntry.getPrefix() + "::" + Hashing.md5(originalId);

	}

	public static String calculateObjId(final String originalId, final String collectedFromId, final ISLookUpService lookupService) throws ISLookUpException {

		return calculateObjId(originalId, getDatasourceInfo(collectedFromId, lookupService));

	}

	private synchronized static DatasourceEntry getDatasourceInfo(final String dsId, final ISLookUpService lookupService) throws ISLookUpException {

		if (StringUtils

				.isBlank(dsId)) { return new DatasourceEntry("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "unknown_____"); }

		if (!cached_datasources.containsKey(dsId)) {

			final String query =

					"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//CONFIGURATION[./DATASOURCE_ORIGINAL_ID='" + dsId

							+ "']/concat(./OFFICIAL_NAME, ' @@@ ', .//FIELD/value[../key='NamespacePrefix'])";

			final String s = lookupService.getResourceProfileByQuery(query);

			final String[] arr = s.split("@@@");

			final DatasourceEntry ds = new DatasourceEntry(dsId, arr[0].trim(), arr[1].trim());

			if (StringUtils.isBlank(ds.getName()) || StringUtils.isBlank(ds.getPrefix())) {

				log.error("Invalid datasource id: " + dsId);

				throw new ISLookUpException("Invalid datasource id: " + dsId);

			} else {

				cached_datasources.put(dsId, ds);

			}

		}

		return cached_datasources.get(dsId);

	}

	private synchronized static Map<String, String> getVocabulary(final String voc, final ISLookUpService lookupService) throws ISLookUpException {

		if (((DateUtils.now() - last_cache_update) < TimeUnit.MINUTES.toMillis(15)) && cached_vocabularies.containsKey(voc)) {

			return cached_vocabularies.get(voc);

		} else {

			final String query = "collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code='" + voc

					+ "']//TERM/concat(@code, ' @@@ ', @english_name)";

			final Map<String, String> map = new HashMap<String, String>();

			for (final String s : lookupService.quickSearchProfile(query)) {

				final String[] arr = s.split("@@@");

				map.put(arr[0].trim(), arr[1].trim());

			}

			cached_vocabularies.put(voc, map);

			last_cache_update = DateUtils.now();

			return map;

		}

	}

	private synchronized static Map<String, String> getContexts(final ISLookUpService lookupService) throws ISLookUpException {

		if (((DateUtils.now() - last_cache_update) > TimeUnit.MINUTES.toMillis(15)) || cached_contexts.isEmpty()) {

			final String query =

					"collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)";

			cached_contexts.clear();

			for (final String s : lookupService.quickSearchProfile(query)) {

				final String[] arr = s.split("@@@");

				cached_contexts.put(arr[0].trim(), arr[1].trim());

			}

			last_cache_update = DateUtils.now();

		}

		return cached_contexts;

	}

}

package eu.dnetlib.msro.openaireplus.api.objects;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import org.apache.commons.lang.StringUtils;

/**

 * Created by michele on 02/12/15.

 */

public class DatasourceEntry {

	private String id;

	private String name;

	private String prefix;

	public DatasourceEntry() {

	}

	public DatasourceEntry(final String id, final String name, final String prefix) {

		this.id = id;

		this.name = name;

		this.prefix = prefix;

	}

	public String getId() {

		return id;

	}

	public void setId(final String id) {

		this.id = id;

	}

	public String getName() {

		return name;

	}

	public void setName(final String name) {

		this.name = name;

	}

	public String getPrefix() {

		return prefix;

	}

	public void setPrefix(final String prefix) {

		this.prefix = prefix;

	}

	public String calculateOpenaireId() {

		if (StringUtils.isNotBlank(id)) {

			final String[] arr = id.split("::");

			if (arr.length == 2) {

				return String.format("%s::%s", arr[0], Hashing.md5(arr[1]));

			}

		}

		return "";

	}

}

package eu.dnetlib.msro.openaireplus.api.objects;

//import io.swagger.annotations.ApiModelProperty;

/**

 * Created by michele on 02/12/15.

 */

public class PidEntry {

	private String type;

	private String value;

	public PidEntry() {

	}

	public PidEntry(final String type, final String value) {

		this.type = type;

		this.value = value;

	}

	//@ApiModelProperty(required = true, value="E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types")

	public String getType() {

		return type;

	}

	public void setType(final String type) {

		this.type = type;

	}

	//@ApiModelProperty(required = true)

	public String getValue() {

		return value;

	}

	public void setValue(final String value) {

		this.value = value;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<!--

#macro(context $ctx)

        <$!ctx.elem id="$!esc.xml($!ctx.id)" label="$!esc.xml($contexts.get($!ctx.id))" #if($!ctx.isRoot()) type="community"#end>

	#foreach($child in $!ctx.children)#context($child)#end

        </$!ctx.elem>

#end

-->

<record xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oaf="http://namespace.openaire.eu/oaf">

    <result>

        <header>

            <dri:objIdentifier>$!esc.xml($!objIdentifier)</dri:objIdentifier>

            <dri:dateOfCollection>$!esc.xml($!dateOfCollection)</dri:dateOfCollection>

            <dri:status>under curation</dri:status>

            <counters/>

        </header>

        <metadata>

            <oaf:entity schemaLocation="http://namespace.openaire.eu/oaf $!oafSchemaLocation">

                <oaf:result>

                    <title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">

						$!esc.xml($!pub.title)

                    </title>

                    <dateofacceptance/>

                    <resulttype classid="$!esc.xml($!pub.type)" classname="$!esc.xml($!resultTypes.get($!pub.type))" schemeid="dnet:result_typologies" schemename="dnet:result_typologies"/>

                    <language classid="$!esc.xml($!pub.language)" classname="$!esc.xml($!languages.get($!pub.language))" schemeid="dnet:languages"

                              schemename="dnet:languages"/>

                    <description>

						$!esc.xml($!pub.description)

                    </description>

                    <country classid="" classname="" schemeid="" schemename=""/>

                    <subject classid="" classname="" schemeid="" schemename=""/>

                    <relevantdate classid="" classname="" schemeid="" schemename=""/>

                    <publisher>$!esc.xml($!pub.publisher)</publisher>

                    <embargoenddate>$!esc.xml($!pub.embargoEndDate)</embargoenddate>

                    <source/>

                    <fulltext/>

                    <format/>

                    <storagedate/>

                    <resourcetype classid="" classname="" schemeid="" schemename=""/>

                    <device/>

                    <size/>

                    <version/>

                    <lastmetadataupdate/>

                    <metadataversionnumber/>

                    <originalId>$!esc.xml($!pub.originalId)</originalId>

                    <collectedfrom name="$!esc.xml($!collectedFrom.name)" id="$!esc.xml($!collectedFrom.calculateOpenaireId())"/>

					#foreach($pid in $!pub.pids)

                        <pid classid="$!esc.xml($!pid.type)" classname="$!esc.xml($!pidTypes.get($!pid.type))" schemeid="dnet:pid_types"

                             schemename="dnet:pid_types">$!esc.xml($pid.value)</pid>

					#end

                    <bestlicense classid="$!esc.xml($!pub.licenseCode)" classname="$!esc.xml($!licenses.get($!pub.licenseCode))"

                                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>

					#foreach($ctx in $util.processContexts($!pub.contexts))

						#context($ctx)

					#end

                    <datainfo>

                        <inferred>false</inferred>

                        <deletedbyinference>false</deletedbyinference>

                        <trust>0.9</trust>

                        <inferenceprovenance/>

                        <provenanceaction classid="user:insert" classname="user:insert" schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>

                    </datainfo>

                    <rels>

						#foreach($author in $!pub.authors)

                            <rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="sysimport:crosswalk:repository">

                                <to class="hasAuthor" scheme="dnet:personroles" type="person"/>

                                <ranking>$velocityCount</ranking>

                                <fullname>$!author</fullname>

                            </rel>

						#end