D-Net

<?xml version="1.0" ?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-directindex-api</artifactId>

	<packaging>jar</packaging>

	<version>2.3.5</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-directindex-api/tags/dnet-directindex-api-2.3.5</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-openaireplus-mapping-utils</artifactId>

			<version>[6.3.24,7.0.0)</version>

			<exclusions>

				<exclusion>

					<groupId>eu.dnetlib</groupId>

					<artifactId>dnet-hadoop-commons</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-index-client</artifactId>

			<version>[2.3.4,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>io.springfox</groupId>

			<artifactId>springfox-swagger2</artifactId>

			<version>${springfox-version}</version>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>1.9.5</version>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity</artifactId>

			<version>1.7</version>

			<exclusions>

				<exclusion>

					<artifactId>antlr</artifactId>

					<groupId>antlr</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.apache.velocity</groupId>

			<artifactId>velocity-tools</artifactId>

			<version>2.0</version>

			<exclusions>

				<exclusion>

					<artifactId>antlr</artifactId>

					<groupId>antlr</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.springframework</groupId>

			<artifactId>spring-context-support</artifactId>

			<version>${spring.version}</version>

		</dependency>

		<dependency>

			<groupId>com.fasterxml.jackson.core</groupId>

			<artifactId>jackson-annotations</artifactId>

			<version>2.9.4</version>

		</dependency>

    </dependencies>

	<properties>

		<springfox-version>2.5.0</springfox-version>

	</properties>

</project>

package eu.dnetlib.openaire.directindex.api;

import java.io.StringWriter;

import java.util.*;

import java.util.stream.Collectors;

import java.util.stream.Collectors;

import java.util.stream.Stream;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.openaire.directindex.objects.DatasourceEntry;

import eu.dnetlib.openaire.directindex.objects.ZenodoContextList;

import org.apache.commons.lang.StringUtils;

import org.springframework.web.client.RestClientException;

import org.springframework.web.client.RestTemplate;

/**

 * Created by michele on 15/01/16.

 */

public class OpenAIRESubmitterUtils {

    private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(OpenAIRESubmitterUtils.class);

    private static final String ZENODO_COMMUNITY = "zenodo.org/communities/";

    private String community_api;

    public OpenAIRESubmitterUtils(String community_api) {

        this.community_api = community_api;

    }

    public Map<String, String> calculateProjectInfo(final String link) {

        final Map<String, String> info = new HashMap<String, String>();

        final String[] arr = link.split("/");

        // info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble

        if (arr.length > 4) {

            final String acronym = arr.length > 7 ? arr[7] : "";

            final String title = arr.length > 6 ? StringUtils.isNotBlank(arr[6]) ? arr[6] : acronym : "";

            final String jurisdiction = arr.length > 5 ? arr[5] : "";

            final String funderId = calculateFunderId(arr[2], arr[3]);

            final String funderShortName = fixFunderShortName(arr[2]);

            final String fundingName = fixFundingName(funderShortName, arr[3]);

            info.put("id", calculateProjectId(arr[2], arr[3], arr[4]));

            info.put("funderShortName", fixFunderShortName(arr[2]));

            info.put("fundingName", fundingName);

            info.put("code", unescape(arr[4]));

            info.put("jurisdiction", jurisdiction);

            info.put("title", title);

            info.put("acronym", acronym);

            info.put("funderId", funderId);

            info.put("funderName", calculateFunderName(arr[2]));

            if (StringUtils.isNotBlank(arr[3])) info.put("fundingId", funderId + "::" + fundingName);

        }

        return info;

    }

    //TODO: remove me when Zenodo ingests the good UKRI projects

    protected String fixFunderShortName(final String funderShortName) {

        switch (funderShortName) {

            case "RCUK":

                return "UKRI";

            default:

                return funderShortName;

        }

    }

    protected String calculateFunderPrefix(final String funderShortName, final String funding) {

        switch (funderShortName.toLowerCase()) {

            case "conicyt":

                return "conicytf____::";

            case "dfg":

                return "dfgf________::";

            case "ec":

                if (funding.equalsIgnoreCase("fp7")) {

                    return "corda_______::";

                } else {

                    return "corda__h2020::";

                }

            case "eea":

                return "euenvagency_::";

            case "hrzz":

            case "mzos":

                return "irb_hr______::";

            case "tara":

                return "taraexp_____::";

            case "tubitak":

                return "tubitakf____::";

            case "rcuk":

                return "ukri________::";

            default:

                String prefix = funderShortName.toLowerCase();

                //ensure we have 12 chars

                while (prefix.length() < 12) prefix += "_";

                return prefix + "::";

        }

    }

    protected String calculateProjectId(final String funderShortName, final String funding, final String code) {

       final String suffix = Hashing.md5(unescape(code));

        final String funderPrefix = calculateFunderPrefix(funderShortName, funding);

        return funderPrefix + suffix;

    }

    private String unescape(String code) {

        return code.replace( "%2F", "/");

    }

    protected String calculateFunderId(final String funderShortName, final String funding) {

        switch (funderShortName.toLowerCase()) {

            case "ec":

                return "ec__________::EC";

            default:

                String fixedFunderShortName = fixFunderShortName(funderShortName);

                String prefix = calculateFunderPrefix(fixedFunderShortName, funding);

                return prefix + fixedFunderShortName.toUpperCase();

        }

    }

    protected String calculateFunderName(final String funderShortName) {

        switch (funderShortName.toLowerCase()) {

            case "aff":

            case "aka":

                return "Academy of Finland";

            case "anr":

                return "French National Research Agency (ANR)";

            case "arc":

                return "Australian Research Council (ARC)";

            case "cihr":

                return "Canadian Institutes of Health Research";

            case "conicyt":

                return "Comisión Nacional de Investigación Científica y Tecnológica";

            case "dfg":

                return "Deutsche Forschungsgemeinschaft";

            case "ec":

                return "European Commission";

            case "eea":

                return "European Environment Agency";

            case "fct":

                return "Fundação para a Ciência e a Tecnologia, I.P.";

            case "fwf":

                return "Austrian Science Fund (FWF)";

            case "gsrt":

                return "General Secretariat of Research and Technology (GSRT)";

            case "hrzz":

                return "Croatian Science Foundation (CSF)";

            case "innoviris":

                return "INNOVIRIS";

            case "mestd":

                return "Ministry of Education, Science and Technological Development of Republic of Serbia";

            case "miur":

                return "Ministero dell'Istruzione dell'Università e della Ricerca";

            case "mzos":

                return "Ministry of Science, Education and Sports of the Republic of Croatia (MSES)";

            case "nhmrc":

                return "National Health and Medical Research Council (NHMRC)";

            case "nih":

                return "National Institutes of Health";

            case "nsf":

                return "National Science Foundation";

            case "nserc":

                return "Natural Sciences and Engineering Research Council of Canada";

            case "nwo":

                return "Netherlands Organisation for Scientific Research (NWO)";

            case "rcuk":

            case "ukri":

                return "UK Research and Innovation";

            case "rif":

            case "rpf":

                return "Research and Innovation Foundation";

            case "rsf":

                return "Russian Science Foundation";

            case "sfi":

                return "Science Foundation Ireland";

            case "sgov":

                return "Gobierno de España";

            case "snsf":

                return "Swiss National Science Foundation";

            case "sshrc":

                return "Social Sciences and Humanities Research Council";

            case "tara":

                return "Tara Expeditions Foundation";

            case "tubitak":

                return "Türkiye Bilimsel ve Teknolojik Araştırma Kurumu";

            case "wt":

                return "Wellcome Trust";

            default:

                log.error("Funder short name '" + funderShortName + "' not managed");

                return "";

        }

    }

    protected String fixFundingName(final String funderShortName, final String fundingName){

        switch(funderShortName){

            case "EC":

                if(fundingName.toLowerCase().startsWith("horizon 2020")) return "H2020";

            default: return fundingName;

        }

    }

    public Collection<String> translateZenodoCommunity(final String community) {

        if (community.contains(ZENODO_COMMUNITY)) {

            String context = community.substring(community.lastIndexOf("/") + 1);

            RestTemplate rt = new RestTemplate();

            try {

                return new HashSet<>(rt.getForObject(community_api + context + "/openairecommunities", ZenodoContextList.class)

                        .getOpenAirecommunitylist());

            } catch (RestClientException rce) {

                log.error("Unable to get object for " + community_api + context + "/openairecommunities");

                log.error(rce.getMessage());

                return new HashSet<>();

            }

        } else return Sets.newHashSet(community);

    }

    private ContextInfo createContextInfo(final String[] arr, final int pos, Map<String, String> labelMap) {

        final StringWriter id = new StringWriter();

        id.write(arr[0]);

        for (int i = 0; i < pos; i++) {

            id.write("::");

            id.write(arr[i + 1]);

        }

        final String label = labelMap.get(id.toString());

        final String elem = (pos == 0) ? "context" : (pos == 1) ? "category" : "concept";

        final ContextInfo info = new ContextInfo(elem, id.toString(), label);

        if ((pos + 1) < arr.length) {

            info.getChildren().add(createContextInfo(arr, pos + 1, labelMap));

        }

        return info;

    }

    public List<ContextInfo> processContexts(final List<String> zenodoCommunities, final Map<String, String> labelMap) {

        return zenodoCommunities.stream().map(c -> translateZenodoCommunity(c))

                .flatMap(coll -> coll.stream())

                .map(ctx -> createContextInfo(ctx.split("::"), 0,  labelMap))

                .filter(info -> StringUtils.isNotBlank(info.getLabel()))

                .collect(Collectors.toList());

    }

    public class ContextInfo {

        private String elem;

        private String id;

        private String label;

        private List<ContextInfo> children = new ArrayList<ContextInfo>();

        public ContextInfo(final String elem,

                           final String id, final String label) {

            this.elem = elem;

            this.id = id;

            this.label = label;

        }

        public String getElem() {

            return elem;

        }

        public void setElem(final String elem) {

            this.elem = elem;

        }

        public String getId() {

            return id;

        }

        public void setId(final String id) {

            this.id = id;

        }

        public List<ContextInfo> getChildren() {

            return children;

        }

        public void setChildren(final List<ContextInfo> children) {

            this.children = children;

        }

        public boolean isRoot() {

            return elem.equals("context");

        }

        public String getLabel() {

            return label;

        }

        public void setLabel(String label) {

            this.label = label;

        }

    }

}

{

  "openaireId":"dedup_wf_001::ab42e8116f4ae3c4021fb7c643a8167a",

  "originalId": "ORIG_ID_TEST",

  "type": "publication",

  "title": "TEST TITLE",

  "authors": [

	"Michele Artini",

	"Claudio Atzori",

	"Alessia Bardi"

  ],

  "publisher": "Test publisher",

  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",

  "language": "ita",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123"

	},

	{

	  "type": "oai",

	  "value": "oai:1234"

	}

  ],

  "accessRightCode": "EMBARGO",

  "licenseCode": "OPEN",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2659",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",

	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"

  ],

  "contexts": [

	"egi::classification::natsc::math::pure",

	"egi::classification::natsc::math::stats"

  ]

}

package eu.dnetlib.openaire.directindex.api;

import java.text.SimpleDateFormat;

import java.util.Date;

import javax.annotation.Resource;

import com.google.gson.Gson;

import eu.dnetlib.common.rmi.DNetRestDocumentation;

import eu.dnetlib.data.index.CloudIndexClientException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.functionality.index.solr.feed.StreamingInputDocumentFactory;

import eu.dnetlib.miscutils.functional.UnaryFunction;

import eu.dnetlib.openaire.directindex.objects.ResultEntry;

import eu.dnetlib.openaire.directindex.objects.ResultEntryToOaf;

import eu.dnetlib.openaire.directindex.utils.OafToIndexRecordFactory;

import org.apache.commons.lang.exception.ExceptionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.common.SolrInputDocument;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.http.HttpStatus;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.*;

/**

 * Created by michele on 11/11/15.

 */

@Controller

@DNetRestDocumentation

public class OpenaireResultSubmitter {

	private static final Log log = LogFactory.getLog(OpenaireResultSubmitter.class);

	@Value(value = "${openaire.api.community}")

	private String community_api;

	@Value(value = "oaf.schema.location")

	private String oafSchemaLocation;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Resource

	private OafToIndexRecordFactory oafToIndexRecordFactory;

	@Resource

	private RecentResultsQueue recentResultsQueue;

	@Resource(name = "openaireplusApisVelocityEngine")

	private VelocityEngine velocityEngine;

	@Resource(name = "indexClientManager")

	private IndexClientManager clientManager;

	@Resource(name = "resultSubmitterService")

	private ResultSubmitterService submitterService;

	@Autowired

	private IndexDSRetriever indexDSRetriever;

	@RequestMapping(value = { "/api/admin/autocommit/active" }, method = RequestMethod.GET)

	public @ResponseBody Boolean getAutocommit() throws DirecIndexApiException {

		return submitterService.isAutocommitactive();

	}

	@RequestMapping(value = { "/api/admin/autocommit/active" }, method = RequestMethod.POST)

	public @ResponseBody Boolean setAutocommit(@RequestParam(value = "active", required = true) final Boolean active) throws DirecIndexApiException {

		submitterService.setAutocommitactive(active);

		log.info(String.format("automatic commit, active '%s', frequency '%s'", submitterService.isAutocommitactive(), submitterService.getCommitfrquency()));

		return submitterService.isAutocommitactive();

	}

	@RequestMapping(value="/api/admin/evictCache", method=RequestMethod.GET)

	@ResponseStatus(value = HttpStatus.OK)

	public void evictCache(){

		indexDSRetriever.evictCache();

	}

	@Deprecated

	@RequestMapping(value = { "/api/publications/feedJson", "/api/results/feedJson" }, method = RequestMethod.POST)

	public @ResponseBody String feedObjectJson(@RequestParam(value = "json", required = true) final String json,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirecIndexApiException {

		log.debug(json);

		final ResultEntry pub = new Gson().fromJson(json, ResultEntry.class);

		return feedObject(pub, commit);

	}

	@RequestMapping(value = { "/api/results/feedObject" }, method = RequestMethod.POST)

	public @ResponseBody String feedResult(@RequestBody final ResultEntry pub,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)

			throws DirecIndexApiException {

		return feed(pub,commit);

	}

	@Deprecated

	@RequestMapping(value = { "/api/publications/feedObject" }, method = RequestMethod.POST)

	public @ResponseBody String feedObject(@RequestBody final ResultEntry pub,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)

			throws DirecIndexApiException {

		return feed(pub, commit);

	}

	@RequestMapping(value = "/api/result/{openaireId}", method = RequestMethod.DELETE)

	public @ResponseBody boolean deleteResultWithOpenaireId(

			@PathVariable(value = "openaireId") final String openaireId,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirecIndexApiException {

		return deleteResult(openaireId);

	}

	@RequestMapping(value = "/api/results", method = RequestMethod.DELETE)

	public @ResponseBody boolean deleteResultWithOriginalId(

			@RequestParam(value = "originalId", required = true) final String originalId,

			@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws Exception {

		final String openaireId = ResultEntryToOaf.calculateOpenaireId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class));

		return deleteResult(openaireId);

	}

	@Deprecated

	@RequestMapping(value = { "/api/publications/deleteObject", "/api/results/delete" }, method = RequestMethod.POST)

	public @ResponseBody boolean deleteResultPost(

			@RequestParam(value = "originalId", required = true) final String originalId,

			@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,

			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws Exception {

		final String openaireId = ResultEntryToOaf.calculateOpenaireId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class));

		return deleteResult(openaireId);

	}

	@Deprecated

	private String feed(final ResultEntry pub, final boolean commit) throws DirecIndexApiException {

		return feed(pub);

	}

	private String feed(final ResultEntry pub) throws DirecIndexApiException {

		log.debug(pub);

		try {

			ResultEntryToOaf toOaf = new ResultEntryToOaf(community_api);

			final IndexDsInfo info = indexDSRetriever.calculateCurrentIndexDsInfo();

			final String oafRecord = toOaf.asOafRecord(pub, velocityEngine, serviceLocator.getService(ISLookUpService.class), oafSchemaLocation);

			final SolrInputDocument solrDocument = prepareSolrDocument(oafRecord, info.getIndexDsId(), oafToIndexRecordFactory.newTransformer(info.getFormat()));

			clientManager.getClient(info).add(solrDocument);

			recentResultsQueue.add(oafRecord);

			return pub.getOpenaireId();

		} catch (final Throwable e) {

			log.error("Error saving record", e);

			log.debug(pub.toString());

			throw new DirecIndexApiException("Error adding publication: " + e.getMessage(), e);

		}

	}

	private SolrInputDocument prepareSolrDocument(String record, String indexDsId, UnaryFunction<String, String> toIndexRecord) throws CloudIndexClientException {

		try {

			StreamingInputDocumentFactory documentFactory = new StreamingInputDocumentFactory();

			String version = (new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'")).format(new Date());

			String indexRecord = toIndexRecord.evaluate(record);

			if (log.isDebugEnabled()) {

				log.debug("***************************************\nSubmitting index record:\n" + indexRecord + "\n***************************************\n");

			}

			return documentFactory.parseDocument(version, indexRecord, indexDsId, "dnetResult");

		} catch (Throwable e) {

			throw new CloudIndexClientException("Error creating solr document", e);

		}

	}

	private boolean deleteResult(final String openaireId) throws DirecIndexApiException {

		try {

			IndexDsInfo info = indexDSRetriever.calculateCurrentIndexDsInfo();

			String query = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", openaireId, openaireId);

			final CloudSolrClient client = clientManager.getClient(info);

			client.deleteByQuery(info.getColl(), query);

			log.info("Deleted result with id: " + openaireId + " from: " + info.getIndexBaseUrl());

			recentResultsQueue.remove(openaireId);

			return true;

		} catch (Throwable e) {

			throw new DirecIndexApiException("Error deleting publication: " + e.getMessage(), e);

		}

	}

	@ExceptionHandler(Exception.class)

	@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)

	public @ResponseBody ErrorMessage handleException(final Exception e) {

		log.error("Error in direct index API", e);

		return new ErrorMessage(e);

	}

	public class ErrorMessage {

		private final String message;

		private final String stacktrace;

		public ErrorMessage(final Exception e) {

			this(e.getMessage(), ExceptionUtils.getStackTrace(e));

		}

		public ErrorMessage(final String message, final String stacktrace) {

			this.message = message;

			this.stacktrace = stacktrace;

		}

		public String getMessage() {

			return this.message;

		}

		public String getStacktrace() {

			return this.stacktrace;

		}

	}

}

package eu.dnetlib.openaire.directindex.api;

import java.io.StringReader;

import java.util.Iterator;

import java.util.List;

import com.mongodb.*;

import com.mongodb.client.model.IndexOptions;

import eu.dnetlib.miscutils.datetime.DateUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

/**

 * Created by michele on 11/11/15.

 */

public class RecentResultsQueue implements Iterable<String> {

	private static final Log log = LogFactory.getLog(RecentResultsQueue.class);

	private DB db;

	private String collection;

	public void init() {

		if (!db.collectionExists(collection)) {

			log.info(String.format("creating collection %s", collection));

			db.createCollection(collection, new BasicDBObject());

		}

		db.getCollection(collection).createIndex(new BasicDBObject("id", 1), new BasicDBObject("background", true));

	}

	@Override

	public Iterator<String> iterator() {

		final DBCursor cursor = db.getCollection(collection).find();

		return new Iterator<String>() {

			@Override

			public boolean hasNext() {

				return cursor.hasNext();

			}

			@Override

			public String next() {

				final DBObject obj = cursor.next();

				return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : "";

			}

			@Override

			public void remove() {

				throw new RuntimeException("NOT IMPLEMENTED");

			}

		};

	}

	synchronized public void add(final String oaf) throws Exception {

		final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']");

		log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection);

		final DBCollection coll = db.getCollection(collection);

		final DBObject obj = BasicDBObjectBuilder.start()

				.append("id", id)

				.append("record", oaf)

				.append("date", DateUtils.now())

				.get();

		coll.update(new BasicDBObject("id", id), obj, true, false);

	}

	public void remove(final List<String> list) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : list) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public void remove(final String... ids) {

		final DBCollection coll = db.getCollection(collection);

		for (final String id : ids) {

			coll.remove(new BasicDBObject("id", id));

		}

	}

	public DB getDb() {

		return db;

	}

	@Required

	public void setDb(final DB db) {

		this.db = db;

	}

	public String getCollection() {

		return collection;

	}

	@Required

	public void setCollection(final String collection) {

		this.collection = collection;

	}

}

{

  "openaireId":"od_______278::d5b39693ac6c2197e8cf48008a982951",

  "originalId": "oai:rudar.ruc.dk:1800/24691",

  "type": "other",

  "title": "Other test record",

  "authors": [

    "Sloan, Patrick Alexander Raaby",

    "Strange, Gustav Valdemar"

  ],

  "publisher": "",

  "description": "This is for testing software",

  "language": "eng",

  "pids": [],

  "licenseCode": "OPEN SOURCE",

  "resourceType": "0000",

  "url": "http://rudar.ruc.dk/handle/1800/24691",

  "collectedFromId": "opendoar____::278",

  "hostedById": "opendoar____::278",

  "linksToProjects": [],

  "contexts": []

}

package eu.dnetlib.openaire.directindex.api;

import eu.dnetlib.functionality.index.utils.ZkServers;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

public class IndexClientManager {

    private static final Log log = LogFactory.getLog(IndexClientManager.class);

    private CloudSolrClient client = null;

    public CloudSolrClient getClient(IndexDsInfo info) {

        if (client == null) {

            log.info(String.format("Initializing solr client (%s) with collection %s", info.getIndexBaseUrl(), info.getColl()));

            ZkServers zk = ZkServers.newInstance(info.getIndexBaseUrl());

            CloudSolrClient.Builder builder = new CloudSolrClient.Builder(zk.getHosts(), zk.getChroot()).withParallelUpdates(true);

            client = builder.build();

        }

        client.setDefaultCollection(info.getColl());

        return client;

    }

}

package eu.dnetlib.openaire.directindex.api;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.cache.annotation.CacheEvict;

import org.springframework.cache.annotation.Cacheable;

import org.springframework.core.io.ClassPathResource;

import javax.annotation.Resource;

import java.io.IOException;

public class IndexDSRetriever {

	private static final Log log = LogFactory.getLog(IndexDSRetriever.class);

	@Value(value = "${openaire.api.directindex.findSolrIndexUrl.xquery}")

	private ClassPathResource findSolrIndexUrl;

	@Value(value = "${openaire.api.directindex.findIndexDsInfo.xquery}")

	private ClassPathResource findIndexDsInfo;

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Cacheable("indexDsInfo")

	public IndexDsInfo calculateCurrentIndexDsInfo() throws IOException, ISLookUpException {

		log.info("Not using cache");

		final String queryUrl = IOUtils.toString(findSolrIndexUrl.getInputStream());

		final String queryDs = IOUtils.toString(findIndexDsInfo.getInputStream());

		final ISLookUpService lu = serviceLocator.getService(ISLookUpService.class);

		final String indexBaseUrl = lu.getResourceProfileByQuery(queryUrl);

		final String idxDs = lu.getResourceProfileByQuery(queryDs);

		if (idxDs.isEmpty()) {

			throw new IllegalStateException(queryDs + "\n\nreturned no results, check IS profiles");

		}

		final String[] arr = idxDs.split("@@@");

		return new IndexDsInfo(indexBaseUrl, arr[0].trim(), arr[1].trim(), arr[2].trim());

	}

	@CacheEvict(value="indexDsInfo", allEntries = true)

	public void evictCache(){

		log.info("Evicting indexDsInfo cache");

	}

}

{

  "authors": [

	"Conradt, Tobias",

	"& members of the ISIMIP project (original data provision), cf. Hempel et al. 2013, https://doi.org/10.5194/esd-4-219-2013"

  ],

  "collectedFromId": "re3data_____::r3d100010468",

  "description": "<p>ISIMIP-2a climate data cutout provided for Sardinia in the framework of SIM4NEXUS & X<\/p>",

  "hostedById": "re3data_____::r3d100010468",

  "licenseCode": "OPEN",

  "linksToProjects": ["info:eu-repo/grantAgreement/EC/H2020/689150//Sustainable Integrated Management FOR the NEXUS of water-land-food-energy-climate for a resource-efficient Europe & beyond/SIM4NEXUS"],

  "originalId": "10.5281/zenodo.1460665",

  "pids": [

	{

	  "type": "oai",

	  "value": "oai:zenodo.org:1460665"

	},

	{

	  "type": "doi",

	  "value": "10.5281/zenodo.1460665"

	}

  ],

  "publisher": "Zenodo & Zenodo",

  "resourceType": "0021",

  "title": "sardinia_tasmax_MIROC-ESM-CHEM_rcp4p5_2006-2099_daily.nc4 & the title has && in it",

  "type": "dataset",

  "url": "https://zenodo.org/record/1460665"

}

package eu.dnetlib.openaire.directindex.api;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import javax.annotation.Resource;

import java.util.concurrent.Executors;

import java.util.concurrent.ScheduledExecutorService;

import java.util.concurrent.TimeUnit;

public class ResultSubmitterService {

    private static final Log log = LogFactory.getLog(ResultSubmitterService.class);

    @Autowired

    private IndexDSRetriever indexDSRetriever;

    @Resource(name = "indexClientManager")

    private IndexClientManager clientManager;

    /**

     * Autocommit feature activation flag

     */

    @Value(value = "${openaire.api.directindex.autocommit.active}")

    private boolean autocommitactive;

    /**

     * Autocommit frequency (Seconds)

     */

    @Value(value = "${openaire.api.directindex.autocommit.frequency}")

    private long commitfrquency = 60;

    private ScheduledExecutorService executor;

    public ResultSubmitterService() {

        executor = Executors.newSingleThreadScheduledExecutor();

        updateCommitSchedule();

    }

    private void updateCommitSchedule() {

        log.info("updating commit schedule");

        executor.scheduleAtFixedRate(() -> {

            if (isAutocommitactive()) {

                try {

                    IndexDsInfo info = indexDSRetriever.calculateCurrentIndexDsInfo();

                    CloudSolrClient client = clientManager.getClient(info);

                    log.info("performing commit on " + info.getColl());

                    client.commit();

                } catch (Throwable e) {

                    log.error("unable to perform commit", e);

                }

            }

        }, 0, getCommitfrquency(), TimeUnit.SECONDS);

    }

    public boolean isAutocommitactive() {

        return autocommitactive;

    }

    public synchronized void setAutocommitactive(boolean autocommitactive) {

        this.autocommitactive = autocommitactive;

    }

    public long getCommitfrquency() {

        return commitfrquency;

    }

    public synchronized void setCommitfrquency(long commitfrquency) {

        this.commitfrquency = commitfrquency;

    }

}

package eu.dnetlib.openaire.directindex.api;

public class IndexDsInfo {

    private final String indexBaseUrl;

    private final String indexDsId;

    private final String format;

    private final String coll;

    public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) {

        this.indexBaseUrl = indexBaseUrl;

        this.indexDsId = indexDsId;

        this.format = format;

        this.coll = coll;

    }

    public String getIndexBaseUrl() {

        return indexBaseUrl;

    }

    public String getIndexDsId() {

        return indexDsId;

    }

    public String getFormat() {

        return format;

    }

    public String getColl() {

        return coll;

    }

    @Override

    public int hashCode() {

        return getColl().hashCode();

    }

    @Override

    public boolean equals(Object other) {

        if (!(other instanceof IndexDsInfo)) return false;

        return getColl().equals(((IndexDsInfo) other).getColl());

    }

}

{

  "authors": ["Milanese, Alessio", "Mende, Daniel", "Zeller, Georg", "Sunagawa, Shinichi"],

  "collectedFromId": "re3data_____::r3d100010468",

  "description": "<p>We simulated ten human gut metagenomic samples to assess the taxonomic quantification accuracy of the mOTUs tool (<a href=\"http: \// motu-tool.org \/\">link</a>). In this directory you can find the metagenomic samples, the gold standard (used to produce them) and the profiles obtained with four metagenomic profiler tools.</p>\\n\\n<p>Check README.txt for more information.</p>",

  "hostedById": "re3data_____::r3d100010468",

  "licenseCode": "OPEN",

  "originalId": "10.5281/zenodo.1473645",

  "pids": [

    {"type": "oai", "value": "oai:zenodo.org:1473645"},

    {"type": "doi", "value": "10.5281/zenodo.1473645"}

  ],

  "publisher": "Zenodo",

  "resourceType": "0021",

  "title": "Simulated Human gut metagenomic samples to benchmark mOTUs v2",

  "type": "dataset",

  "url": "https://zenodo.org/record/1473645",

  "version": "2"

}

package eu.dnetlib.openaire.directindex.api;

import eu.dnetlib.common.rmi.RMIException;

public class DirecIndexApiException extends RMIException {

	/**

	 *

	 */

	private static final long serialVersionUID = -3888037031334809448L;

	public DirecIndexApiException(final String string) {

		super(string);

	}

	public DirecIndexApiException(final String string, final Throwable exception) {

		super(string, exception);

	}

	public DirecIndexApiException(final Throwable exception) {

		super(exception);

	}

}

{"authors":["test"],

  "collectedFromId":"re3data_____::r3d100010468",

  "contexts":["https://zenodo.org/communities/oac-ni"],

  "description":"<p>test</p>",

  "hostedById":"re3data_____::r3d100010468",

  "licenseCode":"OPEN",

  "originalId":"oai:zenodo.org:315784",

  "pids":[{"type":"oai","value":"oai:zenodo.org:315784"},

    {"type":"doi","value":"10.5072/zenodo.315784"}],"resourceType":"0020","title":"test","type":"other","url":"https://zenodo.org/record/315784"}

{"authors": ["Dag Haug", "Marius J\xf8hndal"],

  "collectedFromId": "re3data_____::r3d100010468",

  "contexts": ["https://sandbox.zenodo.org/communities/oac-ni","pippo", "egi"],

  "description": "<p>Official releases of the PROIEL treebank of ancient Indo-European languages</p>",

  "hostedById": "re3data_____::r3d100010468",

  "licenseCode": "OPEN",

  "originalId": "10.5281/zenodo.11003",

  "pids": [{"type": "oai", "value": "oai:zenodo.org:11003"},

  {"type": "doi", "value": "10.5281/zenodo.11003"}],

  "publisher": "Zenodo",

  "resourceType": "0021",

  "title": "proiel-treebank: 20140723 version",

  "type": "dataset",

  "url": "https://zenodo.org/record/11003"}

### Root Level ###

log4j.rootLogger=WARN, CONSOLE

### Configuration for the CONSOLE appender ###

log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender

log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout

log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c - %m%n

org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger

### Application Level ###

log4j.logger.eu.dnetlib=INFO

log4j.logger.eu.dnetlib.msro.openaireplus.workflows.nodes.contexts=DEBUG

{

  "originalId": "ORIG_ID_12345",

  "title": "TEST TITLE WITH Greek Characters",

  "authors": [

	"αβγδεζηθικλμ νξοπρσςτυφχψω",

	"ΑΒΓΔΕΖΗΘΙΚΛ ΜΝΞΟΠΡΣΤΥΦΧΨΩ"

  ],

  "publisher": "âââââââ",

  "description": "Αν περιμένατε να βρίσκεται εδώ μια σελίδα και δεν υπάρχει, η σελίδα μπορεί να μην εμφανίζεται λόγω καθυστέρησης στην ανανέωση της βάσης δεδομένων, ή μπορεί να έχει διαγραφεί. (Δείτε την γρήγορη διαγραφή σελίδων για πιθανούς λόγους). Μπορείτε να δοκιμάστε την λειτουργία εκκαθάρισης, και να ελέγξετε το αρχείο διαγραφών.",

  "language": "ell",

  "pids": [

	{

	  "type": "doi",

	  "value": "10.000/xyz-123-gr"

	}

  ],

  "licenseCode": "EMBARGO",

  "embargoEndDate": "2018-02-02",

  "resourceType": "0001",

  "url": "http://test.it/xyz",

  "collectedFromId": "opendoar____::2659",

  "hostedById": "opendoar____::2367",

  "linksToProjects": [

	"info:eu-repo/grantAgreement/EC/FP7/123456/EU//Test"

  ],

  "contexts": [

	"egi::classification::natsc::math::stats"

  ]

}

package eu.dnetlib.openaire.directindex.objects;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.miscutils.functional.string.EscapeXml;

import eu.dnetlib.openaire.directindex.api.DirecIndexApiException;

import eu.dnetlib.openaire.directindex.api.OpenAIRESubmitterUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.ui.velocity.VelocityEngineUtils;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.concurrent.TimeUnit;

public class ResultEntryToOaf {

    private static final Log log = LogFactory.getLog(ResultEntryToOaf.class);

    private static long last_cache_update = 0;

    private static final Map<String, Map<String, String>> cached_vocabularies = new HashMap<>();

    private static final Map<String, DatasourceEntry> cached_datasources = new HashMap<>();

    private static final Map<String, String> cached_contexts = new HashMap<>();

    private OpenAIRESubmitterUtils utils;

    public ResultEntryToOaf(final String community_api){

        utils = new OpenAIRESubmitterUtils(community_api);

    }

    public String asOafRecord(final ResultEntry entry, final VelocityEngine ve,

                              final ISLookUpService lookupService,

                              final String oafSchemaLocation) throws Exception {

        if (StringUtils.isBlank(entry.getOriginalId())

                && StringUtils

                .isBlank(entry.getOpenaireId())) { throw new DirecIndexApiException("One of the following fields is required: originalId or openaireId"); }

        if (StringUtils.isBlank(entry.getTitle())) { throw new DirecIndexApiException("A required field is missing: title"); }

        if (StringUtils.isBlank(entry.getUrl())) { throw new DirecIndexApiException("A required field is missing: url"); }

        if (StringUtils.isBlank(entry.getLicenseCode()) && StringUtils.isBlank(entry.getAccessRightCode())) { throw new DirecIndexApiException("A required field is missing: accessRightCode"); }

        if (StringUtils.isBlank(entry.getResourceType())) { throw new DirecIndexApiException("A required field is missing: resourceType"); }

        if (StringUtils.isBlank(entry.getCollectedFromId())) { throw new DirecIndexApiException("A required field is missing: collectedFromId"); }

        if (StringUtils.isBlank(entry.getType())) { throw new DirecIndexApiException("A required field is missing: type"); }

        final DatasourceEntry collectedFromEntry = getDatasourceInfo(entry.getCollectedFromId(), lookupService);

        final DatasourceEntry hostedByEntry = getDatasourceInfo(entry.getHostedById(), lookupService);

        if (StringUtils.isBlank(entry.getOpenaireId())) {

            entry.setOpenaireId(calculateOpenaireId(entry.getOriginalId(), collectedFromEntry));

        }

        if (!entry.getOpenaireId()

                .matches("^\\w{12}::\\w{32}$")) { throw new DirecIndexApiException(

                "Invalid openaireId: " + entry.getOpenaireId() + " - regex ^\\w{12}::\\w{32}$ not matched"); }

        final List<OpenAIRESubmitterUtils.ContextInfo> contextInfos = utils.processContexts(entry.getContexts(), getContexts(lookupService));

        final Map<String, Object> model = new HashMap<>();

        model.put("esc", new EscapeXml());

        model.put("util",utils);

        model.put("pub", entry);

        model.put("objIdentifier", entry.getOpenaireId());

        model.put("oafSchemaLocation", oafSchemaLocation);

        model.put("resultTypes", getVocabulary("dnet:result_typologies", lookupService));

        model.put("rights", getVocabulary("dnet:access_modes", lookupService));

        model.put("resourceTypes", getVocabulary("dnet:publication_resource", lookupService));

        model.put("pidTypes", getVocabulary("dnet:pid_types", lookupService));

        model.put("languages", getVocabulary("dnet:languages", lookupService));

        model.put("contexts", getContexts(lookupService));

        model.put("contextInfo",contextInfos);

        model.put("dateOfCollection", new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'").format(new Date()));

        model.put("collectedFrom", collectedFromEntry);

        model.put("hostedBy", hostedByEntry);

        return VelocityEngineUtils.mergeTemplateIntoString(ve, "/eu/dnetlib/openaire/directindex/indexRecord.xml.vm", "UTF-8", model);

    }

    private synchronized static DatasourceEntry getDatasourceInfo(final String dsId, final ISLookUpService lookupService) throws ISLookUpException {

        if (StringUtils

                .isBlank(dsId)) { return new DatasourceEntry("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "unknown_____"); }

        if (!cached_datasources.containsKey(dsId)) {

            final String query =

                    "collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//CONFIGURATION[./DATASOURCE_ORIGINAL_ID='" + dsId

                            + "']/concat(./OFFICIAL_NAME, ' @@@ ', .//FIELD/value[../key='NamespacePrefix'])";

            final String s = lookupService.getResourceProfileByQuery(query);

            final String[] arr = s.split("@@@");

            final DatasourceEntry ds = new DatasourceEntry(dsId, arr[0].trim(), arr[1].trim());

            if (StringUtils.isBlank(ds.getName()) || StringUtils.isBlank(ds.getPrefix())) {

                log.error("Invalid datasource id: " + dsId);

                throw new ISLookUpException("Invalid datasource id: " + dsId);

            } else {

                cached_datasources.put(dsId, ds);

            }

        }

        return cached_datasources.get(dsId);

    }

    private synchronized static Map<String, String> getVocabulary(final String voc, final ISLookUpService lookupService) throws ISLookUpException {

        if (DateUtils.now() - last_cache_update < TimeUnit.MINUTES.toMillis(15) && cached_vocabularies.containsKey(voc)) {

            return cached_vocabularies.get(voc);

        } else {

            final String query = "collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code='" + voc

                    + "']//TERM/concat(@code, ' @@@ ', @english_name)";

            final Map<String, String> map = new HashMap<>();

            for (final String s : lookupService.quickSearchProfile(query)) {

                final String[] arr = s.split("@@@");

                map.put(arr[0].trim(), arr[1].trim());

            }

            cached_vocabularies.put(voc, map);

            last_cache_update = DateUtils.now();

            return map;

        }

    }

    private synchronized static Map<String, String> getContexts(final ISLookUpService lookupService) throws ISLookUpException {

        if (DateUtils.now() - last_cache_update > TimeUnit.MINUTES.toMillis(15) || cached_contexts.isEmpty()) {

            final String query =

                    "collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community' or .//context/@type='ri']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)";

            cached_contexts.clear();

            for (final String s : lookupService.quickSearchProfile(query)) {

                final String[] arr = s.split("@@@");

                cached_contexts.put(arr[0].trim(), arr[1].trim());

            }

            last_cache_update = DateUtils.now();

        }

        return cached_contexts;

    }

    private static String calculateOpenaireId(final String originalId, final DatasourceEntry collectedFromEntry) {

        return collectedFromEntry.getPrefix() + "::" + Hashing.md5(originalId);

    }

    public static String calculateOpenaireId(final String originalId, final String collectedFromId, final ISLookUpService lookupService)

            throws ISLookUpException {

        return calculateOpenaireId(originalId, getDatasourceInfo(collectedFromId, lookupService));

    }

    public OpenAIRESubmitterUtils getUtils() {

        return utils;

    }

    public void setUtils(OpenAIRESubmitterUtils utils) {

        this.utils = utils;

    }

}

package eu.dnetlib.openaire.directindex.objects;

import com.google.gson.Gson;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.miscutils.functional.string.EscapeXml;

import eu.dnetlib.openaire.directindex.api.DirecIndexApiException;

import eu.dnetlib.openaire.directindex.api.OpenAIRESubmitterUtils;

import io.swagger.annotations.ApiModelProperty;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.velocity.app.VelocityEngine;

import org.springframework.ui.velocity.VelocityEngineUtils;

import java.text.SimpleDateFormat;

import java.util.*;

import java.util.concurrent.TimeUnit;

/**

 * Created by michele on 02/12/15.

 */

public class ResultEntry {

	private String openaireId;

	private String originalId;

	private String title;

	private List<String> authors = new ArrayList<>();

	private String publisher;

	private String description;

	private String language;

	private List<PidEntry> pids = new ArrayList<>();

	/**

	 * @Deprecated: use accessRightCode

	 */

	@Deprecated

	private String licenseCode;

	private String accessRightCode;

	private String embargoEndDate;

	/**

	 * One of publication, dataset, software, other. Default value is publication.

	 */

	private String type = "publication";

	private String resourceType;

	private String url;

	private String collectedFromId;

	private String hostedById;

	// String according to the EGI context profile, example: egi::classification::natsc::math

	private List<String> contexts = new ArrayList<>();

	// String according to openaire guidelines:

	// info:eu-repo/grantAgreement/Funder/FundingProgram/ProjectID/[Jurisdiction]/[ProjectName]/[ProjectAcronym]

	private List<String> linksToProjects = new ArrayList<>();

	private static final Log log = LogFactory.getLog(ResultEntry.class);

	public ResultEntry() {}

	public String getOpenaireId() {

		return openaireId;

	}

	public void setOpenaireId(final String openaireId) {

		this.openaireId = openaireId;

	}

	public String getOriginalId() {

		return originalId;

	}

	public void setOriginalId(final String originalId) {

		this.originalId = originalId;

	}

	@ApiModelProperty(required = true)

	public String getTitle() {

		return title;

	}

	public void setTitle(final String title) {

		this.title = title;

	}

	public List<String> getAuthors() {

		return authors;

	}

	public void setAuthors(final List<String> authors) {

		this.authors = authors;

	}

	public String getPublisher() {

		return publisher;

	}

	public void setPublisher(final String publisher) {

		this.publisher = publisher;

	}

	public String getDescription() {

		return description;

	}

	public void setDescription(final String description) {

		this.description = description;

	}

	@ApiModelProperty(value = "ISO Alpha-3 code. E.g. 'eng', 'ita'")

	public String getLanguage() {

		return language;

	}

	public void setLanguage(final String language) {

		this.language = language;

	}

	public List<PidEntry> getPids() {

		return pids;

	}

	public void setPids(final List<PidEntry> pids) {

		this.pids = pids;

	}

	@Deprecated

	@ApiModelProperty(required = false, allowableValues = "OPEN, CLOSED, RESTRICTED, EMBARGO, UNKNOWN, OTHER, OPEN SOURCE")

	public String getLicenseCode() {