D-Net

package eu.dnetlib.data.collector.plugins.excel;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.plugins.HttpCSVCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

/**

 * Created by miriam on 10/05/2017.

 */

public class ReadExcelPlugin extends AbstractCollectorPlugin{

	private static final Log log = LogFactory.getLog(ReadExcelPlugin.class);

	@Autowired

	HttpCSVCollectorPlugin httpCSVCollectorPlugin;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		Read r = new Read(interfaceDescriptor);

		r.setCollector(httpCSVCollectorPlugin);

		try {

			return r.parseFile();

		}catch(Exception e){

			log.error("Error importing excel file");

			throw new CollectorServiceException(e);

		}

	}

}

package eu.dnetlib.data.collector.plugins.datasources;

import java.io.IOException;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.plugins.HttpConnector;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.io.IOUtils;

import org.springframework.beans.factory.annotation.Autowired;

/**

 * Plugin to collect metadata record about data repositories from re3data.

 * <p>

 * Documentation on re3data API: http://service.re3data.org/api/doc.

 * </p>

 * <p>

 * BaseURL: http://service.re3data.org

 * </p>

 * <p>

 * API to get the list of repos: baseURL + /api/v1/repositories

 * </p>

 * <p>

 * API to get a repository: baseURL + content of link/@href of the above list

 * </p>

 *

 * @author alessia

 *

 */

public class Re3DataCollectorPlugin extends AbstractCollectorPlugin {

	private String repositoryListPath = "/api/v1/repositories";

	@Autowired

	private HttpConnector httpConnector;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		String repositoryListURL = interfaceDescriptor.getBaseUrl() + repositoryListPath;

		String input;

		try {

			input = httpConnector.getInputSource(repositoryListURL);

			return new Re3DataRepositoriesIterator(IOUtils.toInputStream(input, "UTF-8"), interfaceDescriptor.getBaseUrl(), getHttpConnector());

		} catch (IOException e) {

			throw new CollectorServiceException(e);

		}

	}

	public String getRepositoryListPath() {

		return repositoryListPath;

	}

	public void setRepositoryListPath(final String repositoryListPath) {

		this.repositoryListPath = repositoryListPath;

	}

	public HttpConnector getHttpConnector() {

		return httpConnector;

	}

	public void setHttpConnector(final HttpConnector httpConnector) {

		this.httpConnector = httpConnector;

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex.SitemapFileIterator;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.io.FileUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.log4j.ConsoleAppender;

import org.apache.log4j.Level;

import org.apache.log4j.Logger;

import org.apache.log4j.PatternLayout;

import java.io.File;

import java.nio.charset.StandardCharsets;

import java.util.HashMap;

import java.util.concurrent.TimeUnit;

public class SchemaOrgMainReactome {

    private static final Log log = LogFactory.getLog(SchemaOrgMainReactome.class);

    public static void main(String[] args) throws Exception {

        ConsoleAppender console = new ConsoleAppender();

        console.setLayout(new PatternLayout("%d [%p|%c|%C{1}] %m%n"));

        console.setThreshold(Level.DEBUG);

        console.activateOptions();

        Logger.getLogger("eu.dnetlib.data.collector.plugins").addAppender(console);

        HashMap<String,String> params = new HashMap<>();

        params.put("consumerBlockPolling", Boolean.toString(true));

        params.put("consumerBlockPollingTimeout", "2");

        params.put("consumerBlockPollingTimeoutUnit", TimeUnit.MINUTES.toString());

        params.put("endpointCharset", StandardCharsets.UTF_8.name());

        params.put("updatedDateFormat", "YYYY-MM-DD");

        params.put("createdDateFormat", "YYYY-MM-DD");

        params.put("publicationDateFormat", "YYYY-MM-DD");

        params.put("contributorFallbackType", DatasetDocument.Contributor.ContributorType.Other.toString());

        params.put("identifierFallbackType", DatasetDocument.Identifier.IdentifierType.Handle.toString());

        params.put("identifierFallbackURL", Boolean.toString(true));

        params.put("identifierMappingARK", "ark, ARK");

        params.put("identifierMappingDOI", "doi, DOI");

        params.put("identifierMappingHandle", "Handle, HANDLE");

        params.put("identifierMappingPURL", "purl, PURL");

        params.put("identifierMappingURN", "urn, URN");

        params.put("identifierMappingURL", "url, URL");

        params.put("repositoryAccessType", "sitemapindex");

        params.put("sitemap_queueSize", "100");

        params.put("sitemap_IndexCharset", StandardCharsets.UTF_8.name());

        params.put("sitemap_FileCharset", StandardCharsets.UTF_8.name());

        params.put("sitemap_FileSchema", SitemapFileIterator.Options.SitemapSchemaType.Text.toString());

        params.put("sitemap_FileType", SitemapFileIterator.Options.SitemapFileType.GZ.toString());

        params.put("sitemap_producerBlockPollingTimeout", "2");

        params.put("sitemap_producerBlockPollingTimeoutUnit", TimeUnit.MINUTES.toString());

        InterfaceDescriptor descriptor = new InterfaceDescriptor();

        descriptor.setId("schema.org - reactome");

        descriptor.setBaseUrl("https://reactome.org/sitemapindex.xml");

        descriptor.setParams(params);

        SchemaOrgPlugin schemaOrgPlugin = new SchemaOrgPlugin();

        Iterable<String> iterable = schemaOrgPlugin.collect(descriptor, null, null);

        String outDir = params.get("repositoryAccessType");

        log.info("saving content in " + outDir);

        File directory = new File(outDir);

        if (directory.exists()) {

            log.info(directory.getAbsolutePath() + " exists, cleaning up");

            FileUtils.deleteDirectory(directory);

        }

        FileUtils.forceMkdir(directory);

        Utils.writeFiles(iterable, outDir);

    }

}

package eu.dnetlib.data.collector.plugins.schemaorg.httpapi.kaggle;

import eu.dnetlib.data.collector.plugins.schemaorg.RepositoryIterable;

import eu.dnetlib.data.collector.plugins.schemaorg.RepositoryQueueIterator;

import eu.dnetlib.data.collector.plugins.schemaorg.httpapi.HttpApiRepositoryIterable;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.json.JSONArray;

import org.json.JSONObject;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.Iterator;

import java.util.NoSuchElementException;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.TimeUnit;

public class KaggleRepositoryIterable implements HttpApiRepositoryIterable {

	private static final Log log = LogFactory.getLog(KaggleRepositoryIterable.class);

	public static class Options {

		private String queryUrl;

		private String queryPagePlaceholder;

		private Charset charset;

		private String responsePropertyTotalDataset;

		private String responsePropertyDatasetList;

		private String responsePropertyDatasetUrl;

		private String responseBaseDatasetUrl;

		private long putTimeout;

		private TimeUnit putTimeoutUnit;

		private RepositoryQueueIterator.Options repositoryQueueIteratorOptions;

		private int queueSize;

		public long getPutTimeout() {

			return putTimeout;

		}

		public void setPutTimeout(long putTimeout) {

			this.putTimeout = putTimeout;

		}

		public TimeUnit getPutTimeoutUnit() {

			return putTimeoutUnit;

		}

		public void setPutTimeoutUnit(TimeUnit putTimeoutUnit) {

			this.putTimeoutUnit = putTimeoutUnit;

		}

		public int getQueueSize() {

			return queueSize;

		}

		public void setQueueSize(int queueSize) {

			this.queueSize = queueSize;

		}

		public String getResponseBaseDatasetUrl() {

			return responseBaseDatasetUrl;

		}

		public void setResponseBaseDatasetUrl(String responseBaseDatasetUrl) {

			this.responseBaseDatasetUrl = responseBaseDatasetUrl;

		}

		public RepositoryQueueIterator.Options getRepositoryQueueIteratorOptions() {

			return repositoryQueueIteratorOptions;

		}

		public void setRepositoryQueueIteratorOptions(RepositoryQueueIterator.Options repositoryQueueIteratorOptions) {

			this.repositoryQueueIteratorOptions = repositoryQueueIteratorOptions;

		}

		public String getResponsePropertyDatasetUrl() {

			return responsePropertyDatasetUrl;

		}

		public void setResponsePropertyDatasetUrl(String responsePropertyDatasetUrl) {

			this.responsePropertyDatasetUrl = responsePropertyDatasetUrl;

		}

		public String getResponsePropertyDatasetList() {

			return responsePropertyDatasetList;

		}

		public void setResponsePropertyDatasetList(String responsePropertyDatasetList) {

			this.responsePropertyDatasetList = responsePropertyDatasetList;

		}

		public String getResponsePropertyTotalDataset() {

			return responsePropertyTotalDataset;

		}

		public void setResponsePropertyTotalDataset(String responsePropertyTotalDataset) {

			this.responsePropertyTotalDataset = responsePropertyTotalDataset;

		}

		public Charset getCharset() {

			return charset;

		}

		public void setCharset(Charset charset) {

			this.charset = charset;

		}

		public String getQueryPagePlaceholder() {

			return queryPagePlaceholder;

		}

		public void setQueryPagePlaceholder(String queryPagePlaceholder) {

			this.queryPagePlaceholder = queryPagePlaceholder;

		}

		public String getQueryUrl() {

			return queryUrl;

		}

		public void setQueryUrl(String queryUrl) {

			this.queryUrl = queryUrl;

		}

	}

	private Options options;

	private ArrayBlockingQueue<String> queue;

	public KaggleRepositoryIterable(Options options) {

		this.options = options;

//		this.currentPage = 1;

//		this.terminated = false;

	}

	public void bootstrap() {

		this.queue = new ArrayBlockingQueue<>(this.options.getQueueSize());

		Thread ft = new Thread(new Harvester() );

		ft.start();

//		ExecutorService executor = Executors.newSingleThreadExecutor();

//		executor.execute(new Harvester());

//		executor.shutdown();

	}

	@Override

	public Iterator<String> iterator() {

		return new RepositoryQueueIterator(this.options.getRepositoryQueueIteratorOptions(), this.queue);

	}

	private class Harvester implements Runnable{

		@Override

		public void run() {

			this.execute();

		}

		private void execute() {

			try {

				int currentPage = 1;

				int totalDatasets = 0;

				int readDatasets = 0;

				while (true) {

					String query = options.getQueryUrl().replace(options.getQueryPagePlaceholder(), Integer.toString(currentPage));

					String response = IOUtils.toString(new URL(query), options.getCharset());

					currentPage += 1;

					JSONObject pageObject = new JSONObject(response);

					totalDatasets = pageObject.optInt(options.getResponsePropertyTotalDataset());

					JSONArray datasets = pageObject.optJSONArray(options.getResponsePropertyDatasetList());

					if (datasets == null || datasets.length() == 0) break;

					readDatasets += datasets.length();

					for (int i = 0; i < datasets.length(); i += 1) {

						JSONObject item = datasets.optJSONObject(i);

						String urlFragment = item.optString(options.getResponsePropertyDatasetUrl());

						if (urlFragment == null || urlFragment.trim().length() == 0) continue;

						String endpoint = String.format("%s%s", options.getResponseBaseDatasetUrl(), urlFragment);

						log.debug("adding endpoint in queue");

						log.debug("queue size: " + queue.size());

						try {

							queue.offer(endpoint, options.getPutTimeout(), options.getPutTimeoutUnit());

						} catch (InterruptedException ex) {

							log.warn(String.format("could not put elements from queue for more than %s %s. breaking", options.getPutTimeout(), options.getPutTimeoutUnit()));

							break;

						}

						log.debug("endpoint added in queue");

						log.debug("queue size: " + queue.size());

					}

					if (readDatasets >= totalDatasets) break;

				}

			} catch (Exception ex) {

				log.error("problem execution harvesting", ex);

			} finally {

				try {

					queue.offer(RepositoryIterable.TerminationHint, options.getPutTimeout(), options.getPutTimeoutUnit());

				} catch (Exception ex) {

					log.fatal("could not add termination hint. the process will not terminate gracefully", ex);

				}

			}

		}

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex;

import eu.dnetlib.data.collector.plugins.schemaorg.RepositoryIterable;

import eu.dnetlib.data.collector.plugins.schemaorg.RepositoryQueueIterator;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.net.URL;

import java.util.Iterator;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.TimeUnit;

public class SitemapIndexRepositoryIterable implements RepositoryIterable {

	private static final Log log = LogFactory.getLog(SitemapIndexRepositoryIterable.class);

	public static class Options {

		private SitemapIndexIterator.Options sitemapIndexIteratorOptions;

		private SitemapFileIterator.Options sitemapFileIteratorOptions;

		private RepositoryQueueIterator.Options repositoryQueueIteratorOptions;

		private long putTimeout;

		private TimeUnit putTimeoutUnit;

		private int queueSize;

		public long getPutTimeout() {

			return putTimeout;

		}

		public void setPutTimeout(long putTimeout) {

			this.putTimeout = putTimeout;

		}

		public TimeUnit getPutTimeoutUnit() {

			return putTimeoutUnit;

		}

		public void setPutTimeoutUnit(TimeUnit putTimeoutUnit) {

			this.putTimeoutUnit = putTimeoutUnit;

		}

		public int getQueueSize() {

			return queueSize;

		}

		public void setQueueSize(int queueSize) {

			this.queueSize = queueSize;

		}

		public RepositoryQueueIterator.Options getRepositoryQueueIteratorOptions() {

			return repositoryQueueIteratorOptions;

		}

		public void setRepositoryQueueIteratorOptions(RepositoryQueueIterator.Options repositoryQueueIteratorOptions) {

			this.repositoryQueueIteratorOptions = repositoryQueueIteratorOptions;

		}

		public SitemapIndexIterator.Options getSitemapIndexIteratorOptions() {

			return sitemapIndexIteratorOptions;

		}

		public void setSitemapIndexIteratorOptions(SitemapIndexIterator.Options sitemapIndexIteratorOptions) {

			this.sitemapIndexIteratorOptions = sitemapIndexIteratorOptions;

		}

		public SitemapFileIterator.Options getSitemapFileIteratorOptions() {

			return sitemapFileIteratorOptions;

		}

		public void setSitemapFileIteratorOptions(SitemapFileIterator.Options sitemapFileIteratorOptions) {

			this.sitemapFileIteratorOptions = sitemapFileIteratorOptions;

		}

	}

	private Options options;

	private ArrayBlockingQueue<String> queue;

	public SitemapIndexRepositoryIterable(Options options) {

		this.options = options;

	}

	public void bootstrap() {

		this.queue = new ArrayBlockingQueue<>(this.options.getQueueSize());

		Thread ft = new Thread(new Harvester() );

		ft.start();

//		ExecutorService executor = Executors.newSingleThreadExecutor();

//		executor.execute(new Harvester());

//		executor.shutdown();

	}

	@Override

	public Iterator<String> iterator() {

		return new RepositoryQueueIterator(this.options.getRepositoryQueueIteratorOptions(), this.queue);

	}

	private class Harvester implements Runnable{

		@Override

		public void run() {

			this.execute();

		}

		private void execute(){

			try {

				SitemapIndexIterator sitemapIndexIterator = new SitemapIndexIterator(options.getSitemapIndexIteratorOptions());

				sitemapIndexIterator.bootstrap();

				while (sitemapIndexIterator.hasNext()) {

					String sitemapFile = sitemapIndexIterator.next();

					if(sitemapFile == null) continue;

					SitemapFileIterator.Options sitemapFileIteratorOptions = (SitemapFileIterator.Options)options.getSitemapFileIteratorOptions().clone();

					sitemapFileIteratorOptions.setFileUrl(new URL(sitemapFile));

					SitemapFileIterator sitemapFileIterator = new SitemapFileIterator(sitemapFileIteratorOptions);

					sitemapFileIterator.bootstrap();

					while(sitemapFileIterator.hasNext()){

						String endpoint = sitemapFileIterator.next();

						if(endpoint == null) continue;;

						log.debug("adding endpoint in queue");

						log.debug("queue size: " + queue.size());

						try {

							queue.offer(endpoint, options.getPutTimeout(), options.getPutTimeoutUnit());

						} catch (InterruptedException ex) {

							log.warn(String.format("could not put elements from queue for more than %s %s. breaking", options.getPutTimeout(), options.getPutTimeoutUnit()));

							break;

						}

						log.debug("endpoint added in queue");

						log.debug("queue size: " + queue.size());

					}

				}

			}catch(Exception ex){

				log.error("problem execution harvesting", ex);

			}

			finally {

				try {

					queue.offer(RepositoryIterable.TerminationHint, options.getPutTimeout(), options.getPutTimeoutUnit());

				} catch (Exception ex) {

					log.fatal("could not add termination hint. the process will not terminate gracefully", ex);

				}

			}

		}

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.w3c.dom.Document;

import org.w3c.dom.NodeList;

import org.xml.sax.InputSource;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathExpression;

import javax.xml.xpath.XPathFactory;

import java.io.*;

import java.net.URL;

import java.nio.charset.Charset;

import java.nio.charset.UnsupportedCharsetException;

import java.util.ArrayList;

import java.util.EnumSet;

import java.util.HashMap;

import java.util.List;

import java.util.zip.GZIPInputStream;

public class Utils {

	private static final Log log = LogFactory.getLog(Utils.class);

	public static List<String> collectAsStrings(String xml, String xpath) throws Exception{

		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();

		DocumentBuilder builder = factory.newDocumentBuilder();

		Document doc = builder.parse(new InputSource(new StringReader(xml)));

		return Utils.collectAsStrings(doc, xpath);

	}

	public static List<String> collectAsStrings(File file, String xpath) throws Exception{

		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();

		DocumentBuilder builder = factory.newDocumentBuilder();

		Document doc = builder.parse(file);

		return Utils.collectAsStrings(doc, xpath);

	}

	public static List<String> collectAsStrings(Document doc, String xpath) throws Exception{

		XPathFactory xPathfactory = XPathFactory.newInstance();

		XPath path = xPathfactory.newXPath();

		XPathExpression expr = path.compile(xpath);

		NodeList nodes = (NodeList) expr.evaluate(doc, XPathConstants.NODESET);

		List<String> values = new ArrayList<>();

		for (int i = 0; i < nodes.getLength(); i++)

			values.add(nodes.item(i).getNodeValue());

		return values;

	}

	public static void decompressGZipTo(File input, File output) throws Exception {

		try (GZIPInputStream in = new GZIPInputStream(new FileInputStream(input))){

			try (FileOutputStream out = new FileOutputStream(output)){

				byte[] buffer = new byte[1024];

				int len;

				while((len = in.read(buffer)) != -1){

					out.write(buffer, 0, len);

				}

			}

		}

	}

	public static String getAsString(HashMap<String,String> map, String key, String defaultValue)

	{

		String value = map.get(key);

		if(value == null) return defaultValue;

		return value;

	}

	public static List<String> getAsStringCsv(HashMap<String,String> map, String key, List<String> defaultValue)

	{

		String value = map.get(key);

		if(value == null) return defaultValue;

		String[] splits = value.split(",");

		List<String> curated = new ArrayList<>();

		for(String item : splits){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item.trim());

		}

		return curated;

	}

	public static int getAsInt(HashMap<String,String> map, String key, int defaultValue)

	{

		String value = map.get(key);

		if(value == null) return defaultValue;

		try {

			return Integer.parseInt(value);

		} catch (NumberFormatException e) {

			return defaultValue;

		}

	}

	public static long getAsLong(HashMap<String,String> map, String key, long defaultValue)

	{

		String value = map.get(key);

		if(value == null) return defaultValue;

		try {

			return Long.parseLong(value);

		} catch (NumberFormatException e) {

			return defaultValue;

		}

	}

	public static <E extends Enum<E>> E getAsEnum(HashMap<String,String> map, String key, E defaultValue, Class<E> clazz) {

		//EnumSet<E> values = EnumSet.allOf(defaultValue.getClass());

		EnumSet<E> values = EnumSet.allOf(clazz);

		String value = map.get(key);

		if (value == null) return defaultValue;

		for(E val : values){

			if(!val.name().equalsIgnoreCase(value)) continue;

			return val;

		}

		return defaultValue;

	}

	public static Boolean getAsBoolean(HashMap<String,String> map, String key, Boolean defaultValue) {

		String value = map.get(key);

		if (value == null) return defaultValue;

		return Boolean.parseBoolean(value);

	}

	public static Charset getAsCharset(HashMap<String,String> map, String key, Charset defaultValue)

	{

		String value = map.get(key);

		if(value == null) return defaultValue;

		try {

			return Charset.forName(value);

		} catch (UnsupportedCharsetException e) {

			return defaultValue;

		}

	}

	public static String RemoteAccessWithRetry(int retryCount, long waitBetweenRetriesMillis, URL endpoint, Charset charset) throws IOException {

		int retry =0;

		while(retry < retryCount) {

			try {

				return IOUtils.toString(endpoint, charset);

			} catch (Exception ex) {

				retry += 1;

				if (retry < retryCount) {

					log.debug("problem accessing url " + endpoint + ". will retry after " + waitBetweenRetriesMillis + " milliseconds");

					try {

						Thread.sleep(waitBetweenRetriesMillis);

					} catch (Exception e) {

					}

				}

				else{

					log.debug("problem accessing url " + endpoint + ". throwing");

					throw  ex;

				}

			}

		}

		return null;

	}

	public static Boolean validateXml(String xml){

		try {

			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();

			DocumentBuilder builder = factory.newDocumentBuilder();

			InputSource is = new InputSource(new StringReader(xml));

			builder.parse(is);

			return true;

		}catch(Exception ex){

			return false;

		}

	}

	public static void writeFiles(final Iterable<String> iterable, final String outDir) throws DocumentException, IOException {

		int skipped = 0;

		int count = 0;

		for(String item : iterable) {

			final org.dom4j.Document doc = new SAXReader().read(new StringReader(item));

			if (StringUtils.isNotBlank(doc.valueOf("/*[local-name() = 'dataset']/*[local-name() = 'identifier']/text()"))) {

				log.info(item);

				String fileName = outDir + "/" + count++;

				try(BufferedWriter w = new BufferedWriter(new FileWriter(fileName))) {

					w.write(item);

				}

				log.info("wrote " + fileName);

			} else {

				skipped++;

			}

			if (skipped % 100 == 0) {

				log.info("skipped so far " + skipped);

			}

			if (count % 100 == 0) {

				log.info("stored so far " + count);

			}

		}

		log.info(String.format("Done! skipped %s, stored %s", skipped, count));

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex;

import eu.dnetlib.data.collector.plugins.schemaorg.Utils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.*;

public class SitemapIndexIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(SitemapIndexIterator.class);

	public static class Options {

		private URL indexUrl;

		private Charset charset;

		public Options(){}

		public Options(URL indexUrl, Charset charset){

			this.indexUrl = indexUrl;

			this.charset = charset;

		}

		public URL getIndexUrl() {

			return indexUrl;

		}

		public void setIndexUrl(URL indexUrl) {

			this.indexUrl = indexUrl;

		}

		public Charset getCharset() {

			return charset;

		}

		public void setCharset(Charset charset) {

			this.charset = charset;

		}

	}

	private Options options;

	private Queue<String> sitemapFiles;

	public SitemapIndexIterator(Options options) {

		this.options = options;

	}

	public void bootstrap() {

		List<String> files = null;

		try {

			log.debug("bootstrapping sitemapindex index access");

			String sitemapIndexPayload = Utils.RemoteAccessWithRetry(3, 5000, this.options.getIndexUrl(), this.options.getCharset());

			log.debug(String.format("sitemapindex payload is: %s", sitemapIndexPayload));

			files = Utils.collectAsStrings(sitemapIndexPayload, "/sitemapindex/sitemap/loc/text()");

			log.debug(String.format("extracted %d sitemapindex files", files.size()));

		}catch(Exception ex){

			log.error("problem bootstrapping sitemapindex index access. returning 0 files", ex);

			files = new ArrayList<>();

		}

		this.sitemapFiles = new PriorityQueue<String>(files);

	}

	@Override

	public boolean hasNext() {

		return !this.sitemapFiles.isEmpty();

	}

	@Override

	public String next() {

		return this.sitemapFiles.poll();

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.json.JSONObject;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.Iterator;

public class EndpointAccessIterator implements Iterator<JSONObject> {

	private static final Log log = LogFactory.getLog(EndpointAccessIterator.class);

	public static class Options {

		private Charset charset;

		public Options(){}

		public Options(Charset charset) {

			this.charset = charset;

		}

		public Charset getCharset() {

			return charset;

		}

		public void setCharset(Charset charset) {

			this.charset = charset;

		}

	}

	private Options options;

	private Iterator<String> repositoryIterator;

	public EndpointAccessIterator(Options options, Iterator<String> repositoryIterator) {

		this.options = options;

		this.repositoryIterator = repositoryIterator;

	}

	@Override

	public boolean hasNext() {

		return this.repositoryIterator.hasNext();

	}

	@Override

	public JSONObject next() {

		String endpoint = this.repositoryIterator.next();

		if(endpoint == null) return null;

		log.debug(String.format("processing: %s", endpoint));

		JSONObject dataset = this.extractDatasetRecord(endpoint);

		return dataset;

	}

	private JSONObject extractDatasetRecord(String endpoint) {

		JSONObject datasetDocument = null;

		try {

			URL urlEndpoint = new URL(endpoint);

			log.debug("downloading endpoint "+urlEndpoint);

			String payload = Utils.RemoteAccessWithRetry(3, 5000, urlEndpoint, this.options.getCharset());

			log.trace("downloaded payload id: "+payload);

			Document doc = Jsoup.parse(payload);

			Elements scriptTags = doc.getElementsByTag("script");

			for (Element scriptTag : scriptTags) {

				if (!scriptTag.hasAttr("type")) continue;

				String scriptType = scriptTag.attr("type");

				if (!scriptType.equalsIgnoreCase("application/ld+json")) continue;

				String data = scriptTag.data();

				JSONObject schemaItem = new JSONObject(data);

				String context = schemaItem.optString("@context");

				String type = schemaItem.optString("@type");

				if (context == null || type == null) continue;

				Boolean isSchemaOrgContext = context.toLowerCase().startsWith("http://schema.org") || context.toLowerCase().startsWith("https://schema.org");

				Boolean isDataset = type.equalsIgnoreCase("dataset");

				if (!isSchemaOrgContext || !isDataset) continue;

				log.debug(String.format("discovered dataset document: %s", schemaItem.toString()));

				datasetDocument = schemaItem;

				break;

			}

		}catch(Exception ex){

			log.error("problem extracting dataset document. returning empty", ex);

			datasetDocument = null;

		}

		if(datasetDocument == null){

			log.debug("did not find any dataset document in endpoint");

		}

		else{

			log.debug("found dataset document in endpoint :"+datasetDocument.toString());

		}

		return datasetDocument;

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import java.util.Iterator;

public interface RepositoryIterable extends Iterable<String> {

	public static String TerminationHint = "df667391-676d-4c0f-9c40-426b1001607a";

}

package eu.dnetlib.data.collector.plugins.httpfilename;

import java.util.*;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.TimeUnit;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.json.JSONObject;

import org.json.XML;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

/**

 * Created by miriam on 04/05/2018.

 */

public class HTTPWithFileNameCollectorIterable implements Iterable<String> {

    private static final Log log = LogFactory.getLog(HTTPWithFileNameCollectorIterable.class);

    private static final String JUNK = "<resource><url>%s</url><DOI>JUNK</DOI></resource>";

    public static final String APP_JSON = "application/json";

    public static final String APP_XML = "application/xml";

    public static final String TEXT_HTML = "text/html";

    private final ArrayBlockingQueue<String> queue = new ArrayBlockingQueue<String>(100);

    private String filterParam;

    int total = 0;

    int filtered = 0;

    public HTTPWithFileNameCollectorIterable(String startUrl, String filter){

        this.filterParam = filter;

        Thread ft = new Thread(new FillMetaQueue(startUrl) );

        ft.start();

    }

    @Override

    public Iterator<String> iterator() {

        return new HttpWithFileNameCollectorIterator(queue);

    }

    private class FillMetaQueue implements Runnable {

        final Connector c = new Connector();

        private final List<String> metas = Collections.synchronizedList(new ArrayList<String>());

        private final List<String> urls = Collections.synchronizedList(new ArrayList<>());

        public FillMetaQueue(String startUrl){

            if(!startUrl.isEmpty()){

                urls.add(startUrl);

            }

        }

        public void fillQueue() {

            String url;

            while((metas.size()>0 || urls.size() > 0 )) {

                log.debug("metas.size() = " + metas.size() + " urls.size() = " + urls.size() + " queue.size() = " +queue.size());

                if (metas.size() > 0) {

                    url = metas.remove(0);

                    try {

                        c.get(url);

                    } catch (CollectorServiceException e) {

                        log.info("Impossible to collect url: " + url + " error: " + e.getMessage());

                    }

                    if(c.isStatusOk()){

                        try {

                            String ret = c.getResponse();

                            if (ret != null && ret.length()>0) {

                                if (!containsFilter(ret))

                                    queue.put(addFilePath(ret, url, url.endsWith(".json")));

                                    //queue.offer(addFilePath(ret, url, url.endsWith(".json")), HttpWithFileNameCollectorIterator.waitTime, TimeUnit.SECONDS);

                                else

                                    filtered++;

                                total++;

                            }

                        } catch (InterruptedException e) {

                            log.info("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );

                        }

                    }

                } else {

                    url = urls.remove(0);

                    try {

                        c.get(url);

                    } catch (CollectorServiceException e) {

                        log.info("Impossible to collect url: " + url + " error: " + e.getMessage());

                    }

                    if(c.isStatusOk()) {

                        if (c.responseTypeContains(TEXT_HTML)){

                            recurFolder(c.getResponse(), url);

                        } else if(c.responseTypeContains(APP_JSON) || c.responseTypeContains(APP_XML)){

                            try {

                                final String element = addFilePath(c.getResponse(), url, c.responseTypeContains(APP_JSON));

                                //queue.offer(element, HttpWithFileNameCollectorIterator.waitTime, TimeUnit.SECONDS);

                                queue.put(element);

                            } catch (InterruptedException e) {

                                log.info("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );

                            }

                        }

                    }

                }

            }

            try {

                //queue.offer(HttpWithFileNameCollectorIterator.TERMINATOR, HttpWithFileNameCollectorIterator.waitTime, TimeUnit.SECONDS);

                queue.put(HttpWithFileNameCollectorIterator.TERMINATOR);

            } catch (InterruptedException e) {

                throw new IllegalStateException(String.format("could not add element to queue for more than %s%s", HttpWithFileNameCollectorIterator.waitTime, TimeUnit.SECONDS), e);

            }

        }

        private boolean containsFilter(String meta){

            if (filterParam == null || filterParam.isEmpty())

                return false;

            String[] filter = filterParam.split(";");

            for(String item:filter){

                if (meta.contains(item))

                    return true;

            }

            return false;

        }

        private String addFilePath(String meta, String url, boolean isJson){

            String path = url.replace("metadata", "pdf");

            try {

                if(isJson)

                    meta = meta.substring(0, meta.length() - 1) + ",'downloadFileUrl':'" + path.substring(0, path.indexOf(".json")) + ".pdf'}";

                else {

                    if (meta.contains("<!DOCTYPE")) {

                        meta = meta.substring(meta.indexOf("<!DOCTYPE"));

                        meta = meta.substring(meta.indexOf(">") + 1);

                    }

                    int index = meta.lastIndexOf("</");

                    meta = meta.substring(0, index) + "<downloadFileUrl>" + path.substring(0, path.indexOf(".xml")) + ".pdf</downloadFileUrl>" + meta.substring(index);

                }

            } catch(Exception ex) {

                log.info("not file with extension .json or .xml");

            }

            if(isJson) {

                try {

                    return XML.toString(new JSONObject("{'resource':" + meta + "}"));

                } catch(Exception e) {

                    log.fatal("Impossible to transform json object to xml \n" + meta + "\n " + e.getMessage() + "\n" + url);

                   // throw new RuntimeException();

                    final String junk = String.format(JUNK, url);

                    log.warn("returning " + junk);

                    return junk;

                }

            }

            return meta;

        }

        private void recurFolder(String text, String url){

            Document doc = Jsoup.parse(text);

            Elements links = doc.select("a");

            for(Element e:links){

                if (!e.text().equals("../")){

                    String file = e.attr("href");

                    if(file.endsWith(".json") || file.endsWith(".xml"))

                        metas.add(url+file);

                    else

                        urls.add(url+file);

                }

            }

        }

        @Override

        public void run() {

            fillQueue();

        }

    }

}

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-collector-plugins</artifactId>

	<version>1.7.14-SNAPSHOT</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-collector-plugins/trunk</developerConnection>

	</scm>

	<build>

		<plugins>

			<plugin>

				<artifactId>maven-assembly-plugin</artifactId>

				<configuration>

					<archive>

						<manifest>

							<mainClass>eu.dnetlib.data.collector.plugins.schemaorg.SchemaOrgMainReactome</mainClass>

						</manifest>

					</archive>

					<descriptorRefs>

						<descriptorRef>jar-with-dependencies</descriptorRef>

					</descriptorRefs>

				</configuration>

			</plugin>

		</plugins>

	</build>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service-rmi</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service</artifactId>

			<version>[3.3.26,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>${google.gson.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-io</groupId>

			<artifactId>commons-io</artifactId>

			<version>${commons.io.version}</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.apache.httpcomponents</groupId>

			<artifactId>httpclient</artifactId>

			<version>4.5</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0, 3.0.0)</version>

			<scope>provided</scope>

		</dependency>

		<dependency>

			<groupId>com.ximpleware</groupId>

			<artifactId>vtd-xml</artifactId>

			<version>[2.12, 3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.9.2</version>

		</dependency>

		<dependency>

			<groupId>org.json</groupId>

			<artifactId>json</artifactId>

			<version>20180813</version>

		 <type>jar</type>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<dependency>

			<groupId>org.apache.poi</groupId>

			<artifactId>poi</artifactId>

			<version>3.16</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->

		<dependency>

			<groupId>org.apache.poi</groupId>

			<artifactId>poi-ooxml</artifactId>

			<version>3.16</version>

		</dependency>

		<dependency>

			<groupId>org.jsoup</groupId>

			<artifactId>jsoup</artifactId>

			<version>1.11.2</version>

		</dependency>

		<dependency>

			<groupId>commons-lang</groupId>

			<artifactId>commons-lang</artifactId>

			<version>2.6</version>

			<scope>compile</scope>

		</dependency>

        <dependency>

            <groupId>org.mockito</groupId>

            <artifactId>mockito-core</artifactId>

            <version>3.3.3</version>

            <scope>test</scope>

        </dependency>

    </dependencies>

</project>

package eu.dnetlib.data.collector.plugins.httpfilename;

import eu.dnetlib.data.collector.plugins.HttpConnector;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

/**

 * Created by miriam on 07/05/2018.

 */

public class Connector extends HttpConnector implements ConnectorInterface  {

    private String response;

    @Override

    public void get(final String requestUrl) throws CollectorServiceException {

        response = getInputSource(requestUrl);

    }

    @Override

    public String getResponse() {

        return response;

    }

    @Override

    public boolean isStatusOk() {

        return (response != null);

    }

    @Override

    public boolean responseTypeContains(String string) {

        String responseType = getResponseType();

        if (responseType != null)

            return responseType.contains(string);

        return false;

    }

}

package eu.dnetlib.data.collector.plugins.datasets;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.gson.JsonArray;

import com.google.gson.JsonElement;

import com.google.gson.JsonObject;

import com.google.gson.JsonParser;

public class ElasticSearchResponse {

	/** The logger. */

	private static final Log log = LogFactory.getLog(ElasticSearchResponse.class);

	private long total;

	private List<String> xmlRecords;

	public static ElasticSearchResponse createNewResponse(final String response) {

		ElasticSearchResponse item = new ElasticSearchResponse();

		if (response == null) {

			log.fatal("Error: null elasticsearch reponse");

			return null;

		}

		JsonElement jElement = new JsonParser().parse(response);

		JsonObject jobject = jElement.getAsJsonObject();

		if (jobject.has("hits")) {

			item.setTotal(jobject.get("hits").getAsJsonObject().get("total").getAsLong());

			JsonElement hits = ((JsonObject) jobject.get("hits")).get("hits");

			JsonArray hitsObject = hits.getAsJsonArray();

			List<String> records = new ArrayList<String>();

			for (JsonElement elem : hitsObject) {

				JsonObject _source = (JsonObject) ((JsonObject) elem).get("_source");

				String xml = _source.get("xml").getAsString();

				records.add(xml);

			}

			item.setXmlRecords(records);

			return item;

		}

		return null;

	}

	/**

	 * @return the xmlRecords

	 */

	public List<String> getXmlRecords() {

		return xmlRecords;

	}

	/**

	 * @param xmlRecords

	 *            the xmlRecords to set

	 */

	public void setXmlRecords(final List<String> xmlRecords) {

		this.xmlRecords = xmlRecords;

	}

	/**

	 * @return the total

	 */

	public long getTotal() {

		return total;

	}

	/**

	 * @param total

	 *            the total to set

	 */

	public void setTotal(final long total) {

		this.total = total;

	}

}

package eu.dnetlib.data.collector.plugins.datasets;

import java.io.IOException;

import java.io.InputStream;

import java.util.Iterator;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringEscapeUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.entity.StringEntity;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

/**

 * The Class JournalIterator.

 */

public class DatasetsIterator implements Iterable<String>, Iterator<String> {

	/** The logger. */

	private static final Log log = LogFactory.getLog(DatasetsIterator.class);

	/** The base url template. */

	private static String BASE_URL_TEMPLATE = "http://ws.pangaea.de/es/pangaea/panmd/_search?_source=xml&size=%d&from=%d";

	/** The journal id. */

	private String journalId = "";

	/** The journal name. */

	private String journalName = "";

	/** The journal issn. */

	private String journalISSN = "";

	/** The openaire datasource. */

	private String openaireDatasource = "";

	/** The total. */

	private long total;

	/** The from. */

	private int from;

	/** The current iterator. */

	private int currentIterator;

	/** The current response. */

	private ElasticSearchResponse currentResponse;

	/** The request. */

	private RequestField request;

	/** The default size. */

	private static int DEFAULT_SIZE = 10;

	private String projectCordaId;

	private static String RECORD_TEMPLATE = "<datasetsRecord><oaf:projectid xmlns:oaf=\"http://namespace.openaire.eu/oaf\">%s</oaf:projectid>"