D-Net

package eu.dnetlib.data.collector.plugins.schemaorg.httpapi;

import eu.dnetlib.data.collector.plugins.schemaorg.RepositoryIterable;

public interface HttpApiRepositoryIterable extends RepositoryIterable {

}

package eu.dnetlib.data.collector.plugins.excel;

/**

 * Created by miriam on 10/05/2017.

 */

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.net.URL;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Iterator;

import eu.dnetlib.data.collector.plugins.HttpCSVCollectorPlugin;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.poi.ss.usermodel.Cell;

import org.apache.poi.ss.usermodel.DataFormatter;

import org.apache.poi.ss.usermodel.Row;

import org.apache.poi.ss.usermodel.Sheet;

import org.apache.poi.ss.usermodel.Workbook;

import org.apache.poi.xssf.usermodel.XSSFWorkbook;

import org.json.*;

import org.apache.commons.io.FileUtils;

public class Read {

	private static final Log log = LogFactory.getLog(Read.class);

	/** The descriptor. */

	private InterfaceDescriptor descriptor;

	/*private final String EXCEL_FILE_URL ="https://pf.fwf.ac.at/en/research-in-practice/project-finder.xlsx?&&&search%5Bcall%5D=&search%5Bdecision_board_ids%5D=&search%5Bend_date%5D=&search%5Binstitute_name%5D=&search%5Blead_firstname%5D=&search%5Blead_lastname%5D=&search%5Bper_page%5D=10&search%5Bproject_number%5D=&search%5Bproject_title%5D=&search%5Bscience_discipline_id%5D=&search%5Bstart_date%5D=&search%5Bstatus_id%5D=&search%5Bwhat%5D=&action=index&controller=projects&locale=en&per_page=10";

	private final String CSV_FILE_PATH = "//Users//miriam//Documents//svn//mirima//FWF//projects_search2017.05.09.5.csv";

	 private final String argument = "{\"replace\":{\"header\":[{\"from\":\"&\",\"to\":\"and\"}],\"body\":[{\"from\":\"\\n\",\"to\":\" \"}]}," +

	  "\"replace_currency\":[{\"from\":\"$\",\"to\":\"€\"}],"

			   + "\"col_currency\":10}"; */

	private Sheet sheet;

	private CSVFileWriter csv_writer = new CSVFileWriter();

	private HashMap<String,String> map_header = new HashMap<String,String>();

	private HashMap<String,String> map_body = new HashMap<String,String>();

	private int header_row;

	private String file_to_save ;

	private boolean replace_currency = false;

	private String from_currency, to_currency;

	private boolean remove_empty, remove_tmp_file;

	private String remove_id;

	private int column_id;

	private int currency_column;

	private int sheet_number;

	private String tmp_file;

	private String argument;

	private String identifier;

	private HttpCSVCollectorPlugin collector;

	public HttpCSVCollectorPlugin getCollector() {

		return collector;

	}

	public void setCollector(HttpCSVCollectorPlugin collector) {

		this.collector = collector;

	}

	public Read(InterfaceDescriptor descriptor){

		this.descriptor = descriptor;

	}

	private static String getCellValue( Cell cell)

	{

		DataFormatter formatter = new DataFormatter();

		String formattedCellValue = formatter.formatCellValue(cell);

		return formattedCellValue;

	}

	private void copyFile() throws IOException{

		FileUtils.copyURLToFile(new URL(descriptor.getBaseUrl()), new File(tmp_file));

	}

	private void parseDescriptor(){

		HashMap<String, String> params = descriptor.getParams();

		argument = params.get("argument");

		header_row = Integer.parseInt(params.get("header_row"));

		tmp_file = params.get("tmp_file");

		remove_empty = (params.get("remove_empty_lines") == "yes");

		remove_id = params.get("remove_lines_with_id");

		column_id = Integer.parseInt(params.get("col_id"));

		remove_tmp_file = (params.get("remove_tmp_file") == "yes");

		sheet_number = Integer.parseInt(params.get("sheet_number"));

		file_to_save = params.get("file_to_save");

	}

	private void init() throws IOException{

		parseDescriptor();

		log.info("Parsing the arguments");

		parseArguments();

		log.info("Copying the file in temp local file");

		copyFile();

		log.info("Extracting the sheet " + sheet_number);

		FileInputStream fis = new FileInputStream(tmp_file);

		Workbook workbook = new XSSFWorkbook(fis);

		sheet = workbook.getSheetAt(sheet_number);

		fis.close();

		if(remove_tmp_file) {

			File f = new File(tmp_file);

			f.delete();

		}

	}

	private void fillMap(JSONObject json, HashMap<String,String> map, String elem){

		try{

			final JSONArray arr = json.getJSONObject("replace").getJSONArray(elem);

			for(Object entry: arr)

				map.put(((JSONObject)entry).getString("from"), ((JSONObject)entry).getString("to"));

		}catch(Throwable e){

			log.error("Problems filling the map for " + elem);

			throw(e);

		}

	}

	private void parseArguments() {

		if (StringUtils.isNotEmpty(argument)){

			try{

				final JSONObject json = new JSONObject(argument);

				JSONObject tmp = json.getJSONObject("replace");

				if(tmp.has("header"))

					fillMap(json, map_header,"header");

				if(tmp.has("body"))

					fillMap(json,map_body,"body");

//				if(json.has("header"))

//					fillMap(json, map_header,"header");

//				if (json.has("body"))

//					fillMap(json,map_body,"body");

				if(json.has("replace_currency"))

				{

					replace_currency = true	;

					from_currency = json.getJSONArray("replace_currency").getJSONObject(0).getString("from");

					to_currency = json.getJSONArray("replace_currency").getJSONObject(0).getString("to");

				}

				if (json.has("col_currency"))

					currency_column = json.getInt("col_currency");

			}catch(Throwable e){

				log.error("Problems while parsing the argument parameter.");

				throw (e);

			}

		}

	}

	private String applyReplace(String row, HashMap<String,String>replace){

		for(String key: replace.keySet()){

			if(row.contains(key))

				row = row.replace(key, replace.get(key));

		}

		return row;

	}

	private void getHeader(){

		Row row = sheet.getRow(header_row);

		Iterator<Cell> cellIterator = row.cellIterator();

		Cell cell;

		String project = "";

		int count = 0;

		while (cellIterator.hasNext()){

			cell = cellIterator.next();

			final String stringCellValue = cell.getStringCellValue();

			project += applyReplace(stringCellValue,map_header) + ";";

			if(count++ == column_id) identifier = applyReplace(stringCellValue,map_header);

		}

		project = project.substring(0, project.length() -1 );

		csv_writer.setHeader(project.split(";"));

	}

	private void getData(){

		Row row;

		Cell cell;

		String tmp;

		Iterator<Cell>cellIterator;

		for(int row_number = header_row + 1; row_number < sheet.getLastRowNum(); row_number++){

			row = sheet.getRow(row_number);

			if (row != null) {

				cellIterator = row.cellIterator();

				int col_number = 0;

				boolean discard_row = false;

				ArrayList<String> al = new ArrayList<String>();

				while (cellIterator.hasNext() && !discard_row) {

					cell = cellIterator.next();

					tmp = getCellValue(cell).trim();

					tmp = tmp.replace("\n"," ");

					if (col_number == column_id &&

							((remove_empty && tmp.trim().equals("")) ||

									(!remove_id.equals("") && tmp.equals(remove_id))))

						discard_row = true;

					if (replace_currency && col_number == currency_column)

						tmp = tmp.replace(from_currency, to_currency);

					al.add(applyReplace(tmp, map_body));

					col_number++;

				}

				if (!discard_row) {

					csv_writer.addProject(al);

				}

			}

		}

	}

	private void writeCSVFile(){

		csv_writer.writeFile(file_to_save);

	}

	private InterfaceDescriptor prepareHTTPCSVDescriptor(){

		InterfaceDescriptor dex = new InterfaceDescriptor();

		dex.setBaseUrl("file://"+file_to_save);

		HashMap<String, String> params = new HashMap<String, String>();

		params.put("separator", descriptor.getParams().get("separator"));

		params.put("identifier",identifier);

		params.put("quote",descriptor.getParams().get("quote"));

		dex.setParams(params);

		return dex;

	}

	public Iterable<String> parseFile() throws Exception{

		init();

		log.info("Getting header elements");

		getHeader();

		log.info("Getting sheet data");

		getData();

		log.info("Writing the csv file");

		writeCSVFile();

		log.info("Preparing to parse csv");

		return collector.collect(prepareHTTPCSVDescriptor(),"","");

	}

}

package eu.dnetlib.data.collector.plugins.excel;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.plugins.HttpCSVCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Required;

/**

 * Created by miriam on 10/05/2017.

 */

public class ReadExcelPlugin extends AbstractCollectorPlugin{

	private static final Log log = LogFactory.getLog(ReadExcelPlugin.class);

	@Autowired

	HttpCSVCollectorPlugin httpCSVCollectorPlugin;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		Read r = new Read(interfaceDescriptor);

		r.setCollector(httpCSVCollectorPlugin);

		try {

			return r.parseFile();

		}catch(Exception e){

			log.error("Error importing excel file");

			throw new CollectorServiceException(e);

		}

	}

}

package eu.dnetlib.data.collector.plugins.datasources;

import java.io.IOException;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.plugins.HttpConnector;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.io.IOUtils;

import org.springframework.beans.factory.annotation.Autowired;

/**

 * Plugin to collect metadata record about data repositories from re3data.

 * <p>

 * Documentation on re3data API: http://service.re3data.org/api/doc.

 * </p>

 * <p>

 * BaseURL: http://service.re3data.org

 * </p>

 * <p>

 * API to get the list of repos: baseURL + /api/v1/repositories

 * </p>

 * <p>

 * API to get a repository: baseURL + content of link/@href of the above list

 * </p>

 *

 * @author alessia

 *

 */

public class Re3DataCollectorPlugin extends AbstractCollectorPlugin {

	private String repositoryListPath = "/api/v1/repositories";

	@Autowired

	private HttpConnector httpConnector;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		String repositoryListURL = interfaceDescriptor.getBaseUrl() + repositoryListPath;

		String input;

		try {

			input = httpConnector.getInputSource(repositoryListURL);

			return new Re3DataRepositoriesIterator(IOUtils.toInputStream(input, "UTF-8"), interfaceDescriptor.getBaseUrl(), getHttpConnector());

		} catch (IOException e) {

			throw new CollectorServiceException(e);

		}

	}

	public String getRepositoryListPath() {

		return repositoryListPath;

	}

	public void setRepositoryListPath(final String repositoryListPath) {

		this.repositoryListPath = repositoryListPath;

	}

	public HttpConnector getHttpConnector() {

		return httpConnector;

	}

	public void setHttpConnector(final HttpConnector httpConnector) {

		this.httpConnector = httpConnector;

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex;

import eu.dnetlib.data.collector.plugins.schemaorg.Utils;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import org.apache.commons.io.FileUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.io.*;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.*;

public class SitemapFileIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(SitemapFileIterator.class);

	public static class Options {

		public enum SitemapFileType{

			Text,

			GZ

		}

		public enum SitemapSchemaType{

			Text,

			Xml

		}

		public Options(){}

		public Options(URL fileUrl, Charset charset, SitemapSchemaType schemaType, SitemapFileType fileType) {

			this.fileUrl = fileUrl;

			this.charset = charset;

			this.schemaType = schemaType;

			this.fileType = fileType;

		}

		private SitemapFileType fileType;

		private SitemapSchemaType schemaType;

		private URL fileUrl;

		private Charset charset;

		public Charset getCharset() {

			return charset;

		}

		public void setCharset(Charset charset) {

			this.charset = charset;

		}

		public URL getFileUrl() {

			return fileUrl;

		}

		public void setFileUrl(URL fileUrl) {

			this.fileUrl = fileUrl;

		}

		public SitemapFileType getFileType() {

			return fileType;

		}

		public void setFileType(SitemapFileType fileType) {

			this.fileType = fileType;

		}

		public SitemapSchemaType getSchemaType() {

			return schemaType;

		}

		public void setSchemaType(SitemapSchemaType schemaType) {

			this.schemaType = schemaType;

		}

		@Override

		public Object clone(){

			Options clone = new Options();

			clone.setCharset(this.getCharset());

			clone.setFileType(this.getFileType());

			clone.setFileUrl(this.getFileUrl());

			clone.setSchemaType(this.getSchemaType());

			return clone;

		}

	}

	private Options options;

	private File downloadedFile;

	private File contentFile;

	private Queue<String> locations;

	public SitemapFileIterator(Options options){

		this.options = options;

	}

	public void bootstrap() {

		LinkedList<String> endpoints = null;

		try {

			log.debug(String.format("bootstrapping sitemapindex file access for sitemapindex %s", this.options.getFileUrl()));

			this.downloadedFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp");

			this.downloadedFile.deleteOnExit();

			FileUtils.copyURLToFile(this.options.getFileUrl(), this.downloadedFile);

			log.debug(String.format("downloaded file: %s has size %d", this.downloadedFile.toString(), this.downloadedFile.length()));

			switch (this.options.getFileType()) {

				case Text: {

					this.contentFile = this.downloadedFile;

					break;

				}

				case GZ: {

					this.contentFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp");

					this.contentFile.deleteOnExit();

					Utils.decompressGZipTo(this.downloadedFile, this.contentFile);

					log.debug(String.format("extracted gz file: %s has size %d", this.contentFile.toString(), this.contentFile.length()));

					break;

				}

				default:

					throw new CollectorServiceException("unrecognized file type " + this.options.getFileType());

			}

			List<String> content = this.collectContentLocations();

			log.debug(String.format("extracted %d sitemapindex endpoints", content.size()));

			endpoints = new LinkedList<>(content);

		}catch(Exception ex){

			log.error(String.format("error processing sitemapindex %s. returning 0 endpoints",this.options.getFileUrl()), ex);

			endpoints = new LinkedList<>();

		}finally {

			if (this.contentFile != null) {

				this.contentFile.delete();

			}

			if (this.downloadedFile != null) {

				this.downloadedFile.delete();

			}

		}

		this.locations = endpoints;

	}

	private List<String> collectContentLocations() throws Exception{

		switch(this.options.getSchemaType()) {

			case Text:{

				return this.collectTextContentLocations();

			}

			case Xml:{

				return this.collectXmlContentLocations();

			}

			default: throw new CollectorServiceException("unrecognized file type "+this.options.getFileType());

		}

	}

	private List<String> collectTextContentLocations() throws Exception {

		log.debug(String.format("reading endpoint locations from text sitemapindex"));

		try (FileInputStream in = new FileInputStream(this.contentFile)) {

			return IOUtils.readLines(in, this.options.getCharset());

		}

	}

	private List<String> collectXmlContentLocations() throws Exception {

		log.debug(String.format("reading endpoint locations from xml sitemapindex"));

		return Utils.collectAsStrings(this.contentFile,"/urlset/url/loc/text()");

	}

	@Override

	public boolean hasNext() {

		return !this.locations.isEmpty();

	}

	@Override

	public String next() {

		return this.locations.poll();

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.util.Iterator;

import java.util.NoSuchElementException;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.TimeUnit;

public class RepositoryQueueIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(RepositoryQueueIterator.class);

	public static class Options {

		private Boolean blockPolling;

		private long pollTimeout;

		private TimeUnit pollTimeoutUnit;

		public Boolean getBlockPolling() {

			return blockPolling;

		}

		public void setBlockPolling(Boolean blockPolling) {

			this.blockPolling = blockPolling;

		}

		public long getPollTimeout() {

			return pollTimeout;

		}

		public void setPollTimeout(long pollTimeout) {

			this.pollTimeout = pollTimeout;

		}

		public TimeUnit getPollTimeoutUnit() {

			return pollTimeoutUnit;

		}

		public void setPollTimeoutUnit(TimeUnit pollTimeoutUnit) {

			this.pollTimeoutUnit = pollTimeoutUnit;

		}

	}

	private ArrayBlockingQueue<String> queue;

	private Options options;

	private boolean hasTerminated;

	public RepositoryQueueIterator(Options options, ArrayBlockingQueue<String> queue) {

		this.options = options;

		this.queue = queue;

		this.hasTerminated = false;

	}

	@Override

	public boolean hasNext() {

		if(this.hasTerminated) return false;

		return true;

	}

	@Override

	public String next() {

		String next = this.poll();

		log.debug("next endpoint to process: " + next);

		if (next != null && next.equalsIgnoreCase(RepositoryIterable.TerminationHint)) {

			log.debug("no more endpoints to process");

			this.hasTerminated = true;

			next = null;

		}

		return next;

	}

	private String poll(){

		String item = null;

		log.debug("retrieving endpoint from queue");

		log.debug("queue size: " + queue.size());

		if(this.options.getBlockPolling()) {

			try {

				item = this.queue.poll(this.options.getPollTimeout(), this.options.getPollTimeoutUnit());

			} catch (InterruptedException ex) {

				log.warn(String.format("could not poll elements from queue for more than %s %s. throwing", this.options.getPollTimeout(), this.options.getPollTimeoutUnit()));

				throw new NoSuchElementException(ex.getMessage());

			}

		}

		else {

			item = this.queue.poll();

		}

		log.debug("retrieved endpoint from queue");

		log.debug("queue size: " + queue.size());

		return item;

	}

}

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-collector-plugins</artifactId>

	<version>1.8.1</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-collector-plugins/tags/dnet-collector-plugins-1.8.1</developerConnection>

	</scm>

	<build>

		<plugins>

			<plugin>

				<artifactId>maven-assembly-plugin</artifactId>

				<configuration>

					<archive>

						<manifest>

							<mainClass>eu.dnetlib.data.collector.plugins.schemaorg.SchemaOrgMainReactome</mainClass>

						</manifest>

					</archive>

					<descriptorRefs>

						<descriptorRef>jar-with-dependencies</descriptorRef>

					</descriptorRefs>

				</configuration>

			</plugin>

		</plugins>

	</build>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service-rmi</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service</artifactId>

			<version>[3.3.26,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>${google.gson.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-io</groupId>

			<artifactId>commons-io</artifactId>

			<version>${commons.io.version}</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.apache.httpcomponents</groupId>

			<artifactId>httpclient</artifactId>

			<version>4.5</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0, 3.0.0)</version>

			<scope>provided</scope>

		</dependency>

		<dependency>

			<groupId>com.ximpleware</groupId>

			<artifactId>vtd-xml</artifactId>

			<version>[2.12, 3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.9.2</version>

		</dependency>

		<dependency>

			<groupId>org.json</groupId>

			<artifactId>json</artifactId>

			<version>20180813</version>

		 <type>jar</type>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<dependency>

			<groupId>org.apache.poi</groupId>

			<artifactId>poi</artifactId>

			<version>3.16</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->

		<dependency>

			<groupId>org.apache.poi</groupId>

			<artifactId>poi-ooxml</artifactId>

			<version>3.16</version>

		</dependency>

		<dependency>

			<groupId>org.jsoup</groupId>

			<artifactId>jsoup</artifactId>

			<version>1.11.2</version>

		</dependency>

		<dependency>

			<groupId>commons-lang</groupId>

			<artifactId>commons-lang</artifactId>

			<version>2.6</version>

			<scope>compile</scope>

		</dependency>

        <dependency>

            <groupId>org.mockito</groupId>

            <artifactId>mockito-core</artifactId>

            <version>3.3.3</version>

            <scope>test</scope>

        </dependency>

    </dependencies>

</project>

package eu.dnetlib.data.collector.plugins.schemaorg;

import java.util.Iterator;

public interface RepositoryIterable extends Iterable<String> {

	public static String TerminationHint = "df667391-676d-4c0f-9c40-426b1001607a";

}

package eu.dnetlib.data.collector.plugins.gtr2;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentHelper;

import org.joda.time.DateTime;

import org.joda.time.format.DateTimeFormat;

import org.joda.time.format.DateTimeFormatter;

import eu.dnetlib.data.collector.plugins.HttpConnector;

import eu.dnetlib.data.collector.rmi.CollectorServiceRuntimeException;

public class Gtr2Helper {

	private static final Log log = LogFactory.getLog(Gtr2Helper.class); // NOPMD by marko on 11/24/08 5:02 PM

	private static final HttpConnector connector = new HttpConnector();

	private static final DateTimeFormatter simpleDateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");

	private static final int MAX_ATTEMPTS = 10;

	public static String cleanURL(final String url) {

		String cleaned = url;

		if (cleaned.contains("gtr.gtr")) {

			cleaned = cleaned.replace("gtr.gtr", "gtr");

		}

		if (cleaned.startsWith("http://")) {

			cleaned = cleaned.replaceFirst("http://", "https://");

		}

		return cleaned;

	}

	public static Document loadURL(final String url) {

		final String cleanUrl = cleanURL(url);

		return loadURL(cleanUrl, 0);

	}

	private static Document loadURL(final String cleanUrl, final int attempt) {

		try {

			log.debug("  * Downloading Url: " + cleanUrl);

			final byte[] bytes = connector.getInputSource(cleanUrl).getBytes("UTF-8");

			return DocumentHelper.parseText(new String(bytes));

		} catch (final Throwable e) {

			log.error("Error dowloading url: " + cleanUrl + ", attempt = " + attempt, e);

			if (attempt < MAX_ATTEMPTS) {

				try {

					Thread.sleep(60000); // I wait for a minute

				} catch (final InterruptedException e1) {

					throw new CollectorServiceRuntimeException("Error dowloading url: " + cleanUrl, e);

				}

				return loadURL(cleanUrl, attempt + 1);

			} else {

				throw new CollectorServiceRuntimeException("Error dowloading url: " + cleanUrl, e);

			}

		}

	}

	public static DateTime parseDate(final String s) {

		return DateTime.parse(s.contains("T") ? s.substring(0, s.indexOf("T")) : s, simpleDateTimeFormatter);

	}

	public static boolean isAfter(final String d, final DateTime fromDate) {

		return StringUtils.isNotBlank(d) && Gtr2Helper.parseDate(d).isAfter(fromDate);

	}

}

package eu.dnetlib.data.collector.plugins.gtr2;

import java.util.Iterator;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public abstract class AbstractGtr2CollectorPlugin extends AbstractCollectorPlugin {

	@Override

	public final Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

		throws CollectorServiceException {

		if (fromDate != null && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) { throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + fromDate); }

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		final String startPage = interfaceDescriptor.getParams().get("startPage");

		final String endPage = interfaceDescriptor.getParams().get("endPage");

		return () -> {

			try {

				return createIterator(baseUrl, fromDate, startPage, endPage);

			} catch (final CollectorServiceException e) {

				throw new RuntimeException(e);

			}

		};

	}

	protected abstract Iterator<String> createIterator(String baseUrl, final String fromDate, String startPage, String endPage)

		throws CollectorServiceException;

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.util.Iterator;

import java.util.concurrent.ArrayBlockingQueue;

public class SchemaOrgIterable implements Iterable<String> {

	private static final Log log = LogFactory.getLog(SchemaOrgIterable.class);

	public static class Options {

		private EndpointAccessIterator.Options endpointAccessOptions;

		private DatasetMappingIterator.Options datasetMappingOptions;

		public EndpointAccessIterator.Options getEndpointAccessOptions() {

			return endpointAccessOptions;

		}

		public void setEndpointAccessOptions(EndpointAccessIterator.Options endpointAccessOptions) {

			this.endpointAccessOptions = endpointAccessOptions;

		}

		public DatasetMappingIterator.Options getDatasetMappingOptions() {

			return datasetMappingOptions;

		}

		public void setDatasetMappingOptions(DatasetMappingIterator.Options datasetMappingOptions) {

			this.datasetMappingOptions = datasetMappingOptions;

		}

	}

	private Options options;

	private RepositoryIterable repository;

	public SchemaOrgIterable(Options options, RepositoryIterable repository){

		this.options = options;

		this.repository = repository;

	}

	@Override

	public Iterator<String> iterator() {

		Iterator<String> repositoryIterator = this.repository.iterator();

		EndpointAccessIterator endpointAccessIterator = new EndpointAccessIterator(options.getEndpointAccessOptions(), repositoryIterator);

		DatasetMappingIterator datasetMappingIterator = new DatasetMappingIterator(options.getDatasetMappingOptions(), endpointAccessIterator);

		return datasetMappingIterator;

	}

}

package eu.dnetlib.data.collector.plugins.httpfilename;

import java.util.*;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.TimeUnit;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.json.JSONObject;

import org.json.XML;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

/**

 * Created by miriam on 04/05/2018.

 */

public class HTTPWithFileNameCollectorIterable implements Iterable<String> {

    private static final Log log = LogFactory.getLog(HTTPWithFileNameCollectorIterable.class);

    private static final String JUNK = "<resource><url>%s</url><DOI>JUNK</DOI></resource>";

    public static final String APP_JSON = "application/json";

    public static final String APP_XML = "application/xml";

    public static final String TEXT_HTML = "text/html";

    private final ArrayBlockingQueue<String> queue = new ArrayBlockingQueue<String>(100);

    private String filterParam;

    int total = 0;

    int filtered = 0;

    public HTTPWithFileNameCollectorIterable(String startUrl, String filter){

        this.filterParam = filter;

        Thread ft = new Thread(new FillMetaQueue(startUrl) );

        ft.start();

    }

    @Override

    public Iterator<String> iterator() {

        return new HttpWithFileNameCollectorIterator(queue);

    }

    private class FillMetaQueue implements Runnable {

        final Connector c = new Connector();

        private final List<String> metas = Collections.synchronizedList(new ArrayList<String>());

        private final List<String> urls = Collections.synchronizedList(new ArrayList<>());

        public FillMetaQueue(String startUrl){

            if(!startUrl.isEmpty()){

                urls.add(startUrl);

            }

        }

        public void fillQueue() {

            String url;

            while((metas.size()>0 || urls.size() > 0 )) {

                log.debug("metas.size() = " + metas.size() + " urls.size() = " + urls.size() + " queue.size() = " +queue.size());

                if (metas.size() > 0) {

                    url = metas.remove(0);

                    try {

                        c.get(url);

                    } catch (CollectorServiceException e) {

                        log.info("Impossible to collect url: " + url + " error: " + e.getMessage());

                    }

                    if(c.isStatusOk()){

                        try {

                            String ret = c.getResponse();

                            if (ret != null && ret.length()>0) {

                                if (!containsFilter(ret))

                                    queue.put(addFilePath(ret, url, url.endsWith(".json")));

                                    //queue.offer(addFilePath(ret, url, url.endsWith(".json")), HttpWithFileNameCollectorIterator.waitTime, TimeUnit.SECONDS);

                                else

                                    filtered++;

                                total++;

                            }

                        } catch (InterruptedException e) {

                            log.info("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );

                        }

                    }

                } else {

                    url = urls.remove(0);

                    try {

                        c.get(url);

                    } catch (CollectorServiceException e) {

                        log.info("Impossible to collect url: " + url + " error: " + e.getMessage());

                    }

                    if(c.isStatusOk()) {

                        if (c.responseTypeContains(TEXT_HTML)){

                            recurFolder(c.getResponse(), url);

                        } else if(c.responseTypeContains(APP_JSON) || c.responseTypeContains(APP_XML)){

                            try {

                                final String element = addFilePath(c.getResponse(), url, c.responseTypeContains(APP_JSON));

                                //queue.offer(element, HttpWithFileNameCollectorIterator.waitTime, TimeUnit.SECONDS);

                                queue.put(element);

                            } catch (InterruptedException e) {

                                log.info("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );

                            }

                        }

                    }

                }

            }

            try {

                //queue.offer(HttpWithFileNameCollectorIterator.TERMINATOR, HttpWithFileNameCollectorIterator.waitTime, TimeUnit.SECONDS);

                queue.put(HttpWithFileNameCollectorIterator.TERMINATOR);

            } catch (InterruptedException e) {

                throw new IllegalStateException(String.format("could not add element to queue for more than %s%s", HttpWithFileNameCollectorIterator.waitTime, TimeUnit.SECONDS), e);

            }

        }

        private boolean containsFilter(String meta){

            if (filterParam == null || filterParam.isEmpty())

                return false;

            String[] filter = filterParam.split(";");

            for(String item:filter){

                if (meta.contains(item))

                    return true;

            }

            return false;

        }

        private String addFilePath(String meta, String url, boolean isJson){

            String path = url.replace("metadata", "pdf");

            try {

                if(isJson)

                    meta = meta.substring(0, meta.length() - 1) + ",'downloadFileUrl':'" + path.substring(0, path.indexOf(".json")) + ".pdf'}";

                else {

                    if (meta.contains("<!DOCTYPE")) {

                        meta = meta.substring(meta.indexOf("<!DOCTYPE"));

                        meta = meta.substring(meta.indexOf(">") + 1);

                    }

                    int index = meta.lastIndexOf("</");

                    meta = meta.substring(0, index) + "<downloadFileUrl>" + path.substring(0, path.indexOf(".xml")) + ".pdf</downloadFileUrl>" + meta.substring(index);

                }

            } catch(Exception ex) {

                log.info("not file with extension .json or .xml");

            }

            if(isJson) {

                try {

                    return XML.toString(new JSONObject("{'resource':" + meta + "}"));

                } catch(Exception e) {

                    log.fatal("Impossible to transform json object to xml \n" + meta + "\n " + e.getMessage() + "\n" + url);

                   // throw new RuntimeException();

                    final String junk = String.format(JUNK, url);

                    log.warn("returning " + junk);

                    return junk;

                }

            }

            return meta;

        }

        private void recurFolder(String text, String url){

            Document doc = Jsoup.parse(text);

            Elements links = doc.select("a");

            for(Element e:links){

                if (!e.text().equals("../")){

                    String file = e.attr("href");

                    if(file.endsWith(".json") || file.endsWith(".xml"))

                        metas.add(url+file);

                    else

                        urls.add(url+file);

                }

            }

        }

        @Override

        public void run() {

            fillQueue();

        }

    }

}

package eu.dnetlib.data.collector.plugins.datasets;

import java.io.IOException;

import java.io.InputStream;

import java.util.Iterator;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringEscapeUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.entity.StringEntity;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

/**

 * The Class JournalIterator.

 */

public class DatasetsIterator implements Iterable<String>, Iterator<String> {

	/** The logger. */

	private static final Log log = LogFactory.getLog(DatasetsIterator.class);

	/** The base url template. */

	private static String BASE_URL_TEMPLATE = "http://ws.pangaea.de/es/pangaea/panmd/_search?_source=xml&size=%d&from=%d";

	/** The journal id. */

	private String journalId = "";

	/** The journal name. */

	private String journalName = "";

	/** The journal issn. */

	private String journalISSN = "";

	/** The openaire datasource. */

	private String openaireDatasource = "";

	/** The total. */

	private long total;

	/** The from. */

	private int from;

	/** The current iterator. */

	private int currentIterator;

	/** The current response. */

	private ElasticSearchResponse currentResponse;

	/** The request. */

	private RequestField request;

	/** The default size. */

	private static int DEFAULT_SIZE = 10;

	private String projectCordaId;

	private static String RECORD_TEMPLATE = "<datasetsRecord><oaf:projectid xmlns:oaf=\"http://namespace.openaire.eu/oaf\">%s</oaf:projectid>"

			+ "<journal name='%s' issn='%s' datasourceid = '%s'/><metadata>%s</metadata></datasetsRecord>";

	/**

	 * Instantiates a new journal iterator.

	 * 

	 * @param request

	 *            the request

	 */

	public DatasetsIterator(final RequestField request, final String projectCordaId, final PangaeaJournalInfo info) {

		this.request = request;

		this.setProjectCordaId(projectCordaId);

		if (info != null) {

			this.setJournalId(info.getJournalId());

			this.setJournalName(StringEscapeUtils.escapeXml(info.getJournalName()));

			this.setJournalISSN(info.getJournalISSN());

			this.setOpenaireDatasource(info.getDatasourceId());

		}

		log.debug("Start Iterator");

	}

	/**

	 * Execute query.

	 * 

	 * @param from

	 *            the from

	 * @param size

	 *            the size

	 * @return the string

	 */

	private String executeQuery(final int from, final int size) {

		log.debug("executing query " + this.request.getQuery().getTerm());

		log.debug(String.format("from:%d size:%d", from, size));

		CloseableHttpResponse response = null;

		InputStream responseBody = null;

		CloseableHttpClient httpclient = HttpClients.createDefault();

		try {

			HttpPost post = new HttpPost(String.format(BASE_URL_TEMPLATE, size, from));

			Gson g = new GsonBuilder().disableHtmlEscaping().create();

			StringEntity entry = new StringEntity(g.toJson(this.request));

			post.setEntity(entry);

			long start = System.currentTimeMillis();

			response = httpclient.execute(post);

			int statusCode = response.getStatusLine().getStatusCode();

			if (statusCode == 200) {

				responseBody = response.getEntity().getContent();

				String s = IOUtils.toString(responseBody);

				log.debug("Request done in " + (System.currentTimeMillis() - start) + " ms");

				responseBody.close();

				return s;

			}

			return null;

		} catch (Exception e) {

			log.error("Error on executing query :" + request.getQuery().getTerm(), e);

			return null;

		} finally {

			try {

				responseBody.close();

				response.close();

				httpclient.close();

			} catch (IOException e) {

				log.error("Can't close connections gracefully", e);

			}

		}

	}

	/**

	 * Gets the journal id.

	 * 

	 * @return the journalId

	 */

	public String getJournalId() {

		return journalId;

	}

	/**

	 * Sets the journal id.

	 * 

	 * @param journalId

	 *            the journalId to set

	 */

	public void setJournalId(final String journalId) {

		this.journalId = journalId;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.Iterator#hasNext()

	 */

	@Override

	public boolean hasNext() {

		return (from + currentIterator) < total;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.Iterator#next()

	 */

	@Override

	public String next() {

		String xml = String.format(RECORD_TEMPLATE, this.projectCordaId, this.journalName, this.journalISSN, this.openaireDatasource, currentResponse

				.getXmlRecords().get(currentIterator));

		currentIterator++;

		if (currentIterator == DEFAULT_SIZE) {

			getNextItem();

		}

		return xml;

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.util.Iterator#remove()

	 */

	@Override

	public void remove() {

		throw new UnsupportedOperationException();

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see java.lang.Iterable#iterator()

	 */

	@Override

	public Iterator<String> iterator() {

		from = 0;

		total = 0;

		getNextItem();

		return this;

	}

	/**

	 * Gets the next item.

	 * 

	 * @return the next item

	 */

	private void getNextItem() {

		from += currentIterator;

		currentResponse = ElasticSearchResponse.createNewResponse(executeQuery(from, DEFAULT_SIZE));

		total = currentResponse == null ? 0 : currentResponse.getTotal();

		log.debug("from : " + from + " total of the request is " + total);

		currentIterator = 0;

	}

	/**

	 * @return the projectCordaId

	 */

	public String getProjectCordaId() {

		return projectCordaId;

	}

	/**

	 * @param projectCordaId

	 *            the projectCordaId to set

	 */

	public void setProjectCordaId(final String projectCordaId) {

		this.projectCordaId = projectCordaId;

	}

	/**

	 * @return the journalName

	 */

	public String getJournalName() {

		return journalName;

	}

	/**

	 * @param journalName

	 *            the journalName to set

	 */

	public void setJournalName(final String journalName) {

		this.journalName = journalName;

	}

	/**

	 * @return the journalISSN

	 */

	public String getJournalISSN() {

		return journalISSN;

	}

	/**

	 * @param journalISSN

	 *            the journalISSN to set

	 */

	public void setJournalISSN(final String journalISSN) {

		this.journalISSN = journalISSN;

	}

	/**

	 * @return the openaireDatasource

	 */

	public String getOpenaireDatasource() {

		return openaireDatasource;

	}

	/**

	 * @param openaireDatasource

	 *            the openaireDatasource to set

	 */

	public void setOpenaireDatasource(final String openaireDatasource) {

		this.openaireDatasource = openaireDatasource;

	}

}

package eu.dnetlib.data.collector.plugins.gtr2;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Iterator;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import java.util.Queue;

import java.util.function.Function;

import org.apache.commons.lang.math.NumberUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

import org.joda.time.DateTime;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.CollectorServiceRuntimeException;

public abstract class Gtr2Iterator implements Iterator<String> {

	public static final int PAGE_SIZE = 20;

	private static final Log log = LogFactory.getLog(Gtr2Iterator.class);

	private final String baseUrl;

	private int currPage;

	private int endPage;

	private boolean incremental = false;

	private DateTime fromDate;

	private final Map<String, String> cache = new HashMap<>();

	private final Queue<String> queue = new LinkedList<>();

	private String nextElement;

	public Gtr2Iterator(final String baseUrl, final String fromDate, final String startPage, final String endPage)

		throws CollectorServiceException {

		this.baseUrl = baseUrl;

		this.currPage = NumberUtils.toInt(startPage, 1);

		this.endPage = NumberUtils.toInt(endPage, Integer.MAX_VALUE);

		this.incremental = StringUtils.isNotBlank(fromDate);

		if (this.incremental) {

			this.fromDate = Gtr2Helper.parseDate(fromDate);