D-Net

package eu.dnetlib.data.collector.plugins.schemaorg;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.io.FileUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.log4j.ConsoleAppender;

import org.apache.log4j.Level;

import org.apache.log4j.Logger;

import org.apache.log4j.PatternLayout;

import java.io.File;

import java.nio.charset.StandardCharsets;

import java.util.HashMap;

import java.util.concurrent.TimeUnit;

public class SchemaOrgMainKaggle {

    private static final Log log = LogFactory.getLog(SchemaOrgMainKaggle.class);

    public static void main(String[] args) throws Exception {

        ConsoleAppender console = new ConsoleAppender();

        console.setLayout(new PatternLayout("%d [%p|%c|%C{1}] %m%n"));

        console.setThreshold(Level.DEBUG);

        console.activateOptions();

        Logger.getLogger("eu.dnetlib.data.collector.plugins").addAppender(console);

        HashMap<String,String> params = new HashMap<>();

        params.put("consumerBlockPolling", Boolean.toString(true));

        params.put("consumerBlockPollingTimeout", "2");

        params.put("consumerBlockPollingTimeoutUnit", TimeUnit.MINUTES.toString());

        params.put("endpointCharset", StandardCharsets.UTF_8.name());

        params.put("updatedDateFormat", "YYYY-MM-DD");

        params.put("createdDateFormat", "YYYY-MM-DD");

        params.put("publicationDateFormat", "YYYY-MM-DD");

        params.put("contributorFallbackType", DatasetDocument.Contributor.ContributorType.Other.toString());

        params.put("identifierFallbackType", DatasetDocument.Identifier.IdentifierType.Handle.toString());

        params.put("identifierFallbackURL", Boolean.toString(true));

        params.put("identifierMappingARK", "ark, ARK");

        params.put("identifierMappingDOI", "doi, DOI");

        params.put("identifierMappingHandle", "Handle, HANDLE");

        params.put("identifierMappingPURL", "purl, PURL");

        params.put("identifierMappingURN", "urn, URN");

        params.put("identifierMappingURL", "url, URL");

        params.put("repositoryAccessType", "httpapi-kaggle");

        params.put("httpapi-kaggle_queueSize", "100");

        params.put("httpapi-kaggle_APICharset", StandardCharsets.UTF_8.name());

        params.put("httpapi-kaggle_queryUrl", "https://www.kaggle.com/datasets_v2.json?sortBy=updated&group=public&page={PAGE}&pageSize=20&size=sizeAll&filetype=fileTypeAll&license=licenseAll");

        params.put("httpapi-kaggle_queryPagePlaceholder", "{PAGE}");

        params.put("httpapi-kaggle_responsePropertyTotalDataset", "totalDatasetListItems");

        params.put("httpapi-kaggle_responsePropertyDatasetList", "datasetListItems");

        params.put("httpapi-kaggle_responsePropertyDatasetUrl", "datasetUrl");

        params.put("httpapi-kaggle_responseBaseDatasetUrl", "https://www.kaggle.com");

        params.put("httpapi-kaggle_producerBlockPollingTimeout", "2");

        params.put("httpapi-kaggle_producerBlockPollingTimeoutUnit", TimeUnit.MINUTES.toString());

        InterfaceDescriptor descriptor = new InterfaceDescriptor();

        descriptor.setId("schema.org - kaggle");

        descriptor.setBaseUrl("https://www.kaggle.com");

        descriptor.setParams(params);

        SchemaOrgPlugin schemaOrgPlugin = new SchemaOrgPlugin();

        Iterable<String> iterable = schemaOrgPlugin.collect(descriptor, null, null);

        String outDir = params.get("repositoryAccessType");

        log.info("saving content in " + outDir);

        File directory = new File(outDir);

        if (directory.exists()) {

            log.info(directory.getAbsolutePath() + " exists, cleaning up");

            FileUtils.deleteDirectory(directory);

        }

        FileUtils.forceMkdir(directory);

        Utils.writeFiles(iterable, outDir);

    }

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.plugins.schemaorg.httpapi.kaggle.KaggleRepositoryIterable;

import eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex.SitemapFileIterator;

import eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex.SitemapIndexIterator;

import eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex.SitemapIndexRepositoryIterable;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.net.MalformedURLException;

import java.net.URL;

import java.nio.charset.StandardCharsets;

import java.util.concurrent.TimeUnit;

public class SchemaOrgPlugin extends AbstractCollectorPlugin {

    private static final Log log = LogFactory.getLog(SchemaOrgPlugin.class);

    public String hello(){

        return "hello";

    }

    @Override

    public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate) throws CollectorServiceException {

        try {

			RepositoryIterable repository = null;

        	String repositoryAccessType = Utils.getAsString(interfaceDescriptor.getParams(), "repositoryAccessType", null);

        	switch(repositoryAccessType) {

				case "sitemapindex": {

					SitemapIndexRepositoryIterable.Options repositoryOptions = this.compileSitemapIndexRepositoryOptions(interfaceDescriptor);

					SitemapIndexRepositoryIterable repositoryIterable = new SitemapIndexRepositoryIterable(repositoryOptions);

					repositoryIterable.bootstrap();

					repository = repositoryIterable;

					break;

				}

				case "httpapi-kaggle": {

					KaggleRepositoryIterable.Options repositoryOptions = this.compileKaggleRepositoryOptions(interfaceDescriptor);

					KaggleRepositoryIterable repositoryIterable = new KaggleRepositoryIterable(repositoryOptions);

					repositoryIterable.bootstrap();

					repository = repositoryIterable;

					break;

				}

				default:

					throw new CollectorServiceException(String.format("unrecognized repository access type ", repositoryAccessType));

			}

			SchemaOrgIterable.Options schemaOrgOptions = this.compileSchemaOrgOptions(interfaceDescriptor);

            SchemaOrgIterable iterable = new SchemaOrgIterable(schemaOrgOptions, repository);

            return iterable;

        } catch (Exception e) {

            throw new CollectorServiceException("Could not create iterator", e);

        }

    }

	private KaggleRepositoryIterable.Options compileKaggleRepositoryOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		KaggleRepositoryIterable.Options kaggleRepositoryOptions = new KaggleRepositoryIterable.Options();

		kaggleRepositoryOptions.setQueueSize(Utils.getAsInt(interfaceDescriptor.getParams(), "httpapi-kaggle_queueSize", 100));

		kaggleRepositoryOptions.setPutTimeout(Utils.getAsLong(interfaceDescriptor.getParams(), "httpapi-kaggle_producerBlockPollingTimeout", 20));

		kaggleRepositoryOptions.setPutTimeoutUnit(Utils.getAsEnum(interfaceDescriptor.getParams(), "httpapi-kaggle_producerBlockPollingTimeoutUnit", TimeUnit.MINUTES, TimeUnit.class));

		kaggleRepositoryOptions.setCharset(Utils.getAsCharset(interfaceDescriptor.getParams(), "httpapi-kaggle_APICharset", StandardCharsets.UTF_8));

		kaggleRepositoryOptions.setQueryUrl(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_queryUrl", null));

		kaggleRepositoryOptions.setQueryPagePlaceholder(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_queryPagePlaceholder", "{PAGE}"));

		kaggleRepositoryOptions.setResponsePropertyTotalDataset(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_responsePropertyTotalDataset", "totalDatasetListItems"));

		kaggleRepositoryOptions.setResponsePropertyDatasetList(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_responsePropertyDatasetList", "datasetListItems"));

		kaggleRepositoryOptions.setResponsePropertyDatasetUrl(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_responsePropertyDatasetUrl", "datasetUrl"));

		kaggleRepositoryOptions.setResponseBaseDatasetUrl(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_responseBaseDatasetUrl", interfaceDescriptor.getBaseUrl()));

		kaggleRepositoryOptions.setRepositoryQueueIteratorOptions(this.compileRepositoryQueueOptions(interfaceDescriptor));

		return kaggleRepositoryOptions;

	}

    private SitemapIndexIterator.Options compileSitemapIndexOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		SitemapIndexIterator.Options sitemapIndexIteratorOptions = new SitemapIndexIterator.Options();

		sitemapIndexIteratorOptions.setCharset(Utils.getAsCharset(interfaceDescriptor.getParams(), "sitemap_IndexCharset", StandardCharsets.UTF_8));

		sitemapIndexIteratorOptions.setIndexUrl(new URL(interfaceDescriptor.getBaseUrl()));

		return sitemapIndexIteratorOptions;

	}

	private SitemapFileIterator.Options compileSitemapFileOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		SitemapFileIterator.Options sitemapFileIteratorOptions = new SitemapFileIterator.Options();

		sitemapFileIteratorOptions.setCharset(Utils.getAsCharset(interfaceDescriptor.getParams(), "sitemap_FileCharset", StandardCharsets.UTF_8));

		sitemapFileIteratorOptions.setSchemaType(Utils.getAsEnum(interfaceDescriptor.getParams(), "sitemap_FileSchema", SitemapFileIterator.Options.SitemapSchemaType.Xml, SitemapFileIterator.Options.SitemapSchemaType.class));

		sitemapFileIteratorOptions.setFileType(Utils.getAsEnum(interfaceDescriptor.getParams(), "sitemap_FileType", SitemapFileIterator.Options.SitemapFileType.Text, SitemapFileIterator.Options.SitemapFileType.class));

		return sitemapFileIteratorOptions;

	}

	private RepositoryQueueIterator.Options compileRepositoryQueueOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		RepositoryQueueIterator.Options repositoryQueueIteratorOptions = new RepositoryQueueIterator.Options();

		repositoryQueueIteratorOptions.setBlockPolling(Utils.getAsBoolean(interfaceDescriptor.getParams(), "consumerBlockPolling", true));

		repositoryQueueIteratorOptions.setPollTimeout(Utils.getAsLong(interfaceDescriptor.getParams(), "consumerBlockPollingTimeout", 2));

		repositoryQueueIteratorOptions.setPollTimeoutUnit(Utils.getAsEnum(interfaceDescriptor.getParams(), "consumerBlockPollingTimeoutUnit", TimeUnit.MINUTES, TimeUnit.class));

		return repositoryQueueIteratorOptions;

	}

	private SitemapIndexRepositoryIterable.Options compileSitemapIndexRepositoryOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		SitemapIndexRepositoryIterable.Options sitemapIndexRepositoryIterableOptions = new SitemapIndexRepositoryIterable.Options();

		sitemapIndexRepositoryIterableOptions.setQueueSize(Utils.getAsInt(interfaceDescriptor.getParams(), "sitemap_queueSize", 100));

		sitemapIndexRepositoryIterableOptions.setPutTimeout(Utils.getAsLong(interfaceDescriptor.getParams(), "sitemap_producerBlockPollingTimeout", 20));

		sitemapIndexRepositoryIterableOptions.setPutTimeoutUnit(Utils.getAsEnum(interfaceDescriptor.getParams(), "sitemap_producerBlockPollingTimeoutUnit", TimeUnit.MINUTES, TimeUnit.class));

		sitemapIndexRepositoryIterableOptions.setRepositoryQueueIteratorOptions(this.compileRepositoryQueueOptions(interfaceDescriptor));

		sitemapIndexRepositoryIterableOptions.setSitemapFileIteratorOptions(this.compileSitemapFileOptions(interfaceDescriptor));

		sitemapIndexRepositoryIterableOptions.setSitemapIndexIteratorOptions(this.compileSitemapIndexOptions(interfaceDescriptor));

		return sitemapIndexRepositoryIterableOptions;

	}

	private EndpointAccessIterator.Options compileEndpointAccessOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		EndpointAccessIterator.Options endpointAccessIteratorOptions = new EndpointAccessIterator.Options();

		endpointAccessIteratorOptions.setCharset(Utils.getAsCharset(interfaceDescriptor.getParams(), "endpointCharset", StandardCharsets.UTF_8));

		return endpointAccessIteratorOptions;

	}

	private DatasetMappingIterator.Options compileDatasetMappingOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		DatasetMappingIterator.Options datasetMappingIteratorOptions = new DatasetMappingIterator.Options();

		DatasetMappingIterator.Options.UpdatedDateOptions datasetMappingIteratorUpdatedDateOptions = new DatasetMappingIterator.Options.UpdatedDateOptions();

		datasetMappingIteratorUpdatedDateOptions.format =Utils.getAsString(interfaceDescriptor.getParams(), "updatedDateFormat", "YYYY-MM-DD");

		datasetMappingIteratorOptions.setUpdatedDateOptions(datasetMappingIteratorUpdatedDateOptions);

		DatasetMappingIterator.Options.CreatedDateOptions datasetMappingIteratorCreatedDateOptions = new DatasetMappingIterator.Options.CreatedDateOptions();

		datasetMappingIteratorCreatedDateOptions.format =Utils.getAsString(interfaceDescriptor.getParams(), "createdDateFormat", "YYYY-MM-DD");

		datasetMappingIteratorOptions.setCreatedDateOptions(datasetMappingIteratorCreatedDateOptions);

		DatasetMappingIterator.Options.PublicationDateOptions datasetMappingIteratorPublicationDateOptions = new DatasetMappingIterator.Options.PublicationDateOptions();

		datasetMappingIteratorPublicationDateOptions.format =Utils.getAsString(interfaceDescriptor.getParams(), "publicationDateFormat", "YYYY-MM-DD");

		datasetMappingIteratorOptions.setPublicationDateOptions(datasetMappingIteratorPublicationDateOptions);

		DatasetMappingIterator.Options.ContributorOptions datasetMappingIteratorContributorOptions = new DatasetMappingIterator.Options.ContributorOptions();

		datasetMappingIteratorContributorOptions.fallbackType =Utils.getAsEnum(interfaceDescriptor.getParams(), "contributorFallbackType",DatasetDocument.Contributor.ContributorType.Other, DatasetDocument.Contributor.ContributorType.class);

		datasetMappingIteratorOptions.setContributorOptions(datasetMappingIteratorContributorOptions);

		DatasetMappingIterator.Options.IdentifierOptions datasetMappingIteratorIdentifierOptions = new DatasetMappingIterator.Options.IdentifierOptions();

		datasetMappingIteratorIdentifierOptions.fallbackType = Utils.getAsEnum(interfaceDescriptor.getParams(), "identifierFallbackType", null, DatasetDocument.Identifier.IdentifierType.class);

		datasetMappingIteratorIdentifierOptions.fallbackURL = Utils.getAsBoolean(interfaceDescriptor.getParams(), "identifierFallbackURL", true);

		datasetMappingIteratorIdentifierOptions.mappingARK = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingARK", null);

		datasetMappingIteratorIdentifierOptions.mappingDOI = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingDOI", null);

		datasetMappingIteratorIdentifierOptions.mappingHandle = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingHandle", null);

		datasetMappingIteratorIdentifierOptions.mappingPURL = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingPURL", null);

		datasetMappingIteratorIdentifierOptions.mappingURL = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingURL", null);

		datasetMappingIteratorIdentifierOptions.mappingURN = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingURN", null);

		datasetMappingIteratorOptions.setIdentifierOptions(datasetMappingIteratorIdentifierOptions);

		return datasetMappingIteratorOptions;

	}

	private SchemaOrgIterable.Options compileSchemaOrgOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		SchemaOrgIterable.Options schemaOrgIterableOptions = new SchemaOrgIterable.Options();

		schemaOrgIterableOptions.setDatasetMappingOptions(this.compileDatasetMappingOptions(interfaceDescriptor));

		schemaOrgIterableOptions.setEndpointAccessOptions(this.compileEndpointAccessOptions(interfaceDescriptor));

		return schemaOrgIterableOptions;

	}

}

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-collector-plugins</artifactId>

	<version>1.6.0</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-collector-plugins/tags/dnet-collector-plugins-1.6.0</developerConnection>

	</scm>

	<build>

		<plugins>

			<plugin>

				<artifactId>maven-assembly-plugin</artifactId>

				<configuration>

					<archive>

						<manifest>

							<mainClass>eu.dnetlib.data.collector.plugins.schemaorg.SchemaOrgMainReactome</mainClass>

						</manifest>

					</archive>

					<descriptorRefs>

						<descriptorRef>jar-with-dependencies</descriptorRef>

					</descriptorRefs>

				</configuration>

			</plugin>

		</plugins>

	</build>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service-rmi</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service</artifactId>

			<version>[3.3.26,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>${google.gson.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-io</groupId>

			<artifactId>commons-io</artifactId>

			<version>${commons.io.version}</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.apache.httpcomponents</groupId>

			<artifactId>httpclient</artifactId>

			<version>4.5</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0, 3.0.0)</version>

			<scope>provided</scope>

		</dependency>

		<dependency>

			<groupId>com.ximpleware</groupId>

			<artifactId>vtd-xml</artifactId>

			<version>[2.12, 3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.9.2</version>

		</dependency>

		<dependency>

			<groupId>org.json</groupId>

			<artifactId>json</artifactId>

			<version>20180813</version>

		 <type>jar</type>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<dependency>

			<groupId>org.apache.poi</groupId>

			<artifactId>poi</artifactId>

			<version>3.16</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->

		<dependency>

			<groupId>org.apache.poi</groupId>

			<artifactId>poi-ooxml</artifactId>

			<version>3.16</version>

		</dependency>

		<dependency>

			<groupId>org.jsoup</groupId>

			<artifactId>jsoup</artifactId>

			<version>1.11.2</version>

		</dependency>

		<dependency>

			<groupId>commons-lang</groupId>

			<artifactId>commons-lang</artifactId>

			<version>2.6</version>

			<scope>compile</scope>

		</dependency>

        <dependency>

            <groupId>org.mockito</groupId>

            <artifactId>mockito-core</artifactId>

            <version>3.3.3</version>

            <scope>test</scope>

        </dependency>

    </dependencies>

</project>

package eu.dnetlib.data.collector.plugins.fairsharing;

import java.io.UnsupportedEncodingException;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.HttpEntity;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.entity.StringEntity;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import org.json.JSONObject;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class FairSharingPlugin extends AbstractCollectorPlugin {

	private static final int PAGE_SIZE = 100;

	private static final Log log = LogFactory.getLog(FairSharingPlugin.class); // NOPMD by marko on 11/24/08 5:02 PM

	// Suggested values:

	// baseUrl = https://api.fairsharing.org

	// XPATH_ID = /record/id

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

		throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		final String login = interfaceDescriptor.getParams().get("login");

		final String password = interfaceDescriptor.getParams().get("password");

		final String authCode = login(baseUrl, login, password);

		log.debug("authCode: " + authCode);

		if (StringUtils.isBlank(authCode)) { throw new CollectorServiceException("Authorization failed: authCode is empty"); }

		return () -> new FairSharingIterator(baseUrl, authCode, PAGE_SIZE);

	}

	private String login(final String baseUrl, final String login, final String password) throws CollectorServiceException {

		final HttpPost req = new HttpPost(baseUrl + "/users/sign_in");

		req.addHeader("Accept", "application/json");

		req.addHeader("Content-Type", "application/json");

		req.setEntity(prepareCredentials(login, password));

		try (final CloseableHttpClient client = HttpClients.createDefault()) {

			try (final CloseableHttpResponse response = client.execute(req)) {

				final String content = IOUtils.toString(response.getEntity().getContent());

				final JSONObject obj = new JSONObject(content);

				return obj.getString("jwt");

			}

		} catch (final Exception e) {

			throw new CollectorServiceException("Error perfoming login", e);

		}

	}

	public HttpEntity prepareCredentials(final String login, final String password) throws CollectorServiceException {

		final JSONObject objUser = new JSONObject();

		objUser.put("login", login);

		objUser.put("password", password);

		final JSONObject objCredentials = new JSONObject();

		objCredentials.put("user", objUser);

		try {

			return new StringEntity(objCredentials.toString());

		} catch (final UnsupportedEncodingException e) {

			throw new CollectorServiceException("Error preparing http entity for login");

		}

	}

}

package eu.dnetlib.data.collector.plugins.fairsharing;

import java.util.Iterator;

import java.util.Queue;

import java.util.concurrent.PriorityBlockingQueue;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import org.json.JSONObject;

import org.json.XML;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

public class FairSharingIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(FairSharingIterator.class); // NOPMD by marko on 11/24/08 5:02 PM

	private final Queue<String> queue = new PriorityBlockingQueue<>();

	private final String baseUrl;

	private final String authCode;

	private final int pageSize;

	private String nextUrl;

	private boolean started;

	public FairSharingIterator(final String baseUrl, final String authCode, final int pageSize) {

		this.baseUrl = baseUrl;

		this.authCode = authCode;

		this.pageSize = pageSize;

		this.started = false;

	}

	private void verifyStarted() {

		if (!this.started) {

			this.started = true;

			try {

				final String url = baseUrl + "/fairsharing_records/?page%5Bnumber%5D=1&page%5Bsize%5D=" + pageSize;

				this.nextUrl = downloadPage(url);

			} catch (final CollectorServiceException e) {

				throw new RuntimeException(e);

			}

		}

	}

	@Override

	public boolean hasNext() {

		synchronized (queue) {

			verifyStarted();

			return !queue.isEmpty();

		}

	}

	@Override

	public String next() {

		synchronized (queue) {

			verifyStarted();

			final String res = queue.poll();

			while (queue.isEmpty() && nextUrl != null && !nextUrl.isEmpty()) {

				try {

					nextUrl = downloadPage(nextUrl);

				} catch (final CollectorServiceException e) {

					throw new RuntimeException(e);

				}

			}

			return res;

		}

	}

	@Override

	public void remove() {}

	private String downloadPage(final String url) throws CollectorServiceException {

		log.debug("Fetching url: " + url);

		final HttpGet req = new HttpGet(url);

		req.addHeader("Accept", "application/json");

		req.addHeader("Content-Type", "application/json");

		req.addHeader("Authorization", "Bearer " + authCode);

		try (final CloseableHttpClient client = HttpClients.createDefault()) {

			try (final CloseableHttpResponse response = client.execute(req)) {

				final String content = IOUtils.toString(response.getEntity().getContent());

				final JSONObject obj = new JSONObject(content);

				obj.getJSONArray("data")

					.forEach(x -> queue.add(XML.toString(x, "record")));

				final JSONObject links = obj.getJSONObject("links");

				return links.isNull("next") ? null : links.getString("next");

			}

		} catch (final Exception e) {

			throw new CollectorServiceException("Error perfoming call fro login", e);

		}

	}

}

package eu.dnetlib.data.collector.plugins.eosc;

import org.apache.commons.lang3.math.NumberUtils;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import org.dom4j.Document;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class EoscServicesPlugin extends AbstractCollectorPlugin {

	// Suggested values:

	// baseUrl = https://api.eosc-portal.eu

	// maxProviders = 10000

	// XPATH_ID = /record/organization/id

	@SuppressWarnings("unchecked")

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

		throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		final long maxProviders = NumberUtils.toLong(interfaceDescriptor.getParams().get("maxProviders"), 10000);

		return () -> httpCall(baseUrl + "/provider/all?quantity=" + maxProviders)

			.selectNodes("/Paging/results/results")

			.stream()

			.map(o -> processProvider(baseUrl, (Node) o))

			.iterator();

	}

	private String processProvider(final String baseUrl, final Node nodeProv) {

		final String orgId = nodeProv.valueOf("./id");

		final Document newDoc = DocumentHelper.createDocument();

		final Element newRoot = DocumentHelper.createElement("record");

		newDoc.setRootElement(newRoot);

		nodeProv.setName("organization");

		newRoot.add(nodeProv.detach());

		final Document docSrvs = httpCall(baseUrl + "/provider/services/" + orgId);

		for (final Object o : docSrvs.selectNodes("/List/item")) {

			final Node nodeSrv = (Node) o;

			nodeSrv.setName("service");

			newRoot.add(nodeSrv.detach());

		}

		return newDoc.asXML();

	}

	private Document httpCall(final String url) {

		final SAXReader reader = new SAXReader();

		final HttpGet req = new HttpGet(url);

		req.addHeader("Accept", "application/xml");

		try (final CloseableHttpClient client = HttpClients.createDefault()) {

			try (final CloseableHttpResponse response = client.execute(req)) {

				return reader.read(response.getEntity().getContent());

			}

		} catch (final Exception e) {

			throw new RuntimeException(e);

		}

	}

}

package eu.dnetlib.data.collector.plugins.httpfilename;

import eu.dnetlib.data.collector.plugins.HttpConnector;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

/**

 * Created by miriam on 07/05/2018.

 */

public class Connector extends HttpConnector implements ConnectorInterface  {

    private String response;

    @Override

    public void get(final String requestUrl) throws CollectorServiceException {

        response = getInputSource(requestUrl);

    }

    @Override

    public String getResponse() {

        return response;

    }

    @Override

    public boolean isStatusOk() {

        return (response != null);

    }

    @Override

    public boolean responseTypeContains(String string) {

        String responseType = getResponseType();

        if (responseType != null)

            return responseType.contains(string);

        return false;

    }

}

/**

 * log.debug(...) equal to  log.trace(...) in the application-logs

 * <p>

 * known bug: at resumptionType 'discover' if the (resultTotal % resultSizeValue) == 0 the collecting fails -> change the resultSizeValue

 */

package eu.dnetlib.data.collector.plugins.rest;

import java.io.InputStream;

import java.io.StringWriter;

import java.io.UnsupportedEncodingException;

import java.net.URL;

import java.net.URLEncoder;

import java.nio.charset.StandardCharsets;

import java.net.HttpURLConnection;

import java.util.Iterator;

import java.util.Map;

import java.util.Queue;

import java.util.concurrent.PriorityBlockingQueue;

import java.util.regex.Pattern;

import java.util.regex.Matcher;

import javax.xml.transform.OutputKeys;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerConfigurationException;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.dom.DOMSource;

import javax.xml.transform.stream.StreamResult;

import javax.xml.xpath.*;

import com.google.common.collect.Maps;

import eu.dnetlib.data.collector.plugins.utils.JsonUtils;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.HttpClients;

import org.w3c.dom.Node;

import org.w3c.dom.NodeList;

import org.xml.sax.InputSource;

/**

 * @author Jochen Schirrwagen, Aenne Loehden, Andreas Czerniak, Alessia Bardi, Miriam Baglioni

 * @date 2020-04-09

 */

public class RestIterator implements Iterator<String> {

    private final String AUTHBASIC = "basic";

    // TODO: clean up the comments of replaced source code

    private static final Log log = LogFactory.getLog(RestIterator.class); // NOPMD by marko on 11/24/08 5:02 PM

    private static final String XML_HEADER = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";

    private static final String EMPTY_XML = XML_HEADER + JsonUtils.wrapName + "></" + JsonUtils.wrapName + ">";

    private JsonUtils jsonUtils;

    private String baseUrl;

    private String resumptionType;

    private String resumptionParam;

    private String resultFormatValue;

    private String queryParams = "";

    private int resultSizeValue;

    private int resumptionInt = 0;            // integer resumption token (first record to harvest)

    private int resultTotal = -1;

    private String resumptionStr = Integer.toString(resumptionInt);  // string resumption token (first record to harvest or token scanned from results)

    private InputStream resultStream;

    private Transformer transformer;

    private XPath xpath;

    private String query;

    private XPathExpression xprResultTotalPath;

    private XPathExpression xprResumptionPath;

    private XPathExpression xprEntity;

    private String queryFormat;

    private String querySize;

    private String authMethod;

    private String authToken;

    private Queue<String> recordQueue = new PriorityBlockingQueue<String>();

    private int discoverResultSize = 0;

    private int pagination = 1;

    /*

    While resultFormatValue is added to the request parameter, this is used to say that the results are retrieved in json.

    useful for cases when the target API expects a resultFormatValue != json, but the results are returned in json.

    An example is the EU Open Data Portal API: resultFormatValue=standard, results are in json format.

     */

    private String resultOutputFormat;

    /*

    Can be used to set additional request headers, like for content negotiation

     */

    private Map<String, String> requestHeaders;

    public RestIterator(

            final String baseUrl,

            final String resumptionType,

            final String resumptionParam,

            final String resumptionXpath,

            final String resultTotalXpath,

            final String resultFormatParam,

            final String resultFormatValue,

            final String resultSizeParam,

            final String resultSizeValueStr,

            final String queryParams,

            final String entityXpath,

            final String authMethod,

            final String authToken,

            final String resultOutputFormat,

            final Map<String, String> requestHeaders

    ) {

        this.jsonUtils = new JsonUtils();

        this.baseUrl = baseUrl;

        this.resumptionType = resumptionType;

        this.resumptionParam = resumptionParam;

        this.resultFormatValue = resultFormatValue;

        this.queryParams = queryParams;

        this.resultSizeValue = Integer.valueOf(resultSizeValueStr);

        this.authMethod = authMethod;

        this.authToken = authToken;

        this.resultOutputFormat = resultOutputFormat;

        this.requestHeaders = requestHeaders != null ? requestHeaders : Maps.newHashMap();

        queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue : "";

        querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : "";

        try {

            initXmlTransformation(resultTotalXpath, resumptionXpath, entityXpath);

        } catch (Exception e) {

            throw new IllegalStateException("xml transformation init failed: " + e.getMessage());

        }

        initQueue();

    }

    private void initXmlTransformation(String resultTotalXpath, String resumptionXpath, String entityXpath)

            throws TransformerConfigurationException, XPathExpressionException {

        transformer = TransformerFactory.newInstance().newTransformer();

        transformer.setOutputProperty(OutputKeys.INDENT, "yes");

        transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "3");

        xpath = XPathFactory.newInstance().newXPath();

        xprResultTotalPath = xpath.compile(resultTotalXpath);

        xprResumptionPath = xpath.compile(StringUtils.isBlank(resumptionXpath) ? "/" : resumptionXpath);

        xprEntity = xpath.compile(entityXpath);

    }

    private void initQueue() {

        if (queryParams.equals("") && querySize.equals("") && queryFormat.equals("")) {

            query = baseUrl;

        } else {

            query = baseUrl + "?" + queryParams + querySize + queryFormat;

        }

        log.info("RestIterator.initQueue():: REST calls starting with " + query);

    }

    private void disconnect() {

        // TODO close inputstream

    }

    /* (non-Javadoc)

     * @see java.util.Iterator#hasNext()

     */

    @Override

    public boolean hasNext() {

        if (recordQueue.isEmpty() && query.isEmpty()) {

            disconnect();

            return false;

        } else {

            return true;

        }

    }

    /* (non-Javadoc)

     * @see java.util.Iterator#next()

     */

    @Override

    public String next() {

        synchronized (recordQueue) {

            while (recordQueue.isEmpty() && !query.isEmpty()) {

                try {

                    log.debug("RestIterator.next():: get Query: " + query);

                    query = downloadPage(query);

                    log.debug("RestIterator.next():: next queryURL from downloadPage(): " + query);

                } catch (CollectorServiceException e) {

                    log.debug("RestIterator.next():: CollectorPlugin.next()-Exception: " + e);

                    throw new RuntimeException(e);

                }

            }

            return recordQueue.poll();

        }

    }

    /*

     * download page and return nextQuery

     */

    private String downloadPage(String query) throws CollectorServiceException {

        String resultJson;

        String resultXml = XML_HEADER;

        String nextQuery = "";

        Node resultNode = null;

        NodeList nodeList = null;

        String qUrlArgument = "";

        int urlOldResumptionSize = 0;

        InputStream theHttpInputStream;

        // modifying request URL

        // check if cursor=* is initial set otherwise add it to the queryParam URL

        if (resumptionType.equalsIgnoreCase("deep-cursor")) {

            log.debug("RestIterator.downloadPage():: check resumptionType deep-cursor and check cursor=*?" + query);

            if (!query.contains("&cursor=")) {

                query += "&cursor=*";

            }

        }

        // find pagination page start number in queryParam and remove before start the first query

        if((resumptionType.toLowerCase().equals("pagination")) && (query.contains("paginationStart="))) {

            final Matcher m = Pattern.compile("paginationStart=([0-9]+)").matcher(query);

            m.find(); // guaranteed to be true for this regex

            String[] pageVal = m.group(0).split("=");

            pagination = Integer.parseInt(pageVal[1]);

            // remove page start number from queryParams

            query = query.replaceFirst("&?paginationStart=[0-9]+", "");

        }

        try {

            URL qUrl = new URL(query);

            log.debug("authMethod :" + authMethod);

            if (this.authMethod == "bearer") {

                log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml);

                requestHeaders.put("Authorization", "Bearer " + authToken);

                //requestHeaders.put("Content-Type", "application/json");

            } else if (AUTHBASIC.equalsIgnoreCase(this.authMethod)) {

                log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml);

                requestHeaders.put("Authorization", "Basic " + authToken);

                //requestHeaders.put("accept", "application/xml");

            }

            HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection();

            conn.setRequestMethod("GET");

            this.setRequestHeader(conn);

            resultStream = conn.getInputStream();

            if ("json".equals(resultOutputFormat)) {

                resultJson = IOUtils.toString(resultStream, "UTF-8");

                resultXml = jsonUtils.convertToXML(resultJson);

                resultStream = IOUtils.toInputStream(resultXml, "UTF-8");

            }

            if (!isEmptyXml(resultXml)) {

                resultNode = (Node) xpath.evaluate("/", new InputSource(resultStream), XPathConstants.NODE);

                nodeList = (NodeList) xprEntity.evaluate(resultNode, XPathConstants.NODESET);

                log.debug("RestIterator.downloadPage():: nodeList.length=" + nodeList.getLength());

                for (int i = 0; i < nodeList.getLength(); i++) {

                    StringWriter sw = new StringWriter();

                    transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw));

                    String toEnqueue = sw.toString();

                    if (toEnqueue == null || StringUtils.isBlank(toEnqueue) || isEmptyXml(toEnqueue)) {

                        log.warn("RestIterator.downloadPage():: The following record resulted in empty item for the feeding queue: " + resultXml);

                    } else {

                        recordQueue.add(sw.toString());

                    }

                }

            } else {

                log.warn("resultXml is equal with emptyXml");

            }

            resumptionInt += resultSizeValue;

            switch (resumptionType.toLowerCase()) {

                case "scan":    // read of resumptionToken , evaluate next results, e.g. OAI, iterate over items

                    resumptionStr = xprResumptionPath.evaluate(resultNode);

                    break;

                case "count":   // begin at one step for all records, iterate over items

                    resumptionStr = Integer.toString(resumptionInt);

                    break;

                case "discover":   // size of result items unknown, iterate over items  (for openDOAR - 201808)

                    if (resultSizeValue < 2) {

                        log.debug("RestIterator.downloadPage().discover:: ode: discover, Param 'resultSizeValue' must greater then 1");

                        throw new CollectorServiceException("Mode: discover, Param 'resultSizeValue' must greater then 1");

                    }

                    log.debug("RestIterator.downloadPage().discover:: resumptionInt="+Integer.toString(resumptionInt)+"; ");

                    qUrlArgument = qUrl.getQuery();

                    if( qUrlArgument != null ) {

                    String[] arrayQUrlArgument = qUrlArgument.split("&");

                    // check if URL arguments given

                    if( arrayQUrlArgument != null ) {

                        for (String arrayUrlArgStr : arrayQUrlArgument) {

                            log.debug("RestIterator.downloadPage/discover:: "+arrayUrlArgStr);

                            if (arrayUrlArgStr.startsWith(resumptionParam)) {

                                String[] resumptionKeyValue = arrayUrlArgStr.split("=");

                                if (isInteger(resumptionKeyValue[1])) {

                                    urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]);

                                    log.debug("RestIterator.downloadPage():discover:: OldResumptionSize from Url (int): " + urlOldResumptionSize);

                                } else {

                                    log.debug("RestIterator.downloadPage().discover:: OldResumptionSize from Url (str): " + resumptionKeyValue[1]);

                                }

                            }

                        }

                    }

                    }

                    log.debug("RestIterator.downloadPage().discover:: nodeList.length=" + nodeList.getLength());

                    if (isEmptyXml(resultXml) || ((nodeList != null) && (nodeList.getLength() < resultSizeValue))

                    ) {

                        // resumptionStr = "";

                        if (nodeList != null) {

                            discoverResultSize += nodeList.getLength();

                        }

                        resultTotal = discoverResultSize;

                    } else {

                        resumptionStr = Integer.toString(resumptionInt);

                        resultTotal = resumptionInt + 1;

                        if (nodeList != null) {

                            discoverResultSize += nodeList.getLength();

                        }

                    }

                    log.debug("RestIterator.downloadPage().discover:: discoverResultSize=" + discoverResultSize);

                    break;

                case "pagination":

                case "page":         // pagination, iterate over page numbers

                    // find start page number

                    pagination += 1;

                    if (nodeList != null) {

                        discoverResultSize += nodeList.getLength();

                    } else {

                        resultTotal = discoverResultSize;

                        pagination = discoverResultSize;

                    }

                    resumptionInt = pagination;

                    resumptionStr = Integer.toString(resumptionInt);

                    log.debug("RestIterator.downloadPage().pagination:: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams + " ; resultTotal: " + resultTotal + " ; discoverResultSize: " + discoverResultSize);

                    break;

                case "deep-cursor":   // size of result items unknown, iterate over items  (for supporting deep cursor in solr)

                    // isn't relevant -- if (resultSizeValue < 2) {throw new CollectorServiceException("Mode: deep-cursor, Param 'resultSizeValue' is less than 2");}

                    resumptionStr = encodeValue(xprResumptionPath.evaluate(resultNode));

                    queryParams = queryParams.replace("&cursor=*", "");

                    // terminating if length of nodeList is 0

                    if ((nodeList != null) && (nodeList.getLength() < discoverResultSize)) {

                        resumptionInt += (nodeList.getLength() + 1 - resultSizeValue);

                    } else {

                        resumptionInt += (nodeList.getLength() - resultSizeValue);    // subtract the resultSizeValue because the iteration is over real length and the resultSizeValue is added before the switch()

                    }

                    discoverResultSize = nodeList.getLength();

                    log.debug("RestIterator.downloadPage().deep-cursor:: resumptionStr=" + resumptionStr + " ; queryParams=" + queryParams + " resumptionLengthIncreased: " + resumptionInt);

                    break;

                default:        // otherwise: abort

                    // resultTotal = resumptionInt;

                    break;

            }

        } catch (Exception e) {

            log.error(e);

            throw new IllegalStateException("collection failed: " + e.getMessage());

        }

        try {

            String resultTotalXpathEval = xprResultTotalPath.evaluate(resultNode);

            log.debug("downloadPage():: resInt: " +resumptionInt + "; resultTotal: " + resultTotal + " ; resultTotalXpath eval.: " + resultTotalXpathEval );

            if ((resultTotal == -1) && (!resultTotalXpathEval.isEmpty())) {

                resultTotal = Integer.parseInt(xprResultTotalPath.evaluate(resultNode));

                if (resumptionType.toLowerCase().equals("page") && !AUTHBASIC.equalsIgnoreCase(authMethod)) {

                    resultTotal += 1;

                }           // to correct the upper bound

                log.info("resultTotal was -1 is now: " + resultTotal);

            }

        } catch (Exception e) {

            log.error(e);

            throw new IllegalStateException("downloadPage() resultTotal couldn't parse: " + e.getMessage());

        }

        log.debug("resultTotal: " + resultTotal + " ; resInt: " + resumptionInt);

        if (resumptionInt <= resultTotal) {

            nextQuery = baseUrl + "?" + queryParams + querySize + "&" + resumptionParam + "=" + resumptionStr + queryFormat;

        } else {

            nextQuery = "";

            // if (resumptionType.toLowerCase().equals("deep-cursor")) { resumptionInt -= 1; }    	// correct the resumptionInt and prevent a NullPointer Exception at mdStore

        }

        log.debug("downloadPage() nextQueryUrl: " + nextQuery);

        return nextQuery;

    }

    private boolean isEmptyXml(String s){

        return EMPTY_XML.equalsIgnoreCase(s);

    }

    private boolean isInteger(String s) {

        boolean isValidInteger = false;

        try {

            Integer.parseInt(s);

            // s is a valid integer

            isValidInteger = true;

        } catch (NumberFormatException ex) {

            // s is not an integer

        }

        return isValidInteger;

    }

    // Method to encode a string value using `UTF-8` encoding scheme

    private String encodeValue(String value) {

        try {

            return URLEncoder.encode(value, StandardCharsets.UTF_8.toString());

        } catch (UnsupportedEncodingException ex) {

            throw new RuntimeException(ex.getCause());

        }

    }

    /**

     * setRequestHeader

     * 

     * setRequestProperty: Sets the general request property. If a property with the key already exists, overwrite its value with the new value.

     * @param conn

     */

    private void setRequestHeader(HttpURLConnection conn) {

        if (requestHeaders != null) {

            for (String key : requestHeaders.keySet()) {

                conn.setRequestProperty(key, requestHeaders.get(key));

            }

            log.debug("Set Request Header with: " + requestHeaders);

        }

    }

    public String getResultFormatValue() {

        return resultFormatValue;

    }

    public String getResultOutputFormat() {

        return resultOutputFormat;

    }

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.json.JSONObject;

import java.net.URL;

import java.time.LocalDate;

import java.time.format.DateTimeFormatter;

import java.util.*;

public class DatasetMappingIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(EndpointAccessIterator.class);

	public static class Options {

		public static class IdentifierOptions{

			public List<String> mappingARK;

			public List<String> mappingDOI;

			public List<String> mappingHandle;

			public List<String> mappingPURL;

			public List<String> mappingURN;

			public List<String> mappingURL;

			public DatasetDocument.Identifier.IdentifierType fallbackType;

			public Boolean fallbackURL;

		}

		public static class ContributorOptions{

			public DatasetDocument.Contributor.ContributorType fallbackType;

		}

		public static class PublicationDateOptions{

			public String format;

		}

		public static class CreatedDateOptions{

			public String format;

		}

		public static class UpdatedDateOptions{

			public String format;

		}

		private IdentifierOptions identifierOptions;

		private PublicationDateOptions publicationDateOptions;

		private ContributorOptions contributorOptions;

		private CreatedDateOptions createdDateOptions;

		private UpdatedDateOptions updatedDateOptions;

		public UpdatedDateOptions getUpdatedDateOptions() {

			return updatedDateOptions;

		}

		public void setUpdatedDateOptions(UpdatedDateOptions updatedDateOptions) {

			this.updatedDateOptions = updatedDateOptions;

		}

		public CreatedDateOptions getCreatedDateOptions() {

			return createdDateOptions;

		}

		public void setCreatedDateOptions(CreatedDateOptions createdDateOptions) {

			this.createdDateOptions = createdDateOptions;

		}

		public ContributorOptions getContributorOptions() {

			return contributorOptions;

		}

		public void setContributorOptions(ContributorOptions contributorOptions) {

			this.contributorOptions = contributorOptions;

		}

		public PublicationDateOptions getPublicationDateOptions() {

			return publicationDateOptions;

		}

		public void setPublicationDateOptions(PublicationDateOptions publicationDateOptions) {

			this.publicationDateOptions = publicationDateOptions;

		}

		public IdentifierOptions getIdentifierOptions() {

			return identifierOptions;

		}

		public void setIdentifierOptions(IdentifierOptions identifierOptions) {

			this.identifierOptions = identifierOptions;

		}

	}

	private Options options;

	private EndpointAccessIterator endpointAccessIterator;

	public DatasetMappingIterator(Options options, EndpointAccessIterator endpointAccessIterator) {

		this.options = options;

		this.endpointAccessIterator = endpointAccessIterator;

	}

	@Override

	public boolean hasNext() {

		return this.endpointAccessIterator.hasNext();

	}

	@Override

	public String next() {

		JSONObject document = this.endpointAccessIterator.next();

		String xml = null;

		if (document == null) {

			log.debug("no document provided to process. returning empty");

			xml = DatasetDocument.emptyXml();

		}

		else {

			log.debug("building document");

			xml = this.buildDataset(document);

			if (!Utils.validateXml(xml)) {

				log.debug("xml not valid. setting to empty");

				xml = null;

			}

			if (xml == null) {

				log.debug("could not build xml. returning empty");

				xml = DatasetDocument.emptyXml();

			}

		}

		//if all else fails

		if(xml == null){

			log.debug("could not build xml. returning empty");

			xml = "<dataset/>";

		}

		log.debug("xml document for dataset is: "+xml);

		return xml;

	}

	private String buildDataset(JSONObject document){

		String xml = null;

		try{

			DatasetDocument dataset = new DatasetDocument();

			dataset.setIdentifiers(this.extractIdentifier(document));

			dataset.setCreators(this.extractCreator(document));

			dataset.setTitles(this.extractTitles(document));

			dataset.setAlternativeTitles(this.extractAlternateTitles(document));

			dataset.setPublishers(this.extractPublisher(document));

			dataset.setPublicationDates(this.extractPublicationDate(document));

			dataset.setSubjects(this.extractSubjects(document));

			dataset.setContributors(this.extractContributors(document));

			dataset.setCreatedDates(this.extractCreatedDate(document));

			dataset.setUpdatedDates(this.extractUpdatedDate(document));

			dataset.setLanguages(this.extractLanguages(document));

			dataset.setResourceTypes(this.extractResourceTypes(document));

			dataset.setAlternateIdentifier(this.extractAlternateIdentifiers(document));

			dataset.setCitations(this.extractCitations(document));

			dataset.setSizes(this.extractSize(document));

			dataset.setFormat(this.extractEncodingFormat(document));

			dataset.setVersion(this.extractVersion(document));

			dataset.setLicenses(this.extractLicense(document));

			dataset.setDescriptions(this.extractDescription(document));

			dataset.setDisambiguatingDescriptions(this.extractDisambiguatingDescription(document));

			dataset.setGeoLocations(this.extractSpatialCoverage(document));

			log.debug("document contains native identifier: : "+(dataset.getIdentifiers()!=null && dataset.getIdentifiers().size() > 0));

			if((dataset.getIdentifiers() == null || dataset.getIdentifiers().size() == 0) &&

					this.options.getIdentifierOptions().fallbackURL){

				log.debug("falling back to url identifier");

				dataset.setIdentifiers(this.extractIdentifierFallbackURL(document));

				log.debug("document contains overridden identifier: : "+(dataset.getIdentifiers()!=null && dataset.getIdentifiers().size() > 0));

			}

			xml = dataset.toXml();

		}

		catch(Exception ex){

			log.error("problem constructing dataset xml. returning empty", ex);

			xml = null;

		}

		return xml;

	}

	private List<DatasetDocument.Identifier> extractIdentifierFallbackURL(JSONObject document){

		List<String> urls = JSONLDUtils.extractString(document, "url");

		ArrayList<DatasetDocument.Identifier> curated = new ArrayList<>();

		for(String item : urls){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(new DatasetDocument.Identifier(DatasetDocument.Identifier.IdentifierType.URL,  item.trim()));

		}

		return curated;

	}

	private List<DatasetDocument.SpatialCoverage> extractSpatialCoverage(JSONObject document){

		List<JSONLDUtils.PlaceInfo> spatials = JSONLDUtils.extractPlaces(document, "spatialCoverage");

		ArrayList<DatasetDocument.SpatialCoverage> curated = new ArrayList<>();

		for(JSONLDUtils.PlaceInfo item : spatials){

			if((item.name == null || item.name.trim().length() == 0) &&

					(item.geoCoordinates == null || item.geoCoordinates.size() == 0) &&

					(item.geoShapes == null || item.geoShapes.size() == 0)) continue;

			List<DatasetDocument.SpatialCoverage.Point> points = new ArrayList<>();

			List<String> boxes = new ArrayList<>();

			if(item.geoCoordinates!=null) {

				for (JSONLDUtils.GeoCoordinatesInfo iter : item.geoCoordinates){

					points.add(new DatasetDocument.SpatialCoverage.Point(iter.latitude, iter.longitude));

				}

			}

			if(item.geoShapes!=null) {

				for (JSONLDUtils.GeoShapeInfo iter : item.geoShapes){

					boxes.add(iter.box);

				}

			}

			curated.add(new DatasetDocument.SpatialCoverage(item.name, points, boxes));

		}

		return curated;

	}

	private List<String> extractDescription(JSONObject document){

		List<String> descriptions = JSONLDUtils.extractString(document, "description");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : descriptions){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<String> extractDisambiguatingDescription(JSONObject document){

		List<String> descriptions = JSONLDUtils.extractString(document, "disambiguatingDescription");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : descriptions){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<DatasetDocument.License> extractLicense(JSONObject document){

		List<JSONLDUtils.LicenseInfo> licenses = JSONLDUtils.extractLicenses(document, "license");

		ArrayList<DatasetDocument.License> curated = new ArrayList<>();

		for(JSONLDUtils.LicenseInfo item : licenses){

			if(item.url == null || item.url.trim().length() == 0) continue;

			curated.add(new DatasetDocument.License(item.name, item.url));

		}

		return curated;

	}

	private List<String> extractVersion(JSONObject document){

		List<String> versions = JSONLDUtils.extractString(document, "version");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : versions){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<String> extractSize(JSONObject document) {

		List<String> sizes = JSONLDUtils.extractSize(document, "distribution");

		HashSet<String> curated = new HashSet<>();

		for (String item : sizes) {

			if (item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return new ArrayList<>(curated);

	}

	private List<String> extractEncodingFormat(JSONObject document){

		List<String> formats = JSONLDUtils.extractEncodingFormat(document, "distribution");

		HashSet<String> curated = new HashSet<>();

		for(String item : formats){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return new ArrayList<>(curated);

	}

	//TODO: Handle different citation types. Currently only urls

	private List<DatasetDocument.Citation> extractCitations(JSONObject document){

		List<JSONLDUtils.CitationInfo> citations = JSONLDUtils.extractCitations(document, "citation");

		ArrayList<DatasetDocument.Citation> curated = new ArrayList<>();

		for(JSONLDUtils.CitationInfo item : citations){

			if(item.url == null || item.url.trim().length() == 0) continue;

			try{

				new URL(item.url);

			}catch (Exception ex){

				continue;

			}

			curated.add(new DatasetDocument.Citation(item.url, DatasetDocument.Citation.CitationIdentifierType.URL));

		}

		return curated;

	}

	private List<DatasetDocument.AlternateIdentifier> extractAlternateIdentifiers(JSONObject document){

		List<String> issns = JSONLDUtils.extractString(document, "issn");

		List<String> urls = JSONLDUtils.extractString(document, "url");

		ArrayList<DatasetDocument.AlternateIdentifier> curated = new ArrayList<>();

		for(String item : issns){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(new DatasetDocument.AlternateIdentifier(item.trim(), "ISSN"));

		}

		for(String item : urls){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(new DatasetDocument.AlternateIdentifier(item.trim(), "URL"));

		}

		return curated;

	}

	private List<DatasetDocument.ResourceType> extractResourceTypes(JSONObject document){

		List<DatasetDocument.ResourceType> resourceTypes = new ArrayList<>();

		resourceTypes.add(new DatasetDocument.ResourceType(DatasetDocument.ResourceType.ResourceTypeGeneralType.Dataset));

		return resourceTypes;

	}

	private List<String> extractLanguages(JSONObject document){

		List<String> languages = JSONLDUtils.extractLanguage(document, "inLanguage");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : languages){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<LocalDate> extractUpdatedDate(JSONObject document){

		List<LocalDate> updatedDates = new ArrayList<>();

		if(this.options.getUpdatedDateOptions() == null || this.options.getUpdatedDateOptions().format == null || this.options.getUpdatedDateOptions().format.length() == 0) return updatedDates;

		DateTimeFormatter formatter = DateTimeFormatter.ofPattern(this.options.getPublicationDateOptions().format);

		List<String> dates = JSONLDUtils.extractString(document, "dateModified");

		for(String updatedDate : dates){

			if(updatedDate == null || updatedDate.trim().length() == 0) continue;

			try {

				LocalDate localDate = LocalDate.parse(updatedDate, formatter);

				updatedDates.add(localDate);

			} catch (Exception e) {

				continue;