D-Net

package eu.dnetlib.data.collector.plugins.schemaorg;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.json.JSONObject;

import java.net.URL;

import java.time.LocalDate;

import java.time.format.DateTimeFormatter;

import java.util.*;

public class DatasetMappingIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(EndpointAccessIterator.class);

	public static class Options {

		public static class IdentifierOptions{

			public List<String> mappingARK;

			public List<String> mappingDOI;

			public List<String> mappingHandle;

			public List<String> mappingPURL;

			public List<String> mappingURN;

			public List<String> mappingURL;

			public DatasetDocument.Identifier.IdentifierType fallbackType;

			public Boolean fallbackURL;

		}

		public static class ContributorOptions{

			public DatasetDocument.Contributor.ContributorType fallbackType;

		}

		public static class PublicationDateOptions{

			public String format;

		}

		public static class CreatedDateOptions{

			public String format;

		}

		public static class UpdatedDateOptions{

			public String format;

		}

		private IdentifierOptions identifierOptions;

		private PublicationDateOptions publicationDateOptions;

		private ContributorOptions contributorOptions;

		private CreatedDateOptions createdDateOptions;

		private UpdatedDateOptions updatedDateOptions;

		public UpdatedDateOptions getUpdatedDateOptions() {

			return updatedDateOptions;

		}

		public void setUpdatedDateOptions(UpdatedDateOptions updatedDateOptions) {

			this.updatedDateOptions = updatedDateOptions;

		}

		public CreatedDateOptions getCreatedDateOptions() {

			return createdDateOptions;

		}

		public void setCreatedDateOptions(CreatedDateOptions createdDateOptions) {

			this.createdDateOptions = createdDateOptions;

		}

		public ContributorOptions getContributorOptions() {

			return contributorOptions;

		}

		public void setContributorOptions(ContributorOptions contributorOptions) {

			this.contributorOptions = contributorOptions;

		}

		public PublicationDateOptions getPublicationDateOptions() {

			return publicationDateOptions;

		}

		public void setPublicationDateOptions(PublicationDateOptions publicationDateOptions) {

			this.publicationDateOptions = publicationDateOptions;

		}

		public IdentifierOptions getIdentifierOptions() {

			return identifierOptions;

		}

		public void setIdentifierOptions(IdentifierOptions identifierOptions) {

			this.identifierOptions = identifierOptions;

		}

	}

	private Options options;

	private EndpointAccessIterator endpointAccessIterator;

	public DatasetMappingIterator(Options options, EndpointAccessIterator endpointAccessIterator) {

		this.options = options;

		this.endpointAccessIterator = endpointAccessIterator;

	}

	@Override

	public boolean hasNext() {

		return this.endpointAccessIterator.hasNext();

	}

	@Override

	public String next() {

		JSONObject document = this.endpointAccessIterator.next();

		String xml = null;

		if (document == null) {

			log.debug("no document provided to process. returning empty");

			xml = DatasetDocument.emptyXml();

		}

		else {

			log.debug("building document");

			xml = this.buildDataset(document);

			if (!Utils.validateXml(xml)) {

				log.debug("xml not valid. setting to empty");

				xml = null;

			}

			if (xml == null) {

				log.debug("could not build xml. returning empty");

				xml = DatasetDocument.emptyXml();

			}

		}

		//if all else fails

		if(xml == null){

			log.debug("could not build xml. returning empty");

			xml = "<dataset/>";

		}

		log.debug("xml document for dataset is: "+xml);

		return xml;

	}

	private String buildDataset(JSONObject document){

		String xml = null;

		try{

			DatasetDocument dataset = new DatasetDocument();

			dataset.setIdentifiers(this.extractIdentifier(document));

			dataset.setCreators(this.extractCreator(document));

			dataset.setTitles(this.extractTitles(document));

			dataset.setAlternativeTitles(this.extractAlternateTitles(document));

			dataset.setPublishers(this.extractPublisher(document));

			dataset.setPublicationDates(this.extractPublicationDate(document));

			dataset.setSubjects(this.extractSubjects(document));

			dataset.setContributors(this.extractContributors(document));

			dataset.setCreatedDates(this.extractCreatedDate(document));

			dataset.setUpdatedDates(this.extractUpdatedDate(document));

			dataset.setLanguages(this.extractLanguages(document));

			dataset.setResourceTypes(this.extractResourceTypes(document));

			dataset.setAlternateIdentifier(this.extractAlternateIdentifiers(document));

			dataset.setCitations(this.extractCitations(document));

			dataset.setSizes(this.extractSize(document));

			dataset.setFormat(this.extractEncodingFormat(document));

			dataset.setVersion(this.extractVersion(document));

			dataset.setLicenses(this.extractLicense(document));

			dataset.setDescriptions(this.extractDescription(document));

			dataset.setDisambiguatingDescriptions(this.extractDisambiguatingDescription(document));

			dataset.setGeoLocations(this.extractSpatialCoverage(document));

			log.debug("document contains native identifier: : "+(dataset.getIdentifiers()!=null && dataset.getIdentifiers().size() > 0));

			if((dataset.getIdentifiers() == null || dataset.getIdentifiers().size() == 0) &&

					this.options.getIdentifierOptions().fallbackURL){

				log.debug("falling back to url identifier");

				dataset.setIdentifiers(this.extractIdentifierFallbackURL(document));

				log.debug("document contains overridden identifier: : "+(dataset.getIdentifiers()!=null && dataset.getIdentifiers().size() > 0));

			}

			xml = dataset.toXml();

		}

		catch(Exception ex){

			log.error("problem constructing dataset xml. returning empty", ex);

			xml = null;

		}

		return xml;

	}

	private List<DatasetDocument.Identifier> extractIdentifierFallbackURL(JSONObject document){

		List<String> urls = JSONLDUtils.extractString(document, "url");

		ArrayList<DatasetDocument.Identifier> curated = new ArrayList<>();

		for(String item : urls){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(new DatasetDocument.Identifier(DatasetDocument.Identifier.IdentifierType.URL,  item.trim()));

		}

		return curated;

	}

	private List<DatasetDocument.SpatialCoverage> extractSpatialCoverage(JSONObject document){

		List<JSONLDUtils.PlaceInfo> spatials = JSONLDUtils.extractPlaces(document, "spatialCoverage");

		ArrayList<DatasetDocument.SpatialCoverage> curated = new ArrayList<>();

		for(JSONLDUtils.PlaceInfo item : spatials){

			if((item.name == null || item.name.trim().length() == 0) &&

					(item.geoCoordinates == null || item.geoCoordinates.size() == 0) &&

					(item.geoShapes == null || item.geoShapes.size() == 0)) continue;

			List<DatasetDocument.SpatialCoverage.Point> points = new ArrayList<>();

			List<String> boxes = new ArrayList<>();

			if(item.geoCoordinates!=null) {

				for (JSONLDUtils.GeoCoordinatesInfo iter : item.geoCoordinates){

					points.add(new DatasetDocument.SpatialCoverage.Point(iter.latitude, iter.longitude));

				}

			}

			if(item.geoShapes!=null) {

				for (JSONLDUtils.GeoShapeInfo iter : item.geoShapes){

					boxes.add(iter.box);

				}

			}

			curated.add(new DatasetDocument.SpatialCoverage(item.name, points, boxes));

		}

		return curated;

	}

	private List<String> extractDescription(JSONObject document){

		List<String> descriptions = JSONLDUtils.extractString(document, "description");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : descriptions){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<String> extractDisambiguatingDescription(JSONObject document){

		List<String> descriptions = JSONLDUtils.extractString(document, "disambiguatingDescription");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : descriptions){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<DatasetDocument.License> extractLicense(JSONObject document){

		List<JSONLDUtils.LicenseInfo> licenses = JSONLDUtils.extractLicenses(document, "license");

		ArrayList<DatasetDocument.License> curated = new ArrayList<>();

		for(JSONLDUtils.LicenseInfo item : licenses){

			if(item.url == null || item.url.trim().length() == 0) continue;

			curated.add(new DatasetDocument.License(item.name, item.url));

		}

		return curated;

	}

	private List<String> extractVersion(JSONObject document){

		List<String> versions = JSONLDUtils.extractString(document, "version");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : versions){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<String> extractSize(JSONObject document) {

		List<String> sizes = JSONLDUtils.extractSize(document, "distribution");

		HashSet<String> curated = new HashSet<>();

		for (String item : sizes) {

			if (item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return new ArrayList<>(curated);

	}

	private List<String> extractEncodingFormat(JSONObject document){

		List<String> formats = JSONLDUtils.extractEncodingFormat(document, "distribution");

		HashSet<String> curated = new HashSet<>();

		for(String item : formats){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return new ArrayList<>(curated);

	}

	//TODO: Handle different citation types. Currently only urls

	private List<DatasetDocument.Citation> extractCitations(JSONObject document){

		List<JSONLDUtils.CitationInfo> citations = JSONLDUtils.extractCitations(document, "citation");

		ArrayList<DatasetDocument.Citation> curated = new ArrayList<>();

		for(JSONLDUtils.CitationInfo item : citations){

			if(item.url == null || item.url.trim().length() == 0) continue;

			try{

				new URL(item.url);

			}catch (Exception ex){

				continue;

			}

			curated.add(new DatasetDocument.Citation(item.url, DatasetDocument.Citation.CitationIdentifierType.URL));

		}

		return curated;

	}

	private List<DatasetDocument.AlternateIdentifier> extractAlternateIdentifiers(JSONObject document){

		List<String> issns = JSONLDUtils.extractString(document, "issn");

		List<String> urls = JSONLDUtils.extractString(document, "url");

		ArrayList<DatasetDocument.AlternateIdentifier> curated = new ArrayList<>();

		for(String item : issns){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(new DatasetDocument.AlternateIdentifier(item.trim(), "ISSN"));

		}

		for(String item : urls){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(new DatasetDocument.AlternateIdentifier(item.trim(), "URL"));

		}

		return curated;

	}

	private List<DatasetDocument.ResourceType> extractResourceTypes(JSONObject document){

		List<DatasetDocument.ResourceType> resourceTypes = new ArrayList<>();

		resourceTypes.add(new DatasetDocument.ResourceType(DatasetDocument.ResourceType.ResourceTypeGeneralType.Dataset));

		return resourceTypes;

	}

	private List<String> extractLanguages(JSONObject document){

		List<String> languages = JSONLDUtils.extractLanguage(document, "inLanguage");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : languages){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<LocalDate> extractUpdatedDate(JSONObject document){

		List<LocalDate> updatedDates = new ArrayList<>();

		if(this.options.getUpdatedDateOptions() == null || this.options.getUpdatedDateOptions().format == null || this.options.getUpdatedDateOptions().format.length() == 0) return updatedDates;

		DateTimeFormatter formatter = DateTimeFormatter.ofPattern(this.options.getPublicationDateOptions().format);

		List<String> dates = JSONLDUtils.extractString(document, "dateModified");

		for(String updatedDate : dates){

			if(updatedDate == null || updatedDate.trim().length() == 0) continue;

			try {

				LocalDate localDate = LocalDate.parse(updatedDate, formatter);

				updatedDates.add(localDate);

			} catch (Exception e) {

				continue;

			}

		}

		return updatedDates;

	}

	private List<LocalDate> extractCreatedDate(JSONObject document){

		List<LocalDate> createdDates = new ArrayList<>();

		if(this.options.getCreatedDateOptions() == null || this.options.getCreatedDateOptions().format == null || this.options.getCreatedDateOptions().format.length() == 0) return createdDates;

		DateTimeFormatter formatter = DateTimeFormatter.ofPattern(this.options.getCreatedDateOptions().format);

		List<String> dates = JSONLDUtils.extractString(document, "dateCreated");

		for(String createdDate : dates){

			if(createdDate == null || createdDate.trim().length() == 0) continue;

			try {

				LocalDate localDate = LocalDate.parse(createdDate, formatter);

				createdDates.add(localDate);

			} catch (Exception e) {

				continue;

			}

		}

		return createdDates;

	}

	private List<DatasetDocument.Contributor> extractContributors(JSONObject document){

		List<JSONLDUtils.PrincipalInfo> editors = JSONLDUtils.extractPrincipal(document, "editor");

		List<JSONLDUtils.PrincipalInfo> funders = JSONLDUtils.extractPrincipal(document, "funder");

		List<JSONLDUtils.PrincipalInfo> producers = JSONLDUtils.extractPrincipal(document, "producer");

		List<JSONLDUtils.PrincipalInfo> sponsors = JSONLDUtils.extractPrincipal(document, "sponsor");

		List<JSONLDUtils.PrincipalInfo> constributors = JSONLDUtils.extractPrincipal(document, "contributor");

		ArrayList<DatasetDocument.Contributor> curated = new ArrayList<>();

		for(JSONLDUtils.PrincipalInfo item : editors){

			if(item.name() == null || item.name().trim().length() == 0) continue;

			curated.add(new DatasetDocument.Contributor(item.name(), item.affiliationNames(), DatasetDocument.Contributor.ContributorType.Editor));

		}

		for(JSONLDUtils.PrincipalInfo item : funders){

			if(item.name() == null || item.name().trim().length() == 0) continue;

			curated.add(new DatasetDocument.Contributor(item.name(), item.affiliationNames(), DatasetDocument.Contributor.ContributorType.Funder));

		}

		for(JSONLDUtils.PrincipalInfo item : producers){

			if(item.name() == null || item.name().trim().length() == 0) continue;

			curated.add(new DatasetDocument.Contributor(item.name(), item.affiliationNames(), DatasetDocument.Contributor.ContributorType.Producer));

		}

		for(JSONLDUtils.PrincipalInfo item : sponsors){

			if(item.name() == null || item.name().trim().length() == 0) continue;

			curated.add(new DatasetDocument.Contributor(item.name(), item.affiliationNames(), DatasetDocument.Contributor.ContributorType.Sponsor));

		}

		for(JSONLDUtils.PrincipalInfo item : constributors){

			if(item.name() == null || item.name().trim().length() == 0) continue;

			DatasetDocument.Contributor.ContributorType type = DatasetDocument.Contributor.ContributorType.Other;

			if(this.options.getContributorOptions()!=null && this.options.getContributorOptions().fallbackType != null) type = this.options.getContributorOptions().fallbackType;

			curated.add(new DatasetDocument.Contributor(item.name(), item.affiliationNames(), type));

		}

		return curated;

	}

	private List<String> extractSubjects(JSONObject document){

		List<String> subjects = JSONLDUtils.extractString(document, "keywords");

		ArrayList<String> curated = new ArrayList<>();

		for(String item : subjects){

			if(item == null || item.trim().length() == 0) continue;

			curated.add(item);

		}

		return curated;

	}

	private List<LocalDate> extractPublicationDate(JSONObject document){

		List<LocalDate> publicationDates = new ArrayList<>();

		if(this.options.getPublicationDateOptions() == null || this.options.getPublicationDateOptions().format == null || this.options.getPublicationDateOptions().format.length() == 0) return publicationDates;

		DateTimeFormatter formatter = DateTimeFormatter.ofPattern(this.options.getPublicationDateOptions().format);

		List<String> dates = JSONLDUtils.extractString(document, "datePublished");

		for(String publicationDate : dates){

			if(publicationDate == null || publicationDate.trim().length() == 0) continue;

			try {

				LocalDate localDate = LocalDate.parse(publicationDate, formatter);

				publicationDates.add(localDate);

			} catch (Exception e) {

				continue;

			}

		}

		return publicationDates;

	}

	private List<String> extractPublisher(JSONObject document){

		List<JSONLDUtils.PrincipalInfo> publishers = JSONLDUtils.extractPrincipal(document, "publisher");

		ArrayList<String> curated = new ArrayList<>();

		for(JSONLDUtils.PrincipalInfo item : publishers){

			if(item.name() == null || item.name().trim().length() == 0) continue;

			curated.add(item.name());

		}

		return curated;

	}

	private List<String> extractTitles(JSONObject document){

		List<String> names = JSONLDUtils.extractString(document, "name");

		List<String> headlines = JSONLDUtils.extractString(document, "headline");

		HashSet<String> titles = new HashSet<>();

		titles.addAll(names);

		titles.addAll(headlines);

		return new ArrayList<>(titles);

	}

	private List<String> extractAlternateTitles(JSONObject document){

		List<String> names = JSONLDUtils.extractString(document, "alternateName");

		List<String> headlines = JSONLDUtils.extractString(document, "alternativeHeadline");

		HashSet<String> titles = new HashSet<>();

		titles.addAll(names);

		titles.addAll(headlines);

		return new ArrayList<>(titles);

	}

	private List<DatasetDocument.Identifier> extractIdentifier(JSONObject document){

		List<DatasetDocument.Identifier> curated = new ArrayList<>();

		List<JSONLDUtils.IdentifierInfo> identifiers = JSONLDUtils.extractIdentifier(document, "identifier");

		for(JSONLDUtils.IdentifierInfo item : identifiers){

			if(item.value == null || item.value.trim().length() == 0) continue;

			if(item.type == null || item.type.trim().length() == 0) {

				if (this.options.getIdentifierOptions().fallbackType == null) continue;

				curated.add(new DatasetDocument.Identifier(this.options.getIdentifierOptions().fallbackType, item.value.trim()));

			}

			else {

				DatasetDocument.Identifier.IdentifierType type = null;

				if(this.options.getIdentifierOptions().mappingARK != null && this.options.getIdentifierOptions().mappingARK.contains(item.type.trim())) type = DatasetDocument.Identifier.IdentifierType.ARK;

				else if(this.options.getIdentifierOptions().mappingDOI != null && this.options.getIdentifierOptions().mappingDOI.contains(item.type.trim())) type = DatasetDocument.Identifier.IdentifierType.DOI;

				else if(this.options.getIdentifierOptions().mappingHandle != null && this.options.getIdentifierOptions().mappingHandle.contains(item.type.trim())) type = DatasetDocument.Identifier.IdentifierType.Handle;

				else if(this.options.getIdentifierOptions().mappingPURL != null && this.options.getIdentifierOptions().mappingPURL.contains(item.type.trim())) type = DatasetDocument.Identifier.IdentifierType.PURL;

				else if(this.options.getIdentifierOptions().mappingURL != null && this.options.getIdentifierOptions().mappingURL.contains(item.type.trim())) type = DatasetDocument.Identifier.IdentifierType.URL;

				else if(this.options.getIdentifierOptions().mappingURN != null && this.options.getIdentifierOptions().mappingURN.contains(item.type.trim())) type = DatasetDocument.Identifier.IdentifierType.URN;

				if(type == null) continue;

				curated.add(new DatasetDocument.Identifier(type, item.value.trim()));

			}

		}

		return curated;

	}

	private List<DatasetDocument.Creator> extractCreator(JSONObject document){

		List<JSONLDUtils.PrincipalInfo> creators = JSONLDUtils.extractPrincipal(document, "creator");

		List<JSONLDUtils.PrincipalInfo> authors = JSONLDUtils.extractPrincipal(document, "author");

		HashSet<String> foundNames = new HashSet<>();

		List<DatasetDocument.Creator> curated = new ArrayList<>();

		for(JSONLDUtils.PrincipalInfo item : creators){

			if(item.name() == null || item.name().trim().length() == 0) continue;

			if(foundNames.contains(item.name())) continue;

			foundNames.add(item.name());

			curated.add(new DatasetDocument.Creator(item.name(), item.affiliationNames()));

		}

		for(JSONLDUtils.PrincipalInfo item : authors){

			if(item.name() == null || item.name().trim().length() == 0) continue;

			if(foundNames.contains(item.name())) continue;

			foundNames.add(item.name());

			curated.add(new DatasetDocument.Creator(item.name(), item.affiliationNames()));

		}

		return curated;

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.io.FileUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.log4j.ConsoleAppender;

import org.apache.log4j.Level;

import org.apache.log4j.Logger;

import org.apache.log4j.PatternLayout;

import java.io.File;

import java.nio.charset.StandardCharsets;

import java.util.HashMap;

import java.util.concurrent.TimeUnit;

public class SchemaOrgMainKaggle {

    private static final Log log = LogFactory.getLog(SchemaOrgMainKaggle.class);

    public static void main(String[] args) throws Exception {

        ConsoleAppender console = new ConsoleAppender();

        console.setLayout(new PatternLayout("%d [%p|%c|%C{1}] %m%n"));

        console.setThreshold(Level.DEBUG);

        console.activateOptions();

        Logger.getLogger("eu.dnetlib.data.collector.plugins").addAppender(console);

        HashMap<String,String> params = new HashMap<>();

        params.put("consumerBlockPolling", Boolean.toString(true));

        params.put("consumerBlockPollingTimeout", "2");

        params.put("consumerBlockPollingTimeoutUnit", TimeUnit.MINUTES.toString());

        params.put("endpointCharset", StandardCharsets.UTF_8.name());

        params.put("updatedDateFormat", "YYYY-MM-DD");

        params.put("createdDateFormat", "YYYY-MM-DD");

        params.put("publicationDateFormat", "YYYY-MM-DD");

        params.put("contributorFallbackType", DatasetDocument.Contributor.ContributorType.Other.toString());

        params.put("identifierFallbackType", DatasetDocument.Identifier.IdentifierType.Handle.toString());

        params.put("identifierFallbackURL", Boolean.toString(true));

        params.put("identifierMappingARK", "ark, ARK");

        params.put("identifierMappingDOI", "doi, DOI");

        params.put("identifierMappingHandle", "Handle, HANDLE");

        params.put("identifierMappingPURL", "purl, PURL");

        params.put("identifierMappingURN", "urn, URN");

        params.put("identifierMappingURL", "url, URL");

        params.put("repositoryAccessType", "httpapi-kaggle");

        params.put("httpapi-kaggle_queueSize", "100");

        params.put("httpapi-kaggle_APICharset", StandardCharsets.UTF_8.name());

        params.put("httpapi-kaggle_queryUrl", "https://www.kaggle.com/datasets_v2.json?sortBy=updated&group=public&page={PAGE}&pageSize=20&size=sizeAll&filetype=fileTypeAll&license=licenseAll");

        params.put("httpapi-kaggle_queryPagePlaceholder", "{PAGE}");

        params.put("httpapi-kaggle_responsePropertyTotalDataset", "totalDatasetListItems");

        params.put("httpapi-kaggle_responsePropertyDatasetList", "datasetListItems");

        params.put("httpapi-kaggle_responsePropertyDatasetUrl", "datasetUrl");

        params.put("httpapi-kaggle_responseBaseDatasetUrl", "https://www.kaggle.com");

        params.put("httpapi-kaggle_producerBlockPollingTimeout", "2");

        params.put("httpapi-kaggle_producerBlockPollingTimeoutUnit", TimeUnit.MINUTES.toString());

        InterfaceDescriptor descriptor = new InterfaceDescriptor();

        descriptor.setId("schema.org - kaggle");

        descriptor.setBaseUrl("https://www.kaggle.com");

        descriptor.setParams(params);

        SchemaOrgPlugin schemaOrgPlugin = new SchemaOrgPlugin();

        Iterable<String> iterable = schemaOrgPlugin.collect(descriptor, null, null);

        String outDir = params.get("repositoryAccessType");

        log.info("saving content in " + outDir);

        File directory = new File(outDir);

        if (directory.exists()) {

            log.info(directory.getAbsolutePath() + " exists, cleaning up");

            FileUtils.deleteDirectory(directory);

        }

        FileUtils.forceMkdir(directory);

        Utils.writeFiles(iterable, outDir);

    }

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.plugins.schemaorg.httpapi.kaggle.KaggleRepositoryIterable;

import eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex.SitemapFileIterator;

import eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex.SitemapIndexIterator;

import eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex.SitemapIndexRepositoryIterable;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.net.MalformedURLException;

import java.net.URL;

import java.nio.charset.StandardCharsets;

import java.util.concurrent.TimeUnit;

public class SchemaOrgPlugin extends AbstractCollectorPlugin {

    private static final Log log = LogFactory.getLog(SchemaOrgPlugin.class);

    public String hello(){

        return "hello";

    }

    @Override

    public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate) throws CollectorServiceException {

        try {

			RepositoryIterable repository = null;

        	String repositoryAccessType = Utils.getAsString(interfaceDescriptor.getParams(), "repositoryAccessType", null);

        	switch(repositoryAccessType) {

				case "sitemapindex": {

					SitemapIndexRepositoryIterable.Options repositoryOptions = this.compileSitemapIndexRepositoryOptions(interfaceDescriptor);

					SitemapIndexRepositoryIterable repositoryIterable = new SitemapIndexRepositoryIterable(repositoryOptions);

					repositoryIterable.bootstrap();

					repository = repositoryIterable;

					break;

				}

				case "httpapi-kaggle": {

					KaggleRepositoryIterable.Options repositoryOptions = this.compileKaggleRepositoryOptions(interfaceDescriptor);

					KaggleRepositoryIterable repositoryIterable = new KaggleRepositoryIterable(repositoryOptions);

					repositoryIterable.bootstrap();

					repository = repositoryIterable;

					break;

				}

				default:

					throw new CollectorServiceException(String.format("unrecognized repository access type ", repositoryAccessType));

			}

			SchemaOrgIterable.Options schemaOrgOptions = this.compileSchemaOrgOptions(interfaceDescriptor);

            SchemaOrgIterable iterable = new SchemaOrgIterable(schemaOrgOptions, repository);

            return iterable;

        } catch (Exception e) {

            throw new CollectorServiceException("Could not create iterator", e);

        }

    }

	private KaggleRepositoryIterable.Options compileKaggleRepositoryOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		KaggleRepositoryIterable.Options kaggleRepositoryOptions = new KaggleRepositoryIterable.Options();

		kaggleRepositoryOptions.setQueueSize(Utils.getAsInt(interfaceDescriptor.getParams(), "httpapi-kaggle_queueSize", 100));

		kaggleRepositoryOptions.setPutTimeout(Utils.getAsLong(interfaceDescriptor.getParams(), "httpapi-kaggle_producerBlockPollingTimeout", 20));

		kaggleRepositoryOptions.setPutTimeoutUnit(Utils.getAsEnum(interfaceDescriptor.getParams(), "httpapi-kaggle_producerBlockPollingTimeoutUnit", TimeUnit.MINUTES, TimeUnit.class));

		kaggleRepositoryOptions.setCharset(Utils.getAsCharset(interfaceDescriptor.getParams(), "httpapi-kaggle_APICharset", StandardCharsets.UTF_8));

		kaggleRepositoryOptions.setQueryUrl(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_queryUrl", null));

		kaggleRepositoryOptions.setQueryPagePlaceholder(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_queryPagePlaceholder", "{PAGE}"));

		kaggleRepositoryOptions.setResponsePropertyTotalDataset(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_responsePropertyTotalDataset", "totalDatasetListItems"));

		kaggleRepositoryOptions.setResponsePropertyDatasetList(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_responsePropertyDatasetList", "datasetListItems"));

		kaggleRepositoryOptions.setResponsePropertyDatasetUrl(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_responsePropertyDatasetUrl", "datasetUrl"));

		kaggleRepositoryOptions.setResponseBaseDatasetUrl(Utils.getAsString(interfaceDescriptor.getParams(), "httpapi-kaggle_responseBaseDatasetUrl", interfaceDescriptor.getBaseUrl()));

		kaggleRepositoryOptions.setRepositoryQueueIteratorOptions(this.compileRepositoryQueueOptions(interfaceDescriptor));

		return kaggleRepositoryOptions;

	}

    private SitemapIndexIterator.Options compileSitemapIndexOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		SitemapIndexIterator.Options sitemapIndexIteratorOptions = new SitemapIndexIterator.Options();

		sitemapIndexIteratorOptions.setCharset(Utils.getAsCharset(interfaceDescriptor.getParams(), "sitemap_IndexCharset", StandardCharsets.UTF_8));

		sitemapIndexIteratorOptions.setIndexUrl(new URL(interfaceDescriptor.getBaseUrl()));

		return sitemapIndexIteratorOptions;

	}

	private SitemapFileIterator.Options compileSitemapFileOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		SitemapFileIterator.Options sitemapFileIteratorOptions = new SitemapFileIterator.Options();

		sitemapFileIteratorOptions.setCharset(Utils.getAsCharset(interfaceDescriptor.getParams(), "sitemap_FileCharset", StandardCharsets.UTF_8));

		sitemapFileIteratorOptions.setSchemaType(Utils.getAsEnum(interfaceDescriptor.getParams(), "sitemap_FileSchema", SitemapFileIterator.Options.SitemapSchemaType.Xml, SitemapFileIterator.Options.SitemapSchemaType.class));

		sitemapFileIteratorOptions.setFileType(Utils.getAsEnum(interfaceDescriptor.getParams(), "sitemap_FileType", SitemapFileIterator.Options.SitemapFileType.Text, SitemapFileIterator.Options.SitemapFileType.class));

		return sitemapFileIteratorOptions;

	}

	private RepositoryQueueIterator.Options compileRepositoryQueueOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		RepositoryQueueIterator.Options repositoryQueueIteratorOptions = new RepositoryQueueIterator.Options();

		repositoryQueueIteratorOptions.setBlockPolling(Utils.getAsBoolean(interfaceDescriptor.getParams(), "consumerBlockPolling", true));

		repositoryQueueIteratorOptions.setPollTimeout(Utils.getAsLong(interfaceDescriptor.getParams(), "consumerBlockPollingTimeout", 2));

		repositoryQueueIteratorOptions.setPollTimeoutUnit(Utils.getAsEnum(interfaceDescriptor.getParams(), "consumerBlockPollingTimeoutUnit", TimeUnit.MINUTES, TimeUnit.class));

		return repositoryQueueIteratorOptions;

	}

	private SitemapIndexRepositoryIterable.Options compileSitemapIndexRepositoryOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		SitemapIndexRepositoryIterable.Options sitemapIndexRepositoryIterableOptions = new SitemapIndexRepositoryIterable.Options();

		sitemapIndexRepositoryIterableOptions.setQueueSize(Utils.getAsInt(interfaceDescriptor.getParams(), "sitemap_queueSize", 100));

		sitemapIndexRepositoryIterableOptions.setPutTimeout(Utils.getAsLong(interfaceDescriptor.getParams(), "sitemap_producerBlockPollingTimeout", 20));

		sitemapIndexRepositoryIterableOptions.setPutTimeoutUnit(Utils.getAsEnum(interfaceDescriptor.getParams(), "sitemap_producerBlockPollingTimeoutUnit", TimeUnit.MINUTES, TimeUnit.class));

		sitemapIndexRepositoryIterableOptions.setRepositoryQueueIteratorOptions(this.compileRepositoryQueueOptions(interfaceDescriptor));

		sitemapIndexRepositoryIterableOptions.setSitemapFileIteratorOptions(this.compileSitemapFileOptions(interfaceDescriptor));

		sitemapIndexRepositoryIterableOptions.setSitemapIndexIteratorOptions(this.compileSitemapIndexOptions(interfaceDescriptor));

		return sitemapIndexRepositoryIterableOptions;

	}

	private EndpointAccessIterator.Options compileEndpointAccessOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		EndpointAccessIterator.Options endpointAccessIteratorOptions = new EndpointAccessIterator.Options();

		endpointAccessIteratorOptions.setCharset(Utils.getAsCharset(interfaceDescriptor.getParams(), "endpointCharset", StandardCharsets.UTF_8));

		return endpointAccessIteratorOptions;

	}

	private DatasetMappingIterator.Options compileDatasetMappingOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		DatasetMappingIterator.Options datasetMappingIteratorOptions = new DatasetMappingIterator.Options();

		DatasetMappingIterator.Options.UpdatedDateOptions datasetMappingIteratorUpdatedDateOptions = new DatasetMappingIterator.Options.UpdatedDateOptions();

		datasetMappingIteratorUpdatedDateOptions.format =Utils.getAsString(interfaceDescriptor.getParams(), "updatedDateFormat", "YYYY-MM-DD");

		datasetMappingIteratorOptions.setUpdatedDateOptions(datasetMappingIteratorUpdatedDateOptions);

		DatasetMappingIterator.Options.CreatedDateOptions datasetMappingIteratorCreatedDateOptions = new DatasetMappingIterator.Options.CreatedDateOptions();

		datasetMappingIteratorCreatedDateOptions.format =Utils.getAsString(interfaceDescriptor.getParams(), "createdDateFormat", "YYYY-MM-DD");

		datasetMappingIteratorOptions.setCreatedDateOptions(datasetMappingIteratorCreatedDateOptions);

		DatasetMappingIterator.Options.PublicationDateOptions datasetMappingIteratorPublicationDateOptions = new DatasetMappingIterator.Options.PublicationDateOptions();

		datasetMappingIteratorPublicationDateOptions.format =Utils.getAsString(interfaceDescriptor.getParams(), "publicationDateFormat", "YYYY-MM-DD");

		datasetMappingIteratorOptions.setPublicationDateOptions(datasetMappingIteratorPublicationDateOptions);

		DatasetMappingIterator.Options.ContributorOptions datasetMappingIteratorContributorOptions = new DatasetMappingIterator.Options.ContributorOptions();

		datasetMappingIteratorContributorOptions.fallbackType =Utils.getAsEnum(interfaceDescriptor.getParams(), "contributorFallbackType",DatasetDocument.Contributor.ContributorType.Other, DatasetDocument.Contributor.ContributorType.class);

		datasetMappingIteratorOptions.setContributorOptions(datasetMappingIteratorContributorOptions);

		DatasetMappingIterator.Options.IdentifierOptions datasetMappingIteratorIdentifierOptions = new DatasetMappingIterator.Options.IdentifierOptions();

		datasetMappingIteratorIdentifierOptions.fallbackType = Utils.getAsEnum(interfaceDescriptor.getParams(), "identifierFallbackType", null, DatasetDocument.Identifier.IdentifierType.class);

		datasetMappingIteratorIdentifierOptions.fallbackURL = Utils.getAsBoolean(interfaceDescriptor.getParams(), "identifierFallbackURL", true);

		datasetMappingIteratorIdentifierOptions.mappingARK = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingARK", null);

		datasetMappingIteratorIdentifierOptions.mappingDOI = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingDOI", null);

		datasetMappingIteratorIdentifierOptions.mappingHandle = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingHandle", null);

		datasetMappingIteratorIdentifierOptions.mappingPURL = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingPURL", null);

		datasetMappingIteratorIdentifierOptions.mappingURL = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingURL", null);

		datasetMappingIteratorIdentifierOptions.mappingURN = Utils.getAsStringCsv(interfaceDescriptor.getParams(), "identifierMappingURN", null);

		datasetMappingIteratorOptions.setIdentifierOptions(datasetMappingIteratorIdentifierOptions);

		return datasetMappingIteratorOptions;

	}

	private SchemaOrgIterable.Options compileSchemaOrgOptions(InterfaceDescriptor interfaceDescriptor) throws MalformedURLException {

		SchemaOrgIterable.Options schemaOrgIterableOptions = new SchemaOrgIterable.Options();

		schemaOrgIterableOptions.setDatasetMappingOptions(this.compileDatasetMappingOptions(interfaceDescriptor));

		schemaOrgIterableOptions.setEndpointAccessOptions(this.compileEndpointAccessOptions(interfaceDescriptor));

		return schemaOrgIterableOptions;

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg;

import org.w3c.dom.Attr;

import org.w3c.dom.Document;

import org.w3c.dom.Element;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.parsers.ParserConfigurationException;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.dom.DOMSource;

import javax.xml.transform.stream.StreamResult;

import java.io.StringWriter;

import java.time.LocalDate;

import java.time.format.DateTimeFormatter;

import java.util.Calendar;

import java.util.Date;

import java.util.List;

public class DatasetDocument {

	private List<Identifier> identifiers;

	private List<Creator> creators;

	private List<String> titles;

	private List<String> alternativeTitles;

	private List<String> publishers;

	private List<LocalDate> publicationDates;

	private List<String> subjects;

	private List<Contributor> contributors;

	private List<LocalDate> createdDates;

	private List<LocalDate> updatedDates;

	private List<String> languages;

	private List<ResourceType> resourceTypes;

	private List<AlternateIdentifier> alternateIdentifier;

	private List<Citation> citations;

	private List<String> sizes;

	private List<String> format;

	private List<String> version;

	private List<License> licenses;

	private List<String> descriptions;

	private List<String> disambiguatingDescriptions;

	private List<SpatialCoverage> geoLocations;

	public List<Identifier> getIdentifiers() {

		return identifiers;

	}

	public void setIdentifiers(List<Identifier> identifiers) {

		this.identifiers = identifiers;

	}

	public List<Creator> getCreators() {

		return creators;

	}

	public void setCreators(List<Creator> creators) {

		this.creators = creators;

	}

	public List<String> getTitles() {

		return titles;

	}

	public void setTitles(List<String> titles) {

		this.titles = titles;

	}

	public List<String> getAlternativeTitles() {

		return alternativeTitles;

	}

	public void setAlternativeTitles(List<String> alternativeTitles) {

		this.alternativeTitles = alternativeTitles;

	}

	public List<String> getPublishers() {

		return publishers;

	}

	public void setPublishers(List<String> publishers) {

		this.publishers = publishers;

	}

	public List<LocalDate> getPublicationDates() {

		return publicationDates;

	}

	public void setPublicationDates(List<LocalDate> publicationDates) {

		this.publicationDates = publicationDates;

	}

	public List<String> getSubjects() {

		return subjects;

	}

	public void setSubjects(List<String> subjects) {

		this.subjects = subjects;

	}

	public List<Contributor> getContributors() {

		return contributors;

	}

	public void setContributors(List<Contributor> contributors) {

		this.contributors = contributors;

	}

	public List<LocalDate> getCreatedDates() {

		return createdDates;

	}

	public void setCreatedDates(List<LocalDate> createdDates) {

		this.createdDates = createdDates;

	}

	public List<LocalDate> getUpdatedDates() {

		return updatedDates;

	}

	public void setUpdatedDates(List<LocalDate> updatedDates) {

		this.updatedDates = updatedDates;

	}

	public List<String> getLanguages() {

		return languages;

	}

	public void setLanguages(List<String> languages) {

		this.languages = languages;

	}

	public List<ResourceType> getResourceTypes() {

		return resourceTypes;

	}

	public void setResourceTypes(List<ResourceType> resourceTypes) {

		this.resourceTypes = resourceTypes;

	}

	public List<AlternateIdentifier> getAlternateIdentifier() {

		return alternateIdentifier;

	}

	public void setAlternateIdentifier(List<AlternateIdentifier> alternateIdentifier) {

		this.alternateIdentifier = alternateIdentifier;

	}

	public List<Citation> getCitations() {

		return citations;

	}

	public void setCitations(List<Citation> citations) {

		this.citations = citations;

	}

	public List<String> getSizes() {

		return sizes;

	}

	public void setSizes(List<String> sizes) {

		this.sizes = sizes;

	}

	public List<String> getFormat() {

		return format;

	}

	public void setFormat(List<String> format) {

		this.format = format;

	}

	public List<String> getVersion() {

		return version;

	}

	public void setVersion(List<String> version) {

		this.version = version;

	}

	public List<License> getLicenses() {

		return licenses;

	}

	public void setLicenses(List<License> licenses) {

		this.licenses = licenses;

	}

	public List<String> getDescriptions() {

		return descriptions;

	}

	public void setDescriptions(List<String> descriptions) {

		this.descriptions = descriptions;

	}

	public List<String> getDisambiguatingDescriptions() {

		return disambiguatingDescriptions;

	}

	public void setDisambiguatingDescriptions(List<String> disambiguatingDescriptions) {

		this.disambiguatingDescriptions = disambiguatingDescriptions;

	}

	public List<SpatialCoverage> getGeoLocations() {

		return geoLocations;

	}

	public void setGeoLocations(List<SpatialCoverage> geoLocations) {

		this.geoLocations = geoLocations;

	}

	private  static String emptyXml;

	private  static Object lockEmptyXml = new Object();

	public static String emptyXml() {

		if(DatasetDocument.emptyXml!=null) return DatasetDocument.emptyXml;

		String xml = null;

		try {

			DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();

			DocumentBuilder docBuilder = docFactory.newDocumentBuilder();

			Document doc = docBuilder.newDocument();

			Element root = doc.createElement("dataset");

			doc.appendChild(root);

			TransformerFactory tf = TransformerFactory.newInstance();

			Transformer transformer = tf.newTransformer();

			StringWriter writer = new StringWriter();

			transformer.transform(new DOMSource(doc), new StreamResult(writer));

			xml = writer.getBuffer().toString();

		}catch(Exception ex){

			xml = "<dataset/>";

		}

		synchronized (DatasetDocument.lockEmptyXml) {

			if (DatasetDocument.emptyXml == null) DatasetDocument.emptyXml = xml;

		}

		return DatasetDocument.emptyXml;

	}

	public String toXml() throws Exception {

		DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();

		DocumentBuilder docBuilder = docFactory.newDocumentBuilder();

		Document doc = docBuilder.newDocument();

		Element root = doc.createElement("dataset");

		doc.appendChild(root);

		if(this.identifiers!=null){

			for(Identifier item : this.identifiers){

				item.toXml(root);

			}

		}

		if(this.creators!=null){

			Element creators = doc.createElement("creators");

			root.appendChild(creators);

			for(Creator item : this.creators){

				item.toXml(creators);

			}

		}

		if(this.titles!=null || this.alternativeTitles!=null){

			Element titles = doc.createElement("titles");

			root.appendChild(titles);

			if(this.titles!=null) {

				for (String item : this.titles) {

					Element title = doc.createElement("title");

					titles.appendChild(title);

					title.appendChild(doc.createTextNode(item));

				}

			}

			if(this.alternativeTitles!=null) {

				for (String item : this.alternativeTitles) {

					Element title = doc.createElement("title");

					titles.appendChild(title);

					title.setAttribute("titleType", "AlternativeTitle");

					title.appendChild(doc.createTextNode(item));

				}

			}

		}

		if(this.publishers!=null){

			for(String item : this.publishers){

				Element publisher = doc.createElement("publisher");

				root.appendChild(publisher);

				publisher.appendChild(doc.createTextNode(item));

			}

		}

		if(this.publicationDates!=null){

			for(LocalDate item : this.publicationDates){

				Element publicationYear = doc.createElement("publicationYear");

				root.appendChild(publicationYear);

				publicationYear.appendChild(doc.createTextNode(Integer.toString(item.getYear())));

			}

		}

		if(this.subjects!=null){

			Element subjects = doc.createElement("subjects");

			root.appendChild(subjects);

			for(String item : this.subjects){

				Element subject = doc.createElement("subject");

				subjects.appendChild(subject);

				subject.appendChild(doc.createTextNode(item));

			}

		}

		if(this.contributors!=null){

			for(Contributor item : this.contributors){

				item.toXml(root);

			}

		}

		if(this.createdDates!=null || this.updatedDates!=null){

			Element dates = doc.createElement("dates");

			root.appendChild(dates);

			DateTimeFormatter formatter = DateTimeFormatter.ofPattern("YYYY-MM-DD");

			if(createdDates!=null) {

				for (LocalDate item : this.createdDates) {

					Element date = doc.createElement("date");

					root.appendChild(date);

					date.setAttribute("dateType", "Created");

					date.appendChild(doc.createTextNode(item.format(formatter)));

				}

			}

			if(updatedDates!=null) {

				for (LocalDate item : this.updatedDates) {

					Element date = doc.createElement("date");

					root.appendChild(date);

					date.setAttribute("dateType", "Updated");

					date.appendChild(doc.createTextNode(item.format(formatter)));

				}

			}

		}

		if(this.languages!=null){

			for(String item : this.languages){

				Element language = doc.createElement("language");

				root.appendChild(language);

				language.appendChild(doc.createTextNode(item));

			}

		}

		if(this.resourceTypes!=null){

			for(ResourceType item : this.resourceTypes){

				item.toXml(root);

			}

		}

		if(this.alternateIdentifier!=null){

			Element alternateIdentifiers = doc.createElement("alternateIdentifiers");

			root.appendChild(alternateIdentifiers);

			for(AlternateIdentifier item : this.alternateIdentifier){

				item.toXml(alternateIdentifiers);

			}

		}

		if(this.citations!=null){

			for(Citation item : this.citations){

				item.toXml(root);

			}

		}

		if(this.sizes!=null){

			Element sizes = doc.createElement("sizes");

			root.appendChild(sizes);

			for(String item : this.sizes){

				Element size = doc.createElement("size");

				sizes.appendChild(size);

				size.appendChild(doc.createTextNode(item));

			}

		}

		if(this.format!=null){

			Element formats = doc.createElement("formats");

			root.appendChild(formats);

			for(String item : this.format){

				Element format = doc.createElement("format");

				formats.appendChild(format);

				format.appendChild(doc.createTextNode(item));

			}

		}

		if(this.version!=null){

			for(String item : this.version){

				Element version = doc.createElement("version");

				root.appendChild(version);

				version.appendChild(doc.createTextNode(item));

			}

		}

		if(this.licenses!=null){

			Element rightsList = doc.createElement("rightsList");

			root.appendChild(rightsList);

			for(License item : this.licenses){

				item.toXml(rightsList);

			}

		}

		if(this.descriptions!=null || this.disambiguatingDescriptions!=null){

			Element descriptions = doc.createElement("descriptions");

			root.appendChild(descriptions);

			if(this.descriptions!=null) {

				for (String item : this.descriptions) {

					Element description = doc.createElement("description");

					descriptions.appendChild(description);

					description.setAttribute("descriptionType", "Abstract");

					description.appendChild(doc.createTextNode(item));

				}

			}

			if(this.disambiguatingDescriptions!=null) {

				for (String item : this.disambiguatingDescriptions) {

					Element description = doc.createElement("description");

					descriptions.appendChild(description);

					description.setAttribute("descriptionType", "Other");

					description.appendChild(doc.createTextNode(item));

				}

			}

		}

		if(this.geoLocations!=null){

			Element geoLocations = doc.createElement("geoLocations");

			root.appendChild(geoLocations);

			for(SpatialCoverage item : this.geoLocations){

				item.toXml(geoLocations);

			}

		}

		TransformerFactory tf = TransformerFactory.newInstance();

		Transformer transformer = tf.newTransformer();

		StringWriter writer = new StringWriter();

		transformer.transform(new DOMSource(doc), new StreamResult(writer));

		String xml = writer.getBuffer().toString();

		return xml;

	}

	public static class SpatialCoverage{

		public static class Point{

			public String latitude;

			public String longitude;

			public Point() {}

			public Point(String latitude, String longitude){

				this.latitude = latitude;

				this.longitude = longitude;

			}

		}

		public String name;

		public List<Point> points;

		public List<String> boxes;

		public SpatialCoverage() {}

		public SpatialCoverage(String name, List<Point> points, List<String> boxes ) {

			this.name = name;

			this.points = points;

			this.boxes = boxes;

		}

		public void toXml(Element parent){

			Element node = parent.getOwnerDocument().createElement("geoLocation");

			parent.appendChild(node);

			if(this.points!=null) {

				for(Point point : this.points) {

					if(point.latitude == null || point.longitude == null) continue;

					Element geoLocationPoint = parent.getOwnerDocument().createElement("geoLocationPoint");

					geoLocationPoint.appendChild(parent.getOwnerDocument().createTextNode(String.format("%s %s", point.latitude, point.longitude)));

					node.appendChild(geoLocationPoint);

				}

			}

			if(this.boxes!=null) {

				for(String box : this.boxes) {

					if(box == null) continue;

					Element geoLocationBox = parent.getOwnerDocument().createElement("geoLocationBox");

					geoLocationBox.appendChild(parent.getOwnerDocument().createTextNode(box));

					node.appendChild(geoLocationBox);

				}

			}

			if(this.name!=null) {

				Element geoLocationPlace = parent.getOwnerDocument().createElement("geoLocationPlace");

				geoLocationPlace.appendChild(parent.getOwnerDocument().createTextNode(this.name));

				node.appendChild(geoLocationPlace);

			}

		}

	}

	public static class License{

		public String name;

		public String url;

		public License() {}

		public License(String name, String url) {

			this.name = name;

			this.url = url;

		}

		public void toXml(Element parent){

			Element node = parent.getOwnerDocument().createElement("rights");

			parent.appendChild(node);

			if(this.url!=null) {

				node.setAttribute("rightsURI", this.url);

			}

			if(this.name!=null) {

				node.appendChild(parent.getOwnerDocument().createTextNode(this.name));

			}

		}

	}

	public static class Citation{

		public enum CitationIdentifierType{

			ARK, arXiv, bibcode, DOI, EAN13, EISSN, Handle, ISBN, ISSN, ISTC, LISSN, LSID, PMID,

			PURL, UPC, URL, URN

		}

		public CitationIdentifierType type;

		public String value;

		public Citation() {}

		public Citation(String value, CitationIdentifierType type) {

			this.value = value;

			this.type = type;

		}

		public void toXml(Element parent){

			Element node = parent.getOwnerDocument().createElement("relatedIdentifier");

			parent.appendChild(node);

			node.setAttribute("relatedIdentifierType", this.type.toString());

			node.setAttribute("relationType", "Cites");

			node.appendChild(parent.getOwnerDocument().createTextNode(this.value));

		}

	}

	public static class Contributor{

		public enum ContributorType{

			ContactPerson, DataCollector, DataCurator, DataManager, Distributor, Editor, Funder, HostingInstitution,

			Producer, ProjectLeader, ProjectManager, ProjectMember, RegistrationAgency, RegistrationAuthority,

			RelatedPerson, Researcher, ResearchGroup, RightsHolder, Sponsor, Supervisor, WorkPackageLeader, Other

		}

		public String name;

		public List<String> affiliations;

		public ContributorType type;

		public Contributor() {

		}

		public Contributor(String name) {

			this.name = name;

		}

		public Contributor(String name, List<String> affiliations) {

			this.name = name;

			this.affiliations = affiliations;

		}

		public Contributor(String name, List<String> affiliations, ContributorType type) {

			this.name = name;

			this.affiliations = affiliations;

			this.type = type;

		}

		public void toXml(Element parent){

			Element node = parent.getOwnerDocument().createElement("contributor");

			parent.appendChild(node);

			node.setAttribute("contributorType", this.type.toString());

			if(this.name!=null) {

				Element contributorName = parent.getOwnerDocument().createElement("contributorName");

				node.appendChild(contributorName);

				contributorName.appendChild(parent.getOwnerDocument().createTextNode(this.name));

			}

			if(this.affiliations!=null) {

				for(String item : this.affiliations) {

					Element affiliation = parent.getOwnerDocument().createElement("affiliation");

					node.appendChild(affiliation);

					affiliation.appendChild(parent.getOwnerDocument().createTextNode(item));

				}

			}

		}

	}

	public static class AlternateIdentifier{

		public String identifier;

		public String type;

		public AlternateIdentifier() {}

		public AlternateIdentifier(String identifier, String type) {

			this.identifier = identifier;

			this.type = type;

		}

		public void toXml(Element parent){

			Element node = parent.getOwnerDocument().createElement("alternateIdentifier");

			parent.appendChild(node);

			if(this.type!=null) {

				node.setAttribute("alternateIdentifierType", this.type);

			}

			if(this.identifier!=null) {

				node.appendChild(parent.getOwnerDocument().createTextNode(this.identifier));

			}

		}

	}

	public static class ResourceType{

		public enum ResourceTypeGeneralType {

			Audiovisual, Collection, Dataset, Event, Image, InteractiveResource, Model, PhysicalObject, Service,

			Software, Sound, Text, Workflow, Other

		}

		public ResourceTypeGeneralType type;

		public ResourceType() {}

		public ResourceType(ResourceTypeGeneralType type) {

			this.type = type;

		}

		public void toXml(Element parent){

			Element node = parent.getOwnerDocument().createElement("resourceType");

			parent.appendChild(node);

			if(this.type!=null) {

				node.setAttribute("resourceTypeGeneral", this.type.toString());

			}

		}

	}

	public static class Creator {

		public String name;

		public List<String> affiliations;

		public Creator() {

		}

		public Creator(String name) {

			this.name = name;

		}

		public Creator(String name, List<String> affiliations) {

			this.name = name;

			this.affiliations = affiliations;

		}

		public void toXml(Element parent){

			Element node = parent.getOwnerDocument().createElement("creator");

			parent.appendChild(node);

			if(this.name!=null) {

				Element creatorName = parent.getOwnerDocument().createElement("creatorName");

				node.appendChild(creatorName);

				creatorName.appendChild(parent.getOwnerDocument().createTextNode(this.name));

			}

			if(this.affiliations!=null) {

				for(String item : this.affiliations) {

					Element affiliation = parent.getOwnerDocument().createElement("affiliation");

					node.appendChild(affiliation);

					affiliation.appendChild(parent.getOwnerDocument().createTextNode(item));

				}

			}

		}

	}

	public static class Identifier {

		public enum IdentifierType {

			ARK, DOI, Handle, PURL, URN, URL

		}

		public String value;

		public IdentifierType type;

		public Identifier() {

		}

		public Identifier(IdentifierType type, String value) {

			this.type = type;

			this.value = value;

		}

		public void toXml(Element parent){

			Element node = parent.getOwnerDocument().createElement("identifier");

			parent.appendChild(node);

			node.setAttribute("identifierType", this.type.toString());

			if(this.value!=null) {

				node.appendChild(parent.getOwnerDocument().createTextNode(this.value));

			}

		}

	}

}

package eu.dnetlib.data.collector.plugins.schemaorg.httpapi;

import eu.dnetlib.data.collector.plugins.schemaorg.RepositoryIterable;

public interface HttpApiRepositoryIterable extends RepositoryIterable {

}

package eu.dnetlib.data.collector.plugins.schemaorg.sitemapindex;

import eu.dnetlib.data.collector.plugins.schemaorg.Utils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.net.URL;

import java.nio.charset.Charset;

import java.util.*;

public class SitemapIndexIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(SitemapIndexIterator.class);

	public static class Options {

		private URL indexUrl;

		private Charset charset;

		public Options(){}

		public Options(URL indexUrl, Charset charset){

			this.indexUrl = indexUrl;

			this.charset = charset;

		}

		public URL getIndexUrl() {

			return indexUrl;

		}