D-Net

package eu.dnetlib.data.collector.plugins.projects.gtr2;

import java.util.Iterator;

import java.util.NoSuchElementException;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.ExecutorService;

import java.util.concurrent.Executors;

import java.util.concurrent.TimeUnit;

import com.ximpleware.AutoPilot;

import com.ximpleware.VTDGen;

import com.ximpleware.VTDNav;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.CollectorServiceRuntimeException;

import eu.dnetlib.enabling.resultset.SizedIterable;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.joda.time.DateTime;

import org.joda.time.format.DateTimeFormat;

import org.joda.time.format.DateTimeFormatter;

import eu.dnetlib.data.collector.plugins.HttpConnector;

/**

 * Created by alessia on 28/11/16.

 */

public class Gtr2ProjectsIterator implements Iterator<String> {

	public static final String TERMINATOR = "ARNOLD";

	public static final int WAIT_END_SECONDS = 600;

	public static final int PAGE_SZIE = 20;

	private static final Log log = LogFactory.getLog(Gtr2ProjectsIterator.class);

	private String queryURL;

	private int total = -1;

	private int startFromPage = 1;

	private int endAtPage;

	private VTDGen vg;

	private VTDNav vn;

	private AutoPilot ap;

	private String namespaces;

	private boolean incremental = false;

	private DateTime fromDate;

	private DateTimeFormatter simpleDateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");

	private ArrayBlockingQueue<String> projects = new ArrayBlockingQueue<String>(20);

	//private boolean finished = false;

	private final ExecutorService es = Executors.newFixedThreadPool(PAGE_SZIE);

	private String nextElement = "<doc></doc>";

	private HttpConnector connector;

	public boolean hasNext() {

			return !nextElement.equals(TERMINATOR);

	}

	@Override

	public String next() {

		try{

			return nextElement;

		}finally{

			try {

				nextElement = projects.poll(WAIT_END_SECONDS, TimeUnit.SECONDS);

			} catch (InterruptedException e) {

				throw new RuntimeException(e);

			}

		}

	}

	@Override

	public void remove() {

		throw new UnsupportedOperationException();

	}

	public Gtr2ProjectsIterator(final String baseUrl, final String fromDate) throws CollectorServiceException {

		prepare(baseUrl, fromDate);

		fillInfo(true);

	}

	public Gtr2ProjectsIterator(final String baseUrl, final String fromDate, final int startFromPage, final int endAtPage) throws CollectorServiceException {

		prepare(baseUrl, fromDate);

		this.setStartFromPage(startFromPage);

		this.setEndAtPage(endAtPage);

		fillInfo(false);

	}

	private void prepare(final String baseUrl, final String fromDate) {

		connector = new HttpConnector();

		queryURL = baseUrl + "/projects";

		vg = new VTDGen();

		this.incremental = StringUtils.isNotBlank(fromDate);

		if (incremental) {

			// I expect fromDate in the format 'yyyy-MM-dd'. See class eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode

			this.fromDate = DateTime.parse(fromDate, simpleDateTimeFormatter);

			log.debug("fromDate string: " + fromDate + " -- parsed: " + this.fromDate.toString());

		}

	}

	private void fillInfo(final boolean all) throws CollectorServiceException {

		try {

			// log.debug("Getting hit count from: " + queryURL);

			byte[] bytes = connector.getInputSource(queryURL).getBytes("UTF-8");

			vg.setDoc(bytes);

			vg.parse(false);

			//vg.parseHttpUrl(queryURL, false);

			initParser();

			String hitCount = vn.toNormalizedString(vn.getAttrVal("totalSize"));

			String totalPages = vn.toNormalizedString(vn.getAttrVal("totalPages"));

			namespaces = "xmlns:ns1=\"" + vn.toNormalizedString(vn.getAttrVal("ns1")) + "\" ";

			namespaces += "xmlns:ns2=\"" + vn.toNormalizedString(vn.getAttrVal("ns2")) + "\" ";

			namespaces += "xmlns:ns3=\"" + vn.toNormalizedString(vn.getAttrVal("ns3")) + "\" ";

			namespaces += "xmlns:ns4=\"" + vn.toNormalizedString(vn.getAttrVal("ns4")) + "\" ";

			namespaces += "xmlns:ns5=\"" + vn.toNormalizedString(vn.getAttrVal("ns5")) + "\" ";

			namespaces += "xmlns:ns6=\"" + vn.toNormalizedString(vn.getAttrVal("ns6")) + "\" ";

			if (all) {

				setEndAtPage(Integer.parseInt(totalPages));

				total = Integer.parseInt(hitCount);

			}

			Thread ft = new Thread(new FillProjectList());

			ft.start();

			log.debug("Expected number of pages: " + (endAtPage - startFromPage + 1));

		} catch (NumberFormatException e) {

			log.error("Cannot set the total count or the number of pages");

			throw new CollectorServiceException(e);

		} catch (Throwable e) {

			throw new CollectorServiceException(e);

		}

	}

	private void initParser() {

		vn = vg.getNav();

		ap = new AutoPilot(vn);

	}

	public String getQueryURL() {

		return queryURL;

	}

	public void setQueryURL(final String queryURL) {

		this.queryURL = queryURL;

	}

	public int getTotal() {

		return total;

	}

	public void setTotal(final int total) {

		this.total = total;

	}

	public int getEndAtPage() {

		return endAtPage;

	}

	public void setEndAtPage(final int endAtPage) {

		this.endAtPage = endAtPage;

		log.debug("Overriding endAtPage to " + endAtPage);

	}

	public VTDGen getVg() {

		return vg;

	}

	public void setVg(final VTDGen vg) {

		this.vg = vg;

	}

	public VTDNav getVn() {

		return vn;

	}

	public void setVn(final VTDNav vn) {

		this.vn = vn;

	}

	public AutoPilot getAp() {

		return ap;

	}

	public void setAp(final AutoPilot ap) {

		this.ap = ap;

	}

	public String getNamespaces() {

		return namespaces;

	}

	public void setNamespaces(final String namespaces) {

		this.namespaces = namespaces;

	}

	public int getStartFromPage() {

		return startFromPage;

	}

	public void setStartFromPage(final int startFromPage) {

		this.startFromPage = startFromPage;

		log.debug("Overriding startFromPage to " + startFromPage);

	}

	private class FillProjectList implements Runnable {

		private boolean morePages = true;

		private int pageNumber = startFromPage;

		@Override

		public void run() {

			String resultPageUrl = "";

			try {

				do {

					resultPageUrl = getNextPageUrl();

					log.debug("Page: " + resultPageUrl);

					// clear VGen before processing the next file

					vg.clear();

					byte[] bytes = connector.getInputSource(resultPageUrl).getBytes("UTF-8");

					vg.setDoc(bytes);

					vg.parse(false);

					//vg.parseHttpUrl(resultPageUrl, false);

					initParser();

					ap.selectXPath("//project");

					int res;

					while ((res = ap.evalXPath()) != -1) {

						final String projectHref = vn.toNormalizedString(vn.getAttrVal("href"));

						Thread t = new Thread(new ParseProject(projectHref));

						t.setName("Thread for " + res);

						es.execute(t);

					}

					ap.resetXPath();

				} while (morePages);

				es.shutdown();

				es.awaitTermination(WAIT_END_SECONDS, TimeUnit.SECONDS);

				projects.put(TERMINATOR);

			} catch (Throwable e) {

				log.error("Exception processing " + resultPageUrl + "\n" + e.getMessage());

			}

		}

		private String getNextPageUrl() {

			String url = queryURL + "?p=" + pageNumber;

			if (pageNumber == endAtPage) {

				morePages = false;

			}

			pageNumber++;

			return url;

		}

	}

	private class ParseProject implements Runnable {

		VTDNav vn1;

		VTDGen vg1;

		private String projectRef;

		public ParseProject(String projectHref) {

			if(projectHref.contains("gtr.gtr")){

				projectHref = projectHref.replace("gtr.gtr","gtr");

			}

			projectRef = projectHref;

			log.debug("strat " + projectRef);

			vg1 = new VTDGen();

			try {

				byte[] bytes = connector.getInputSource(projectRef).getBytes("UTF-8");

				vg1.setDoc(bytes);

				vg1.parse(false);

				//vg1.parseHttpUrl(projectRef, false);

				vn1 = vg1.getNav();

			}catch(Throwable e){

				log.error("Exception processing " + projectRef + "\n" + e.getMessage());

			}

			log.debug("end " + projectRef);

		}

		private int projectsUpdate(String attr) throws CollectorServiceException {

			try {

				int index = vn1.getAttrVal(attr);

				if (index != -1) {

					String d = vn1.toNormalizedString(index);

					DateTime recordDate = DateTime.parse(d.substring(0, d.indexOf("T")), simpleDateTimeFormatter);

					// updated or created after the last time it was collected

					if (recordDate.isAfter(fromDate)) {

						log.debug("New project to collect");

						return index;

					}

					return -1;

				}

				return index;

			} catch (Throwable e) {

				throw new CollectorServiceException(e);

			}

		}

		private String collectProject() throws CollectorServiceException {

			try {

				int p = vn1.getAttrVal("href");

				final String projectHref = vn1.toNormalizedString(p);

				log.debug("collecting project at " + projectHref);

				Gtr2Helper gtr2Helper = new Gtr2Helper();

				String projectPackage = gtr2Helper.processProject(vn1, namespaces);

				return projectPackage;

			} catch (Throwable e) {

				throw new CollectorServiceException(e);

			}

		}

		private boolean add(String attr) throws CollectorServiceException {

			return projectsUpdate(attr) != -1;

		}

		@Override

		public void run() {

			log.debug("Getting project info from " + projectRef);

			try {

				if (!incremental || (incremental && (add("created") || add("updated")))) {

					projects.put(collectProject());

					log.debug("Project enqueued " + projectRef);

				}

			} catch (Throwable e) {

				log.error("Error on ParseProject " + e.getMessage());

				throw new CollectorServiceRuntimeException(e);

			}

		}

	}

}

package eu.dnetlib.data.collector.plugins.projects.grist;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

/**

 * Plugin to collect metadata record about projects and fundings via the europePMC GRIST API (e.g. WT projects).

 * <p>

 * Documentation on GRIST API: http://europepmc.org/GristAPI.

 * </p>

 * <p>

 * BaseURL: http://www.ebi.ac.uk/europepmc/GristAPI/rest/get/query=ga:"Wellcome Trust"&resultType=core

 * where resultType=core asks for the complete information (including abstracts).

 * The results returned by the API are XMLs.

 * </p>

 * <p>

 * Pagination: use parameter 'page'. When the response contains empty 'RecordList', it means we reached the end.

 * </p>

 *

 * @author alessia

 */

public class GristCollectorPlugin extends AbstractCollectorPlugin {

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		//baseURL: http://www.ebi.ac.uk/europepmc/GristAPI/rest/get/query=ga:%22Wellcome%20Trust%22&resultType=core

		return new GristProjectsIterable(interfaceDescriptor.getBaseUrl());

	}

}

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-collector-plugins</artifactId>

	<version>1.7.8</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-collector-plugins/tags/dnet-collector-plugins-1.7.8</developerConnection>

	</scm>

	<build>

		<plugins>

			<plugin>

				<artifactId>maven-assembly-plugin</artifactId>

				<configuration>

					<archive>

						<manifest>

							<mainClass>eu.dnetlib.data.collector.plugins.schemaorg.SchemaOrgMainReactome</mainClass>

						</manifest>

					</archive>

					<descriptorRefs>

						<descriptorRef>jar-with-dependencies</descriptorRef>

					</descriptorRefs>

				</configuration>

			</plugin>

		</plugins>

	</build>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service-rmi</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service</artifactId>

			<version>[3.3.26,4.0.0)</version>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>${google.gson.version}</version>

		</dependency>

		<dependency>

			<groupId>commons-io</groupId>

			<artifactId>commons-io</artifactId>

			<version>${commons.io.version}</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.apache.httpcomponents</groupId>

			<artifactId>httpclient</artifactId>

			<version>4.5</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0, 3.0.0)</version>

			<scope>provided</scope>

		</dependency>

		<dependency>

			<groupId>com.ximpleware</groupId>

			<artifactId>vtd-xml</artifactId>

			<version>[2.12, 3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.9.2</version>

		</dependency>

		<dependency>

			<groupId>org.json</groupId>

			<artifactId>json</artifactId>

			<version>20180813</version>

		 <type>jar</type>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-lang3</artifactId>

			<version>3.5</version>

		</dependency>

		<dependency>

			<groupId>org.apache.poi</groupId>

			<artifactId>poi</artifactId>

			<version>3.16</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->

		<dependency>

			<groupId>org.apache.poi</groupId>

			<artifactId>poi-ooxml</artifactId>

			<version>3.16</version>

		</dependency>

		<dependency>

			<groupId>org.jsoup</groupId>

			<artifactId>jsoup</artifactId>

			<version>1.11.2</version>

		</dependency>

		<dependency>

			<groupId>commons-lang</groupId>

			<artifactId>commons-lang</artifactId>

			<version>2.6</version>

			<scope>compile</scope>

		</dependency>

        <dependency>

            <groupId>org.mockito</groupId>

            <artifactId>mockito-core</artifactId>

            <version>3.3.3</version>

            <scope>test</scope>

        </dependency>

    </dependencies>

</project>

/**

 * 

 */

package eu.dnetlib.data.collector.plugins.rest;

import com.google.gson.Gson;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.lang3.StringUtils;

import org.json.JSONObject;

import java.util.Map;

/**

 * @author 	js, Andreas Czerniak

 * @date 	2020-04-09

 *

 */

public class RestCollectorPlugin extends AbstractCollectorPlugin {

	@Override

	public Iterable<String> collect(InterfaceDescriptor ifDescriptor, String arg1, String arg2)

			throws CollectorServiceException {

		final String baseUrl = ifDescriptor.getBaseUrl();

		final String resumptionType = ifDescriptor.getParams().get("resumptionType");

		final String resumptionParam = ifDescriptor.getParams().get("resumptionParam");

		final String resumptionXpath = ifDescriptor.getParams().get("resumptionXpath");

		final String resultTotalXpath = ifDescriptor.getParams().get("resultTotalXpath");

		final String resultFormatParam = ifDescriptor.getParams().get("resultFormatParam");

		final String resultFormatValue = ifDescriptor.getParams().get("resultFormatValue");

		final String resultSizeParam = ifDescriptor.getParams().get("resultSizeParam");

		final String resultSizeValue = (StringUtils.isBlank(ifDescriptor.getParams().get("resultSizeValue"))) ? "100" : ifDescriptor.getParams().get("resultSizeValue");

        final String queryParams = ifDescriptor.getParams().get("queryParams");

		final String entityXpath = ifDescriptor.getParams().get("entityXpath");

		final String authMethod = ifDescriptor.getParams().get("authMethod");

		final String authToken = ifDescriptor.getParams().get("authToken");

		final String requestHeaderMap = ifDescriptor.getParams().get("requestHeaderMap");

		Gson gson = new Gson();

		Map<String, String> requestHeaders = gson.fromJson(requestHeaderMap, Map.class);

		if (StringUtils.isBlank(baseUrl)) {throw new CollectorServiceException("Param 'baseUrl' is null or empty");}

		if (StringUtils.isBlank(resumptionType)) {throw new CollectorServiceException("Param 'resumptionType' is null or empty");}

		if (StringUtils.isBlank(resumptionParam)) {throw new CollectorServiceException("Param 'resumptionParam' is null or empty");}

		// if (StringUtils.isBlank(resumptionXpath)) {throw new CollectorServiceException("Param 'resumptionXpath' is null or empty");}

		// if (StringUtils.isBlank(resultTotalXpath)) {throw new CollectorServiceException("Param 'resultTotalXpath' is null or empty");}

		// resultFormatParam can be emtpy because some Rest-APIs doesn't like this argument in the query

		//if (StringUtils.isBlank(resultFormatParam)) {throw new CollectorServiceException("Param 'resultFormatParam' is null, empty or whitespace");}

		if (StringUtils.isBlank(resultFormatValue)) {throw new CollectorServiceException("Param 'resultFormatValue' is null or empty");}

		// if (StringUtils.isBlank(resultSizeParam)) {throw new CollectorServiceException("Param 'resultSizeParam' is null or empty");}

		// prevent resumptionType: discover -- if (Integer.valueOf(resultSizeValue) <= 1) {throw new CollectorServiceException("Param 'resultSizeValue' is less than 2");}

		// queryParams could be empty like for DRIS+ API from euroCRIS

		//if (StringUtils.isBlank(queryParams)) {throw new CollectorServiceException("Param 'queryParams' is null or empty");}

		if (StringUtils.isBlank(entityXpath)) {throw new CollectorServiceException("Param 'entityXpath' is null or empty");}

		String resFormat = ifDescriptor.getParams().get("resultOutputFormat");

		final String resultOutputFormat = StringUtils.isNotBlank(resFormat) ? resFormat.toLowerCase() : resultFormatValue.toLowerCase();

		return () -> new RestIterator(

				baseUrl,

				resumptionType,

				resumptionParam,

				resumptionXpath,

				resultTotalXpath,

				resultFormatParam,

				resultFormatValue,

				resultSizeParam,

                resultSizeValue,

				queryParams,

				entityXpath,

				authMethod,

				authToken,

				resultOutputFormat, requestHeaders);

	}

}

package eu.dnetlib.data.collector.plugins.utils;

import eu.dnetlib.data.collector.plugins.oai.engine.XmlCleaner;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class JsonUtils {

    private static final Log log = LogFactory.getLog(JsonUtils.class);

    public static final String wrapName = "recordWrap";

    /**

     * convert in JSON-KeyName 'whitespace(s)' to '_' and '/' to '_', '(' and ')' to ''

     * check W3C XML syntax: https://www.w3.org/TR/2006/REC-xml11-20060816/#sec-starttags for valid tag names

     * and work-around for the JSON to XML converting of org.json.XML-package.

     *

     * known bugs:     doesn't prevent     "key name":" ["sexy name",": penari","erotic dance"],

     *

     * @param jsonInput

     * @return convertedJsonKeynameOutput

     */

    public String syntaxConvertJsonKeyNames(String jsonInput) {

        log.trace("before convertJsonKeyNames: " + jsonInput);

        // pre-clean json - rid spaces of element names (misinterpreted as elements with attributes in xml)

        // replace ' 's in JSON Namens with '_'

        while (jsonInput.matches(".*\"([^\"]*)\\s+([^\"]*)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"([^\"]*)\\s+([^\"]*)\":", "\"$1_$2\":");

        }

        // replace forward-slash (sign '/' ) in JSON Names with '_'

        while (jsonInput.matches(".*\"([^\"]*)/([^\"]*)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"([^\"]*)/([^\"]*)\":", "\"$1_$2\":");

        }

        // replace '(' in JSON Names with ''

        while (jsonInput.matches(".*\"([^\"]*)[(]([^\"]*)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"([^\"]*)[(]([^\"]*)\":", "\"$1$2\":");

        }

        // replace ')' in JSON Names with ''

        while (jsonInput.matches(".*\"([^\"]*)[)]([^\"]*)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"([^\"]*)[)]([^\"]*)\":", "\"$1$2\":");

        }

        // add prefix of startNumbers in JSON Keynames with 'n_'

        while (jsonInput.matches(".*\"([^\"][0-9])([^\"]*)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"([^\"][0-9])([^\"]*)\":", "\"n_$1$2\":");

        }

        // add prefix of only numbers in JSON Keynames with 'm_'

        while (jsonInput.matches(".*\"([0-9]+)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"([0-9]+)\":", "\"m_$1\":");

        }

        // replace ':' between number like '2018-08-28T11:05:00Z' in JSON keynames with ''

        while (jsonInput.matches(".*\"([^\"]*[0-9]):([0-9][^\"]*)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"([^\"]*[0-9]):([0-9][^\"]*)\":", "\"$1$2\":");

        }

        // replace ',' in JSON Keynames with '.' to prevent , in xml tagnames.

        //            while (jsonInput.matches(".*\"([^\"]*),([^\"]*)\":.*")) {

        //                jsonInput = jsonInput.replaceAll("\"([^\"]*),([^\"]*)\":", "\"$1.$2\":");

        //            }

        // replace '=' in JSON Keynames with '-'

        while (jsonInput.matches(".*\"([^\"]*)=([^\"]*)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"([^\"]*)=([^\"]*)\":", "\"$1-$2\":");

        }

        // replace '@' in JSON Keynames with 'oat_'

        while (jsonInput.matches(".*\"@([^\"]*)\":.*")) {

            jsonInput = jsonInput.replaceAll("\"@([^\"]*)\":", "\"oat_$1\":");

        }

        log.trace("after syntaxConvertJsonKeyNames: " + jsonInput);

        return jsonInput;

    }

    /**

     *

     * https://www.w3.org/TR/REC-xml/#charencoding shows character enoding in entities

     *          *

     * @param bufferStr - XML string

     * @return

     */

    public String cleanUnwantedJsonCharsInXmlTagnames(String bufferStr) {

        while (bufferStr.matches(".*<([^<>].*),(.)>.*")) {

            bufferStr = bufferStr.replaceAll("<([^<>.*),(.*)>", "<$1$2>");

        }

        // replace [#x10-#x1f] with ''

        //            while (bufferStr.matches(".*&#x1[0-9a-f].*")) {

        //                bufferStr = bufferStr.replaceAll("&#x1([0-9a-fA-F])", "");

        //            }

        return bufferStr;

    }

    public String convertToXML(final String jsonRecord){

        String resultXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";

        log.trace("before convertToXML: " + jsonRecord);

        org.json.JSONObject jsonObject = new org.json.JSONObject(syntaxConvertJsonKeyNames(jsonRecord));

        resultXml += org.json.XML.toString(jsonObject, wrapName); // wrap xml in single root element

        log.trace("before inputStream: " + resultXml);

        resultXml = XmlCleaner.cleanAllEntities(resultXml);

        log.trace("after cleaning and end of convertToXML: " + resultXml);

        return resultXml;

    }

}

package eu.dnetlib.data.collector.plugins.researchfi;

import java.util.HashSet;

import java.util.Set;

import org.dom4j.DocumentException;

import org.dom4j.DocumentHelper;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class ResearchFiCollectorPluginTest {

	private final ResearchFiCollectorPlugin plugin = new ResearchFiCollectorPlugin();

	@Before

	public void setUp() throws Exception {}

	@Test

	@Ignore

	public final void testCollect() throws CollectorServiceException, DocumentException {

		final InterfaceDescriptor iface = new InterfaceDescriptor();

		iface.setBaseUrl("https://research.fi/api/rest/v1/funding-decisions?FunderName=AKA&FundingStartYearFrom=2022");

		iface.setProtocol("research_fi");

		iface.getParams().put("auth_url", "https://researchfi-auth.2.rahtiapp.fi/realms/publicapi/protocol/openid-connect/token");

		iface.getParams().put("auth_client_id", "");

		iface.getParams().put("auth_client_secret", "");

		int count = 0;

		final Set<String> ids = new HashSet<>();

		for (final String s : plugin.collect(iface, null, null)) {

			if (count == 0) {

				System.out.println("First: " + s);

			}

			count++;

			final String id = DocumentHelper.parseText(s).valueOf("/recordWrap/funderProjectNumber");

			if (ids.contains(id)) {

				System.out.println("Id already present: " + id);

			}

			ids.add(id);

		}

		System.out.println("Total records: " + count);

		System.out.println("Total identifiers: " + ids.size());

	}

}

package eu.dnetlib.data.collector.plugins.doiresolver;

import eu.dnetlib.data.collector.plugins.filesystem.FileSystemIterator;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.io.IOException;

import java.nio.file.Files;

import java.nio.file.Paths;

import java.util.Iterator;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.TimeUnit;

import java.util.stream.Stream;

public class DOIResolverIterator implements Iterator<String> {

    private static final Log log = LogFactory.getLog(DOIResolverIterator.class);

    protected static final String STARTER = "FIRE";

    protected static final String TERMINATOR = "ARNOLD";

    protected static final String BAD_TERMINATOR = "BAD";

    protected static final String UNRESOLVED = "UNRESOLVED";

    protected static long TIMEOUT = 5;

    protected static TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS;

    /** Path to the dir that contains the files, each a csv with a list of DOIs, one per line. **/

    private String baseDir;

    private String fromDate;

    private ArrayBlockingQueue<String> queue;

    private CrossrefResolver crossrefResolver;

    public DOIResolverIterator(final String baseDir, final CrossrefResolver crossrefResolver, final String fromDate) {

        this.baseDir = baseDir;

        this.fromDate = fromDate;

        this.queue = new ArrayBlockingQueue<>(100);

        this.crossrefResolver = crossrefResolver;

        init();

    }

    private void init(){

        log.info("Init");

        new Thread(() -> {

            try{

                final FileSystemIterator fsi = new FileSystemIterator(baseDir, "csv", fromDate);

                // put first item in the queue

                if(queue.offer(STARTER)) {

                    // read the file, ask the resolvers, put results in a shared queue

                    //whatever exceptions, add terminator to the queue

                    while (fsi.hasNext()) {

                        String filePath = fsi.next();

                        try (Stream<String> stream = Files.lines(Paths.get(filePath))) {

                            stream.forEach(doi -> {

                                try {

                                    String resolved = resolve(doi);

                                    if(!resolved.equals(UNRESOLVED)) queue.offer(resolved, TIMEOUT, TIMEOUT_UNIT);

                                } catch (InterruptedException e) {

                                    log.error("DOI processing aborted, cannot offer resolved doi: "+doi+" . Did the consumer die?");

                                    log.error(e);

                                    queue.offer(BAD_TERMINATOR);

                                }

                            });

                        } catch (IOException e) {

                            log.error("DOI processing aborted");

                            log.error(e);

                            queue.offer(BAD_TERMINATOR);

                        }

                    }

                }

            } catch (Exception e) {

                log.error("DOI processing aborted");

                log.error(e);

                queue.offer(BAD_TERMINATOR);

            }

            queue.offer(TERMINATOR);

            log.info("Finished processing DOI list");

        }

        ).start();

    }

    private String resolve(final String doi){

       log.debug("Resolving "+doi);

       log.debug("Crossref...");

       String record = crossrefResolver.resolve(cleanDOI(doi));

       if(StringUtils.isNotBlank(record)) return record;

       else {

           //try another resolver

           log.debug("Resolver returned blank item");

       }

       return UNRESOLVED;

    }

    /**

     * Returns the identifier part of the DOI only.

     * @param doi

     * @return the DOI

     */

    protected String cleanDOI(final String doi){

       return doi.replace("http://dx.doi.org/", "").replace("https://dx.doi.org/", "")

               .replace("https://doi.org/", "").replace("http://doi.org/", "");

    }

    @Override

    public boolean hasNext() {

       return doHasNext();

    }

    private boolean doHasNext(){

        //If I get a null value, the queue is currently empty. so we wait for something

        String element = queue.peek();

        while(element == null) {

            try {

                log.debug("Sleeping while waiting for something in the queue");

                Thread.sleep(1000);

                element = queue.peek();

            } catch (InterruptedException e) {

                e.printStackTrace();

            }

        }

        log.debug("Found in queue element: "+element);

        switch(element){

            case TERMINATOR:

            case BAD_TERMINATOR:

                return false;

            case STARTER:

            case UNRESOLVED: //although they should not be inserted at all in the queue

                queue.poll();

                return doHasNext();

            default:

                return true;

        }

    }

    @Override

    public String next() {

        return queue.poll();

    }

    public String getBaseDir() {

        return baseDir;

    }

    public void setBaseDir(String baseDir) {

        this.baseDir = baseDir;

    }

    public CrossrefResolver getCrossrefResolver() {

        return crossrefResolver;

    }

    public void setCrossrefResolver(CrossrefResolver crossrefResolver) {

        this.crossrefResolver = crossrefResolver;

    }

}

package eu.dnetlib.data.collector.plugins.datacite;

import java.io.ByteArrayOutputStream;

import java.io.IOException;

import java.net.URL;

import java.util.ArrayDeque;

import java.util.Iterator;

import java.util.Objects;

import java.util.Queue;

import java.util.zip.DataFormatException;

import java.util.zip.Inflater;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.data.collector.plugins.datacite.schema.DataciteSchema;

import eu.dnetlib.data.collector.plugins.datacite.schema.Result;

import eu.dnetlib.data.collector.plugins.oai.engine.XmlCleaner;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class DataciteESIterator implements Iterator<String> {

    private static final Log log = LogFactory.getLog(DataciteESIterator.class);

    private final long timestamp;

    private String scrollId;

    private Queue<String> currentPage;

    private final Gson g =  new GsonBuilder().create();

    private String baseURL;

    private static final String START_PATH = "new_scan";

    private static final String NEXT_PATH = "scan/%s";

    public DataciteESIterator(long timestamp, String baseUrl) throws Exception {

        this.timestamp = timestamp;

        this.baseURL = baseUrl;

        currentPage = new ArrayDeque<>();

        startRequest();

    }

    protected static String decompression(final Result r) {

        return decompression(r.getBody().getBytes());

    }

    protected static String decompression(final byte[] bodyBytes){

        try {

            byte[] byteArray = Base64.decodeBase64(bodyBytes);

            Inflater decompresser = new Inflater();

            decompresser.setInput(byteArray);

            ByteArrayOutputStream bos = new ByteArrayOutputStream(byteArray.length);

            byte[] buffer = new byte[8192];

            while (!decompresser.finished()) {

                int size = decompresser.inflate(buffer);

                bos.write(buffer, 0, size);

            }

            byte[] unzippeddata = bos.toByteArray();

            decompresser.end();

            return new String(unzippeddata);

        } catch (DataFormatException e) {

            log.warn("Exception when decompressing: "+e.getMessage());

            return null;

        }

    }

    private void fillQueue(final String hits) {

        if (StringUtils.isBlank(hits) || "[]".equalsIgnoreCase(hits.trim()))

            return;

        try {

            DataciteSchema datacitepage = g.fromJson(hits, DataciteSchema.class);

            this.scrollId = datacitepage.getScrollId();

            datacitepage.getResult().stream().map(DataciteESIterator::decompression).filter(Objects::nonNull).forEach(this.currentPage::add);

        } catch (Throwable e) {

            System.out.println(hits);

            e.printStackTrace();

        }

    }

    private void startRequest() throws Exception {

        String url = baseURL+"/"+START_PATH;

        final URL startUrl = new URL(timestamp >0 ? url + "?timestamp="+timestamp : url);

        fillQueue(IOUtils.toString(startUrl.openStream()));

    }

    private void getNextPage() throws IOException {

        String url = baseURL+"/"+NEXT_PATH;

        final URL startUrl = new URL(String.format(url,scrollId));

        fillQueue(IOUtils.toString(startUrl.openStream()));

    }

    @Override

    public boolean hasNext() {

        return  currentPage.size() >0;

    }

    @Override

    public String next() {

        if (currentPage.size() == 0) {

            return null;

        }

        String nextItem = currentPage.remove();

        if (currentPage.size() == 0) {

            try {

                getNextPage();

            } catch (Throwable e) {

                throw new RuntimeException(e);

            }

        }

        return XmlCleaner.cleanAllEntities(nextItem);

    }

    public String getBaseURL() {

        return baseURL;

    }

    public void setBaseURL(final String baseURL) {

        this.baseURL = baseURL;

    }

}

package eu.dnetlib.data.collector.plugins.doiresolver;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

public interface DOIResolver {

    String resolve(String doi);

    void setBaseURL(String baseURL);

}

<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">

    <sitemap>

        <loc>file:target/test-classes/eu/dnetlib/data/collector/plugins/schemaorg/sitemap_file.xml</loc>

    </sitemap>

</sitemapindex>

4148;FondTara;Fondation Tara Expeditions;tara________::1

60;QUEEN;Quaternary Environment of the Eurasian North;corda_______::304178

4106;EPOCA;European Project on Ocean Acidification;corda_______::211384

4119;HERMIONE;Hotspot Ecosystem Research and Mans Impact On European Seas;corda_______::226354

4122;HYPOX;In situ monitoring of oxygen depletion in hypoxic ecosystems of coastal and open seas and land-locked water bodies;corda_______::226213

4127;CoralFISH;Ecosystem based management of corals, fish and fisheries in the deep waters of Europe and beyond;corda_______::213144

4129;ice2sea;ice2sea;corda_______::226375

4138;ECO2;Sub-seabed CO2 Storage: Impact on Marine Ecosystems;corda_______::265847

4142;MedSeA;Mediterranean Sea Acidification in a Changing Climate;corda_______::265103

4145;DARCLIFE;Deep subsurface Archaea: carbon cycle, life strategies, and role in sedimentary ecosystems;corda_______::247153

4147;EURO-BASIN;Basin Scale Analysis, Synthesis and Integration;corda_______::264933

4154;Past4Future;Climate Change: Learning from the past climate;corda_______::243908

4172;CARBOCHANGE;Changes in the carbon uptake and emissions by oceans in a changing climate;corda_______::264879

4175;ERA-CLIM;European Reanalysis of Global Climate Observations;corda_______::265229

4181;PAGE21;Changing Permafrost in the Arctic and its Global Effects in the 21st Century;corda_______::282700

4182;MicroB3;MicroB3 - Microbial Biodiversity, Bioinformatics and Biotechnology;corda_______::308299

package eu.dnetlib.data.collector.plugins.datacite.schema;

import com.google.gson.annotations.Expose;

import com.google.gson.annotations.SerializedName;

public class Result {

    @SerializedName("body")

    @Expose

    private String body;

    @SerializedName("id")

    @Expose

    private String id;

    @SerializedName("originalId")

    @Expose

    private String originalId;

    @SerializedName("timestamp")

    @Expose

    private Integer timestamp;

    public String getBody() {

        return body;

    }

    public void setBody(String body) {

        this.body = body;

    }

    public String getId() {

        return id;

    }

    public void setId(String id) {

        this.id = id;

    }

    public String getOriginalId() {

        return originalId;

    }

    public void setOriginalId(String originalId) {

        this.originalId = originalId;

    }

    public Integer getTimestamp() {

        return timestamp;

    }

    public void setTimestamp(Integer timestamp) {

        this.timestamp = timestamp;

    }

}

<ns2:projects xmlns:ns1="http://gtr.rcuk.ac.uk/gtr/api"

              xmlns:ns2="http://gtr.rcuk.ac.uk/gtr/api/project"

              xmlns:ns3="http://gtr.rcuk.ac.uk/gtr/api/project/outcome"

              xmlns:ns4="http://gtr.rcuk.ac.uk/gtr/api/organisation"

              xmlns:ns5="http://gtr.rcuk.ac.uk/gtr/api/person" xmlns:ns6="http://gtr.rcuk.ac.uk/gtr/api/fund"

              ns1:page="1" ns1:size="20" ns1:totalPages="3417" ns1:totalSize="68323">

	<ns2:project ns1:id="E178742B-571B-498F-8402-122F17C47546"

	             ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/projects/E178742B-571B-498F-8402-122F17C47546"

	             ns1:created="2016-11-11T20:42:55Z">

		<ns1:links test="ciao">

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/persons/CB8C3733-D17E-46A8-9E7C-D5A76F36612A"

					ns1:rel="PI_PER"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/persons/95D46800-A4DF-40AC-8FD3-7EAF5194B22C"

					ns1:rel="COI_PER"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/organisations/8319F78A-DCBD-49F6-BE00-78E1CD75CDA9"

					ns1:rel="LEAD_ORG"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/funds/B794CB74-BD85-452B-8030-69BD4AF82CE9"

					ns1:rel="FUND" ns1:start="2007-06-01T00:00:00+01:00"

					ns1:end="2010-05-31T00:00:00+01:00"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/outcomes/publications/2A19F274-8BAF-4AFC-865C-638BFB5FBDB9"

					ns1:rel="PUBLICATION"/>

		</ns1:links>

		<ns2:identifiers>

			<ns2:identifier ns2:type="RCUK">BB/E021409/1</ns2:identifier>

		</ns2:identifiers>

		<ns2:title> A multicellular 3D stem cell model to define the role of stroma in epithelial

			differentiation </ns2:title>

		<ns2:status>Closed</ns2:status>

		<ns2:grantCategory>Research Grant</ns2:grantCategory>

		<ns2:leadOrganisationDepartment>Biology</ns2:leadOrganisationDepartment>

		<ns2:abstractText> In aging men the disorders of prostate are a major medical problem.

			Benign prostatic hyperplasia and cancer are increasingly prevalent. To find cures for

			these diseases it is essential to understand how the prostate grows and functions

			normally. All organs have their own population of stem cells which grow and develop into

			a variety of cells which communicate to form correct organ architecture and function.

			This occurs as a result of signals from the stem cell's own genes but also from signals

			provided by neighbouring cells, known as stroma. In the prostate, how this occurs is

			unknown. We propose to develop a model to grow gland-like structures from adult stem

			cells in the laboratory. The model will be employed to understand how stromal cells

			influence prostate cellular architecture. We aim to identify proteins which act as

			signals from the stroma to change epithelial shape. The shape of a cell has important

			effects on cell function. These experiments will increase our knowledge of how tissues

			develop and function. Development of tissue-like models based on human cells will

			provide a valuable gap between results from animal models and human clinical studies, to

			help understand the basic mechanisms of human physiology and disease. Such model systems

			will reduce the need for animal experimentation, which is currently the best way to

			investigate complex cell interactions in tissues. We anticipate the model will aid

			university directed research into human differentiation and disease mechanisms, but also

			for the pharmaceutical industry to screen new drugs for efficacy and safety in humans

			before trial. </ns2:abstractText>

		<ns2:techAbstractText> Recent advances in our lab have resulted in the isolation of human

			adult prostate stem cells and the development of 3D models of prostatic acini from basal

			cells. Results from 3D modelling indicate that stroma is important for epithelial

			morphogenesis and differentiation. Importantly, stromal cultures increase epithelial

			cell polarity and columnar cell shape. Using electron microscopy and RT-PCR our

			preliminary data has found that these morphological effects are accompanied by increased

			desmosomal expression. We now wish to develop our tissue engineering to produce a 3D

			model of prostatic acini using a homogeneous population of stem cells. A stem cell model

			will allow the study of full epithelial differentiation and the stem cell niche. It is

			important to model the prostate with human cells because the mouse prostate has a

			different anatomy, cell structure and protein function, and does not develop equivalent

			diseases to humans. The model will be used to investigate our hypothesis that 'stroma

			signals to control epithelial cell shape and polarity'. We will confirm which desmosomal

			isoforms are present in prostate epithelial acini and which are upregulated by stromal

			cultures, using Western Blotting and real time PCR. Upregulated desmosomal isoforms will

			be used as markers for epithelial cell polarity and shape. A differential gene

			expression profile will be generated from stroma grown with epithelial acini in 3D

			culture and stroma grown in 3D culture without acini, using microarray analysis.

			Candidate stromal genes will be identified that signal to upregulate epithelial polarity

			(desmosomal expression) and their function will be confirmed using siRNA knockdown

			studies. This is a novel pathway for epithelial cell differentiation which has not been

			studied before. </ns2:techAbstractText>

		<ns2:healthCategories/>

		<ns2:researchActivities/>

		<ns2:researchSubjects/>

		<ns2:researchTopics/>

	</ns2:project>

	<ns2:project ns1:id="E37C97C5-7489-4205-834F-151D05B7E07A"

	             ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/projects/E37C97C5-7489-4205-834F-151D05B7E07A"

	             ns1:created="2016-11-11T20:42:55Z">

		<ns1:links test="helo2">

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/persons/AFFB5A85-DAC7-48F2-AE07-952481073BAA"

					ns1:rel="PI_PER"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/persons/DFA58FA2-CCD6-445F-B2BC-E830C23FA563"

					ns1:rel="COI_PER"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/persons/6E4302E7-A895-4FF7-AE8C-26C2478A82E6"

					ns1:rel="COI_PER"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/organisations/30A429E3-83B7-4E41-99C0-14A144F07DFE"

					ns1:rel="LEAD_ORG"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/organisations/FADDC755-1F45-47D7-8591-F183B7160CC2"

					ns1:rel="PP_ORG"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/organisations/E82E4BC6-2839-4E7A-82CA-88B033E53B45"

					ns1:rel="PP_ORG"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/organisations/F8B807ED-ACF2-4724-9DC6-291B77059637"

					ns1:rel="PP_ORG"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/funds/B7A4A2BB-1530-4846-8F2B-891EEBFF3F5F"

					ns1:rel="FUND" ns1:start="2013-10-01T00:00:00+01:00" ns1:end="2017-11-30T00:00:00Z"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/outcomes/keyfindings/45480FAD-42D9-4948-ADE2-1B161F6BF481"

					ns1:rel="KEY_FINDING"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/outcomes/disseminations/6E8F7769-1A48-4BDD-8248-6B5938CA3495"

					ns1:rel="DISSEMINATION"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/outcomes/impactsummaries/01F42709-4FC3-4267-9E2A-08591C7950F8"

					ns1:rel="IMPACT_SUMMARY"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/outcomes/publications/76690C2C-FEF9-42AC-A4A2-B09338B33C45"

					ns1:rel="PUBLICATION"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/outcomes/publications/7B0D46AB-374C-4327-9EFE-5CA9683560FB"

					ns1:rel="PUBLICATION"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/outcomes/publications/03415E7B-3BE0-4207-857D-B595F52E2C65"

					ns1:rel="PUBLICATION"/>

			<ns1:link

					ns1:href="http://gtr.rcuk.ac.uk:80/gtr/api/outcomes/publications/2D27AB50-6985-4A61-83C6-54A102DFCFB8"

					ns1:rel="PUBLICATION"/>

		</ns1:links>

		<ns2:identifiers>

			<ns2:identifier ns2:type="RCUK">NE/K001906/1</ns2:identifier>

		</ns2:identifiers>

		<ns2:title> Biogeochemistry, macronutrient and carbon cycling in the benthic layer </ns2:title>

		<ns2:status>Active</ns2:status>

		<ns2:grantCategory>Research Grant</ns2:grantCategory>

		<ns2:leadOrganisationDepartment>School of Ocean and Earth

			Science</ns2:leadOrganisationDepartment>

		<ns2:abstractText> The coasts and shelf seas that surround us have been the focal point of

			human prosperity and well-being throughout our history and, consequently, have had a

			disproportionate effect on our culture. The societal importance of the shelf seas

			extends beyond food production to include biodiversity, carbon cycling and storage,

			waste disposal, nutrient cycling, recreation and renewable energy. Yet, as increasing

			proportions of the global population move closer to the coast, our seas have become

			progressively eroded by human activities, including overfishing, pollution, habitat

			disturbance and climate change. This is worrying because the condition of the seabed,

			biodiversity and human society are inextricably linked. Hence, there is an urgent need

			to understand the relative sensitivities of a range of shelf habitats so that human

			pressures can be managed more effectively to ensure the long-term sustainability of our

			seas and provision of societal benefits. Achieving these aims is not straightforward, as

			the capacity of the seabed to provide the goods and services we rely upon depends on the

			type of substrate (rock, gravel, sand, mud) and local conditions; some habitats are

			naturally dynamic and relatively insensitive to disturbance, while others are

			comparatively stable and vulnerable to change. This makes it very difficult to assess

			habitat sensitivities or make general statements about what benefits we can expect from

			our seas in the future. Recently, NERC and DEFRA have initiated a major new research

			programme on Shelf Sea Biogeochemistry that will improve knowledge about these issues.

			In response to this call, we have assembled a consortium of leading scientists that

			includes microbiologists, ecologists, physical oceanographers, biogeochemists,

			mathematical modellers and policy advisors. With assistance from organisations like

			CEFAS, Marine Scotland and AFBI, they will carry out a series of research cruises around

			the UK that will map the sensitivity and status of seabed habitats based on their

			physical condition, the microbial and faunal communities that inhabit them, and the size

			and dynamics of the nitrogen and carbon pools found there. The latest marine

			technologies will measure the amount of mixing and flow rates just above the seabed, as

			well as detailed seabed topography. These measurements will allow better understanding

			of the physical processes responsible for movement and mixing of sediment, nutrient, and

			carbon. At the same time, cores will be retrieved containing the microbial and faunal

			communities and their activity and behaviour will be linked to specific biogeochemical

			responses. Highly specialised autonomous vehicles, called landers, will also measure

			nutrient concentrations and fluxes at the seabed. Components of the system can then be

			experimentally manipulated to mimic scenarios of change, such as changing hydrodynamics,

			disturbance or components of climate change. This will be achieved in the field by

			generating different flow regimes using a submerged flume or, in the laboratory, using

			intact sediment communities exposed to different levels of CO2, temperature and oxygen.

			By measuring the biogeochemical response and behaviour of the microbial and faunal

			communities to these changes, we will generate an understanding of what may happen if

			such changes did occur across our shelf seas. We will use all of this information to

			assess the relative vulnerability of areas of the UK seabed by overlaying the

			observation and experimental results over maps of various human pressures, which will be

			of value to planners and policymakers. Mathematical models will test future scenarios of

			change, such as opening or closing vulnerable areas to fishing or anticipated changes in

			the factors that control nutrient and carbon stocks. This will be valuable in exploring

			different responses to external pressures and for deciding which management measures

			should be put in place to preserve our shelf seas for future generations </ns2:abstractText>

		<ns2:potentialImpact> Commercial private sector and the knowledge economy: new and

			innovative methodologies, equipment and techniques, and combined state-of-the-art

			technologies (>2.3 million in-kind, see JeS) will assess what the primary physical

			and biogeochemical controls of shelf productivity are up to shelf sea scales. Since many

			interests rely on the marine environment, beneficiaries will be varied. By sharing

			expertise and knowledge, a world-leading manufacturer of microsensors and microscale

			instrumentation and an internationally recognized marine environmental data acquisition

			company will benefit from exploitable opportunities, e.g. new visualisation tools that

			enable holistic understanding of large-scale ecosystem processes. Policy professionals,

			governmental and devolved governmental organisations: The importance of shelf seas to

			society extends beyond fisheries to wider issues, such as biodiversity, carbon cycling

			and storage, waste disposal, nutrient cycling, and renewable energy resources.

			Consortium expertise will contribute to these UK priority challenges. The UK Marine

			& Coastal Access Act (MCAA), UK Climate Change Act, EU Habitats Directive and EU

			Marine Strategy Framework Directive (MSFD) support sustainable use of the marine

			environment. They also support the UK vision for achieving 'clean, healthy, safe

			productive and biologically diverse ocean and seas' (UK Marine Science Strategy). We

			will provide a coherent framework for sound evidence based-science in support of these

			policy instruments and statutory requirements. For example, the MSFD aims to achieve

			Good Environmental Status in EU marine waters by 2020, but we lack understanding of the

			magnitude and synchronicity of change in SSEs. Our research will directly inform

			Descriptor 1 (biological diversity) and 6 (seabed integrity) for a wide range of

			sediment habitats over time, which is important because the determination of good

			environmental status may have to be adapted over time (addressed in Task 2) "in vie

			of the dynamic nature of marine ecosystems and their natural variability, and given that

			the pressures and impacts on them may vary with the evolvement of different patterns of

			human activity and the impact of climate change" (MSFD). Our work will also inform

			environmental monitoring programmes: OSPARs Joint Assessment and Monitoring programme,

			the Eutrophication Monitoring Programme and The Clean Seas Environment Monitoring

			Programme (CSEMP, led by consortium member CEFAS). Task 1-3 complement the outcomes of

			CESEMP and provide scientific evidence to OSPAR. Similarly, experimental scenarios and

			modelling approaches will provide needed information for (i) the EU Water Framework

			Directive (the requirement for 'good chemical and ecological status' by 2015 does not

			account for climate change) and, (ii) the UK White Paper for MCAA (it is unclear how

			commitments to "look ahead at the predicted impacts of climate change on the marine

			environment, how marine activities will contribute towards it, and how they are affected

			by it" will be achieved). Finally, other EU instruments, such as the Habitats

			Directive (introduced in 1992), the EU Common Fisheries Policy (revised in 2002) and

			national legislation such as the UK MCAA and Scottish Marine Act, assume that removal

			(or control) of direct pressures will result in ecosystem recovery and/or species

			persistence. Our programme includes experimental scenarios and modelling approaches to

			provide further information on the vulnerability of SSEs in environmental futures under

			multiple pressures (Task 3). Our outputs will also help NERC meet its science theme

			challenges. Public, wider community: active engagement with a variety of organisations

			is detailed in Pathways to Impact (PtI). Skills& training: In addition to academic

			progression, early career researchers will gain experience and receive mentoring in

			running a large interdisciplinary programme, as well as training in communication skills

			and scientific methodology </ns2:potentialImpact>

		<ns2:healthCategories/>

		<ns2:researchActivities/>

		<ns2:researchSubjects>

			<ns2:researchSubject>

				<ns2:id>138395</ns2:id>

				<ns2:text>Marine environments</ns2:text>

				<ns2:percentage>75</ns2:percentage>

			</ns2:researchSubject>

			<ns2:researchSubject>

				<ns2:id>46902</ns2:id>

				<ns2:text>Geosciences</ns2:text>

				<ns2:percentage>15</ns2:percentage>

			</ns2:researchSubject>

			<ns2:researchSubject>

				<ns2:id>13097</ns2:id>

				<ns2:text>Ecol, biodivers.&amp; systematics</ns2:text>

				<ns2:percentage>5</ns2:percentage>

			</ns2:researchSubject>

			<ns2:researchSubject>

				<ns2:id>33851</ns2:id>

				<ns2:text>Microbial sciences</ns2:text>

				<ns2:percentage>5</ns2:percentage>

			</ns2:researchSubject>

		</ns2:researchSubjects>

		<ns2:researchTopics>

			<ns2:researchTopic>

				<ns2:id>21005</ns2:id>

				<ns2:text>Sediment/Sedimentary Processes</ns2:text>

				<ns2:percentage>15</ns2:percentage>

			</ns2:researchTopic>

			<ns2:researchTopic>

				<ns2:id>143045</ns2:id>

				<ns2:text>Ecosystem Scale Processes</ns2:text>

				<ns2:percentage>15</ns2:percentage>

			</ns2:researchTopic>

			<ns2:researchTopic>

				<ns2:id>63200</ns2:id>

				<ns2:text>Biogeochemical Cycles</ns2:text>

				<ns2:percentage>60</ns2:percentage>

			</ns2:researchTopic>

			<ns2:researchTopic>

				<ns2:id>108367</ns2:id>

				<ns2:text>Community Ecology</ns2:text>

				<ns2:percentage>5</ns2:percentage>

			</ns2:researchTopic>

			<ns2:researchTopic>

				<ns2:id>80410</ns2:id>

				<ns2:text>Responses to environment</ns2:text>

				<ns2:percentage>5</ns2:percentage>

			</ns2:researchTopic>

		</ns2:researchTopics>

	</ns2:project>

</ns2:projects>

package eu.dnetlib.data.collector.plugins.excel;

/**

 * Created by miriam on 10/05/2017.

 */

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.net.URL;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Iterator;

import eu.dnetlib.data.collector.plugins.HttpCSVCollectorPlugin;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.poi.ss.usermodel.Cell;

import org.apache.poi.ss.usermodel.DataFormatter;

import org.apache.poi.ss.usermodel.Row;

import org.apache.poi.ss.usermodel.Sheet;

import org.apache.poi.ss.usermodel.Workbook;

import org.apache.poi.xssf.usermodel.XSSFWorkbook;

import org.json.*;

import org.apache.commons.io.FileUtils;

public class Read {