D-Net

package eu.dnetlib.data.collector.plugins.projects.grist;

import java.util.Iterator;

import java.util.List;

import java.util.Set;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import org.apache.commons.io.IOUtils;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

/**

 * GristProjectsIterable Tester.

 *

 * @author alessia

 * @version 1.0

 * @since <pre>Apr 22, 2016</pre>

 */

@Ignore

public class GristProjectsIterableTest {

	private String baseUrl = "http://www.ebi.ac.uk/europepmc/GristAPI/rest/get/query=ga:%22Wellcome%20Trust%22&resultType=core";

	private GristProjectsIterable iterable;

	private Iterator<String> it;

	private SAXReader reader;

	@Before

	public void before() throws Exception {

		iterable = new GristProjectsIterable(baseUrl);

		it = iterable.iterator();

		reader = new SAXReader();

	}

	/**

	 * Method: hasNext()

	 */

	@Test

	public void testHasNext() throws Exception {

		assertTrue(it.hasNext());

	}

	/**

	 * Method: next()

	 */

	@Test

	public void testNext() throws Exception {

		assertNotNull(it.next());

	}

	/**

	 * Method: remove()

	 */

	@Test(expected = UnsupportedOperationException.class)

	public void testRemove() throws Exception {

		it.remove();

	}

	@Test

	public void iterateToNextPage() {

		for (int maxInPage = 25; maxInPage > 0; maxInPage--) {

			it.next();

		}

		if (it.hasNext()) {

			System.out.println(it.next());

		}

	}

	@Test

	public void checkProjectIdentifiers() throws DocumentException {

		List<String> identifiers = Lists.newArrayList();

		Iterator<String> it2 = iterable.iterator();

		while (it2.hasNext()) {

			String id = parseId(it2.next());

			if (identifiers.contains(id)) {

				System.out.println("Found duplicate identifier: " + id);

			}

			identifiers.add(id);

		}

		int listSize = identifiers.size();

		System.out.println("List of ids has " + listSize + " identifiers");

		System.out.println("Now putting ids into a set to delete duplicates");

		Set<String> set = Sets.newHashSet(identifiers);

		int setSize = set.size();

		System.out.println("Set of ids has " + setSize + " identifiers");

		System.out.println();

	}

	private String parseId(String record) throws DocumentException {

		Document doc = reader.read(IOUtils.toInputStream(record));

		return doc.selectSingleNode("//Grant/Id").getText();

	}

}

log4j.logger.eu.dnetlib.data.collector.plugins.projects.grist=DEBUG

package eu.dnetlib.data.collector.plugins.projects.grist;

import java.io.IOException;

import java.net.MalformedURLException;

import java.net.URL;

import java.util.Iterator;

import java.util.List;

import java.util.Queue;

import java.util.concurrent.PriorityBlockingQueue;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.CollectorServiceRuntimeException;

import eu.dnetlib.enabling.resultset.SizedIterable;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Element;

import org.dom4j.io.SAXReader;

public class GristProjectsIterable implements SizedIterable<String> {

	private static final Log log = LogFactory.getLog(GristProjectsIterable.class); // NOPMD by marko on 11/24/08 5:02 PM

	private String queryURL;

	private int total;

	private SAXReader reader;

	public GristProjectsIterable(String baseURL) throws CollectorServiceException {

		queryURL = baseURL;

		reader = new SAXReader();

		total = getTotalCount();

	}

	@Override

	public int getNumberOfElements() {

		return total;

	}

	private int getTotalCount() throws CollectorServiceException {

		try {

			URL pageUrl = new URL(queryURL);

			log.debug("Getting hit count from: " + pageUrl.toString());

			String resultPage = IOUtils.toString(pageUrl);

			Document doc = reader.read(IOUtils.toInputStream(resultPage));

			String hitCount = doc.selectSingleNode("/Response/HitCount").getText();

			return Integer.parseInt(hitCount);

		} catch (NumberFormatException e) {

			log.warn("Cannot set the total count from '/Response/HitCount'");

		} catch (DocumentException e) {

			throw new CollectorServiceException(e);

		} catch (MalformedURLException e) {

			throw new CollectorServiceException(e);

		} catch (IOException e) {

			throw new CollectorServiceException(e);

		}

		return -1;

	}

	@Override

	public Iterator<String> iterator() {

		return new Iterator<String>() {

			private Queue<String> projects = new PriorityBlockingQueue<String>();

			private boolean morePages = true;

			private int pageNumber = 0;

			private SAXReader reader = new SAXReader();

			//The following is for debug only

			private int nextCounter = 0;

			@Override

			public boolean hasNext() {

				try {

					fillProjectListIfNeeded();

				} catch (CollectorServiceException e) {

					throw new CollectorServiceRuntimeException(e);

				}

				return !projects.isEmpty();

			}

			@Override

			public String next() {

				nextCounter++;

				log.debug(String.format("Calling next %s times. projects queue has %s elements", nextCounter, projects.size()));

				try {

					fillProjectListIfNeeded();

					return projects.poll();

				} catch (CollectorServiceException e) {

					throw new CollectorServiceRuntimeException(e);

				}

			}

			@Override

			public void remove() {

				throw new UnsupportedOperationException();

			}

			private boolean fillProjectListIfNeeded() throws CollectorServiceException {

				if (morePages && projects.isEmpty()) {

					String resultPage = getNextPage();

					Document doc = null;

					try {

						doc = reader.read(IOUtils.toInputStream(resultPage));

						List<Element> records = doc.selectNodes("//RecordList/Record");

						if (records != null && !records.isEmpty()) {

							for (Element p : records) {

								projects.add(p.asXML());

							}

							return true;

						} else {

							log.info("No more projects to read at page nr. " + pageNumber);

							morePages = false;

							return false;

						}

					} catch (DocumentException e) {

						throw new CollectorServiceException(e);

					}

				} else return false;

			}

			private String getNextPage() {

				pageNumber++;

				try {

					URL pageUrl = new URL(queryURL + "&page=" + pageNumber);

					log.debug("Getting page at: " + pageUrl.toString());

					return IOUtils.toString(pageUrl);

				} catch (Exception e) {

					throw new CollectorServiceRuntimeException("Error on page " + pageNumber, e);

				}

			}

		};

	}

}

package eu.dnetlib.data.collector.plugins.projects.grist;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

/**

 * Plugin to collect metadata record about projects and fundings via the europePMC GRIST API (e.g. WT projects).

 * <p>

 * Documentation on GRIST API: http://europepmc.org/GristAPI.

 * </p>

 * <p>

 * BaseURL: http://www.ebi.ac.uk/europepmc/GristAPI/rest/get/query=ga:"Wellcome Trust"&resultType=core

 * where resultType=core asks for the complete information (including abstracts).

 * The results returned by the API are XMLs.

 * </p>

 * <p>

 * Pagination: use parameter 'page'. When the response contains empty 'RecordList', it means we reached the end.

 * </p>

 *

 * @author alessia

 */

public class GristCollectorPlugin extends AbstractCollectorPlugin {

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		//baseURL: http://www.ebi.ac.uk/europepmc/GristAPI/rest/get/query=ga:%22Wellcome%20Trust%22&resultType=core

		return new GristProjectsIterable(interfaceDescriptor.getBaseUrl());

	}

}

	<bean id="gristPlugin" class="eu.dnetlib.data.collector.plugins.projects.grist.GristCollectorPlugin">

			<bean class="eu.dnetlib.data.collector.rmi.ProtocolDescriptor" p:name="gristProjects"/>