D-Net

package eu.dnetlib.data.collector.filesystem.csv;

import java.net.URISyntaxException;

import java.util.HashMap;

import org.junit.Assert;

import org.junit.Test;

import eu.dnetlib.data.collector.plugins.HttpCSVCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class HTTPCSVCollectorPluginTest {

	@Test

	public void testCSVHeader() throws URISyntaxException, CollectorServiceException {

		InterfaceDescriptor descr = new InterfaceDescriptor();

		HashMap<String, String> params = new HashMap<String, String>();

		params.put("separator", ",");

		params.put("quote", "\"");

		params.put("identifier", "ISSN");

		descr.setBaseUrl("http://doaj.org/csv");

		descr.setParams(params);

		HttpCSVCollectorPlugin plugin = new HttpCSVCollectorPlugin();

		int i = 0;

		for (String s : plugin.collect(descr, null, null)) {

			Assert.assertTrue(s.length() > 0);

			i++;

		}

		Assert.assertTrue(i > 0);

	}

}

package eu.dnetlib.data.collector.plugins;

import java.io.InputStreamReader;

import java.io.Reader;

import java.net.URL;

import java.util.Iterator;

import java.util.List;

import java.util.Set;

import org.apache.commons.csv.CSVFormat;

import org.apache.commons.csv.CSVParser;

import org.apache.commons.csv.CSVRecord;

import org.dom4j.Document;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

import com.google.common.base.Function;

import com.google.common.collect.Iterators;

import com.google.common.collect.Lists;

import eu.dnetlib.data.collector.plugin.CollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

/**

 * The Class HttpCSVCollectorPlugin.

 */

public class HttpCSVCollectorPlugin implements CollectorPlugin {

	/**

	 * The Class HTTPCSVIterator.

	 */

	class HTTPCSVIterator implements Iterable<String> {

		/** The descriptor. */

		private InterfaceDescriptor descriptor;

		/**

		 * Instantiates a new HTTPCSV iterator.

		 *

		 * @param descriptor

		 *            the descriptor

		 */

		public HTTPCSVIterator(final InterfaceDescriptor descriptor) {

			this.descriptor = descriptor;

		}

		/**

		 * Iterator.

		 *

		 * @return the iterator

		 */

		@SuppressWarnings("resource")

		@Override

		public Iterator<String> iterator() {

			CSVParser parser;

			Reader reader;

			try {

				String separator = descriptor.getParams().get("separator");

				final String identifier = descriptor.getParams().get("identifier");

				String quote = descriptor.getParams().get("quote");

				URL url = new URL(descriptor.getBaseUrl());

				url.openConnection();

				reader = new InputStreamReader(url.openStream());

				CSVFormat format = CSVFormat.EXCEL.withHeader().withDelimiter(separator.charAt(0)).withQuote(quote.charAt(0));

				parser = new CSVParser(reader, format);

				final Set<String> headers = parser.getHeaderMap().keySet();

				Function<CSVRecord, String> function = new Function<CSVRecord, String>() {

					@Override

					public String apply(final CSVRecord input) {

						Document document = DocumentHelper.createDocument();

						Element root = document.addElement("csvRecord");

						for (String key : headers) {

							Element row = root.addElement("column");

							row.addAttribute("name", key).addText(input.get(key));

							if (key.equals(identifier)) {

								row.addAttribute("isID", "true");

							}

						}

						return document.asXML();

					}

				};

				Iterator<String> result = Iterators.transform(parser.iterator(), function);

				return result;

			} catch (Exception e) {

				return null;

			}

		}

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.collector.plugin.CollectorPlugin#collect(eu.dnetlib.data.collector.rmi.InterfaceDescriptor, java.lang.String,

	 * java.lang.String)

	 */

	@Override

	public Iterable<String> collect(final InterfaceDescriptor descriptor, final String fromDate, final String untilDate) throws CollectorServiceException {

		return new HTTPCSVIterator(descriptor);

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.data.collector.plugin.CollectorPlugin#getProtocol()

	 */

	@Override

	public String getProtocol() {

		return "HttpCSVCollectorPlugin";

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.data.collector.plugin.CollectorPlugin#listNameParameters()

	 */

	@Override

	public List<String> listNameParameters() {

		return Lists.newArrayList("separator", "identifier", "quote");

	}

}

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-modular-collector-service/trunk</developerConnection>

	</scm>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-csv</artifactId>

			<version>1.0</version>

		</dependency>