D-Net

package eu.dnetlib.data.collector.plugins;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileReader;

import java.io.IOException;

import java.net.MalformedURLException;

import java.net.URL;

import java.util.Iterator;

import java.util.List;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

import com.google.common.collect.Lists;

import eu.dnetlib.data.collector.plugin.CollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class FileCSVCollectorPlugin implements CollectorPlugin {

	private static final Log log = LogFactory.getLog(FileCSVCollectorPlugin.class);

	class FileCSVIterator implements Iterator<String> {

		private String next;

		private BufferedReader reader;

		private String separator;

		public FileCSVIterator(final BufferedReader reader, final String separator) {

			this.reader = reader;

			this.separator = separator;

			next = calculateNext();

		}

		@Override

		public boolean hasNext() {

			return next != null;

		}

		@Override

		public String next() {

			String s = next;

			next = calculateNext();

			return s;

		}

		private String calculateNext() {

			try {

				Document document = DocumentHelper.createDocument();

				Element root = document.addElement("csvRecord");

				String newLine = reader.readLine();

				// FOR SOME FILES IT RETURN NULL ALSO IF THE FILE IS NOT READY DONE

				if (newLine == null) {

					newLine = reader.readLine();

				}

				if (newLine == null) {

					log.info("there is no line, closing RESULT SET");

					reader.close();

					return null;

				}

				String[] currentRow = newLine.split(separator);

				if (currentRow != null) {

					for (int i = 0; i < currentRow.length; i++) {

						String hAttribute = headers == null ? "column" + i : headers[i];

						Element row = root.addElement("column");

						if (i == identifierNumber) {

							row.addAttribute("isID", "true");

						}

						row.addAttribute("name", hAttribute).addText(currentRow[i]);

					}

					return document.asXML();

				}

			} catch (IOException e) {

				log.error("Error calculating next csv element", e);

			}

			return null;

		}

		@Override

		public void remove() {

			throw new UnsupportedOperationException();

		}

	}

	private String[] headers = null;

	private int identifierNumber;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String header = interfaceDescriptor.getParams().get("header");

		final String separator = StringEscapeUtils.unescapeJava(interfaceDescriptor.getParams().get("separator"));

		identifierNumber = Integer.parseInt(interfaceDescriptor.getParams().get("identifier"));

		URL u = null;

		try {

			u = new URL(interfaceDescriptor.getBaseUrl());

		} catch (MalformedURLException e1) {

			throw new CollectorServiceException(e1);

		}

		final String baseUrl = u.getPath();

		log.info("base URL = " + baseUrl);

		try {

			final BufferedReader br = new BufferedReader(new FileReader(new File(baseUrl)));

			if ((header != null) && ("true".equals(header.toLowerCase()))) {

				headers = br.readLine().split(separator);

			}

			return new Iterable<String>() {

				@Override

				public Iterator<String> iterator() {

					FileCSVIterator it = new FileCSVIterator(br, separator);

					return it;

				}

			};

		} catch (Exception e) {

			throw new CollectorServiceException(e);

		}

	}

	@Override

	public String getProtocol() {

		return "fileCSV";

	}

	@Override

	public List<String> listNameParameters() {

		return Lists.newArrayList("header", "separator", "identifier", "quote");

	}

}