D-Net

package eu.dnetlib.data.collector.plugins;

import java.io.BufferedReader;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStreamReader;

import java.net.MalformedURLException;

import java.net.URL;

import java.util.Iterator;

import org.apache.commons.io.input.BOMInputStream;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class FileCSVCollectorPlugin extends AbstractCollectorPlugin {

	private static final Log log = LogFactory.getLog(FileCSVCollectorPlugin.class);

	class FileCSVIterator implements Iterator<String> {

		private String next;

		private BufferedReader reader;

		private String separator;

		private String quote;

		public FileCSVIterator(final BufferedReader reader, final String separator, final String quote) {

			this.reader = reader;

			this.separator = separator;

			this.quote = quote;

			next = calculateNext();

		}

		@Override

		public boolean hasNext() {

			return next != null;

		}

		@Override

		public String next() {

			final String s = next;

			next = calculateNext();

			return s;

		}

		private String calculateNext() {

			try {

				final Document document = DocumentHelper.createDocument();

				final Element root = document.addElement("csvRecord");

				String newLine = reader.readLine();

				// FOR SOME FILES IT RETURN NULL ALSO IF THE FILE IS NOT READY DONE

				if (newLine == null) {

					newLine = reader.readLine();

				}

				if (newLine == null) {

					log.info("there is no line, closing RESULT SET");

					reader.close();

					return null;

				}

				final String[] currentRow = newLine.split(separator);

				if (currentRow != null) {

					for (int i = 0; i < currentRow.length; i++) {

						final String hAttribute = (headers != null) && (i < headers.length) ? headers[i] : "column" + i;

						final Element row = root.addElement("column");

						if (i == identifierNumber) {

							row.addAttribute("isID", "true");

						}

						final String value = StringUtils.isBlank(quote) ? currentRow[i] : StringUtils.strip(currentRow[i], quote);

						row.addAttribute("name", hAttribute).addText(value);

					}

					return document.asXML();

				}

			} catch (final IOException e) {

				log.error("Error calculating next csv element", e);

			}

			return null;

		}

		@Override

		public void remove() {

			throw new UnsupportedOperationException();

		}

	}

	private String[] headers = null;

	private int identifierNumber;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String header = interfaceDescriptor.getParams().get("header");

		final String separator = StringEscapeUtils.unescapeJava(interfaceDescriptor.getParams().get("separator"));

		final String quote = interfaceDescriptor.getParams().get("quote");

		identifierNumber = Integer.parseInt(interfaceDescriptor.getParams().get("identifier"));

		URL u = null;

		try {

			u = new URL(interfaceDescriptor.getBaseUrl());

		} catch (final MalformedURLException e1) {

			throw new CollectorServiceException(e1);

		}

		final String baseUrl = u.getPath();

		log.info("base URL = " + baseUrl);

		try {

			final BufferedReader br = new BufferedReader(new InputStreamReader(new BOMInputStream(new FileInputStream(baseUrl))));

			if ((header != null) && "true".equals(header.toLowerCase())) {

				final String[] tmpHeader = br.readLine().split(separator);

				if (StringUtils.isNotBlank(quote)) {

					int i = 0;

					headers = new String[tmpHeader.length];

					for (final String h : tmpHeader) {

						headers[i] = StringUtils.strip(h, quote);

						i++;

					}

				} else headers = tmpHeader;

			}

			return new Iterable<String>() {

				@Override

				public Iterator<String> iterator() {

					return new FileCSVIterator(br, separator, quote);

				}

			};

		} catch (final Exception e) {

			throw new CollectorServiceException(e);

		}

	}

}