D-Net

package eu.dnetlib.data.collector.plugins;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileReader;

import java.io.IOException;

import java.net.MalformedURLException;

import java.net.URL;

import java.util.Iterator;

import eu.dnetlib.rmi.data.CollectorServiceException;

import eu.dnetlib.rmi.data.InterfaceDescriptor;

import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin;

import org.apache.commons.lang3.StringEscapeUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

public class FileCSVCollectorPlugin extends AbstractCollectorPlugin {

	private static final Log log = LogFactory.getLog(FileCSVCollectorPlugin.class);

	private String[] headers = null;

	private int identifierNumber;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String header = interfaceDescriptor.getParams().get("header");

		final String separator = StringEscapeUtils.unescapeJava(interfaceDescriptor.getParams().get("separator"));

		identifierNumber = Integer.parseInt(interfaceDescriptor.getParams().get("identifier"));

		URL u = null;

		try {

			u = new URL(interfaceDescriptor.getBaseUrl());

		} catch (MalformedURLException e1) {

			throw new CollectorServiceException(e1);

		}

		final String baseUrl = u.getPath();

		log.info("base URL = " + baseUrl);

		try {

			final BufferedReader br = new BufferedReader(new FileReader(new File(baseUrl)));

			if (header != null && "true".equals(header.toLowerCase())) {

				headers = br.readLine().split(separator);

			}

			return new Iterable<String>() {

				@Override

				public Iterator<String> iterator() {

					return new FileCSVIterator(br, separator);

				}

			};

		} catch (Exception e) {

			throw new CollectorServiceException(e);

		}

	}

	class FileCSVIterator implements Iterator<String> {

		private String next;

		private BufferedReader reader;

		private String separator;

		public FileCSVIterator(final BufferedReader reader, final String separator) {

			this.reader = reader;

			this.separator = separator;

			next = calculateNext();

		}

		@Override

		public boolean hasNext() {

			return next != null;

		}

		@Override

		public String next() {

			String s = next;

			next = calculateNext();

			return s;

		}

		private String calculateNext() {

			try {

				Document document = DocumentHelper.createDocument();

				Element root = document.addElement("csvRecord");

				String newLine = reader.readLine();

				// FOR SOME FILES IT RETURN NULL ALSO IF THE FILE IS NOT READY DONE

				if (newLine == null) {

					newLine = reader.readLine();

				}

				if (newLine == null) {

					log.info("there is no line, closing RESULT SET");

					reader.close();

					return null;

				}

				String[] currentRow = newLine.split(separator);

				if (currentRow != null) {

					for (int i = 0; i < currentRow.length; i++) {

						String hAttribute = headers != null && i < headers.length ? headers[i] : "column" + i;

						Element row = root.addElement("column");

						if (i == identifierNumber) {

							row.addAttribute("isID", "true");

						}

						row.addAttribute("name", hAttribute).addText(currentRow[i]);

					}

					return document.asXML();

				}

			} catch (IOException e) {

				log.error("Error calculating next csv element", e);

			}

			return null;

		}

		@Override

		public void remove() {

			throw new UnsupportedOperationException();

		}

	}

}