D-Net

package eu.dnetlib.data.collector.plugins.sftp;

import java.io.OutputStream;

import java.net.URI;

import java.net.URISyntaxException;

import java.util.*;

import com.jcraft.jsch.*;

import eu.dnetlib.data.collector.rmi.CollectorServiceRuntimeException;

import org.apache.commons.io.output.ByteArrayOutputStream;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.joda.time.DateTime;

import org.joda.time.format.DateTimeFormat;

import org.joda.time.format.DateTimeFormatter;

/**

 * Created by andrea on 11/01/16.

 */

public class SftpIterator implements Iterator<String> {

    private static final Log log = LogFactory.getLog(SftpIterator.class);

    private static final int MAX_RETRIES = 5;

    private static final int DEFAULT_TIMEOUT = 30000;

    private static final long BACKOFF_MILLIS = 10000;

    private String baseUrl;

    private String sftpURIScheme;

    private String sftpServerAddress;

    private String remoteSftpBasePath;

    private String username;

    private String password;

    private boolean isRecursive;

    private Set<String> extensionsSet;

	private boolean incremental;

    private Session sftpSession;

    private ChannelSftp sftpChannel;

    private Queue<String> queue;

	private DateTime fromDate = null;

	private DateTimeFormatter simpleDateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd");

	public SftpIterator(String baseUrl, String username, String password, boolean isRecursive, Set<String> extensionsSet, String fromDate) {

		this.baseUrl = baseUrl;

        this.username = username;

        this.password = password;

        this.isRecursive = isRecursive;

        this.extensionsSet = extensionsSet;

		this.incremental = StringUtils.isNotBlank(fromDate);

		if (incremental) {

			//I expect fromDate in the format 'yyyy-MM-dd'. See class eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode .

			this.fromDate = DateTime.parse(fromDate, simpleDateTimeFormatter);

			log.debug("fromDate string: " + fromDate + " -- parsed: " + this.fromDate.toString());

		}

		try {

            URI sftpServer = new URI(baseUrl);

            this.sftpURIScheme = sftpServer.getScheme();

            this.sftpServerAddress = sftpServer.getHost();

            this.remoteSftpBasePath = sftpServer.getPath();

        } catch (URISyntaxException e) {

            throw new CollectorServiceRuntimeException("Bad syntax in the URL " + baseUrl);

        }

        connectToSftpServer();

        initializeQueue();

    }

    private void connectToSftpServer() {

        JSch jsch = new JSch();

        try {

            JSch.setConfig("StrictHostKeyChecking", "no");

            sftpSession = jsch.getSession(username, sftpServerAddress);

            sftpSession.setPassword(password);

            sftpSession.connect();

            Channel channel = sftpSession.openChannel(sftpURIScheme);

            channel.connect();

            sftpChannel = (ChannelSftp) channel;

	        String pwd = sftpChannel.pwd();

	        log.debug("PWD from server: " + pwd);

	        String fullPath = pwd + remoteSftpBasePath;

	        sftpChannel.cd(fullPath);

	        log.debug("PWD from server 2 after 'cd " + fullPath + "' : " + sftpChannel.pwd());

	        log.info("Connected to SFTP server " + sftpServerAddress);

        } catch (JSchException e) {

            throw new CollectorServiceRuntimeException("Unable to connect to remote SFTP server.", e);

        } catch (SftpException e) {

            throw new CollectorServiceRuntimeException("Unable to access the base remote path on the SFTP server.", e);

        }

    }

    private void disconnectFromSftpServer() {

        sftpChannel.exit();

        sftpSession.disconnect();

    }

    private void initializeQueue() {

        queue = new LinkedList<String>();

	    log.info(String.format("SFTP collector plugin collecting from %s with recursion = %s, incremental = %s with fromDate=%s", remoteSftpBasePath,

			    isRecursive,

			    incremental, fromDate));

	    listDirectoryRecursive(".", "");

    }

    private void listDirectoryRecursive(final String parentDir, final String currentDir) {

        String dirToList = parentDir;

	    if (StringUtils.isNotBlank(currentDir)) {

		    dirToList += "/" + currentDir;

        }

	    log.debug("PARENT DIR: " + parentDir);

	    log.debug("DIR TO LIST: " + dirToList);

	    try {

            Vector<ChannelSftp.LsEntry> ls = sftpChannel.ls(dirToList);

            for (ChannelSftp.LsEntry entry : ls) {

                String currentFileName = entry.getFilename();

                if (currentFileName.equals(".") || currentFileName.equals("..")) {

                    // skip parent directory and directory itself

                    continue;

                }

                SftpATTRS attrs = entry.getAttrs();

                if (attrs.isDir()) {

                    if (isRecursive) {

                        listDirectoryRecursive(dirToList, currentFileName);

                    }

                } else {

                    // test the file for extensions compliance and, just in case, add it to the list.

                    for (String ext : extensionsSet) {

                        if (currentFileName.endsWith(ext)) {

	                        //test if the file has been changed after the last collection date:

	                        if (incremental) {

		                        int mTime = attrs.getMTime();

		                        //int times are values reduced by the milliseconds, hence we multiply per 1000L

		                        DateTime dt = new DateTime(mTime * 1000L);

		                        if (dt.isAfter(fromDate)) {

			                        queue.add(currentFileName);

			                        log.debug(currentFileName + " has changed and must be re-collected");

		                        } else {

			                        if (log.isDebugEnabled()) {

				                        log.debug(currentFileName + " has not changed since last collection");

			                        }

		                        }

	                        } else {

		                        //if it is not incremental, just add it to the queue

		                        queue.add(currentFileName);

	                        }

                        }

                    }

                }

            }

        } catch (SftpException e) {

            throw new CollectorServiceRuntimeException("Cannot list the sftp remote directory", e);

        }

    }

    @Override

    public boolean hasNext() {

        if (queue.isEmpty()) {

            disconnectFromSftpServer();

            return false;

        } else {

            return true;

        }

    }

    @Override

    public String next() {

        String nextRemotePath = queue.remove();

        int nRepeat = 0;

	    String fullPathFile = nextRemotePath;

	    while (nRepeat < MAX_RETRIES) {

            try {

                OutputStream baos = new ByteArrayOutputStream();

                sftpChannel.get(nextRemotePath, baos);

	            if (log.isDebugEnabled()) {

		            fullPathFile = sftpChannel.pwd() + "/" + nextRemotePath;

		            log.debug(String.format("Collected file from SFTP: %s%s", sftpServerAddress, fullPathFile));

	            }

	            return baos.toString();

            } catch (SftpException e) {

                nRepeat++;

	            log.warn(String.format("An error occurred [%s] for %s%s, retrying.. [retried %s time(s)]", e.getMessage(), sftpServerAddress, fullPathFile,

			            nRepeat));

	            // disconnectFromSftpServer();

                try {

                    Thread.sleep(BACKOFF_MILLIS);

                } catch (InterruptedException e1) {

                    log.error(e1);

                }

            }

        }

	    throw new CollectorServiceRuntimeException(

			    String.format("Impossible to retrieve FTP file %s after %s retries. Aborting FTP collection.", fullPathFile, nRepeat));

    }

    @Override

    public void remove() {

        throw new UnsupportedOperationException();

    }

}

package eu.dnetlib.data.collector.plugins.ftp;

import com.google.common.base.Splitter;

import com.google.common.collect.Sets;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.springframework.beans.factory.annotation.Required;

import java.util.Iterator;

import java.util.Set;

/**

 *

 * @author Author: Andrea Mannocci

 *

 */

public class FtpCollectorPlugin extends AbstractCollectorPlugin {

	private FtpIteratorFactory ftpIteratorFactory;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		final String username = interfaceDescriptor.getParams().get("username");

		final String password = interfaceDescriptor.getParams().get("password");

		final String recursive = interfaceDescriptor.getParams().get("recursive");

		final String extensions = interfaceDescriptor.getParams().get("extensions");

		if ((baseUrl == null) || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'baseurl' is null or empty"); }

		if ((username == null) || username.isEmpty()) { throw new CollectorServiceException("Param 'username' is null or empty"); }

		if ((password == null) || password.isEmpty()) { throw new CollectorServiceException("Param 'password' is null or empty"); }

		if ((recursive == null) || recursive.isEmpty()) { throw new CollectorServiceException("Param 'recursive' is null or empty"); }

		if ((extensions == null) || extensions.isEmpty()) { throw new CollectorServiceException("Param 'extensions' is null or empty"); }

		return new Iterable<String>() {

			boolean isRecursive = "true".equals(recursive);

			Set<String> extensionsSet = parseSet(extensions);

			@Override

			public Iterator<String> iterator() {

				return getFtpIteratorFactory().newIterator(baseUrl, username, password, isRecursive, extensionsSet);

			}

			private Set<String> parseSet(final String extensions) {

				return Sets.newHashSet(Splitter.on(",").omitEmptyStrings().trimResults().split(extensions));

			}

		};

	}

	public FtpIteratorFactory getFtpIteratorFactory() {

		return ftpIteratorFactory;

	}

	@Required

	public void setFtpIteratorFactory(final FtpIteratorFactory ftpIteratorFactory) {

		this.ftpIteratorFactory = ftpIteratorFactory;

	}

}

package eu.dnetlib.data.collector.plugins.filesfrommetadata;

import java.util.List;

import java.util.Map;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import eu.dnetlib.data.collector.functions.ParamValuesFunction;

import eu.dnetlib.data.collector.rmi.ProtocolParameterValue;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

/**

 * Created by alessia on 17/12/15.

 */

public class PopulateFileDownloadBasePath implements ParamValuesFunction {

	private static final Log log = LogFactory.getLog(PopulateFileDownloadBasePath.class);

	@Autowired

	private UniqueServiceLocator serviceLocator;

	@Value("${services.objectstore.basePathList.xquery}")

	private String xQueryForObjectStoreBasePath;

	@Override

	public List<ProtocolParameterValue> findValues(final String s, final Map<String, String> map) {

		try {

			return Lists.transform(serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xQueryForObjectStoreBasePath),

					new Function<String, ProtocolParameterValue>() {

						@Override

						public ProtocolParameterValue apply(final String s) {

							return new ProtocolParameterValue(s, s);

						}

					});

		} catch (ISLookUpException e) {

			log.error("Cannot read Object store service properties", e);

		}

		return Lists.newArrayList();

	}

	public UniqueServiceLocator getServiceLocator() {

		return serviceLocator;

	}

	public void setServiceLocator(final UniqueServiceLocator serviceLocator) {

		this.serviceLocator = serviceLocator;

	}

	public String getxQueryForObjectStoreBasePath() {

		return xQueryForObjectStoreBasePath;

	}

	public void setxQueryForObjectStoreBasePath(final String xQueryForObjectStoreBasePath) {

		this.xQueryForObjectStoreBasePath = xQueryForObjectStoreBasePath;

	}

}

package eu.dnetlib.data.collector.plugins.oaisets;

import java.util.Iterator;

import org.springframework.beans.factory.annotation.Required;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class OaiSetsCollectorPlugin extends AbstractCollectorPlugin {

	private OaiSetsIteratorFactory oaiSetsIteratorFactory;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		if (baseUrl == null || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'baseurl' is null or empty"); }

		return new Iterable<String>() {

			@Override

			public Iterator<String> iterator() {

				return oaiSetsIteratorFactory.newIterator(baseUrl);

			}

		};

	}

	public OaiSetsIteratorFactory getOaiSetsIteratorFactory() {

		return oaiSetsIteratorFactory;

	}

	@Required

	public void setOaiSetsIteratorFactory(final OaiSetsIteratorFactory oaiSetsIteratorFactory) {

		this.oaiSetsIteratorFactory = oaiSetsIteratorFactory;

	}

}

package eu.dnetlib.data.collector.plugins.filesystem;

import java.io.IOException;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.util.Iterator;

import java.util.Set;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.collect.Iterators;

import com.google.common.collect.Sets;

/**

 * Class enabling lazy & recursive iteration of a filesystem tree. The iterator iterates over file paths.

 *

 * @author Andrea

 *

 */

public class FileSystemIterator implements Iterator<String> {

	/** The logger */

	private static final Log log = LogFactory.getLog(FileSystemIterator.class);

	private Set<String> extensions = Sets.newHashSet();

	private Iterator<Path> pathIterator;

	private String current;

	public FileSystemIterator(final String baseDir, final String extensions) {

		if(StringUtils.isNotBlank(extensions)) {

			this.extensions = Sets.newHashSet(extensions.split(","));

		}

		try {

			this.pathIterator = Files.newDirectoryStream(Paths.get(baseDir)).iterator();

			this.current = walkTillNext();

		} catch (IOException e) {

			log.error("Cannot initialize File System Iterator. Is this path correct? " + baseDir);

			throw new RuntimeException("Filesystem collection error.", e);

		}

	}

	@Override

	public boolean hasNext() {

		return current != null;

	}

	@Override

	public synchronized String next() {

		String pivot = new String(current);

		current = walkTillNext();

		log.debug("Returning: " + pivot);

		return pivot;

	}

	@Override

	public void remove() {}

	/**

	 * Walk the filesystem recursively until it finds a candidate. Strategies: a) For any directory found during the walk, an iterator is

	 * built and concat to the main one; b) Any file is checked against admitted extensions

	 *

	 * @return the next element to be returned by next call of this.next()

	 */

	private synchronized String walkTillNext() {

		while (pathIterator.hasNext()) {

			Path nextFilePath = pathIterator.next();

			if (Files.isDirectory(nextFilePath)) {

				// concat

				try {

					pathIterator = Iterators.concat(pathIterator, Files.newDirectoryStream(nextFilePath).iterator());

					log.debug("Adding folder iterator: " + nextFilePath.toString());

				} catch (IOException e) {

					log.error("Cannot create folder iterator! Is this path correct? " + nextFilePath.toString());

					return null;

				}

			} else {

				if (extensions.isEmpty() || extensions.contains(FilenameUtils.getExtension(nextFilePath.toString()))) {

					log.debug("Returning: " + nextFilePath.toString());

					return nextFilePath.toString();

				}

			}

		}

		return null;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<version>1.0.0</version>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-modular-collector-service</artifactId>

	<packaging>jar</packaging>

	<version>3.3.11</version>

	<scm>

		<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-modular-collector-service/tags/dnet-modular-collector-service-3.3.11</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-modular-collector-service-rmi</artifactId>

			<version>[1.3.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-resultset-service</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>cnr-blackboard-common</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>javax.servlet</groupId>

			<artifactId>javax.servlet-api</artifactId>

			<version>${javax.servlet.version}</version>

			<scope>provided</scope>

		</dependency>

		<dependency>

			<groupId>net.sf.opencsv</groupId>

			<artifactId>opencsv</artifactId>

			<version>2.0</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>commons-net</groupId>

			<artifactId>commons-net</artifactId>

			<version>3.3</version>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-compress</artifactId>

			<version>1.6</version>

		</dependency>

		<dependency>

			<groupId>org.mockito</groupId>

			<artifactId>mockito-core</artifactId>

			<version>${mockito.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>commons-httpclient</groupId>

			<artifactId>commons-httpclient</artifactId>

			<version>3.1</version>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>gson</artifactId>

			<version>${google.gson.version}</version>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<artifactId>commons-csv</artifactId>

			<version>1.4</version>

		</dependency>

		<dependency>

			<groupId>com.jcraft</groupId>

			<artifactId>jsch</artifactId>

			<version>0.1.53</version>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.9.2</version>

		</dependency>

		<dependency>

			<groupId>org.json</groupId>

			<artifactId>json</artifactId>

			<version>20160810</version>

		</dependency>

		<dependency>

			<groupId>com.ximpleware</groupId>

			<artifactId>vtd-xml</artifactId>

			<version>2.13.2</version>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.data.collector.plugins;

import java.io.*;

import java.net.URL;

import java.util.Iterator;

import java.util.Set;

import com.google.common.collect.Iterators;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.apache.commons.csv.CSVFormat;

import org.apache.commons.csv.CSVParser;

import org.apache.commons.io.input.BOMInputStream;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentHelper;

import org.dom4j.Element;

/**

 * The Class HttpCSVCollectorPlugin.

 */

public class HttpCSVCollectorPlugin extends AbstractCollectorPlugin {

	private static final Log log = LogFactory.getLog(HttpCSVCollectorPlugin.class);

	public static final String UTF8_BOM = "\uFEFF";

	/**

	 * The Class HTTPCSVIterator.

	 */

	class HTTPCSVIterator implements Iterable<String> {

		/** The descriptor. */

		private InterfaceDescriptor descriptor;

		/**

		 * Instantiates a new HTTPCSV iterator.

		 *

		 * @param descriptor

		 *            the descriptor

		 */

		public HTTPCSVIterator(final InterfaceDescriptor descriptor) {

			this.descriptor = descriptor;

		}

		/**

		 * Iterator.

		 *

		 * @return the iterator

		 */

		@SuppressWarnings("resource")

		@Override

		public Iterator<String> iterator() {

			try {

				final String separator = descriptor.getParams().get("separator");

				final String identifier = descriptor.getParams().get("identifier");

				final String quote = descriptor.getParams().get("quote");

				final URL url = new URL(descriptor.getBaseUrl());

				long nLines = 0;

				// FIX

				// This code should skip the lines with invalid quotes

				final File tempFile = File.createTempFile("csv-", ".tmp");

				try (InputStream is = url.openConnection().getInputStream();

						BOMInputStream bomIs = new BOMInputStream(is);

						BufferedReader reader = new BufferedReader(new InputStreamReader(bomIs));

						FileWriter fw = new FileWriter(tempFile)) {

					String line;

					while ((line = reader.readLine()) != null) {

						if (StringUtils.isBlank(quote) || (quote.charAt(0) != '"') || verifyQuotes(line, separator.charAt(0))) {

							fw.write(line);

							fw.write("\n");

							nLines++;

						}

					}

				}

				// END FIX

				final CSVFormat format = CSVFormat.EXCEL

						.withHeader()

						.withDelimiter(separator.equals("\\t") || StringUtils.isBlank(separator) ? '\t' : separator.charAt(0))

						.withQuote(StringUtils.isBlank(quote) ? null : quote.charAt(0))

						.withTrim();

				final CSVParser parser = new CSVParser(new FileReader(tempFile), format);

				final Set<String> headers = parser.getHeaderMap().keySet();

				final long nRecords = nLines - 1;

				return Iterators.transform(parser.iterator(), input -> {

					try {

						final Document document = DocumentHelper.createDocument();

						final Element root = document.addElement("csvRecord");

						for (final String key : headers) {

							final Element row = root.addElement("column");

							row.addAttribute("name", key).addText(input.get(key));

							if (key.equals(identifier)) {

								row.addAttribute("isID", "true");

							}

						}

						return document.asXML();

					} finally {

						System.out.println(tempFile.getAbsolutePath());

						if (parser.getRecordNumber() == nRecords) {

							System.out.println("DELETING " + tempFile.getAbsolutePath());

							tempFile.delete();

						}

					}

				});

			} catch (final Exception e) {

				log.error("Error iterating csv lines", e);

				return null;

			}

		}

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.collector.plugin.CollectorPlugin#collect(eu.dnetlib.data.collector.rmi.InterfaceDescriptor, java.lang.String,

	 * java.lang.String)

	 */

	@Override

	public Iterable<String> collect(final InterfaceDescriptor descriptor, final String fromDate, final String untilDate) throws CollectorServiceException {

		return new HTTPCSVIterator(descriptor);

	}

	public boolean verifyQuotes(final String line, final char separator) {

		final char[] cs = line.trim().toCharArray();

		boolean inField = false;

		boolean skipNext = false;

		for (int i = 0; i < cs.length; i++) {

			if (skipNext) {

				skipNext = false;

			} else if (inField) {

				if ((cs[i] == '\"') && ((i == (cs.length - 1)) || (cs[i + 1] == separator))) {

					inField = false;

				} else if ((cs[i] == '\"') && (i < (cs.length - 1))) {

					if ((cs[i + 1] == '\"')) {

						skipNext = true;

					} else {

						log.warn("Skipped invalid line: " + line);

						return false;

					}

				}

			} else {

				if ((cs[i] == '\"') && ((i == 0) || (cs[i - 1] == separator))) {

					inField = true;

				}

			}

		}

		if (inField) {

			log.warn("Skipped invalid line: " + line);

			return false;

		}

		return true;

	}

}

package eu.dnetlib.data.collector.plugins.csv;

import java.net.URISyntaxException;

import java.util.HashMap;

import eu.dnetlib.data.collector.plugins.HttpCSVCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import org.junit.Test;

import static org.junit.Assert.assertFalse;

import static org.junit.Assert.assertTrue;

public class HTTPCSVCollectorPluginTest {

	private String FILE_URL = HTTPCSVCollectorPluginTest.class.getResource("testCSVwithBOM.csv").toString();

	final HttpCSVCollectorPlugin plugin = new HttpCSVCollectorPlugin();

	@Test

	public void testCSVHeader() throws URISyntaxException, CollectorServiceException {

		final InterfaceDescriptor descr = new InterfaceDescriptor();

		final HashMap<String, String> params = new HashMap<String, String>();

		params.put("separator", ",");

		params.put("quote", "\"");

		params.put("identifier", "ID");

		descr.setBaseUrl(FILE_URL);

		descr.setParams(params);

		int i = 0;

		for (final String s : plugin.collect(descr, null, null)) {

			assertTrue(s.length() > 0);

			System.out.println(s);

			i++;

		}

		System.out.println(i);

		assertTrue(i > 0);

	}

	@Test

	public void testVerifyQuotesOk(){

		String correct = "\"5\",\"Il Padrino\",\"EEEEEEEE \"\"ZZZZZ\"\" EEEEEEEEEE\",1970";

		assertTrue(plugin.verifyQuotes(correct, ','));

	}

	@Test

	public void testVerifyQuotesWRONG(){

		String correct = "5\",\"Il Padrino\",\"EEEEEEEE \"ZZZZZ\" EEEEEEEEEE\",1970";

		assertFalse(plugin.verifyQuotes(correct, ','));

	}

	@Test

	public void testSNSF(){

		String s = "\"8773\";\"3101-008773\";\"EMBO workshop on structure, function and regulation of membrane transport proteins\";\"\";\"Rossier Bernard C.\";\"Scientific Conferences\";\"Science communication\";\"Département de Pharmacologie & Toxicologie Faculté de Biologie et de Médecine Université de Lausanne\";\"Université de Lausanne - LA\";\"30103\";\"Cellular Biology, Cytology\";\"Biology and Medicine;Basic Biological Research\";\"01.04.1987\";\"30.09.1987\";\"10000.00\";\"\";\"30103\"" ;

		assertTrue(plugin.verifyQuotes(s, ';'));

	}

	@Test

	public void testSNSF2(){

		String s = "\"11000\";\"4021-011000\";\"Literarische und nationale Erziehung : Schweizerisches Selbstverständnis in der Literatur für Kinder und Jugend- liche\";\"\";\"Tschirky Rosmarie\";\"NRP 21 Cultural Diversity and National Identity\";\"Programmes;National Research Programmes (NRPs)\";\"Schweiz. Inst. für Kinder- und Jugendmedien\";\"Universität Zürich - ZH\";\"10501\";\"German and English languages and literature\";\"Human and Social Sciences;Linguistics and literature, philosophy\";\"10501\";\"01.10.1986\";\"31.03.1990\";\"308807.00\";\"\"";

		assertTrue(plugin.verifyQuotes(s, ';'));

	}

	@Test

	public void testSNSFInvalid(){

		String s = "\"35918\";\"1113-035918\";\"Entwicklung eines dreisprachigen Thesaurus des schweizerischen Rechts zur Unterstützung der Suche in Volltextdatenbanken.\";\"\";\"Verein \"Schweizerische Juristische Datenbank\"\";\"Project funding (Div. I-III)\";\"Project funding\";\"Verein \"\"Schweizerische Juristische Datenbank\"\"\";\"NPO (Biblioth., Museen, Verwalt.) - NPO\";\"10205\";\"Legal sciences\";\"Human and Social Sciences;Economics, law\";\"10205\";\"01.12.1992\";\"31.03.1995\";\"500366.00\";\"\"";

		assertFalse(plugin.verifyQuotes(s, ';'));

	}

}

package eu.dnetlib.data.collector.plugins.csv;

import java.net.URISyntaxException;

import java.net.URL;

import java.util.HashMap;

import org.junit.Assert;

import org.junit.Test;

import eu.dnetlib.data.collector.plugins.FileCSVCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class CSVCollectorPluginTest {

	@Test

	public void testCSVHeader() throws URISyntaxException, CollectorServiceException {

		URL resource = CSVCollectorPluginTest.class.getResource("/eu/dnetlib/data/collector/filesystem/csv/input.tsv");

		InterfaceDescriptor descr = new InterfaceDescriptor();

		HashMap<String, String> params = new HashMap<String, String>();

		params.put("header", "TrUe");

		params.put("separator", "\t");

		params.put("identifier", "56");

		descr.setBaseUrl(resource.toString());

		descr.setParams(params);

		FileCSVCollectorPlugin plugin = new FileCSVCollectorPlugin();

		int i = 0;

		for (String s : plugin.collect(descr, null, null)) {

			Assert.assertTrue(s.length() > 0);

			i++;

			System.out.println(s);

			break;

		}

		Assert.assertTrue(i > 0);

	}

	@Test

	public void testTSVQuote() throws URISyntaxException, CollectorServiceException {

		URL resource = CSVCollectorPluginTest.class.getResource("/eu/dnetlib/data/collector/filesystem/csv/input-quoted.tsv");

		InterfaceDescriptor descr = new InterfaceDescriptor();

		HashMap<String, String> params = new HashMap<String, String>();

		params.put("header", "true");

		params.put("separator", ";");

		params.put("identifier", "0");

		params.put("quote", "\\\"");

		descr.setBaseUrl(resource.toString());

		descr.setParams(params);

		FileCSVCollectorPlugin plugin = new FileCSVCollectorPlugin();

		int i = 0;

		for (String s : plugin.collect(descr, null, null)) {

			Assert.assertTrue(s.length() > 0);

			i++;

			System.out.println(s);

			break;

		}

		Assert.assertTrue(i > 0);

	}

}

package eu.dnetlib.data.collector.plugins.oai;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import static org.mockito.Mockito.verify;

import static org.mockito.Mockito.when;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Iterator;

import java.util.List;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.internal.verification.Times;

import org.mockito.junit.MockitoJUnitRunner;

import com.google.common.base.Joiner;

import com.google.common.collect.Lists;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import eu.dnetlib.data.collector.rmi.ProtocolDescriptor;

import eu.dnetlib.data.collector.rmi.ProtocolParameter;

@RunWith(MockitoJUnitRunner.class)

public class OaiCollectorPluginTest {

	private OaiCollectorPlugin oai;

	@Mock

	private OaiIteratorFactory oaiIteratorFactory;

	private List<String> elements = Lists.newArrayList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12");

	private Iterator<String> oaiIterator1 = elements.subList(0, 3).iterator();

	private Iterator<String> oaiIterator2 = elements.subList(3, 7).iterator();

	private Iterator<String> oaiIterator3 = elements.subList(7, elements.size()).iterator();

	private static final String BASE_URL = "http://oai.test.it/oai";

	private static final String FORMAT = "oai_dc";

	private static final String PROTOCOL = "OAI";

	private static final String SET_1 = "set01";

	private static final String SET_2 = "set02";

	private static final String SET_3 = "set03";

	@Before

	public void setUp() {

		oai = new OaiCollectorPlugin();

		oai.setOaiIteratorFactory(oaiIteratorFactory);

		oai.setProtocolDescriptor(new ProtocolDescriptor(PROTOCOL, new ArrayList<ProtocolParameter>()));

		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_1, null, null)).thenReturn(oaiIterator1);

		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_2, null, null)).thenReturn(oaiIterator2);

		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_3, null, null)).thenReturn(oaiIterator3);

	}

	public void test() {

		oai = new OaiCollectorPlugin();

	}

	@Test

	public void testGetProtocol() {

		assertEquals(PROTOCOL, oai.getProtocol());

	}

	@Test

	public void testCollect() throws Exception {

		final InterfaceDescriptor iface = new InterfaceDescriptor();

		iface.setId("123");

		iface.setProtocol(PROTOCOL);

		iface.setBaseUrl(BASE_URL);

		iface.setParams(new HashMap<String, String>());

		iface.getParams().put("format", FORMAT);

		iface.getParams().put("set", Joiner.on(", ").join(SET_1, SET_2, SET_3));

		final Iterable<String> records = oai.collect(iface, null, null);

		assertNotNull(records);

		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_1, null, null);

		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_2, null, null);

		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_3, null, null);

		int count = 0;

		for (String s : records) {

			System.out.println("RECORD: " + s);

			assertEquals("" + count, s);

			count++;

		}

		assertEquals(elements.size(), count);

		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_1, null, null);

		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_2, null, null);

		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_3, null, null);

	}

}

package eu.dnetlib.data.collector.plugins.oai.engine;

import java.util.HashMap;

import java.util.HashSet;

import java.util.Map;

import java.util.Set;

import java.util.regex.Pattern;

/**

 * @author jochen

 *

 */

public class XmlCleaner {

	/**

	 * Pattern for numeric entities.

	 */

	private static Pattern validCharacterEntityPattern = Pattern.compile("^&#x?\\d{2,4};"); //$NON-NLS-1$

	//	    private static Pattern validCharacterEntityPattern = Pattern.compile("^&#?\\d{2,4};"); //$NON-NLS-1$

	private static Pattern invalidControlCharPattern = Pattern.compile("");

	/**

	 * Pattern that negates the allowable XML 4 byte unicode characters. Valid

	 * are: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] |

	 * [#x10000-#x10FFFF]

	 */

	private static Pattern invalidCharacterPattern = Pattern.compile("[^\t\r\n\u0020-\uD7FF\uE000-\uFFFD]"); //$NON-NLS-1$

	// Map entities to their unicode equivalent

	private static Set<String> goodEntities = new HashSet<String>();

	private static Map<String, String> badEntities = new HashMap<String, String>();

	static {

		// pre-defined XML entities

		goodEntities.add("""); //$NON-NLS-1$ // quotation mark

		goodEntities.add("&"); //$NON-NLS-1$ // ampersand

		goodEntities.add("<"); //$NON-NLS-1$ // less-than sign

		goodEntities.add(">"); //$NON-NLS-1$ // greater-than sign

		// control entities

		//badEntities.put("", "");

		badEntities.put("", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("€", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("‚", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("ƒ", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("„", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("…", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("†", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("‡", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("ˆ", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("‰", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("Š", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("‹", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("Œ", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("Ž", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("‘", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("’", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("“", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("”", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("•", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("–", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("—", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("˜", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("™", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("š", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("›", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("œ", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("ž", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		badEntities.put("Ÿ", " "); //$NON-NLS-1$ //$NON-NLS-2$ // illegal HTML character

		// misc entities

		badEntities.put("€", "\u20AC"); //$NON-NLS-1$ //$NON-NLS-2$ // euro

		badEntities.put("‘", "\u2018"); //$NON-NLS-1$ //$NON-NLS-2$ // left single quotation mark

		badEntities.put("’", "\u2019"); //$NON-NLS-1$ //$NON-NLS-2$ // right single quotation mark

		// Latin 1 entities

		badEntities.put(" ", "\u00A0"); //$NON-NLS-1$ //$NON-NLS-2$ // no-break space

		badEntities.put("¡", "\u00A1"); //$NON-NLS-1$ //$NON-NLS-2$ // inverted exclamation mark

		badEntities.put("¢", "\u00A2"); //$NON-NLS-1$ //$NON-NLS-2$ // cent sign

		badEntities.put("£", "\u00A3"); //$NON-NLS-1$ //$NON-NLS-2$ // pound sign

		badEntities.put("¤", "\u00A4"); //$NON-NLS-1$ //$NON-NLS-2$ // currency sign

		badEntities.put("¥", "\u00A5"); //$NON-NLS-1$ //$NON-NLS-2$ // yen sign

		badEntities.put("¦", "\u00A6"); //$NON-NLS-1$ //$NON-NLS-2$ // broken vertical bar

		badEntities.put("§", "\u00A7"); //$NON-NLS-1$ //$NON-NLS-2$ // section sign

		badEntities.put("¨", "\u00A8"); //$NON-NLS-1$ //$NON-NLS-2$ // diaeresis

		badEntities.put("©", "\u00A9"); //$NON-NLS-1$ //$NON-NLS-2$ // copyright sign

		badEntities.put("ª", "\u00AA"); //$NON-NLS-1$ //$NON-NLS-2$ // feminine ordinal indicator

		badEntities.put("«", "\u00AB"); //$NON-NLS-1$ //$NON-NLS-2$ // left-pointing double angle quotation mark

		badEntities.put("¬", "\u00AC"); //$NON-NLS-1$ //$NON-NLS-2$ // not sign

		badEntities.put("­", "\u00AD"); //$NON-NLS-1$ //$NON-NLS-2$ // soft hyphen

		badEntities.put("®", "\u00AE"); //$NON-NLS-1$ //$NON-NLS-2$ // registered sign

		badEntities.put("¯", "\u00AF"); //$NON-NLS-1$ //$NON-NLS-2$ // macron

		badEntities.put("°", "\u00B0"); //$NON-NLS-1$ //$NON-NLS-2$ // degree sign

		badEntities.put("±", "\u00B1"); //$NON-NLS-1$ //$NON-NLS-2$ // plus-minus sign

		badEntities.put("²", "\u00B2"); //$NON-NLS-1$ //$NON-NLS-2$ // superscript two

		badEntities.put("³", "\u00B3"); //$NON-NLS-1$ //$NON-NLS-2$ // superscript three

		badEntities.put("´", "\u00B4"); //$NON-NLS-1$ //$NON-NLS-2$ // acute accent

		badEntities.put("µ", "\u00B5"); //$NON-NLS-1$ //$NON-NLS-2$ // micro sign

		badEntities.put("¶", "\u00B6"); //$NON-NLS-1$ //$NON-NLS-2$ // pilcrow sign

		badEntities.put("·", "\u00B7"); //$NON-NLS-1$ //$NON-NLS-2$ // middle dot

		badEntities.put("¸", "\u00B8"); //$NON-NLS-1$ //$NON-NLS-2$ // cedilla

		badEntities.put("¹", "\u00B9"); //$NON-NLS-1$ //$NON-NLS-2$ // superscript one

		badEntities.put("º", "\u00BA"); //$NON-NLS-1$ //$NON-NLS-2$ // masculine ordinal indicator

		badEntities.put("»", "\u00BB"); //$NON-NLS-1$ //$NON-NLS-2$ // right-pointing double angle quotation mark

		badEntities.put("¼", "\u00BC"); //$NON-NLS-1$ //$NON-NLS-2$ // vulgar fraction one quarter

		badEntities.put("½", "\u00BD"); //$NON-NLS-1$ //$NON-NLS-2$ // vulgar fraction one half

		badEntities.put("¾", "\u00BE"); //$NON-NLS-1$ //$NON-NLS-2$ // vulgar fraction three quarters

		badEntities.put("¿", "\u00BF"); //$NON-NLS-1$ //$NON-NLS-2$ // inverted question mark

		badEntities.put("À", "\u00C0"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter A with grave

		badEntities.put("Á", "\u00C1"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter A with acute

		badEntities.put("Â", "\u00C2"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter A with circumflex

		badEntities.put("Ã", "\u00C3"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter A with tilde

		badEntities.put("Ä", "\u00C4"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter A with diaeresis

		badEntities.put("Å", "\u00C5"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter A with ring above

		badEntities.put("Æ", "\u00C6"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter AE

		badEntities.put("Ç", "\u00C7"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter C with cedilla

		badEntities.put("È", "\u00C8"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter E with grave

		badEntities.put("É", "\u00C9"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter E with acute

		badEntities.put("Ê", "\u00CA"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter E with circumflex

		badEntities.put("Ë", "\u00CB"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter E with diaeresis

		badEntities.put("Ì", "\u00CC"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter I with grave

		badEntities.put("Í", "\u00CD"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter I with acute

		badEntities.put("Î", "\u00CE"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter I with circumflex

		badEntities.put("Ï", "\u00CF"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter I with diaeresis

		badEntities.put("Ð", "\u00D0"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter ETH

		badEntities.put("Ñ", "\u00D1"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter N with tilde

		badEntities.put("Ò", "\u00D2"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter O with grave

		badEntities.put("Ó", "\u00D3"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter O with acute

		badEntities.put("Ô", "\u00D4"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter O with circumflex

		badEntities.put("Õ", "\u00D5"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter O with tilde

		badEntities.put("Ö", "\u00D6"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter O with diaeresis

		badEntities.put("×", "\u00D7"); //$NON-NLS-1$ //$NON-NLS-2$ // multiplication sign

		badEntities.put("Ø", "\u00D8"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter O with stroke

		badEntities.put("Ù", "\u00D9"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter U with grave

		badEntities.put("Ú", "\u00DA"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter U with acute

		badEntities.put("Û", "\u00DB"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter U with circumflex

		badEntities.put("Ü", "\u00DC"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter U with diaeresis

		badEntities.put("Ý", "\u00DD"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter Y with acute

		badEntities.put("Þ", "\u00DE"); //$NON-NLS-1$ //$NON-NLS-2$ // latin capital letter THORN

		badEntities.put("ß", "\u00DF"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter sharp s

		badEntities.put("à", "\u00E0"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter a with grave

		badEntities.put("á", "\u00E1"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter a with acute

		badEntities.put("â", "\u00E2"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter a with circumflex

		badEntities.put("ã", "\u00E3"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter a with tilde

		badEntities.put("ä", "\u00E4"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter a with diaeresis

		badEntities.put("å", "\u00E5"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter a with ring above

		badEntities.put("æ", "\u00E6"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter ae

		badEntities.put("ç", "\u00E7"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter c with cedilla

		badEntities.put("è", "\u00E8"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter e with grave

		badEntities.put("é", "\u00E9"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter e with acute

		badEntities.put("ê", "\u00EA"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter e with circumflex

		badEntities.put("ë", "\u00EB"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter e with diaeresis

		badEntities.put("ì", "\u00EC"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter i with grave

		badEntities.put("í", "\u00ED"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter i with acute

		badEntities.put("î", "\u00EE"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter i with circumflex

		badEntities.put("ï", "\u00EF"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter i with diaeresis

		badEntities.put("ð", "\u00F0"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter eth

		badEntities.put("ñ", "\u00F1"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter n with tilde

		badEntities.put("ò", "\u00F2"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter o with grave

		badEntities.put("ó", "\u00F3"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter o with acute

		badEntities.put("ô", "\u00F4"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter o with circumflex

		badEntities.put("õ", "\u00F5"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter o with tilde

		badEntities.put("ö", "\u00F6"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter o with diaeresis

		badEntities.put("÷", "\u00F7"); //$NON-NLS-1$ //$NON-NLS-2$ // division sign

		badEntities.put("ø", "\u00F8"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter o with stroke

		badEntities.put("ù", "\u00F9"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter u with grave

		badEntities.put("ú", "\u00FA"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter u with acute

		badEntities.put("û", "\u00FB"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter u with circumflex

		badEntities.put("ü", "\u00FC"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter u with diaeresis

		badEntities.put("ý", "\u00FD"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter y with acute

		badEntities.put("þ", "\u00FE"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter thorn

		badEntities.put("ÿ", "\u00FF"); //$NON-NLS-1$ //$NON-NLS-2$ // latin small letter y with diaeresis

	}

	/**

	 * For each entity in the input that is not allowed in XML, replace the

	 * entity with its unicode equivalent or remove it. For each instance of a

	 * bare &, replace it with &amp;<br/>

	 * XML only allows 4 entities: &, ", < and >.

	 *

	 * @param broken

	 *            the string to handle entities

	 * @return the string with entities appropriately fixed up

	 */

	static public String cleanAllEntities(final String broken) {

		if (broken == null) {

			return null;

		}

		String working = invalidControlCharPattern.matcher(broken).replaceAll("");

		working = invalidCharacterPattern.matcher(working).replaceAll("");

		int cleanfrom = 0;

		while (true) {

			int amp = working.indexOf('&', cleanfrom);

			// If there are no more amps then we are done

			if (amp == -1) {

				break;

			}

			// Skip references of the kind &#ddd;

			if (validCharacterEntityPattern.matcher(working.substring(amp)).find()) {

				cleanfrom = working.indexOf(';', amp) + 1;

				continue;

			}

			int i = amp + 1;

			while (true) {

				// if we are at the end of the string then just escape the '&';

				if (i >= working.length()) {

					return working.substring(0, amp) + "&" + working.substring(amp + 1); //$NON-NLS-1$

				}

				// if we have come to a ; then we have an entity

				// If it is something that xml can't handle then replace it.

				char c = working.charAt(i);

				if (c == ';') {

					final String entity = working.substring(amp, i + 1);

					final String replace = handleEntity(entity);

					working = working.substring(0, amp) + replace + working.substring(i + 1);

					break;

				}

				// Did we end an entity without finding a closing ;

				// Then treat it as an '&' that needs to be replaced with &

				if (!Character.isLetterOrDigit(c)) {

					working = working.substring(0, amp) + "&" + working.substring(amp + 1); //$NON-NLS-1$

					amp = i + 4; // account for the 4 extra characters

					break;

				}

				i++;

			}

			cleanfrom = amp + 1;

		}

		if (Pattern.compile("<<").matcher(working).find()) {

			working = working.replaceAll("<<", "&lt;&lt;");

		}

		if (Pattern.compile(">>").matcher(working).find()) {

			working = working.replaceAll(">>", ">>");

		}

		return working;

	}

	/**

	 * Replace entity with its unicode equivalent, if it is not a valid XML

	 * entity. Otherwise strip it out. XML only allows 4 entities: &,

	 * ", < and >.

	 *

	 * @param entity

	 *            the entity to be replaced

	 * @return the substitution for the entity, either itself, the unicode

	 *         equivalent or an empty string.

	 */

	private static String handleEntity(final String entity) {

		if (goodEntities.contains(entity)) {

			return entity;

		}

		final String replace = (String) badEntities.get(entity);

		if (replace != null) {

			return replace;

		}

		return replace != null ? replace : "";

	}

}

package eu.dnetlib.data.collector.plugins.oai;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

public class HttpConnectorTest {

	private HttpConnector connector;

	private static final String URL = "https://researchdata.ands.org.au/registry/services/oai?verb=Identify";

	@Before

	public void setUp() {

		connector = new HttpConnector();

		connector.initTrustManager();

		connector.setMaxNumberOfRetry(1);

	}

	@Test

	@Ignore

	public void testGetInputSource() throws CollectorServiceException {

		System.out.println(connector.getInputSource(URL));

	}

}

package eu.dnetlib.data.collector.plugins.oai;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

public class OaiIteratorTest {

	private static final String BASE_URL = "http://oai.d.efg.research-infrastructures.eu/oai.do";

	private static final String FORMAT = "oai_dc";

	private static final String SET = "d937bab1-d44c-44aa-bf7d-df5312a3b623";

	private OaiIterator oai;

	@Before

	public void setUp() {

		HttpConnector httpConnector = new HttpConnector();

		httpConnector.initTrustManager();

		oai = new OaiIterator(BASE_URL, FORMAT, SET, null, null, httpConnector);

	}

	@Test

	@Ignore

	public void test() {

		int count = 0;

		while (oai.hasNext()) {

			oai.next();

			count++;

		}

		System.out.println("TOTAL: " + count);

	}

}

package eu.dnetlib.data.collector.plugins.oai;

import java.util.Iterator;

import java.util.List;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.base.Function;

import com.google.common.base.Splitter;

import com.google.common.collect.Iterables;

import com.google.common.collect.Iterators;

import com.google.common.collect.Lists;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class OaiCollectorPlugin extends AbstractCollectorPlugin {

	private static final String FORMAT_PARAM = "format";

	private static final String OAI_SET_PARAM = "set";

	private OaiIteratorFactory oaiIteratorFactory;

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		final String mdFormat = interfaceDescriptor.getParams().get(FORMAT_PARAM);

		final String setParam = interfaceDescriptor.getParams().get(OAI_SET_PARAM);

		final List<String> sets = Lists.newArrayList();

		if (setParam != null) {

			sets.addAll(Lists.newArrayList(Splitter.on(",").omitEmptyStrings().trimResults().split(setParam)));

		}

		if (sets.isEmpty()) {

			// If no set is defined, ALL the sets must be harvested

			sets.add("");

		}

		if (baseUrl == null || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'baseurl' is null or empty"); }

		if (mdFormat == null || mdFormat.isEmpty()) { throw new CollectorServiceException("Param 'mdFormat' is null or empty"); }

		if (fromDate != null && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) { throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + fromDate); }

		if (untilDate != null && !untilDate.matches("\\d{4}-\\d{2}-\\d{2}")) { throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + untilDate); }

		return new Iterable<String>() {

			@SuppressWarnings("unchecked")

			@Override

			public Iterator<String> iterator() {

				final Iterable<Iterator<String>> iter = Iterables.transform(sets, new Function<String, Iterator<String>>() {

					@Override

					public Iterator<String> apply(final String set) {

						return oaiIteratorFactory.newIterator(baseUrl, mdFormat, set, fromDate, untilDate);

					}

				});

				return Iterators.concat(Iterables.toArray(iter, Iterator.class));

			}

		};

	}

	public OaiIteratorFactory getOaiIteratorFactory() {

		return oaiIteratorFactory;

	}

	@Required

	public void setOaiIteratorFactory(final OaiIteratorFactory oaiIteratorFactory) {

		this.oaiIteratorFactory = oaiIteratorFactory;

	}

}

package eu.dnetlib.data.collector.plugins.httplist;

import java.util.Iterator;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class HttpListCollectorPlugin extends AbstractCollectorPlugin {

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		final String listAddress = interfaceDescriptor.getParams().get("listUrl");

		return new Iterable<String>() {

			@Override

			public Iterator<String> iterator() {

				return new HttpListIterator(baseUrl, listAddress);

			}

		};

	}

}

package eu.dnetlib.data.collector.plugins.oai;

import java.io.StringReader;

import java.io.UnsupportedEncodingException;

import java.net.URLEncoder;

import java.util.Iterator;

import java.util.Queue;

import java.util.concurrent.PriorityBlockingQueue;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import eu.dnetlib.data.collector.plugins.oai.engine.HttpConnector;

import eu.dnetlib.data.collector.plugins.oai.engine.XmlCleaner;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

public class OaiIterator implements Iterator<String> {

	private static final Log log = LogFactory.getLog(OaiIterator.class); // NOPMD by marko on 11/24/08 5:02 PM

	private Queue<String> queue = new PriorityBlockingQueue<String>();

	private SAXReader reader = new SAXReader();

	private String baseUrl;

	private String set;

	private String mdFormat;

	private String fromDate;

	private String untilDate;

	private String token;

	private boolean started;