/ - Diff - D-Net - D-Net project tracking tool

     import eu.dnetlib.data.collector.rmi.CollectorServiceException;
     import eu.dnetlib.data.information.collectionservice.rmi.CollectionServiceException;
     // TODO: Auto-generated Javadoc
     /**
      * The Class MongoDumpIterable.
      */
-...
     	/**
     	 * Instantiates a new mongo dump iterable.
+    	 *
     	 * @param inputFile
     	 *            the input file
     	 * @throws CollectionServiceException
     	 *             the collection service exception
     	 * @param inputFile the input file
     	 * @throws CollectionServiceException the collection service exception
     	 */
     	public MongoDumpIterable(final File inputFile) throws CollectorServiceException {
     		try {

modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/filesystem/FileSystemIterator.java
16	16	import com.google.common.collect.Sets;
17	17
18	18	/**
19		* Class enabling lazy & recursive iteration of a filesystem tree. The iterator iterates over file paths.
	19	* Class enabling lazy and recursive iteration of a filesystem tree. The iterator iterates over file paths.
20	20	*
21	21	* @author Andrea
22	22	*

     import org.dom4j.io.SAXReader;
     import eu.dnetlib.data.collector.plugins.HttpConnector;
     import eu.dnetlib.data.collector.plugins.oai.engine.XmlCleaner;
     import eu.dnetlib.data.collector.rmi.CollectorServiceException;
     import eu.dnetlib.data.collector.plugins.oai.engine.XmlCleaner;
     public class OaiIterator implements Iterator<String> {
     	private static final Log log = LogFactory.getLog(OaiIterator.class); // NOPMD by marko on 11/24/08 5:02 PM

     	/**
     	 * For each entity in the input that is not allowed in XML, replace the
     	 * entity with its unicode equivalent or remove it. For each instance of a
     	 * bare &, replace it with &amp;<br/>
     	 * XML only allows 4 entities: &amp;amp;, &amp;quot;, &amp;lt; and &amp;gt;.
     	 * bare {@literal &}, replace it with {@literal &amp;<br/>}
     	 * XML only allows 4 entities: {@literal &amp;amp;}, {@literal &amp;quot;}, {@literal &amp;lt;} and {@literal &amp;gt;}.
+    	 *
     	 * @param broken
     	 *            the string to handle entities

     package eu.dnetlib.data.collector.plugins.oai;
     import java.util.Iterator;
     import java.util.List;
     import org.springframework.beans.factory.annotation.Required;
     import com.google.common.base.Function;
     import com.google.common.base.Splitter;
     import com.google.common.collect.Iterables;
     import com.google.common.collect.Iterators;
     import com.google.common.collect.Lists;
     import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;
     import eu.dnetlib.data.collector.rmi.CollectorServiceException;
     import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;
     import org.springframework.beans.factory.annotation.Required;
     public class OaiCollectorPlugin extends AbstractCollectorPlugin {
-...
     		if (untilDate != null && !untilDate.matches("\\d{4}-\\d{2}-\\d{2}")) { throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + untilDate); }
     		return new Iterable<String>() {
     			@SuppressWarnings("unchecked")
     			@Override
     			public Iterator<String> iterator() {
     				final Iterable<Iterator<String>> iter = Iterables.transform(sets, new Function<String, Iterator<String>>() {
     					@Override
     					public Iterator<String> apply(final String set) {
     						return oaiIteratorFactory.newIterator(baseUrl, mdFormat, set, fromDate, untilDate);
+    					}
     				});
     				return Iterators.concat(Iterables.toArray(iter, Iterator.class));
+    			}
     		};
     		return () -> Iterators.concat(
     				sets.stream()
     					.map(set -> oaiIteratorFactory.newIterator(baseUrl, mdFormat, set, fromDate, untilDate))
     					.iterator());
+    	}
     	public OaiIteratorFactory getOaiIteratorFactory() {

     import java.io.IOException;
     import java.io.InputStream;
     import java.net.CookieHandler;
     import java.net.CookieManager;
     import java.net.CookiePolicy;
     import java.net.HttpURLConnection;
     import java.net.URL;
     import java.net.*;
     import java.security.GeneralSecurityException;
     import java.security.KeyManagementException;
     import java.security.NoSuchAlgorithmException;
     import java.security.cert.CertificateException;
     import java.security.cert.X509Certificate;
     import java.util.List;
     import java.util.Map;
     import javax.net.ssl.HttpsURLConnection;
     import javax.net.ssl.SSLContext;
     import javax.net.ssl.TrustManager;
     import javax.net.ssl.X509TrustManager;
     import eu.dnetlib.data.collector.plugin.CollectorPluginErrorLogList;
     import eu.dnetlib.data.collector.rmi.CollectorServiceException;
     import org.apache.commons.io.IOUtils;
     import org.apache.commons.lang3.math.NumberUtils;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import eu.dnetlib.data.collector.plugin.CollectorPluginErrorLogList;
     import eu.dnetlib.data.collector.rmi.CollectorServiceException;
     /**
      * @author jochen, michele, andrea
+     *
-...
+        }
     	/**
     	 * @param requestUrl
     	 * Given the URL returns the content via HTTP GET
+    	 *
     	 * @param requestUrl the URL
     	 * @return the content of the downloaded resource
     	 * @throws CollectorServiceException
     	 * @throws CollectorServiceException when retrying more than maxNumberOfRetry times
     	 */
     	public String getInputSource(final String requestUrl) throws CollectorServiceException {
     		return attemptDownload(requestUrl, 1, new CollectorPluginErrorLogList());
-...
     				final HttpURLConnection urlConn = (HttpURLConnection) new URL(requestUrl).openConnection();
     				urlConn.setInstanceFollowRedirects(false);
     				urlConn.setReadTimeout(readTimeOut * 1000);
                                     urlConn.addRequestProperty("User-Agent", userAgent);
     				urlConn.addRequestProperty("User-Agent", userAgent);
     				if (log.isDebugEnabled()) {
     					logHeaderFields(urlConn);
-...
     					return attemptDownload(requestUrl, retryNumber + 1, errorList);
     				} else if ((urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_PERM) || (urlConn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP)) {
     					final String newUrl = obtainNewLocation(urlConn.getHeaderFields());
     					log.info("The requested url has been moved to " + newUrl);
     					log.debug("The requested url has been moved to " + newUrl);
     					errorList.add(String.format("%s %s. Moved to: %s", urlConn.getResponseCode(), urlConn.getResponseMessage(), newUrl));
     					urlConn.disconnect();
     					return attemptDownload(newUrl, retryNumber + 1, errorList);
-...
     	/**
     	 * register for https scheme; this is a workaround and not intended for the use in trusted environments
+    	 *
     	 * @throws NoSuchAlgorithmException
     	 * @throws KeyManagementException
     	 */
     	public void initTrustManager() {
     		final X509TrustManager tm = new X509TrustManager() {

     	/**
     	 * Get all beans implementing the CollectorPlugin interface.
+    	 *
     	 * @return
     	 * @return the set of eu.dnetlib.data.collector.plugin.CollectorPlugin(s)
     	 */
     	public Collection<CollectorPlugin> getAll() {
     		return beanFactory.getBeansOfType(CollectorPlugin.class).values();
-...
     	/**
     	 * Get given CollectorPlugin or throws exception.
+    	 *
     	 * @param protocol
     	 * @return
     	 * @throws CollectorServiceException
     	 * @param protocol the given protocol
     	 * @return a CollectorPlugin compatible with the given protocol
     	 * @throws CollectorServiceException when no suitable plugin is found
     	 */
     	public CollectorPlugin get(final String protocol) throws CollectorServiceException {
     		for (CollectorPlugin cp : getAll()) {

Project

General

Profile

D-Net

Revision 51556

Added by Claudio Atzori about 6 years ago