D-Net

package eu.dnetlib.data.collector.plugins.oaisets;

import java.util.HashMap;

import javax.annotation.Resource;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)

@ContextConfiguration(value = "applicationContext-OaiSetsCollectorPluginTest.xml")

public class OaiSetsCollectorPluginTest {

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

@Ignore

public class HttpConnectorTest {

	private static final String URL_MISCONFIGURED_SERVER = "https://www.alexandria.unisg.ch/cgi/oai2?verb=Identify";

	private static final String URL_GOODSNI_SERVER = "https://air.unimi.it/oai/openaire?verb=Identify";

	@Before

	public void setUp() {

		connector = new HttpConnector();

		System.out.println(connector.getInputSource(URL_MISCONFIGURED_SERVER));

	}

	public void testMisconfiguredServers2() throws IOException {

		HttpURLConnection urlConn = (HttpURLConnection) new URL(URL_MISCONFIGURED_SERVER).openConnection();

		urlConn.getResponseMessage();

	}

	@Test

	public void testGoodServers() throws CollectorServiceException {

		System.out.println(connector.getInputSource(URL_GOODSNI_SERVER));

	}

import java.io.IOException;

import java.util.Iterator;

import org.apache.commons.io.FileUtils;

import org.junit.Before;

import org.junit.Ignore;

public class HttpListIteratorTest {

	// Under test

	private Iterator<String> iter;

	@Before

	public void setUp() throws Exception {

	}

	@Test

import java.io.BufferedInputStream;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

public class HttpCollectorPlugin extends AbstractSplittedRecordPlugin {

	@Override

	protected BufferedInputStream getBufferedInputStream(final String baseUrl) throws CollectorServiceException {

	}

}

import java.io.InputStream;

import java.net.*;

import java.security.GeneralSecurityException;

import java.security.cert.X509Certificate;

import java.util.List;

import java.util.Map;

/**

 * @author jochen, michele, andrea

 */

public class HttpConnector {

	private int defaultDelay = 120; // seconds

	private int readTimeOut = 120; // seconds

	private String userAgent = "Mozilla/5.0 (compatible; OAI; +http://www.openaire.eu)";

	/**

	 * Given the URL returns the content via HTTP GET

	 *

	 * @throws CollectorServiceException when retrying more than maxNumberOfRetry times

	 */

	public String getInputSource(final String requestUrl) throws CollectorServiceException {

		return attemptDownload(requestUrl, 1, new CollectorPluginErrorLogList());

	}

			throws CollectorServiceException {

		if (retryNumber > maxNumberOfRetry) { throw new CollectorServiceException("Max number of retries exceeded. Cause: \n " + errorList); }

					errorList.add("503 Service Unavailable");

					urlConn.disconnect();

					return attemptDownload(requestUrl, retryNumber + 1, errorList);

					final String newUrl = obtainNewLocation(urlConn.getHeaderFields());

					log.debug("The requested url has been moved to " + newUrl);

					errorList.add(String.format("%s %s. Moved to: %s", urlConn.getResponseCode(), urlConn.getResponseMessage(), newUrl));

				} else {

					input = urlConn.getInputStream();

					responseType = urlConn.getContentType();

				}

			} catch (IOException e) {

				log.error("error while retrieving from http-connection occured: " + requestUrl, e);

				Thread.sleep(defaultDelay * 1000);

				errorList.add(e.getMessage());

				return attemptDownload(requestUrl, retryNumber + 1, errorList);

			}

		} catch (InterruptedException e) {

			throw new CollectorServiceException(e);

	private int obtainRetryAfter(final Map<String, List<String>> headerMap) {

		for (String key : headerMap.keySet()) {

		}

		return -1;

	}

		final X509TrustManager tm = new X509TrustManager() {

			@Override

			@Override

			@Override

			public X509Certificate[] getAcceptedIssuers() {

		}

	}

	public int getMaxNumberOfRetry() {

		return maxNumberOfRetry;

	}

		this.readTimeOut = readTimeOut;

	}

}

package eu.dnetlib.data.collector.plugins.httplist;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class HttpListCollectorPlugin extends AbstractCollectorPlugin {

	@Override

	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)

			throws CollectorServiceException {

		final String baseUrl = interfaceDescriptor.getBaseUrl();

		final String listAddress = interfaceDescriptor.getParams().get("listUrl");

	}

}

import java.io.StringReader;

import java.util.Iterator;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

public class HttpListIterator implements Iterator<String> {

	private String baseUrl;

	private String currentLine;

	private BufferedReader reader;

		try {

			this.reader = new BufferedReader(new StringReader(download(listAddress)));

			this.currentLine = reader.readLine();

		} catch (Exception e) {

	private String download(final String url) {

		try {

		}

	}

	@Override

	public void remove() {}

}

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

		<relativePath />

	</parent>

	<modelVersion>4.0.0</modelVersion>

		<dependency>

			<groupId>commons-net</groupId>

			<artifactId>commons-net</artifactId>

		</dependency>

		<dependency>

			<groupId>org.apache.commons</groupId>

			<version>${mockito.version}</version>

			<scope>test</scope>

		</dependency>

		<dependency>

		</dependency>

		<dependency>

			<groupId>com.google.code.gson</groupId>

			<artifactId>vtd-xml</artifactId>

			<version>2.13.2</version>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.data.objectstore.modular;

import com.google.common.base.Function;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.ResponseBody;

@Controller

public class ObjectStoreConsistency {

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet45-parent</artifactId>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<groupId>eu.dnetlib</groupId>

			<scope>test</scope>

		</dependency>

		<dependency>

			<artifactId>commons-net</artifactId>

		</dependency>

	</dependencies>

</project>