D-Net

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-download-plugins</artifactId>

	<version>1.0.1</version>

	<scm>

	  <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-download-plugins/tags/dnet-download-plugins-1.0.1</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service</artifactId>

			<version>[1.0.0,2.0.0)</version>

		</dependency>

		<dependency>

			<groupId>org.jsoup</groupId>

			<artifactId>jsoup</artifactId>

			<version>1.7.2</version>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.3</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

/**

 * The Class ELisDownloadPlugin.

 */

public class ELisDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(ELisDownloadPlugin.class);

	/**

	 * Extract url.

	 *

	 * @param url

	 *            the url

	 * @return the string

	 */

	private String extractURL(final String url) {

		try {

			Document doc = Jsoup.connect(url).get();

			Elements links = doc.select("a[href$=.pdf]");

			for (Element link : links) {

				String linkvalue = link.attr("abs:href");

				if (!linkvalue.toLowerCase().contains("thumbnailversion")) return linkvalue;

			}

			return null;

		} catch (Exception e) {

			log.error("Error on extract URL", e);

			return null;

		}

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "ELisDownloadPlugin";

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) return null;

		String url = input.getOriginalUrl();

		if ((url == null) || (url.trim().length() == 0)) return input;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) return input;

		for (String s : urls) {

			String newURL = extractURL(s);

			if (newURL != null) {

				input.setOriginalUrl(s);

				input.setUrl(newURL);

				return input;

			}

		}

		input.setOriginalUrl(null);

		input.setUrl(null);

		return input;

	}

}

package eu.dnetlib.download.plugin;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

public class FollowPDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(FollowPDFLinkPlugins.class);

	/**

	 * Extract url.

	 *

	 * @param url

	 *            the url

	 * @return the string

	 */

	private String extractURL(final String url) {

		try {

			URL startURL = new URL(url);

			HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

			conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

			HttpURLConnection.setFollowRedirects(true);

			String location = url;

			if ((conn.getResponseCode() >= 300) && (conn.getResponseCode() < 400)) {

				location = conn.getHeaderFields().get("Location").get(0);

				conn.disconnect();

			}

			Document doc = Jsoup.connect(location).get();

			Elements links = doc.select("a[href$=.pdf]");

			for (Element link : links) {

				String linkvalue = link.attr("abs:href");

				return linkvalue;

			}

			return null;

		} catch (Exception e) {

			log.error("Error on extract URL", e);

			return null;

		}

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "FollowPDFLinkPlugins";

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) return null;

		String url = input.getOriginalUrl();

		if ((url == null) || (url.trim().length() == 0)) return input;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) return input;

		for (String s : urls) {

			String newURL = extractURL(s);

			if (newURL != null) {

				input.setOriginalUrl(s);

				input.setUrl(newURL);

				return input;

			}

		}

		input.setOriginalUrl(null);

		input.setUrl(null);

		return input;

	}

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

// TODO: Auto-generated Javadoc

/**

 * The Class EuropePMC.

 */

public class EuropePMC extends AbstractDownloadPlugin implements DownloadPlugin {

	/** The base path. */

	private String basePath;

	// //*[local-name()='metadata']//*[local-name()='identifier' and ./@identifierType='pmc']/text()

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "europePMCDownloadPlugin";

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		String url = input.getOriginalUrl();

		if ((url == null) || (url.trim().length() == 0)) return input;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) return input;

		input.setFileName(input.getIdItemMetadata());

		Boolean added = false;

		for (String s : urls) {

			if (s.startsWith("http")) {

				input.setOriginalUrl(s);

			} else if (s.startsWith("PMC")) {

				String correctUrl = s.replace("PMC", "");

				added = true;

				String path = PathRetreiver.getInstance(null).getPathForPMCID(Integer.parseInt(correctUrl));

				if (path != null) {

					input.setUrl("file://" + path);

				} else {

					input.setUrl(path);

				}

			}

		}

		if (added == false) {

			input.setOriginalUrl(null);

			input.setUrl(null);

		}

		return input;

	}

	/**

	 * Gets the base path.

	 *

	 * @return the basePath

	 */

	public String getBasePath() {

		return basePath;

	}

	/**

	 * Sets the base path.

	 *

	 * @param basePath

	 *            the basePath to set

	 */

	public void setBasePath(final String basePath) {

		this.basePath = basePath;

	}

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

public class ArxivDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) return null;

		String url = input.getOriginalUrl();

		if ((url == null) || (url.trim().length() == 0)) return input;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) return input;

		for (String s : urls) {

			if (s.startsWith("http")) {

				input.setOriginalUrl(s);

				String correctUrl = s.replace("abs", "pdf");

				correctUrl += ".pdf";

				input.setUrl(correctUrl);

			}

		}

		return input;

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	@Override

	public String getPluginName() {

		return "ArxivDownloadPlugin";

	}

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

// TODO: Auto-generated Javadoc

/**

 * The Class EasyPDFDownloadPlugin.

 */

public class EasyPDFDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) return null;

		if (input == null) return null;

		String url = input.getOriginalUrl();

		if ((url == null) || (url.trim().length() == 0)) return input;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) return input;

		for (String s : urls) {

			if (s.trim().endsWith(".pdf")) {

				input.setOriginalUrl(s);

				input.setUrl(s);

				return input;

			}

		}

		input.setOriginalUrl(null);

		input.setUrl(null);

		return input;

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "easyPDFDownloadPlugin";

	}

}

package eu.dnetlib.download.plugin;

import org.joda.time.DateTime;

import org.joda.time.Days;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadItem.OpenAccessValues;

public abstract class AbstractDownloadPlugin {

	public DownloadItem checkOpenAccess(final DownloadItem input) {

		if (input != null) {

			OpenAccessValues openAccess = OpenAccessValues.valueOf(input.getOpenAccess());

			switch (openAccess) {

			case OPEN:

				return input;

			case CLOSED:

			case RESTRICTED:

			case UNKNOWN:

				return null;

			case EMBARGO:

				if (input.getEmbargoDate() == null) return null;

				DateTime embargoDate = new DateTime(input.getEmbargoDate());

				DateTime today = new DateTime();

				Days days = Days.daysBetween(embargoDate, today);

				if (days.getDays() <= 0) return input;

				return null;

			}

		}

		return null;

	}

}

package eu.dnetlib.download.plugin;

import java.io.File;

import java.io.FileFilter;

import java.util.ArrayList;

import java.util.Collections;

import java.util.Comparator;

import java.util.List;

import org.apache.commons.lang.StringUtils;

// TODO: Auto-generated Javadoc

/**

 * The Class PathRetreiver.

 */

public class PathRetreiver {

	/** The instance. */

	private static PathRetreiver instance;

	/**

	 * Gets the single instance of PathRetreiver.

	 *

	 * @param base_path

	 *            the base_path

	 * @return single instance of PathRetreiver

	 */

	public static PathRetreiver getInstance(final String base_path) {

		if (instance == null) {

			instance = new PathRetreiver();

			instance.setBase_path(base_path);

		}

		return instance;

	}

	/** The base_path. */

	private String base_path;

	/** The values. */

	private List<InfoPath> values;

	/**

	 * Bootstrap.

	 */

	private void bootstrap() {

		values = new ArrayList<InfoPath>();

		File basePath = new File(this.base_path);

		File[] selectedFiles = basePath.listFiles(new FileFilter() {

			@Override

			public boolean accept(final File pathname) {

				return pathname.isDirectory();

			}

		});

		for (File f : selectedFiles) {

			String lower = StringUtils.substringAfter(StringUtils.substringBefore(f.getName(), "_"), "PMC");

			String upper = StringUtils.substringAfter(StringUtils.substringAfter(f.getName(), "_"), "PMC");

			String path = f.getPath();

			InfoPath i = new InfoPath();

			i.setLower(Integer.parseInt(lower));

			i.setUpper(Integer.parseInt(upper));

			i.setPath(path);

			values.add(i);

		}

		Collections.sort(values, new Comparator<InfoPath>() {

			@Override

			public int compare(final InfoPath o1, final InfoPath o2) {

				if (o1.getLower() < o2.getLower()) return -1;

				else if (o1.getLower() < o2.getLower()) return 0;

				else return 1;

			}

		});

	}

	/**

	 * Gets the path for pmcid.

	 *

	 * @param pmcID

	 *            the pmc id

	 * @return the path for pmcid

	 */

	public String getPathForPMCID(final int pmcID) {

		if (values == null) {

			bootstrap();

		}

		for (int i = 0; i < values.size(); i++) {

			if (pmcID < values.get(i).getLower()) {

				if (i == 0) return null;

				File f = new File(values.get(i - 1).getPath() + "/" + pmcID + ".xml");

				String s = null;

				if (f.exists()) {

					s = f.getPath();

				}

				return s;

			}

		}

		return null;

	}

	/**

	 * Sets the base_path.

	 *

	 * @param base_path

	 *            the new base_path

	 */

	public void setBase_path(final String base_path) {

		this.base_path = base_path;

	}

	/**

	 * Gets the base_path.

	 *

	 * @return the base_path

	 */

	public String getBase_path() {

		return this.base_path;

	}

}

package eu.dnetlib.download.plugin;

/**

 * The Class InfoPath.

 */

public class InfoPath {

	/** The lower. */

	private int lower;

	/** The upper. */

	private int upper;

	/** The path. */

	private String path;

	/**

	 * Gets the lower.

	 *

	 * @return the lower

	 */

	public int getLower() {

		return lower;

	}

	/**

	 * Sets the lower.

	 *

	 * @param lower

	 *            the new lower

	 */

	public void setLower(final int lower) {

		this.lower = lower;

	}

	/**

	 * Gets the upper.

	 *

	 * @return the upper

	 */

	public int getUpper() {

		return upper;

	}

	/**

	 * Sets the upper.

	 *

	 * @param upper

	 *            the new upper

	 */

	public void setUpper(final int upper) {

		this.upper = upper;

	}

	/**

	 * Gets the path.

	 *

	 * @return the path

	 */

	public String getPath() {

		return path;

	}

	/**

	 * Sets the path.

	 *

	 * @param path

	 *            the new path

	 */

	public void setPath(final String path) {

		this.path = path;

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xmlns:util="http://www.springframework.org/schema/util"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd

                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<bean id= "easyPDFDownloadPlugin" class="eu.dnetlib.download.plugin.EasyPDFDownloadPlugin"/>

	<bean id= "arxivDownloadPlugin" class="eu.dnetlib.download.plugin.ArxivDownloadPlugin"/>

	<bean id= "europePMCDownloadPlugin" class="eu.dnetlib.download.plugin.EuropePMC"/>

	<bean id= "elisDownloadPlugin" class="eu.dnetlib.download.plugin.ELisDownloadPlugin"/>

	<bean id="followPDFLinkPlugins" class="eu.dnetlib.download.plugin.FollowPDFLinkPlugins"/>

</beans>

import org.joda.time.DateTime;

import org.joda.time.Days;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

public class DateIntervalTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void test() {

		DateTime beforeDate = new DateTime(2010, 01, 1, 0, 0);

		DateTime now = new DateTime();

		Days day = Days.daysBetween(beforeDate, now);

		Assert.assertTrue(day.getDays() > 0);

	}

}

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-download-plugins/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-download-plugins"}