D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-download-plugins/trunk/", "deploy_repository": "dnet4-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", "name": "dnet-download-plugins"}

import org.joda.time.DateTime;

import org.joda.time.Days;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

public class DateIntervalTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void test() {

		DateTime beforeDate = new DateTime(2010, 01, 1, 0, 0);

		DateTime now = new DateTime();

		Days day = Days.daysBetween(beforeDate, now);

		Assert.assertTrue(day.getDays() > 0);

	}

}

package eu.dnetlib.download.plugin;

import junit.framework.Assert;

import org.junit.Before;

import org.junit.Test;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.data.download.rmi.DownloadItem;

public class ArxivImportFromFileTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void testREgEx() {

		String regExp = "\\d{4}\\.\\d{4}";

		String input = "1308.0021";

		Assert.assertTrue(input.matches(regExp));

	}

	@Test

	public void testPlugin() {

		DownloadItem item = new DownloadItem();

		item.setOpenAccess("OPEN");

		String[] myList = new String[] { "http://arxiv.org/abs/1308.0001", "b" };

		Gson g = new GsonBuilder().disableHtmlEscaping().create();

		item.setUrl(g.toJson(myList));

		ArxivImportFromFile plugin = new ArxivImportFromFile();

		plugin.setBasePath("/test/path");

		plugin.retrieveUrl(item);

		Assert.assertNotNull(item);

	}

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

// TODO: Auto-generated Javadoc

/**

 * The Class EuropePMC.

 */

public class EuropePMC extends AbstractDownloadPlugin implements DownloadPlugin {

	private static final Log log = LogFactory.getLog(EuropePMC.class);

	/** The base path. */

	private String basePath;

	// //*[local-name()='metadata']//*[local-name()='identifier' and ./@identifierType='pmc']/text()

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "europePMCDownloadPlugin";

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		String url = input.getOriginalUrl();

		if (url == null || url.trim().length() == 0) return input;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		log.debug(String.format("urls is %s", url));

		if (urls == null || urls.size() == 0) return input;

		input.setFileName(input.getIdItemMetadata());

		Boolean added = false;

		for (String s : urls) {

			if (s.startsWith("http")) {

				log.debug(String.format("found url starting with http replace original URL with %s", s));

				input.setOriginalUrl(s);

			} else if (s.startsWith("PMC")) {

				String correctUrl = s.replace("PMC", "");

				log.debug(String.format("found url starting with PMC %s", correctUrl));

				added = true;

				String path = PathRetreiver.getInstance(basePath).getPathForPMCID(Integer.parseInt(correctUrl));

				if (path != null) {

					input.setUrl("file://" + path);

				} else {

					input.setUrl(path);

				}

			}

		}

		if (added == false) {

			input.setOriginalUrl(null);

			input.setUrl(null);

		}

		return input;

	}

	/**

	 * Gets the base path.

	 *

	 * @return the basePath

	 */

	public String getBasePath() {

		return basePath;

	}

	/**

	 * Sets the base path.

	 *

	 * @param basePath

	 *            the basePath to set

	 */

	@Override

	public void setBasePath(final String basePath) {

		this.basePath = basePath;

	}

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

public class ArxivDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) { return null; }

		String url = input.getOriginalUrl();

		if (url == null || url.trim().length() == 0) { return input; }

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if (urls == null || urls.size() == 0) { return input; }

		for (String s : urls) {

			if (s.startsWith("http")) {

				input.setOriginalUrl(s);

				String correctUrl = s.replace("abs", "pdf");

				correctUrl += ".pdf";

				input.setUrl(correctUrl);

			}

		}

		return input;

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	@Override

	public String getPluginName() {

		return "ArxivDownloadPlugin";

	}

	@Override

	public void setBasePath(final String basePath) {

	}

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

/**

 * The Class EasyPDFDownloadPlugin.

 */

public class EasyPDFDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) { return null; }

		if (input == null) { return null; }

		String url = input.getOriginalUrl();

		if (url == null || url.trim().length() == 0) { return input; }

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if (urls == null || urls.size() == 0) { return input; }

		for (String s : urls) {

			if (s.trim().endsWith(".pdf")) {

				input.setOriginalUrl(s);

				input.setUrl(s);

				return input;

			}

		}

		input.setOriginalUrl(null);

		input.setUrl(null);

		return input;

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "easyPDFDownloadPlugin";

	}

	@Override

	public void setBasePath(final String basePath) {

		// TODO Auto-generated method stub

	}

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

/**

 * The Class ELisDownloadPlugin.

 */

public class ELisDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(ELisDownloadPlugin.class);

	/**

	 * Extract url.

	 *

	 * @param url

	 *            the url

	 * @return the string

	 */

	private String extractURL(final String url) {

		try {

			Document doc = Jsoup.connect(url).get();

			Elements links = doc.select("a[href$=.pdf]");

			for (Element link : links) {

				String linkvalue = link.attr("abs:href");

				if (!linkvalue.toLowerCase().contains("thumbnailversion")) { return linkvalue; }

			}

			return null;

		} catch (Exception e) {

			log.error("Error on extract URL", e);

			return null;

		}

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "ELisDownloadPlugin";

	}

	/*

	 * (non-Javadoc)

	 * 

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) { return null; }

		String url = input.getOriginalUrl();

		if (url == null || url.trim().length() == 0) { return input; }

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if (urls == null || urls.size() == 0) { return input; }

		for (String s : urls) {

			String newURL = extractURL(s);

			if (newURL != null) {

				input.setOriginalUrl(s);

				input.setUrl(newURL);

				return input;

			}

		}

		input.setOriginalUrl(null);

		input.setUrl(null);

		return input;

	}

	@Override

	public void setBasePath(final String basePath) {

		// TODO Auto-generated method stub

	}

}

package eu.dnetlib.download.plugin;

import java.io.File;

import java.io.FileFilter;

import java.util.ArrayList;

import java.util.Collections;

import java.util.Comparator;

import java.util.List;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class PathRetreiver.

 */

public class PathRetreiver {

	private static final Log log = LogFactory.getLog(PathRetreiver.class);

	/** The instance. */

	private static PathRetreiver instance;

	/**

	 * Gets the single instance of PathRetreiver.

	 *

	 * @param base_path

	 *            the base_path

	 * @return single instance of PathRetreiver

	 */

	public static PathRetreiver getInstance(final String base_path) {

		if (instance == null) {

			instance = new PathRetreiver();

			instance.setBase_path(base_path);

		}

		return instance;

	}

	/** The base_path. */

	private String base_path;

	/** The values. */

	private List<InfoPath> values;

	/**

	 * Bootstrap.

	 */

	private void bootstrap() {

		values = new ArrayList<InfoPath>();

		File basePath = new File(this.base_path);

		File[] selectedFiles = basePath.listFiles(new FileFilter() {

			@Override

			public boolean accept(final File pathname) {

				return pathname.isDirectory();

			}

		});

		for (File f : selectedFiles) {

			String lower = StringUtils.substringAfter(StringUtils.substringBefore(f.getName(), "_"), "PMC");

			String upper = StringUtils.substringAfter(StringUtils.substringAfter(f.getName(), "_"), "PMC");

			String path = f.getPath();

			InfoPath i = new InfoPath();

			i.setLower(Integer.parseInt(lower));

			i.setUpper(Integer.parseInt(upper));

			i.setPath(path);

			values.add(i);

		}

		Collections.sort(values, new Comparator<InfoPath>() {

			@Override

			public int compare(final InfoPath o1, final InfoPath o2) {

				if (o1.getLower() < o2.getLower()) return -1;

				else if (o1.getLower() < o2.getLower()) return 0;

				else return 1;

			}

		});

		if (log.isDebugEnabled()) {

			for (InfoPath p : values) {

				log.debug(String.format("%s -- %s : %s", p.getLower(), p.getUpper(), p.getPath()));

			}

		}

	}

	/**

	 * Gets the path for pmcid.

	 *

	 * @param pmcID

	 *            the pmc id

	 * @return the path for pmcid

	 */

	public String getPathForPMCID(final int pmcID) {

		if (values == null) {

			bootstrap();

		}

		for (int i = 0; i < values.size(); i++) {

			if (pmcID < values.get(i).getLower()) {

				if (i == 0) return null;

				String currentPath = values.get(i - 1).getPath() + "/" + pmcID + ".xml";

				File f = new File(currentPath);

				log.debug(String.format("try to search in path %s", currentPath));

				String s = null;

				if (f.exists()) {

					s = f.getPath();

					log.debug(String.format("found in %s", s));

				} else {

					log.debug(String.format("not found in %s", s));

				}

				return s;

			}

		}

		log.debug(String.format("PMC with ID: %s not found", pmcID));

		return null;

	}

	/**

	 * Sets the base_path.

	 *

	 * @param base_path

	 *            the new base_path

	 */

	public void setBase_path(final String base_path) {

		this.base_path = base_path;

	}

	/**

	 * Gets the base_path.

	 *

	 * @return the base_path

	 */

	public String getBase_path() {

		return this.base_path;

	}

}

package eu.dnetlib.download.plugin;

import java.io.File;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.util.List;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

public class ArxivImportFromFile extends AbstractDownloadPlugin implements DownloadPlugin {

	private static final Log log = LogFactory.getLog(ArxivImportFromFile.class); // NOPMD by marko on 11/24/08 5:02 PM

	/** The base path. */

	private String basePath;

	@Override

	public DownloadItem retrieveUrl(final DownloadItem item) {

		if (checkOpenAccess(item) == null) return null;

		final String baseURLs = item.getUrl();

		final List<String> urlsList = new Gson().fromJson(baseURLs, List.class);

		for (final String baseURL : urlsList) {

			if (baseURL.isEmpty() == false && baseURL.trim().startsWith("http://") == true) {

				final String name = StringUtils.substringAfter(baseURL, "abs/").trim();

				if (name == null) {

					item.setUrl(null);

					return item;

				}

				final String fileURL = createPath(name);

				if (StringUtils.isBlank(fileURL)) {

					item.setUrl(null);

					return item;

				}

				final File f = new File(fileURL);

				if (f.exists()) {

					if (log.isDebugEnabled()) {

						log.debug("found path associated to " + item.getIdItemMetadata() + " with path : " + fileURL);

					}

					item.setUrl("file://" + fileURL);

				} else {

					if (log.isDebugEnabled()) {

						log.debug("NOT found path associated to " + item.getIdItemMetadata());

					}

					item.setUrl(null);

				}

				item.setOriginalUrl(baseURL);

				return item;

			} else {

				item.setUrl(null);

			}

		}

		return item;

	}

	public String createPath(final String name) {

		final String regExp = "\\d{4}\\.\\d{4}";

		if (name.matches(regExp)) {

			final String[] values = name.split("\\.");

			final Path bsPath = Paths.get(basePath);

			final Path filePath = Paths.get(String.format("%s/%s.pdf", values[0], name));

			final String fileURL = bsPath.resolve(filePath).toString();

			return fileURL;

		} else {

			if (name.contains("/")) {

				final String[] values = name.split("/");

				if (values.length != 2) return null;

				if (values[1].length() > 4) {

					final String middle = values[1].substring(0, 4);

					final Path bsPath = Paths.get(basePath);

					final Path filePath = Paths.get(String.format("%s/%s.pdf", middle, name.replace("/", "")));

					final String fileURL = bsPath.resolve(filePath).toString();

					return fileURL;

				}

			}

		}

		return null;

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> items) {

		return Iterables.transform(items, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	@Override

	public String getPluginName() {

		return "ArxivImportFromFile";

	}

	/**

	 * @return the basePath

	 */

	public String getBasePath() {

		return basePath;

	}

	/**

	 * @param basePath

	 *            the basePath to set

	 */

	@Override

	public void setBasePath(final String basePath) {

		this.basePath = basePath;

	}

}

package eu.dnetlib.download.plugin;

/**

 * The Class InfoPath.

 */

public class InfoPath {

	/** The lower. */

	private int lower;

	/** The upper. */

	private int upper;

	/** The path. */

	private String path;

	/**

	 * Gets the lower.

	 *

	 * @return the lower

	 */

	public int getLower() {

		return lower;

	}

	/**

	 * Sets the lower.

	 *

	 * @param lower

	 *            the new lower

	 */

	public void setLower(final int lower) {

		this.lower = lower;

	}

	/**

	 * Gets the upper.

	 *

	 * @return the upper

	 */

	public int getUpper() {

		return upper;

	}

	/**

	 * Sets the upper.

	 *

	 * @param upper

	 *            the new upper

	 */

	public void setUpper(final int upper) {

		this.upper = upper;

	}

	/**

	 * Gets the path.

	 *

	 * @return the path

	 */

	public String getPath() {

		return path;

	}

	/**

	 * Sets the path.

	 *

	 * @param path

	 *            the new path

	 */

	public void setPath(final String path) {

		this.path = path;

	}

}

/**

 *

 */

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

/**

 * The Class EasyPDFDownloadPlugin.

 */

public class DLibPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) return null;

		if (input == null) return null;

		String url = input.getOriginalUrl();

		if ((url == null) || (url.trim().length() == 0)) return input;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) return input;

		for (String s : urls) {

			if (s.trim().endsWith(".html")) {

				input.setOriginalUrl(s);

				input.setUrl(s);

				return input;

			}

		}

		input.setOriginalUrl(null);

		input.setUrl(null);

		return input;

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "DLIBDownloadPlugin";

	}

	@Override

	public void setBasePath(final String basePath) {

		// TODO Auto-generated method stub

	}

}

package eu.dnetlib.download.plugin;

import java.util.List;

import org.joda.time.DateTime;

import org.joda.time.Days;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadItem.OpenAccessValues;

// TODO: Auto-generated Javadoc

/**

 * The Class AbstractDownloadPlugin.

 */

public abstract class AbstractDownloadPlugin {

	/** The regular expression. */

	protected List<String> regularExpression;

	/**

	 * Check open access.

	 *

	 * @param input the input

	 * @return the download item

	 */

	public DownloadItem checkOpenAccess(final DownloadItem input) {

		if (input != null) {

			OpenAccessValues openAccess = OpenAccessValues.valueOf(input.getOpenAccess());

			switch (openAccess) {

			case OPEN:

				return input;

			case CLOSED:

			case RESTRICTED:

			case UNKNOWN:

				return null;

			case EMBARGO:

				if (input.getEmbargoDate() == null) return null;

				DateTime embargoDate = new DateTime(input.getEmbargoDate());

				DateTime today = new DateTime();

				Days days = Days.daysBetween(embargoDate, today);

				if (days.getDays() <= 0) return input;

				return null;

			}

		}

		return null;

	}

	/**

	 * Gets the regular expression.

	 *

	 * @return the regular expression

	 */

	public List<String> getRegularExpression() {

		return regularExpression;

	}

	/**

	 * Sets the regular expression.

	 *

	 * @param regularExpression the new regular expression

	 */

	public void setRegularExpression(final List<String> regularExpression) {

		this.regularExpression = regularExpression;

	}

}

package eu.dnetlib.download.plugin;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

public class FollowPDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

	/** The Constant log. */

	private static final Log log = LogFactory.getLog(FollowPDFLinkPlugins.class);

	/**

	 * Extract url.

	 *

	 * @param url

	 *            the url

	 * @return the string

	 */

	private String extractURL(final String url) {

		try {

			URL startURL = new URL(url);

			HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

			conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manully.

			HttpURLConnection.setFollowRedirects(true);

			String location = url;

			if ((conn.getResponseCode() >= 300) && (conn.getResponseCode() < 400)) {

				location = conn.getHeaderFields().get("Location").get(0);

				conn.disconnect();

			}

			Document doc = Jsoup.connect(location).get();

			Elements links = doc.select("a[href$=.pdf]");

			for (Element link : links) {

				String linkvalue = link.attr("abs:href");

				if (regularExpression!= null) {

					for (String regex: regularExpression) {

						if (linkvalue.matches(regex))

							return linkvalue;

					}

				} else

					return linkvalue;

			}

			return null;

		} catch (Exception e) {

			log.error("Error on extract URL", e);

			return null;

		}

	}

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		return "FollowPDFLinkPlugins";

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		if (checkOpenAccess(input) == null) return null;

		String url = input.getOriginalUrl();

		if ((url == null) || (url.trim().length() == 0)) return input;

		@SuppressWarnings("unchecked")

		List<String> urls = new Gson().fromJson(url, ArrayList.class);

		if ((urls == null) || (urls.size() == 0)) return input;

		for (String s : urls) {

			String newURL = extractURL(s);

			if (newURL != null) {

				input.setOriginalUrl(s);

				input.setUrl(newURL);

				return input;

			}

		}

		input.setOriginalUrl(null);

		input.setUrl(null);

		return input;

	}

	@Override

	public void setBasePath(final String basePath) {

		// TODO Auto-generated method stub

	}

}

/**

 *

 */

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

/**

 * @author sandro

 *

 */

public class HALPdfDocumentPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	private static final Log log = LogFactory.getLog(HALPdfDocumentPlugin.class); // NOPMD by marko on 11/24/08 5:02 PM

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

	 */

	@Override

	public String getPluginName() {

		// TODO Auto-generated method stub

		return "HALPdfDocumentPlugin";

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retireveUrls(java.lang.Iterable)

	 */

	@Override

	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) {

		try{

			if (checkOpenAccess(input) == null) return null;

			if (input == null) return null;

			String url = input.getOriginalUrl();

			if ((url == null) || (url.trim().length() == 0)) return input;

			@SuppressWarnings("unchecked")

			List<String> urls = new Gson().fromJson(url, ArrayList.class);

			if ((urls == null) || (urls.size() == 0)) return input;

			for (String s : urls) {

				if (s.trim().toLowerCase().endsWith("document")) {

					input.setOriginalUrl(s);

					input.setUrl(s);

					return input;

				}

			}

			input.setOriginalUrl(null);

			input.setUrl(null);

			return input;

		} catch (Throwable e) {

			log.error("Error on retreiving URL",e);

			input.setOriginalUrl(null);

			input.setUrl(null);

			return input;

		}

	}

	/**

	 * {@inheritDoc}

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#setBasePath(java.lang.String)

	 */

	@Override

	public void setBasePath(final String arg0) {

		// TODO Auto-generated method stub

	}

}

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"

	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"

	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"

	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"

	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"

	xmlns:util="http://www.springframework.org/schema/util"

	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd

                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd

                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd

                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd

                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd

                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd

                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">

	<bean id= "easyPDFDownloadPlugin" class="eu.dnetlib.download.plugin.EasyPDFDownloadPlugin"/>

	<bean id= "arxivDownloadPlugin" class="eu.dnetlib.download.plugin.ArxivDownloadPlugin"/>

	<bean id= "europePMCDownloadPlugin" class="eu.dnetlib.download.plugin.EuropePMC"/>

	<bean id= "elisDownloadPlugin" class="eu.dnetlib.download.plugin.ELisDownloadPlugin"/>

	<bean id="followPDFLinkPlugins" class="eu.dnetlib.download.plugin.FollowPDFLinkPlugins"/>

	<bean id="ArxivImportFromFile" class="eu.dnetlib.download.plugin.ArxivImportFromFile"/>	

	<bean id="DLIBDownloadPlugin" class="eu.dnetlib.download.plugin.DLibPlugin"/>	

	<bean id="HALPdfDocumentPlugin" class="eu.dnetlib.download.plugin.HALPdfDocumentPlugin"/>

</beans>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<parent>

		<groupId>eu.dnetlib</groupId>

		<artifactId>dnet-parent</artifactId>

		<version>1.0.0</version>

	</parent>

	<groupId>eu.dnetlib</groupId>

	<artifactId>dnet-download-plugins</artifactId>

	<version>2.1.9</version>

	<scm>

	  <developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet40/modules/dnet-download-plugins/tags/dnet-download-plugins-2.1.9</developerConnection>

	</scm>

	<dependencies>

		<dependency>

			<groupId>eu.dnetlib</groupId>

			<artifactId>dnet-download-service</artifactId>

			<version>[2.0.0,3.0.0)</version>

		</dependency>

		<dependency>

			<groupId>org.jsoup</groupId>

			<artifactId>jsoup</artifactId>

			<version>1.7.2</version>

		</dependency>

		<dependency>

			<groupId>joda-time</groupId>

			<artifactId>joda-time</artifactId>

			<version>2.3</version>

		</dependency>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>${junit.version}</version>

			<scope>test</scope>

		</dependency>

	</dependencies>

</project>