/ - Diff - D-Net - D-Net project tracking tool

     	/** The xpath metadata id. */
     	private String xpathMetadataId;
     	/** The xpath open access. */
     	private String xpathOpenAccess;
     	/** The xpath embargo date. */
     	private String xpathEmbargoDate;
     	/** The result set client factory. */
     	@Autowired
     	private ResultSetClientFactory resultSetClientFactory;
-...
     	/*
     	 * (non-Javadoc)
+    	 *
+    	 *
     	 * @see eu.dnetlib.msro.workflows.nodes.SimpleJobNode#execute(com.googlecode.sarasvati.NodeToken)
     	 */
     	@Override
     	protected String execute(final NodeToken token) throws Exception {
     		final W3CEndpointReference inputEpr = (new EPRUtils()).getEpr(token.getEnv().getAttribute(inputEprParam));
     		Iterable<String> input = resultSetClientFactory.getClient(inputEpr);
     		Iterable<String> extractedUrls = Iterables.transform(input, new UrlExtractor(xpath, xpathMetadataId));
     		Iterable<String> extractedUrls = Iterables.transform(input, new UrlExtractor(xpath, xpathMetadataId, xpathOpenAccess, xpathEmbargoDate));
     		W3CEndpointReference eprUrls = resultSetFactory.createIterableResultSet(extractedUrls);
     		token.getEnv().setAttribute(getOutputEprParam(), eprUrls.toString());
     		return Arc.DEFAULT_ARC;
-...
     	/**
     	 * Gets the xpath.
+    	 *
+    	 *
     	 * @return the xpath
     	 */
     	public String getXpath() {
-...
     	/**
     	 * Sets the xpath.
+    	 *
+    	 *
     	 * @param xpath
     	 *            the xpath to set
     	 */
-...
     	/**
     	 * Gets the xpath metadata id.
+    	 *
+    	 *
     	 * @return the xpathMetadataId
     	 */
     	public String getXpathMetadataId() {
-...
     	/**
     	 * Sets the xpath metadata id.
+    	 *
+    	 *
     	 * @param xpathMetadataId
     	 *            the xpathMetadataId to set
     	 */
-...
     	/**
     	 * Gets the result set client factory.
+    	 *
+    	 *
     	 * @return the resultSetClientFactory
     	 */
     	public ResultSetClientFactory getResultSetClientFactory() {
-...
     	/**
     	 * Sets the result set client factory.
+    	 *
+    	 *
     	 * @param resultSetClientFactory
     	 *            the resultSetClientFactory to set
     	 */
-...
     	/**
     	 * Gets the result set factory.
+    	 *
+    	 *
     	 * @return the resultSetFactory
     	 */
     	public IterableResultSetFactory getResultSetFactory() {
-...
     	/**
     	 * Sets the result set factory.
+    	 *
+    	 *
     	 * @param resultSetFactory
     	 *            the resultSetFactory to set
     	 */
-...
     	/**
     	 * Gets the output epr param.
+    	 *
+    	 *
     	 * @return the outputEprParam
     	 */
     	public String getOutputEprParam() {
-...
     	/**
     	 * Sets the output epr param.
+    	 *
+    	 *
     	 * @param outputEprParam
     	 *            the outputEprParam to set
     	 */
-...
+    	}
     	/**
     	 * Gets the input epr param.
+    	 *
     	 * @return the inputEprParam
     	 */
     	public String getInputEprParam() {
-...
+    	}
     	/**
     	 * Sets the input epr param.
+    	 *
     	 * @param inputEprParam
     	 *            the inputEprParam to set
     	 */
-...
     		this.inputEprParam = inputEprParam;
+    	}
     	/**
     	 * Gets the xpath open access.
+    	 *
     	 * @return the xpath open access
     	 */
     	public String getXpathOpenAccess() {
     		return xpathOpenAccess;
+    	}
     	/**
     	 * Sets the xpath open access.
+    	 *
     	 * @param xpathOpenAccess
     	 *            the new xpath open access
     	 */
     	public void setXpathOpenAccess(final String xpathOpenAccess) {
     		this.xpathOpenAccess = xpathOpenAccess;
+    	}
     	/**
     	 * Gets the xpath embargo date.
+    	 *
     	 * @return the xpath embargo date
     	 */
     	public String getXpathEmbargoDate() {
     		return xpathEmbargoDate;
+    	}
     	/**
     	 * Sets the xpath embargo date.
+    	 *
     	 * @param xpathEmbargoDate
     	 *            the new xpath embargo date
     	 */
     	public void setXpathEmbargoDate(final String xpathEmbargoDate) {
     		this.xpathEmbargoDate = xpathEmbargoDate;
+    	}
+    }

     	/*
     	 * (non-Javadoc)
+    	 *
+    	 *
     	 * @see eu.dnetlib.msro.workflows.nodes.BlackboardJobNode#getXqueryForServiceId(com.googlecode.sarasvati.NodeToken)
     	 */
     	@Override
-...
     	/*
     	 * (non-Javadoc)
+    	 *
+    	 *
     	 * @see eu.dnetlib.msro.workflows.nodes.BlackboardJobNode#prepareJob(eu.dnetlib.enabling.tools.blackboard.BlackboardJob,
     	 * com.googlecode.sarasvati.NodeToken)
     	 */
-...
     			@Override
     			protected void populateEnv(final Env env, final Map<String, String> responseParams) {
     				env.setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", responseParams.get("total"));
+    			}
     		};

     import javax.xml.xpath.XPathExpression;
     import javax.xml.xpath.XPathFactory;
     import org.apache.commons.lang.StringUtils;
     import org.apache.commons.logging.Log;
     import org.apache.commons.logging.LogFactory;
     import org.joda.time.DateTime;
     import org.joda.time.format.DateTimeFormat;
     import org.joda.time.format.DateTimeFormatter;
     import org.w3c.dom.Document;
     import org.w3c.dom.NodeList;
-...
     import eu.dnetlib.data.download.rmi.DownloadItem;
     // TODO: Auto-generated Javadoc
     /**
      * The Class UrlExtractor.
      */
     public class UrlExtractor implements Function<String, String> {
     	/** The Constant log. */
     	private static final Log log = LogFactory.getLog(UrlExtractor.class);
     	/** The xpath. */
     	private String xpath;
     	/** The xpath url. */
     	private String xpathURL;
     	/** The xpath. */
     	private String xpathMetadataID;
     	public UrlExtractor(final String xpath, final String xpathMetadataID) {
     		this.xpath = xpath;
     	/** The xpath open access. */
     	private String xpathOpenAccess;
     	/** The xpath embargo date. */
     	private String xpathEmbargoDate;
     	/**
     	 * Instantiates a new url extractor.
+    	 *
     	 * @param xpath
     	 *            the xpath
     	 * @param xpathMetadataID
     	 *            the xpath metadata id
     	 */
     	public UrlExtractor(final String xpath, final String xpathMetadataID, final String xpathOpenAccess, final String xpathEmbargoDate) {
     		this.xpathURL = xpath;
     		this.xpathMetadataID = xpathMetadataID;
     		this.xpathOpenAccess = xpathOpenAccess;
     		this.xpathEmbargoDate = xpathEmbargoDate;
+    	}
     	/*
-...
     	@Override
     	public String apply(final String input) {
     		try {
     			DownloadItem di = new DownloadItem();
     			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
     			DocumentBuilder builder;
-...
     			Document doc = builder.parse(new ByteArrayInputStream(input.getBytes()));
     			XPathFactory xPathFactory = XPathFactory.newInstance();
     			XPath myXpath = xPathFactory.newXPath();
     			XPathExpression expression = myXpath.compile(xpath);
     			XPathExpression expression = myXpath.compile(xpathURL);
     			Object values = expression.evaluate(doc, XPathConstants.NODESET);
     			di.setUrl(getNodes((NodeList) values));
     			di.setOriginalUrl(getNodes((NodeList) values));
     			if (xpathOpenAccess != null) {
     				expression = myXpath.compile(xpathOpenAccess);
     				String openAccess = expression.evaluate(doc);
     				di.setOpenAccess(openAccess);
+    			}
     			expression = myXpath.compile(xpathEmbargoDate);
     			String embargoDate = expression.evaluate(doc);
     			if (!StringUtils.isEmpty(embargoDate)) {
     				try {
     					DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd");
     					DateTime dt = fmt.parseDateTime(embargoDate);
     					di.setEmbargoDate(dt.toDate());
     				} catch (Exception pe) {}
+    			}
     			expression = myXpath.compile(xpathMetadataID);
     			String extracted_metadataId = expression.evaluate(doc);
     			di.setIdItemMetadata(extracted_metadataId);
     			// di.setFileName(extracted_metadataId);
     			return di.toJSON();
     		} catch (Exception e) {
     			log.error("OPSSS... Something bad happen on evaluating ", e);
-...
+    	}
     	/**
     	 * Gets the nodes.
+    	 *
     	 * @param nodes
     	 *            the nodes
     	 * @return the nodes
     	 */
     	private String getNodes(final NodeList nodes) {
     		List<String> extracted_Url = new ArrayList<String>();
     		if (nodes != null) {
-...
+    	}
     	/**
     	 * Gets the xpath.
+    	 *
     	 * @return the xpath
     	 * Gets the xpath metadata id.
+    	 *
     	 * @return the xpathMetadataID
     	 */
     	public String getXpath() {
     		return xpath;
     	public String getXpathMetadataID() {
     		return xpathMetadataID;
+    	}
     	/**
     	 * Sets the xpath.
+    	 *
     	 * @param xpath
     	 *            the xpath to set
     	 * Sets the xpath metadata id.
+    	 *
     	 * @param xpathMetadataID
     	 *            the xpathMetadataID to set
     	 */
     	public void setXpath(final String xpath) {
     		this.xpath = xpath;
     	public void setXpathMetadataID(final String xpathMetadataID) {
     		this.xpathMetadataID = xpathMetadataID;
+    	}
     	/**
     	 * @return the xpathMetadataID
     	 * Gets the xpath url.
+    	 *
     	 * @return the xpath url
     	 */
     	public String getXpathMetadataID() {
     		return xpathMetadataID;
     	public String getXpathURL() {
     		return xpathURL;
+    	}
     	/**
     	 * @param xpathMetadataID
     	 *            the xpathMetadataID to set
     	 * Sets the xpath url.
+    	 *
     	 * @param xpathURL
     	 *            the new xpath url
     	 */
     	public void setXpathMetadataID(final String xpathMetadataID) {
     		this.xpathMetadataID = xpathMetadataID;
     	public void setXpathURL(final String xpathURL) {
     		this.xpathURL = xpathURL;
+    	}
     	/**
     	 * Gets the xpath open access.
+    	 *
     	 * @return the xpath open access
     	 */
     	public String getXpathOpenAccess() {
     		return xpathOpenAccess;
+    	}
     	/**
     	 * Sets the xpath open access.
+    	 *
     	 * @param xpathOpenAccess
     	 *            the new xpath open access
     	 */
     	public void setXpathOpenAccess(final String xpathOpenAccess) {
     		this.xpathOpenAccess = xpathOpenAccess;
+    	}
     	/**
     	 * Gets the xpath embargo date.
+    	 *
     	 * @return the xpath embargo date
     	 */
     	public String getXpathEmbargoDate() {
     		return xpathEmbargoDate;
+    	}
     	/**
     	 * Sets the xpath embargo date.
+    	 *
     	 * @param xpathEmbargoDate
     	 *            the new xpath embargo date
     	 */
     	public void setXpathEmbargoDate(final String xpathEmbargoDate) {
     		this.xpathEmbargoDate = xpathEmbargoDate;
+    	}
+    }

     			<version>${junit.version}</version>
     			<scope>test</scope>
     		</dependency>
     		<dependency>
     			<groupId>joda-time</groupId>
     			<artifactId>joda-time</artifactId>
     			<version>2.3</version>
     		</dependency>
     	</dependencies>
     	<properties>

     import org.joda.time.DateTime;
     import org.joda.time.Days;
     import org.junit.Assert;
     import org.junit.Before;
     import org.junit.Test;
     public class DateIntervalTest {
     	@Before
     	public void setUp() throws Exception {}
     	@Test
     	public void test() {
     		DateTime beforeDate = new DateTime(2010, 01, 1, 0, 0);
     		DateTime now = new DateTime();
     		Days day = Days.daysBetween(beforeDate, now);
     		Assert.assertTrue(day.getDays() > 0);
+    	}
+    }

     import java.util.ArrayList;
     import java.util.List;
     import com.google.common.base.Function;
     import com.google.common.collect.Iterables;
     import com.google.gson.Gson;
     import eu.dnetlib.data.download.rmi.DownloadItem;
-...
     		return "europePMCDownloadPlugin";
+    	}
     	@Override
     	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {
     		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {
     			@Override
     			public DownloadItem apply(final DownloadItem input) {
     				return retrieveUrl(input);
+    			}
     		});
+    	}
     	/*
     	 * (non-Javadoc)
+    	 *
-...
     	 */
     	@Override
     	public DownloadItem retrieveUrl(final DownloadItem input) {
     		String url = input.getOriginalUrl();
     		if ((url == null) || (url.trim().length() == 0)) return input;
     		@SuppressWarnings("unchecked")

     import java.util.ArrayList;
     import java.util.List;
     import com.google.common.base.Function;
     import com.google.common.collect.Iterables;
     import com.google.gson.Gson;
     import eu.dnetlib.data.download.rmi.DownloadItem;
-...
     	@Override
     	public DownloadItem retrieveUrl(final DownloadItem input) {
     		if (checkOpenAccess(input) == null) return null;
     		String url = input.getOriginalUrl();
     		if ((url == null) || (url.trim().length() == 0)) return input;
     		@SuppressWarnings("unchecked")
-...
+    	}
     	@Override
     	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {
     		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {
     			@Override
     			public DownloadItem apply(final DownloadItem input) {
     				return retrieveUrl(input);
+    			}
     		});
+    	}
     	@Override
     	public String getPluginName() {
     		return "ArxivDownloadPlugin";
+    	}

     import java.util.ArrayList;
     import java.util.List;
     import com.google.common.base.Function;
     import com.google.common.collect.Iterables;
     import com.google.gson.Gson;
     import eu.dnetlib.data.download.rmi.DownloadItem;
-...
     	/*
     	 * (non-Javadoc)
+    	 *
+    	 *
     	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)
     	 */
     	@Override
     	public DownloadItem retrieveUrl(final DownloadItem input) {
     		if (checkOpenAccess(input) == null) return null;
     		if (input == null) return null;
     		String url = input.getOriginalUrl();
-...
     		List<String> urls = new Gson().fromJson(url, ArrayList.class);
     		if ((urls == null) || (urls.size() == 0)) return input;
     		for (String s : urls) {
     			if (s.trim().endsWith(".pdf")) {
     				input.setOriginalUrl(s);
     				input.setUrl(s);
-...
     		return input;
+    	}
     	@Override
     	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {
     		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {
     			@Override
     			public DownloadItem apply(final DownloadItem input) {
     				return retrieveUrl(input);
+    			}
     		});
+    	}
     	/*
     	 * (non-Javadoc)
+    	 *
+    	 *
     	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()
     	 */
     	@Override

     import org.jsoup.nodes.Element;
     import org.jsoup.select.Elements;
     import com.google.common.base.Function;
     import com.google.common.collect.Iterables;
     import com.google.gson.Gson;
     import eu.dnetlib.data.download.rmi.DownloadItem;
-...
+    	}
     	@Override
     	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {
     		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {
     			@Override
     			public DownloadItem apply(final DownloadItem input) {
     				return retrieveUrl(input);
+    			}
     		});
+    	}
     	/*
     	 * (non-Javadoc)
+    	 *
+    	 *
     	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()
     	 */
     	@Override
-...
     	/*
     	 * (non-Javadoc)
+    	 *
+    	 *
     	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)
     	 */
     	@Override
     	public DownloadItem retrieveUrl(final DownloadItem input) {
     		if (checkOpenAccess(input) == null) return null;
     		String url = input.getOriginalUrl();
     		if ((url == null) || (url.trim().length() == 0)) return input;

     package eu.dnetlib.download.plugin;
     import com.google.common.base.Function;
     import com.google.common.collect.Iterables;
     import org.joda.time.DateTime;
     import org.joda.time.Days;
     import eu.dnetlib.data.download.rmi.DownloadItem;
     import eu.dnetlib.data.download.rmi.DownloadPlugin;
     import eu.dnetlib.data.download.rmi.DownloadItem.OpenAccessValues;
     public abstract class AbstractDownloadPlugin implements DownloadPlugin {
     public abstract class AbstractDownloadPlugin {
     	@Override
     	public Iterable<DownloadItem> retireveUrls(final Iterable<DownloadItem> urls) {
     		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {
     	public DownloadItem checkOpenAccess(final DownloadItem input) {
     		if (input != null) {
     			OpenAccessValues openAccess = OpenAccessValues.valueOf(input.getOpenAccess());
     			switch (openAccess) {
     			case OPEN:
     				return input;
     			case CLOSED:
     			case RESTRICTED:
     			case UNKNOWN:
     				return null;
     			case EMBARGO:
     				if (input.getEmbargoDate() == null) return null;
     				DateTime embargoDate = new DateTime(input.getEmbargoDate());
     				DateTime today = new DateTime();
     				Days days = Days.daysBetween(embargoDate, today);
     				if (days.getDays() <= 0) return input;
     				return null;
+    			}
     			@Override
     			public DownloadItem apply(final DownloadItem input) {
     				return retrieveUrl(input);
+    			}
     		});
+    		}
     		return null;
+    	}
+    }

     			<artifactId>jsoup</artifactId>
     			<version>1.7.2</version>
     		</dependency>
     		<dependency>
     			<groupId>joda-time</groupId>
     			<artifactId>joda-time</artifactId>
     			<version>2.3</version>
     		</dependency>
     		<dependency>
     			<groupId>junit</groupId>
     			<artifactId>junit</artifactId>
     			<version>${junit.version}</version>
     			<scope>test</scope>
     		</dependency>
     	</dependencies>
     </project>

Project

General

Profile

D-Net

Revision 31406

Added by Sandro La Bruzzo almost 10 years ago