Index: modules/uoa-resource-discovery/trunk/lib
===================================================================
--- modules/uoa-resource-discovery/trunk/lib	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/lib	(revision 47463)

Property changes on: modules/uoa-resource-discovery/trunk/lib
___________________________________________________________________
Added: svn:ignore
## -0,0 +1 ##
+ivy
Index: modules/uoa-resource-discovery/trunk/bin
===================================================================
--- modules/uoa-resource-discovery/trunk/bin	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/bin	(revision 47463)

Property changes on: modules/uoa-resource-discovery/trunk/bin
___________________________________________________________________
Added: svn:ignore
## -0,0 +1 ##
+main
Index: modules/uoa-resource-discovery/trunk/build/build.properties
===================================================================
--- modules/uoa-resource-discovery/trunk/build/build.properties	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/build/build.properties	(revision 47463)
@@ -0,0 +1,6 @@
+source.dirs		= main
+
+project.name		= uoa-resource-discovery
+project.version		= 0.0.7
+project.label		=
+
Index: modules/uoa-resource-discovery/trunk/build/ivysettings.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/build/ivysettings.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/build/ivysettings.xml	(revision 47463)
@@ -0,0 +1,6 @@
+<ivysettings>
+
+  <include file="${yvy.build.dir}/ivysettings.xml"/>
+  <settings defaultResolver="yvy-resolver" default="yvy.default.cache" defaultCacheDir="${yvy.cache.dir}"/>
+
+</ivysettings>
Index: modules/uoa-resource-discovery/trunk/pom.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/pom.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/pom.xml	(revision 47463)
@@ -0,0 +1,87 @@
+<?xml version="1.0" ?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<parent>
+		<groupId>eu.dnetlib</groupId>
+		<artifactId>dnet-parent-backward-compatibility</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>eu.dnetlib</groupId>
+	<artifactId>uoa-resource-discovery</artifactId>
+	<packaging>jar</packaging>
+	<version>0.0.8-SNAPSHOT</version>
+	<dependencies>
+		<dependency>
+			<groupId>apache</groupId>
+			<artifactId>commons-logging</artifactId>
+			<version>1.0.4</version>
+		</dependency>
+		<dependency>
+			<groupId>org.w3c</groupId>
+			<artifactId>tidy</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>[0.0.0,)</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>net.matuschek</groupId>
+			<artifactId>jobo</artifactId>
+			<version>[1.4,2.0)</version>
+		</dependency>
+		<dependency>
+			<groupId>DLS</groupId>
+			<artifactId>jOAI</artifactId>
+			<version>[2.0.9.3,2.0.10.0)</version>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>unibi-commons</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>unibi-data-utility-featureextraction-plugins</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+		<dependency>
+			<groupId>com.thoughtworks</groupId>
+			<artifactId>xstream</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+		<dependency>
+			<groupId>org.cyberneko</groupId>
+			<artifactId>nekohtml</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+		<dependency>
+			<groupId>com.jira</groupId>
+			<artifactId>heritrix-commons</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+		<dependency>
+			<groupId>com.jira</groupId>
+			<artifactId>heritrix-modules</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+		<dependency>
+			<groupId>com.googlecode</groupId>
+			<artifactId>kryo</artifactId>
+			<version>1.04</version>
+		</dependency>
+		<dependency>
+			<groupId>edu.indiana</groupId>
+			<artifactId>xpp3</artifactId>
+			<version>[0.0.0,)</version>
+		</dependency>
+	</dependencies>
+</project>
Index: modules/uoa-resource-discovery/trunk/test/junit/gr/uoa/di/resourcediscovery/test/Sample.java
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/gr/uoa/di/resourcediscovery/test/Sample.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/gr/uoa/di/resourcediscovery/test/Sample.java	(revision 47463)
@@ -0,0 +1,36 @@
+package gr.uoa.di.resourcediscovery.test;
+
+import gr.uoa.di.resourcediscovery.MethodProvider;
+import gr.uoa.di.resourcediscovery.MethodProviderFileStorageImpl;
+import gr.uoa.di.resourcediscovery.methods.ResourceDiscoveryMethod;
+import gr.uoa.di.resourcediscovery.methods.XPathAndCrawl;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.BasicConfigurator;
+
+public class Sample {
+	
+	public static void main(String[] args) throws Exception {
+		BasicConfigurator.configure();
+		
+		URL conUrl = new URL("http://rudie.di.uoa.gr:8080/files/");
+		
+		MethodProvider provider = new MethodProviderFileStorageImpl("method-map.xml");
+		ResourceDiscoveryMethod method = provider.getMethod(conUrl);
+		
+		if(method == null) {
+			List<String> mimeTypes = new ArrayList<String>();
+			mimeTypes.add("application/pdf");
+
+			XPathAndCrawl xpath = new XPathAndCrawl(mimeTypes, "http://rudie.di.uoa.gr:8080/robots.txt");
+
+			method = xpath;
+		}
+		
+		System.out.println("resources found in: " + method.getResources(conUrl, provider));
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/Toolkit.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/Toolkit.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/Toolkit.java	(revision 47463)
@@ -0,0 +1,189 @@
+package gr.uoa.di.resourcediscovery;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+
+import org.apache.log4j.Logger;
+
+public class Toolkit {
+	
+	static transient Logger logger = Logger.getLogger(Toolkit.class);
+	static int timeout = 10000;
+
+	static public String makeAbsolute(String url, URL connectionUrl) throws MalformedURLException {
+		return new URL(connectionUrl, url).toString();
+	}
+
+	static public String makeRelative(URL connectionUrl) throws MalformedURLException {
+		return connectionUrl.getPath();
+	}
+	
+	static public String getRedirectedUrl(String resourceURL, long sleepMillis) throws IOException, MalformedURLException {
+		URL url = null;
+
+		try {
+			url = new URL(resourceURL);
+		} catch (MalformedURLException mue) {
+			logger.error("Error opening first url", mue);
+			throw mue;
+		}
+
+		HttpURLConnection.setFollowRedirects(false);
+
+		HttpURLConnection conn = null;
+		try {
+			Thread.sleep(sleepMillis);
+			conn = (HttpURLConnection) url.openConnection();
+			conn.setConnectTimeout(timeout);
+			conn.setReadTimeout(timeout);
+			conn.setAllowUserInteraction(false);         
+			conn.setDoOutput(true);
+		} catch (ClassCastException ex) {
+			throw new MalformedURLException();
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
+
+		conn.setRequestMethod("HEAD");
+
+		try {
+			conn = openConnectionCheckRedirects(conn, sleepMillis);
+		} catch (Exception ex) {
+			throw new MalformedURLException();
+		}
+
+		try {
+			Thread.sleep(sleepMillis);
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
+		int statusCode = conn.getResponseCode();
+		if (statusCode == 503) {
+			logger.error("Url " + conn.getURL() + " reported status code 503. Please increase the crawler's sleep time.");
+			conn.disconnect();
+
+			throw new IOException("Url " + conn.getURL() + " reported status code 503. Please increase the crawler's sleep time.");
+		} else if (conn.getResponseCode() >= 400) {
+			// Client or server error received
+			logger.error("Url " + conn.getURL() + " seems to be unreachable (response code:"+statusCode+"). If this url is not of importance you can ignore this error.");
+			conn.disconnect();
+
+			throw new IOException("Url " + conn.getURL() + " seems to be unreachable (response code:"+statusCode+"). If this url is not of importance you can ignore this error.");
+		} else {
+			return conn.getURL().toString();
+		}
+	}
+
+	static public String getMimeType(String resourceURL, long sleepMillis) throws IOException, MalformedURLException {
+		URL url = null;
+
+		try {
+			url = new URL(resourceURL);
+		} catch (MalformedURLException mue) {
+			logger.debug("Error getting mime type" + mue);
+			throw mue;
+		}
+
+		HttpURLConnection.setFollowRedirects(false);
+
+		HttpURLConnection conn = null;
+		try {
+			Thread.sleep(sleepMillis);
+			conn = (HttpURLConnection) url.openConnection();
+			conn.setConnectTimeout(timeout);
+			conn.setReadTimeout(timeout);
+			conn.setAllowUserInteraction(false);         
+			conn.setDoOutput(true);
+		} catch (ClassCastException ex) {
+			throw new MalformedURLException();
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
+
+		conn.setRequestMethod("HEAD");
+
+		try {
+			conn = openConnectionCheckRedirects(conn, sleepMillis);
+		} catch (Exception ex) {
+			throw new MalformedURLException();
+		}
+
+		try {
+			Thread.sleep(sleepMillis);
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
+		int statusCode = conn.getResponseCode();
+		if (statusCode == 503) {
+			logger.error("WARNING: Url " + conn.getURL() + " reported status code 503. Please increase the crawler's sleep time.");
+			conn.disconnect();
+
+			return "unknown";
+		} else if (conn.getResponseCode() >= 400) {
+			// Client or server error received
+			logger.error("WARNING: Url " + conn.getURL() + " seems to be unreachable (response code:"+statusCode+"). If this url is not of importance you can ignore this error.");
+			conn.disconnect();
+
+			return "unknown";
+		} else {
+			String mimeType = conn.getContentType();
+
+			logger.debug("mime type for " + conn.getURL() + ": " + mimeType);
+			logger.debug("response code was: " + statusCode);
+			conn.disconnect();
+			if (mimeType == null)
+				mimeType = "unknown";
+			return mimeType.replaceAll(";.*", "").trim();
+		}
+	}
+
+	static public HttpURLConnection openConnectionCheckRedirects(URLConnection c, long sleepMillis) throws IOException {
+		boolean redir;
+		int redirects = 0;
+
+		do {
+			redir = false;
+			if (c instanceof HttpURLConnection) {
+				HttpURLConnection http = (HttpURLConnection) c;
+				try {
+					Thread.sleep(sleepMillis);
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				}
+				int stat = http.getResponseCode();
+
+				if (stat >= 300 && stat <= 307 && stat != 306 && stat != HttpURLConnection.HTTP_NOT_MODIFIED) {
+					URL base = http.getURL();
+					String loc = http.getHeaderField("Location");
+					URL target = null;
+					if (loc != null) {
+						target = new URL(base, loc);
+					}
+					http.disconnect();
+					// Redirection should be allowed only for HTTP and HTTPS
+					// and should be limited to 5 redirections at most.
+					if (target == null || !(target.getProtocol().equals("http") || target.getProtocol().equals("https")) || redirects >= 5) {
+						throw new IOException("Redirection should be allowed only for HTTP and HTTPS and should be limited to 5 redirections at most.");
+					}
+					redir = true;
+					try {
+						Thread.sleep(sleepMillis);
+					} catch (InterruptedException e) {
+						e.printStackTrace();
+					}
+					c = target.openConnection();
+					c.setConnectTimeout(timeout);
+					c.setReadTimeout(timeout);
+					c.setAllowUserInteraction(false);         
+					c.setDoOutput(true);
+					redirects++;
+				}
+			}
+		} while (redir);
+
+		return (HttpURLConnection) c;
+	}
+}
Index: modules/uoa-resource-discovery/trunk/testbin
===================================================================
--- modules/uoa-resource-discovery/trunk/testbin	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/testbin	(revision 47463)

Property changes on: modules/uoa-resource-discovery/trunk/testbin
___________________________________________________________________
Added: svn:ignore
## -0,0 +1 ##
+junit
Index: modules/uoa-resource-discovery/trunk/test/junit/TestResourceDiscoverer.java
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/TestResourceDiscoverer.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/TestResourceDiscoverer.java	(revision 47463)
@@ -0,0 +1,33 @@
+import java.util.Calendar;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import eu.dnetlib.data.utility.resource_discovery.crawler.Crawler;
+import eu.dnetlib.data.utility.resource_discovery.crawler.ResourceExtractor;
+import eu.dnetlib.data.utility.resource_discovery.plugin.crawler.ResourceDiscoverer;
+
+
+public class TestResourceDiscoverer {
+
+	@BeforeClass
+	public static void Config() {
+		//BasicConfigurator.configure();
+	}
+	
+	@Test
+	public void test() throws Exception {
+		String url = "http://elib.uni-stuttgart.de/opus/volltexte/1999/1/";
+		Calendar cal = Calendar.getInstance();
+		System.out.println(cal.getTime());
+		ResourceDiscoverer discoverer = new ResourceDiscoverer();
+		System.out.println(discoverer.getResources(url));
+		cal = Calendar.getInstance();
+		System.out.println(cal.getTime());
+		Crawler crawler = new Crawler();
+		ResourceExtractor extractor = new ResourceExtractor();
+		System.out.println(extractor.extractResource(crawler.getLinks(url)));
+		cal = Calendar.getInstance();
+		System.out.println(cal.getTime());
+	}
+}
Index: modules/uoa-resource-discovery/trunk/test/junit/record3.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/record3.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/record3.xml	(revision 47463)
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+  <record rank="0.4618802">
+    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">
+      <header>
+        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/26815</dri:objIdentifier>
+        <dri:dateOfCollection>2009-12-30T00:13:45Z</dri:dateOfCollection>
+      </header>
+
+      <metadata>
+        <dr:CobjContentSynthesis/>
+        <dr:CobjTypology>Textual</dr:CobjTypology>
+        <dr:CobjIdentifier>Logic Group Preprint Series 172 (2008)</dr:CobjIdentifier>
+        <dr:CobjModel>OAI</dr:CobjModel>
+        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>
+        <dr:CobjDescriptionSynthesis/>
+
+        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>
+        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>
+        <dr:repositoryCountry>NL</dr:repositoryCountry>
+        <dr:repositoryInstitution/>
+        <dc:creator>Hollenberg, M.</dc:creator>
+        <dc:title>Equational axioms of test algebra</dc:title>
+
+        <dc:subject>Wijsbegeerte</dc:subject>
+        <dr:CobjCategory>0000</dr:CobjCategory>
+        <dc:language>eng</dc:language>
+        <dc:dateAccepted>1996-12-09</dc:dateAccepted>
+        <dc:identifier>http://igitur-archive.library.uu.nl/lg/2008-0326-201100/UUindex.html</dc:identifier>
+        <dc:publisher/>
+
+        <dc:source/>
+        <dc:contributor/>
+        <dc:relation/>
+        <dc:description>We present a complete axiomatization of test algebra ([24,18,29]), the two-sorted algebraic variant of Propositional Dynamic Logic (PDL,[21,7]). The axiomatization consists of adding a finite number of equations to any axiomatization of Kleene algebra ([15,26,17,4]) and algebraic translations of the Segerberg ([27]) axioms for PDL. Kleene algebras are not finitely axiomatizable ([25,6]), so our result does not give us a finite axiomatization of test algebra: in fact, no finite equational axiomatization exists. We also present a single-sorted version of test algebra, using the notion of dynamic negation ([9,2,11]), to which the previous results carry over.</dc:description>
+      </metadata>
+    </result>
+  </record>
Index: modules/uoa-resource-discovery/trunk/test/junit/record2.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/record2.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/record2.xml	(revision 47463)
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>  
+  <record rank="0.4898979">
+    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">
+      <header>
+        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/29575</dri:objIdentifier>
+
+        <dri:dateOfCollection>2009-12-30T00:16:01Z</dri:dateOfCollection>
+      </header>
+      <metadata>
+        <dr:CobjContentSynthesis/>
+        <dr:CobjTypology>Textual</dr:CobjTypology>
+        <dr:CobjIdentifier>Journal of Personality and Social Psychology 89, 696-716 (2005)</dr:CobjIdentifier>
+        <dr:CobjModel>OAI</dr:CobjModel>
+
+        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>
+        <dr:CobjDescriptionSynthesis/>
+        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>
+        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>
+        <dr:repositoryCountry>NL</dr:repositoryCountry>
+        <dr:repositoryInstitution/>
+        <dc:creator>Wicherts, J.M.</dc:creator>
+
+        <dc:creator>Dolan, C.V.</dc:creator>
+        <dc:creator>Hessen, D.J.</dc:creator>
+        <dc:title>Stereotype threat and group differences in test performance: A question of measurement invariance.</dc:title>
+        <dc:subject>ethnic differences</dc:subject>
+        <dc:subject>sex differences</dc:subject>
+        <dc:subject>measurement invariance</dc:subject>
+
+        <dc:subject>Sociale Wetenschappen</dc:subject>
+        <dc:subject>stereotype threat</dc:subject>
+        <dc:subject>test performance</dc:subject>
+        <dr:CobjCategory>0001</dr:CobjCategory>
+        <dc:language>eng</dc:language>
+        <dc:dateAccepted>2005-01-01</dc:dateAccepted>
+
+        <dc:identifier>http://igitur-archive.library.uu.nl/fss/2008-0807-201603/UUindex.html</dc:identifier>
+        <dc:publisher>American Psychological Association</dc:publisher>
+        <dc:source/>
+        <dc:contributor/>
+        <dc:relation>0022-3514</dc:relation>
+        <dc:description>Studies into the effects of stereotype threat (ST) on test performance have shed new light on race and sex differences in achievement and intelligence test scores. In this article,the authors relate ST theory to the psychometric concept of measurement invariance and show that ST effects may be viewed as a source of measurement bias. As such,ST effects are detectable by means of multigroup confirmatory factor analysis. This enables research into the generalizability of ST effects to real-life or high-stakes testing. The modeling approach is described in detail and applied to 3 experiments in which the amount of ST for minorities and women was manipulated. Results indicate that ST results in measurement bias of intelligence and mathematics tests.</dc:description>
+      </metadata>
+
+    </result>
+  </record>
Index: modules/uoa-resource-discovery/trunk/test/junit/record6.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/record6.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/record6.xml	(revision 47463)
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>  
+  <record rank="0.42426404">
+    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">
+      <header>
+        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/36190</dri:objIdentifier>
+
+        <dri:dateOfCollection>2009-12-30T00:23:45Z</dri:dateOfCollection>
+      </header>
+      <metadata>
+        <dr:CobjContentSynthesis/>
+        <dr:CobjTypology>Textual</dr:CobjTypology>
+        <dr:CobjIdentifier/>
+        <dr:CobjModel>OAI</dr:CobjModel>
+
+        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>
+        <dr:CobjDescriptionSynthesis/>
+        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>
+        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>
+        <dr:repositoryCountry>NL</dr:repositoryCountry>
+        <dr:repositoryInstitution/>
+        <dc:creator>Junte, R.D.</dc:creator>
+
+        <dc:title>Acaricide resistance in the blue cattle tick in South Africa; A comparison of three assays for determining tick resistance</dc:title>
+        <dc:subject>Acaricide resistance, 3 South African Provinces, blue cattle tick, Riphicephalus (Boophilus) decoloratus, comparison, Adult Immersion Test, Shaw Larval Test, Larval Packet Test, cattle, resistant</dc:subject>
+        <dc:subject>Diergeneeskunde</dc:subject>
+        <dc:subject>Diergeneeskunde</dc:subject>
+        <dr:CobjCategory>0007</dr:CobjCategory>
+        <dc:language>eng</dc:language>
+
+        <dc:dateAccepted>2007-08-21</dc:dateAccepted>
+        <dc:identifier>http://igitur-archive.library.uu.nl/student-theses/2009-1015-200152/UUindex.html</dc:identifier>
+        <dc:publisher/>
+        <dc:source/>
+        <dc:contributor>Dr. E. van Dalen, Prof. Dr. F. Jongejan</dc:contributor>
+        <dc:relation/>
+        <dc:description>The susceptibility of engorged females and larvae of Boophilus decoloratus ticks,- collected from cattle on breeding farms in Kwazula Natal, Eastern Cape and Limpopo provinces in South Africa-, for amitraz, cypermetrin and chlorfenvinphos. Was examined by means of the Adult Immersion test (AIT), the Shaw Larval Immersion Test (SLIT) and the Larval Packet Test (LPT). The results indicated resistance of Boophilus ticks to cypermetrin on all nine farms examined. On six farms ticks had developed resistance against amitraz demonstrated by all three tests procedures. On one farm ticks were still susceptible for amitraz in all tests, whereas in 2 remaining farms tests were not in agreement. Furthermore ticks were found susceptible for chlorfinvenphos on three farms in all three tests, whereas in 5 farms showed they appeared to be resistant. In general the results obtained with the different tests were in agreement.</dc:description>
+
+      </metadata>
+    </result>
+  </record>
Index: modules/uoa-resource-discovery/trunk/test/junit/record5.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/record5.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/record5.xml	(revision 47463)
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>  
+  <record rank="0.42426404">
+    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">
+      <header>
+        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/34935</dri:objIdentifier>
+        <dri:dateOfCollection>2009-12-30T00:22:10Z</dri:dateOfCollection>
+
+      </header>
+      <metadata>
+        <dr:CobjContentSynthesis/>
+        <dr:CobjTypology>Textual</dr:CobjTypology>
+        <dr:CobjIdentifier/>
+        <dr:CobjModel>OAI</dr:CobjModel>
+        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>
+
+        <dr:CobjDescriptionSynthesis/>
+        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>
+        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>
+        <dr:repositoryCountry>NL</dr:repositoryCountry>
+        <dr:repositoryInstitution/>
+        <dc:creator>Hoofd, M.G.V. van het</dc:creator>
+        <dc:title>Exercise and depressionafter stroke, a systematic review Shuttle Walk Test in patients who suffered a stroke, a feasibility study</dc:title>
+
+        <dc:subject>exercise</dc:subject>
+        <dc:subject>Fysiotherapiewetenschap</dc:subject>
+        <dc:subject>feasibility study</dc:subject>
+        <dc:subject>systematic review Shuttle Walk Test in patients who suffered a stroke, a feasibility study: stroke</dc:subject>
+        <dc:subject>Exercise and depressionafter stroke, a systematic review: stroke</dc:subject>
+        <dc:subject>Shuttle Walk Test</dc:subject>
+
+        <dc:subject>Geneeskunde</dc:subject>
+        <dc:subject>depression</dc:subject>
+        <dc:subject>aerobic capacity</dc:subject>
+        <dr:CobjCategory>0007</dr:CobjCategory>
+        <dc:language>eng</dc:language>
+        <dc:dateAccepted>2009-06-30</dc:dateAccepted>
+
+        <dc:identifier>http://igitur-archive.library.uu.nl/student-theses/2009-0807-200622/UUindex.html</dc:identifier>
+        <dc:publisher/>
+        <dc:source/>
+        <dc:contributor>Brussel, M. van</dc:contributor>
+        <dc:contributor>Port, I.G.L. van de</dc:contributor>
+        <dc:contributor>Takken, T.</dc:contributor>
+        <dc:relation/>
+
+        <dc:description>Exercise and depression after stroke Purpose-Aim of this review is to summarize the evidence from (randomized) controlled trials regarding the effects of exercise on depression or depressive symptoms in patients who had suffered a stroke. Methods-Studies that included patients who suffered a stroke and measured an outcome concerning depression were systematically reviewed. After determining the methodological quality by the Pedro-scale, a best evidence synthesis was applied. Results-Two out of seven studies showed significant differences between both groups, in favor of the intervention group. Best evidence synthesis showed insufficient evidence for positive effects of exercise on depression in patients who suffered a stroke. Conclusion-From the studies included in the present review it cannot be concluded that exercise interventions had a positive effect on depression in patients who suffered a stroke. Key Words: stroke ... depression ... exercise ... systematic review Shuttle Walk Test in patients who suffered a stroke Objective-To evaluate the feasibility of the Shuttle Walk Test (SWT) and the Shuttle Run Test for children with cerebral palsy at GMFCS level II (SRT-II) in patients who suffered a stroke. Methods-Fifteen patients who suffered a stroke completed both the SWT and SRT-II to evaluate aerobic capacity. Results-Significant differences were found in maximum heart rate and test duration in favor of the SRT-II. No significant difference was found in perceived exertion. Conclusion-The SRT-II is more feasible to assess aerobic capacity in patients who suffered a stroke compared to the SWT. Key Words: stroke ... aerobic capacity ... feasibility study ... Shuttle Walk Test Nederlandse samenvatting De invloed van bewegen op depressie bij mensen met een CVA Doel-Dit review heeft als doel om te onderzoeken wat de invloed van bewegen is op depressie bij mensen met een CVA. Methode-Een systematische zoektocht is gedaan naar (gerandomiseerde), gecontroleerde studies waarin pati..nten met een CVA ge..ncludeerd waren en depressie als uitkomstmaat gemeten werd. Nadat de methodologische kwaliteit werd bepaald met de PEDro-schaal, is een best evidence synthese opgesteld. Resultaten-Twee van zeven studies lieten een significant, positief effect zien van bewegen op depressie bij mensen met een CVA. De best evidence synthese toonde onvoldoende bewijs voor positieve effecten van bewegen bij mensen met een CVA. Conclusie-Uit dit review kan niet geconcludeerd worden dat bewegen een positief effect heeft op depressie bij mensen met een CVA. Trefwoorden: CVA ... depressie ... bewegen ... systematisch review Een shuttle wandeltest bij mensen met een CVA Doel-In deze studie wordt de toepasbaarheid onderzocht van de Shuttle Wandel Test (SWT) en de Shuttle Run Test voor kinderen met een cerebrale parese met GMFCS-niveau II (SRT-II) bij mensen met een cerebrovasculair accident (CVA). Methode-Vijftien pati..nten met een CVA voerden de SWT en de SRT-II uit om het maximale inspanningsvermogen te meten. Resultaten-Significante verschillen werden gevonden wat betreft maximale hartslag en testduur tussen beide testen ten gunste van SRT-II. Er werd geen significant verschil gevonden in ervaren vermoeidheid (Borg-schaal). Conclusie-De SRT-II is beter toepasbaar om het maximale inspanningsvermogen te meten van mensen met een CVA dan de SWT. Trefwoorden: CVA ... maximaal inspanningsvermogen ... haalbaarheidsstudie ... Shuttle Walk Test</dc:description>
+      </metadata>
+    </result>
+  </record>
Index: modules/uoa-resource-discovery/trunk/test/junit/MyResultDao.java
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/MyResultDao.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/MyResultDao.java	(revision 47463)
@@ -0,0 +1,37 @@
+import java.util.ArrayList;
+import java.util.List;
+
+import eu.dnetlib.data.utility.featureextraction.dao.IResultDao;
+
+
+public class MyResultDao implements IResultDao {
+	
+	ArrayList<String> results;
+	
+	public MyResultDao() {
+		this.results = new ArrayList<String>();
+	}
+
+	@Override
+	public void addResults(List<String> results) {
+		for(String result : results)
+			this.results.add(result);
+	}
+
+	@Override
+	public void close() {
+		// TODO Auto-generated method stub
+
+	}
+
+	@Override
+	public int getNumberOfElements() {
+		return results.size();
+	}
+
+	@Override
+	public List<String> getResults(int from, int to) {
+		return results.subList(from-1, to);
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/test/junit/record8.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/record8.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/record8.xml	(revision 47463)
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>    
+  <record rank="0.42426404">
+    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">
+      <header>
+        <dri:objIdentifier>9dffbf71-6914-40fe-b110-8e41a977ba90_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dro.dur.ac.uk.OAI2:1856</dri:objIdentifier>
+        <dri:dateOfCollection>2009-12-29T22:13:24Z</dri:dateOfCollection>
+      </header>
+      <metadata>
+
+        <dr:CobjContentSynthesis/>
+        <dr:CobjTypology>Textual</dr:CobjTypology>
+        <dr:CobjIdentifier>Remedios, R. and Ritchie, K. and Lieberman, D. A. (2005) 'I used to like it but now I don't : the effect of the transfer test in Northern Ireland on pupils' intrinsic motivation.', British journal of educational psychology., 75 (3). pp. 435-452.</dr:CobjIdentifier>
+        <dr:CobjModel>OAI</dr:CobjModel>
+        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>
+        <dr:CobjDescriptionSynthesis/>
+        <dr:repositoryName>Durham Research Online</dr:repositoryName>
+
+        <dr:repositoryLink>http://dro.dur.ac.uk</dr:repositoryLink>
+        <dr:repositoryCountry>UK</dr:repositoryCountry>
+        <dr:repositoryInstitution/>
+        <dc:creator>Remedios, R.</dc:creator>
+        <dc:creator>Lieberman, D. A.</dc:creator>
+        <dc:creator>Ritchie, K.</dc:creator>
+
+        <dc:title>I used to like it but now I don't : the effect of the transfer test in Northern Ireland on pupils' intrinsic motivation.</dc:title>
+        <dc:subject/>
+        <dr:CobjCategory>0001</dr:CobjCategory>
+        <dc:language>eng</dc:language>
+        <dc:dateAccepted>2005-09-01</dc:dateAccepted>
+        <dc:identifier/>
+        <dc:publisher>British Psychological Society</dc:publisher>
+
+        <dc:source/>
+        <dc:contributor/>
+        <dc:relation>http://dro.dur.ac.uk/1856/</dc:relation>
+        <dc:relation>http://dx.doi.org/10.1348/000709904X24771</dc:relation>
+        <dc:description>Background. Research has suggested that the pressure of exams could undermine pupils' interest in their subjects, but almost all of this research has been conducted in laboratory settings. The Transfer Test in Northern Ireland provides an unusual opportunity to assess the effects of exam pressure in real life because some 10- and 11-year-olds sit a Transfer Test to be admitted to grammar school while others are not tested until they are 14. Aim. To assess the effect of exams on pupils' interest in their subjects both during the period before the exam and after the results are known. Sample. The sample comprised 66 pupils preparing to sit the Transfer Test and 55 not preparing for the test. Method. Pupils' interest in their school subjects was assessed by questionnaires administered 2 weeks before the Transfer Test and then again 2 weeks after the results were announced. Results. Surprisingly, prior to sitting the test, there was no significant difference in motivation between the test and no-test pupils. However, after sitting the test, the motivation of the test pupils decreased significantly relative to their no-test counterparts, despite the fact that most achieved the grades they needed for admission to grammar school. Conclusions. Exams provide a valuable tool for assessing academic progress, but under some circumstances they can reduce pupils' interest in the subjects they are studying. </dc:description>
+      </metadata>
+    </result>
+
+  </record>
Index: modules/uoa-resource-discovery/trunk/test/junit/record1.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/record1.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/record1.xml	(revision 47463)
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+  <record rank="0.99999994">
+    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">
+      <header>
+        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/34916</dri:objIdentifier>
+        <dri:dateOfCollection>2009-12-30T00:22:07Z</dri:dateOfCollection>
+      </header>
+      <metadata>
+
+        <dr:CobjContentSynthesis/>
+        <dr:CobjTypology>Textual</dr:CobjTypology>
+        <dr:CobjIdentifier/>
+        <dr:CobjModel>OAI</dr:CobjModel>
+        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>
+        <dr:CobjDescriptionSynthesis/>
+        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>
+
+        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>
+        <dr:repositoryCountry>NL</dr:repositoryCountry>
+        <dr:repositoryInstitution/>
+        <dc:creator>Westerlaken, J.</dc:creator>
+        <dc:title>Test</dc:title>
+        <dc:subject>test</dc:subject>
+
+        <dc:subject>Geneeskunde</dc:subject>
+        <dc:subject>Verplegingswetenschap</dc:subject>
+        <dr:CobjCategory>0007</dr:CobjCategory>
+        <dc:language>dut/nla</dc:language>
+        <dc:dateAccepted>2009-07-31</dc:dateAccepted>
+        <dc:identifier>http://igitur-archive.library.uu.nl/dissertations/2005-1018-200018/index.htm</dc:identifier>
+
+        <dc:publisher/>
+        <dc:source/>
+        <dc:contributor>Teat, test</dc:contributor>
+        <dc:contributor>Test, Test</dc:contributor>
+        <dc:relation/>
+        <dc:description>Test</dc:description>
+      </metadata>
+
+    </result>
+  </record>
Index: modules/uoa-resource-discovery/trunk/test/junit/SuperTester.java
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/SuperTester.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/SuperTester.java	(revision 47463)
@@ -0,0 +1,28 @@
+import java.util.Vector;
+
+import org.apache.log4j.BasicConfigurator;
+import org.junit.Test;
+
+import eu.dnetlib.data.utility.resource_discovery.crawler.Crawler;
+import eu.dnetlib.data.utility.resource_discovery.crawler.ResourceExtractor;
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+
+public class SuperTester {
+
+	@Test
+	public void TestCrawlingAndExtraction() throws Exception {
+		BasicConfigurator.configure();
+		
+		Crawler crawler = new Crawler();
+		ResourceExtractor extractor = new ResourceExtractor();
+		
+		String idUrl = UrlFilter.resolveRedirections("http://www.di.uoa.gr");
+		System.out.println("Now processing " + idUrl);
+		Vector<String> urls = crawler.getLinks(idUrl);
+		System.out.println("Retrieved links are: "+ urls);
+		System.out.println("Resources seem to be available in: " + extractor.extractResource(urls));
+		System.out.println();
+	}
+	
+}
Index: modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MethodProviderFileStorageImpl.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MethodProviderFileStorageImpl.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MethodProviderFileStorageImpl.java	(revision 47463)
@@ -0,0 +1,65 @@
+package gr.uoa.di.resourcediscovery;
+
+import gr.uoa.di.resourcediscovery.methods.ResourceDiscoveryMethod;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.URL;
+import java.util.HashMap;
+
+import com.thoughtworks.xstream.XStream;
+
+public class MethodProviderFileStorageImpl implements MethodProvider {
+
+	private String pathToFile = null;
+
+	HashMap<URL, ResourceDiscoveryMethod> map = new HashMap<URL, ResourceDiscoveryMethod>();
+
+	public MethodProviderFileStorageImpl() {
+
+	}
+
+	@SuppressWarnings("unchecked")
+	public MethodProviderFileStorageImpl(String pathToFile) throws FileNotFoundException {
+		XStream xstream = new XStream();
+		if(!(new File(pathToFile).exists()))
+			map = new  HashMap<URL, ResourceDiscoveryMethod>();
+		else
+			map = (HashMap<URL, ResourceDiscoveryMethod>) xstream.fromXML(new FileReader(new File(pathToFile)));
+		this.pathToFile = pathToFile;
+	}
+
+	@Override
+	public ResourceDiscoveryMethod getMethod(URL baseUrl) throws MalformedConfigurationException, UnknownMethodException, IOException {
+		baseUrl = new URL(Toolkit.getRedirectedUrl(baseUrl.toString(), 500));
+		ResourceDiscoveryMethod ret = map.get(new URL(baseUrl.getProtocol()+"://"+baseUrl.getHost()));
+		return ret;
+	}
+
+	@Override
+	public void setMethod(URL baseUrl, ResourceDiscoveryMethod method) {
+		map.put(baseUrl, method);
+		try {
+			store();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	public String getPathToFile() {
+		return pathToFile;
+	}
+
+	public void setPathToFile(String pathToFile) {
+		this.pathToFile = pathToFile;
+	}
+
+	public void store() throws IOException {
+		XStream xstream = new XStream();
+		xstream.toXML(map, new FileWriter(new File(pathToFile)));
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/XPathAndCrawl.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/XPathAndCrawl.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/XPathAndCrawl.java	(revision 47463)
@@ -0,0 +1,471 @@
+package gr.uoa.di.resourcediscovery.methods;
+
+import gr.uoa.di.resourcediscovery.MalformedConfigurationException;
+import gr.uoa.di.resourcediscovery.MethodProvider;
+import gr.uoa.di.resourcediscovery.Toolkit;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+import org.archive.modules.net.RobotsDirectives;
+import org.archive.modules.net.Robotstxt;
+import org.cyberneko.html.parsers.DOMParser;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.traversal.DocumentTraversal;
+import org.w3c.dom.traversal.NodeFilter;
+import org.w3c.dom.traversal.NodeIterator;
+import org.xml.sax.SAXException;
+
+public class XPathAndCrawl implements ResourceDiscoveryMethod {
+
+	transient Logger logger = Logger.getLogger(XPathAndCrawl.class);
+
+	private boolean resolveFrames = true;
+	private boolean skipFirstPage = false;
+	private long sleepMillis = 100;
+	private boolean ignoreRobotsTxt = false;
+	private String agentName = "OpenAIRE_Harvester";
+	private List<String> mimeTypes = new ArrayList<String>();
+	private boolean fallback = true;
+	private String robotstxtUrl = null;
+
+	transient private Robotstxt robot = null;
+	transient private RobotsDirectives directives = null;
+
+	private List<String> xpaths = new ArrayList<String>();
+
+	public XPathAndCrawl() {
+		this.ignoreRobotsTxt = true;
+	}
+
+	// you need one per repository!
+	public XPathAndCrawl(List<String> mimeTypes, String robotstxtUrl) throws FileNotFoundException, IOException {
+		this.mimeTypes.addAll(mimeTypes);
+
+		if (robotstxtUrl != null) {
+			URL url = new URL(robotstxtUrl);
+			try {
+				BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
+				this.robot = new Robotstxt(in);
+				this.directives = this.robot.getDirectivesFor(agentName);
+			} catch (FileNotFoundException ex) {
+				logger.debug("Robots.txt was not found at " + robotstxtUrl);
+				ignoreRobotsTxt = true;
+			}
+		} else {
+			ignoreRobotsTxt = true;
+		}
+	}
+
+	public void setRobotstxt(String robotstxtUrl) throws FileNotFoundException, IOException {
+		this.robotstxtUrl = robotstxtUrl;
+		if (robotstxtUrl != null) {
+			URL url = new URL(robotstxtUrl);
+			try {
+				BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
+				this.robot = new Robotstxt(in);
+				this.directives = this.robot.getDirectivesFor(agentName);
+			} catch (FileNotFoundException ex) {
+				logger.debug("Robots.txt was not found at " + robotstxtUrl);
+				ignoreRobotsTxt = true;
+			}
+		} else {
+			ignoreRobotsTxt = true;
+		}
+	}
+
+	public String getRobotstxtUrl() {
+		return robotstxtUrl;
+	}
+
+	@Override
+	public List<String> getResources(URL upageUrl, MethodProvider provider) throws SAXException, IOException {
+
+		String pageUrl = upageUrl.toString();
+		
+		logger.debug("Known xpaths: "+this.xpaths);
+		
+		pageUrl = Toolkit.getRedirectedUrl(pageUrl, this.sleepMillis);
+		
+		logger.debug("Resolved possible redirections. Url: "+pageUrl);
+		
+		List<String> ret = new ArrayList<String>();
+		List<String> urls = new ArrayList<String>();
+		urls.add(pageUrl);
+		
+		// check if url is a redirection
+		
+		
+		if(this.mimeTypes.contains(Toolkit.getMimeType(pageUrl, this.sleepMillis))) {
+			ret.add(Toolkit.makeAbsolute(pageUrl, new URL(pageUrl)));
+			return ret;
+		}
+
+		if (this.resolveFrames) {
+			DOMParser parser = new DOMParser();
+			parser.parse(pageUrl);
+			Document doc = parser.getDocument();
+			urls.addAll(resolveFrames(doc, new URL(pageUrl)));
+			logger.debug("urls after resolving frames: " + urls);
+		}
+
+		if (this.skipFirstPage) {
+			List<String> addme = new ArrayList<String>();
+			for (String url : urls) {
+				DOMParser parser = new DOMParser();
+				parser.parse(url);
+				Document doc = parser.getDocument();
+				addme.addAll(oneDepthDown(doc, new URL(url)));
+			}
+
+			urls.remove(pageUrl);
+
+			if (this.resolveFrames) {
+				for (String url : urls) {
+					DOMParser parser = new DOMParser();
+					parser.parse(url);
+					Document doc = parser.getDocument();
+					addme.addAll(resolveFrames(doc, new URL(url)));
+				}
+			}
+
+			urls.addAll(addme);
+			logger.debug("urls after skipping 1st page and resolving frames: " + urls);
+		}
+
+		for (String url : urls) {
+			logger.debug("looking for resource in: " + url);
+			try {
+				url = Toolkit.makeAbsolute(url, new URL(pageUrl));
+			} catch (Exception e) {
+				e.printStackTrace();
+				continue;
+			}
+			URL startingUrl = new URL(url);
+
+			if (!this.ignoreRobotsTxt)
+				if (!this.directives.allows(Toolkit.makeRelative(startingUrl))) {
+					logger.debug("Skipping " + startingUrl + ". Disallowed by robots.txt directives.");
+					continue;
+				}
+
+			if (this.xpaths.size() == 0) {
+				logger.debug("No xpath information, crawling");
+				// this for the first time
+				DOMParser parser = new DOMParser();
+				parser.parse(startingUrl.toString());
+				Document doc = parser.getDocument();
+
+				List<Node> resourceNodes = findNodesWithResource(doc, startingUrl);
+
+				for (Node resourceNode : resourceNodes) {
+					String xp = getXpathToRoot(resourceNode);
+					xpaths.add(xp);
+					logger.debug(xp);
+				}
+				
+				try {
+					URL methodUrl = new URL(pageUrl);
+					provider.setMethod(new URL(methodUrl.getProtocol()+"://"+methodUrl.getHost()), this);
+				} catch(MalformedConfigurationException e) {
+					logger.error("Error updating xpath information", e);
+				}
+
+				for (String xp : xpaths) {
+					String resourceUrl = getResourceUrl(xp, doc, startingUrl);
+					if (resourceUrl != null) {
+						logger.debug(resourceUrl);
+						ret.add(resourceUrl);
+					}
+				}
+			} else {
+				// this is for the rest of the pages of the repo
+				DOMParser parser = new DOMParser();
+				parser.parse(startingUrl.toString());
+				Document doc = parser.getDocument();
+
+				for (String xp : xpaths) {
+					String resourceUrl = getResourceUrl(xp, doc, startingUrl);
+					if (resourceUrl != null) {
+						logger.debug(resourceUrl);
+						ret.add(resourceUrl);
+					}
+				}
+			}
+		}
+
+		if (ret.size() == 0 && this.fallback) {
+			// if no xpath contained the resource, try to find it and add
+			// all the xpaths
+			for (String url : urls) {
+				logger.debug("looking for resource in (not found in xpath): " + url);
+
+				try {
+					url = Toolkit.makeAbsolute(url, new URL(pageUrl));
+				} catch (Exception e) {
+					e.printStackTrace();
+					continue;
+				}
+				URL startingUrl = new URL(url);
+
+				if (!this.ignoreRobotsTxt)
+					if (!this.directives.allows(Toolkit.makeRelative(startingUrl))) {
+						logger.debug("Skipping " + startingUrl + ". Disallowed by robots.txt directives.");
+						continue;
+					}
+
+				DOMParser parser = new DOMParser();
+				parser.parse(startingUrl.toString());
+				Document doc = parser.getDocument();
+				List<Node> resourceNodes = findNodesWithResource(doc, startingUrl);
+				for (Node resourceNode : resourceNodes) {
+					String xp = getXpathToRoot(resourceNode);
+					xpaths.add(xp);
+					logger.debug(xp);
+				}
+				
+				try {
+					URL methodUrl = new URL(pageUrl);
+					provider.setMethod(new URL(methodUrl.getProtocol()+"://"+methodUrl.getHost()), this);
+				} catch(MalformedConfigurationException e) {
+					logger.error("Error updating xpath information", e);
+				}
+				
+				for (String xp : xpaths) {
+					String resourceUrl = getResourceUrl(xp, doc, startingUrl);
+					if (resourceUrl != null) {
+						logger.debug(resourceUrl);
+						ret.add(resourceUrl);
+					}
+				}
+			}
+		}
+
+		return ret;
+	}
+
+	private List<String> resolveFrames(Document doc, URL connectionUrl) {
+		List<String> ret = new ArrayList<String>();
+
+		DocumentTraversal traversal = (DocumentTraversal) doc;
+
+		NodeIterator iterator = null;
+		try {
+			iterator = traversal.createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, null, true);
+		} catch (Exception e) {
+			e.printStackTrace();
+			return ret;
+		}
+
+		for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) {
+			if (n.getNodeName().equals("FRAME") || n.getNodeName().equals("IFRAME")) {
+				String url = n.getAttributes().getNamedItem("src").getNodeValue();
+				try {
+					url = Toolkit.makeAbsolute(url, connectionUrl);
+					ret.add(url);
+				} catch (MalformedURLException ex) {
+					continue;
+				}
+			}
+		}
+		return ret;
+	}
+
+	private List<String> oneDepthDown(Document doc, URL connectionUrl) throws IOException {
+		List<String> ret = new ArrayList<String>();
+
+		DocumentTraversal traversal = (DocumentTraversal) doc;
+
+		NodeIterator iterator = null;
+		try {
+			iterator = traversal.createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, null, true);
+		} catch (Exception e) {
+			e.printStackTrace();
+			return ret;
+		}
+
+		for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) {
+			if (n.getNodeName().equals("A")) {
+				String url = n.getAttributes().getNamedItem("href").getNodeValue();
+				try {
+					url = Toolkit.makeAbsolute(url, connectionUrl);
+					if (Toolkit.getMimeType(url, this.sleepMillis).trim().contains("text/html"))
+						ret.add(url);
+				} catch (MalformedURLException ex) {
+					continue;
+				}
+			}
+		}
+		return ret;
+	}
+
+	private String getXpathToRoot(Node node) {
+		String xpath = "";
+		do {
+			if (node.getNodeName().equals("HTML")) {
+				int before = 1;
+				while ((node = node.getPreviousSibling()) != null)
+					before++;
+				return "/HTML["+before+"]" + xpath;
+			}
+			int before = 0;
+			Node current = node;
+			while ((current = current.getPreviousSibling()) != null)
+				if (current.getNodeName().equals(node.getNodeName()))
+					before++;
+			xpath = "/" + node.getNodeName() + "[" + (before + 1) + "]" + xpath;
+		} while ((node = node.getParentNode()) != null);
+		return xpath;
+	}
+
+	private List<Node> findNodesWithResource(Document doc, URL connectionUrl) throws IOException {
+		List<Node> ret = new ArrayList<Node>();
+
+		DocumentTraversal traversal = (DocumentTraversal) doc;
+
+		NodeIterator iterator = null;
+		try {
+			iterator = traversal.createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, null, true);
+		} catch (Exception e) {
+			e.printStackTrace();
+			return ret;
+		}
+
+		for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) {
+			if (n.getNodeName().equals("A")) {
+				String url = null;
+				try {
+					url = n.getAttributes().getNamedItem("href").getNodeValue();
+				} catch(NullPointerException e) {
+					// anchor without href
+					continue;
+				}
+				if (url == null)
+					continue;
+				try {
+					url = Toolkit.makeAbsolute(url, connectionUrl);
+					if (this.mimeTypes.contains(Toolkit.getMimeType(url, this.sleepMillis).trim()))
+						ret.add(n);
+				} catch (MalformedURLException ex) {
+					continue;
+				}
+			}
+		}
+		return ret;
+	}
+
+	private String getResourceUrl(String xpath, Document doc, URL url) throws MalformedURLException {
+		try {
+			Node current = doc.getFirstChild();
+			String[] elements = xpath.split("/");
+			for (String element : elements) {
+				if (element.trim().equals(""))
+					continue;
+				int position = Integer.parseInt(element.substring(element.indexOf('[')).replaceAll("\\[", "").replaceAll("\\]", ""));
+				String name = element.substring(0, element.indexOf('['));
+				int found = 0;
+				do {
+					if (current.getNodeName().equals(name)) {
+						found++;
+						if (found == position) {
+							current = current.getFirstChild();
+							break;
+						}
+					}
+				} while ((current = current.getNextSibling()) != null);
+
+			}
+			String ret = current.getParentNode().getAttributes().getNamedItem("href").getNodeValue();
+			return Toolkit.makeAbsolute(ret, url);
+		} catch (Exception e) {
+			return null;
+		}
+	}
+
+	private Object readResolve() throws IOException {
+		if (robotstxtUrl != null) {
+			URL url = new URL(robotstxtUrl);
+			BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
+			this.robot = new Robotstxt(in);
+			this.directives = this.robot.getDirectivesFor(agentName);
+		} else {
+			ignoreRobotsTxt = true;
+		}
+		logger = Logger.getLogger(XPathAndCrawl.class);
+		return this;
+	}
+
+	public boolean isResolveFrames() {
+		return resolveFrames;
+	}
+
+	public void setResolveFrames(boolean resolveFrames) {
+		this.resolveFrames = resolveFrames;
+	}
+
+	public boolean isSkipFirstPage() {
+		return skipFirstPage;
+	}
+
+	public void setSkipFirstPage(boolean skipFirstPage) {
+		this.skipFirstPage = skipFirstPage;
+	}
+
+	public long getSleepMillis() {
+		return sleepMillis;
+	}
+
+	public void setSleepMillis(long sleepMillis) {
+		this.sleepMillis = sleepMillis;
+	}
+
+	public List<String> getMimeTypes() {
+		return mimeTypes;
+	}
+
+	public void setMimeTypes(List<String> mimeTypes) {
+		this.mimeTypes = mimeTypes;
+	}
+
+	public List<String> getXpaths() {
+		return xpaths;
+	}
+
+	public void setXpaths(List<String> xpaths) {
+		this.xpaths = xpaths;
+	}
+
+	public void setIgnoreRobotsTxt(boolean ignoreRobotsTxt) {
+		this.ignoreRobotsTxt = ignoreRobotsTxt;
+	}
+
+	public boolean isIgnoreRobotsTxt() {
+		return ignoreRobotsTxt;
+	}
+
+	public void setAgentName(String agentName) {
+		this.agentName = agentName;
+		this.directives = this.robot.getDirectivesFor(agentName);
+	}
+
+	public String getAgentName() {
+		return agentName;
+	}
+
+	public void setFallback(boolean fallback) {
+		this.fallback = fallback;
+	}
+
+	public boolean isFallback() {
+		return fallback;
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/ResourceDiscoveryMethod.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/ResourceDiscoveryMethod.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/ResourceDiscoveryMethod.java	(revision 47463)
@@ -0,0 +1,14 @@
+package gr.uoa.di.resourcediscovery.methods;
+
+import gr.uoa.di.resourcediscovery.MethodProvider;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.List;
+
+import org.xml.sax.SAXException;
+
+public interface ResourceDiscoveryMethod {
+
+	public List<String> getResources(URL upageUrl, MethodProvider provider) throws SAXException, IOException;
+}
Index: modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/URLTransformation.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/URLTransformation.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/methods/URLTransformation.java	(revision 47463)
@@ -0,0 +1,53 @@
+package gr.uoa.di.resourcediscovery.methods;
+
+import gr.uoa.di.resourcediscovery.MethodProvider;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+public class URLTransformation implements ResourceDiscoveryMethod {
+
+	private String regex = null, replacement = "";
+	private String addToEnd = "";
+
+	@Override
+	public List<String> getResources(URL upageUrl, MethodProvider provider) {
+		String pageUrl = upageUrl.toString();
+		String trsf = pageUrl;
+		if (regex != null && !regex.trim().equals(""))
+			trsf = pageUrl.replaceAll(regex, replacement);
+
+		trsf = trsf + addToEnd;
+
+		List<String> ret = new ArrayList<String>();
+		ret.add(trsf);
+
+		return ret;
+	}
+
+	public String getRegex() {
+		return regex;
+	}
+
+	public void setRegex(String regex) {
+		this.regex = regex;
+	}
+
+	public String getAddToEnd() {
+		return addToEnd;
+	}
+
+	public void setAddToEnd(String addToEnd) {
+		this.addToEnd = addToEnd;
+	}
+
+	public String getReplacement() {
+		return replacement;
+	}
+
+	public void setReplacement(String replacement) {
+		this.replacement = replacement;
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MethodProvider.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MethodProvider.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MethodProvider.java	(revision 47463)
@@ -0,0 +1,13 @@
+package gr.uoa.di.resourcediscovery;
+
+import gr.uoa.di.resourcediscovery.methods.ResourceDiscoveryMethod;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public interface MethodProvider {
+
+	public ResourceDiscoveryMethod getMethod(URL baseUrl) throws MalformedConfigurationException, UnknownMethodException, MalformedURLException, IOException;
+	public void setMethod(URL baseUrl, ResourceDiscoveryMethod method) throws MalformedConfigurationException;
+}
Index: modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/UnknownMethodException.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/UnknownMethodException.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/UnknownMethodException.java	(revision 47463)
@@ -0,0 +1,6 @@
+package gr.uoa.di.resourcediscovery;
+
+public class UnknownMethodException extends Exception {
+	private static final long serialVersionUID = 760327436365242998L;
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MalformedConfigurationException.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MalformedConfigurationException.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/gr/uoa/di/resourcediscovery/MalformedConfigurationException.java	(revision 47463)
@@ -0,0 +1,7 @@
+package gr.uoa.di.resourcediscovery;
+
+public class MalformedConfigurationException extends Exception {
+
+	private static final long serialVersionUID = 8557374776080985539L;
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/api/ResourceDiscoveryAPI.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/api/ResourceDiscoveryAPI.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/api/ResourceDiscoveryAPI.java	(revision 47463)
@@ -0,0 +1,247 @@
+package eu.dnetlib.data.utility.resource_discovery.api;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import net.matuschek.http.DownloadRuleSet;
+import net.matuschek.http.HttpException;
+import net.matuschek.spider.WebRobot;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.log4j.BasicConfigurator;
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.data.utility.resource_discovery.crawler.Crawler;
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+public class ResourceDiscoveryAPI {
+
+	private Logger logger = Logger.getLogger(ResourceDiscoveryAPI.class);
+	private int maxDepth = 1;
+	private List<String> mimeTypes = null;
+	private List<String> urls = null;
+	private boolean ignoreRobotsTxt = false;
+	private Set<URL> blacklistedHosts = new HashSet<URL>();
+	private int sleep_time = 0;
+	private boolean getFirstOnly = false;
+	
+	public ResourceDiscoveryAPI() {
+
+	}
+	
+	public ResourceDiscoveryAPI(List<String> urls, List<String> mimeTypes) {
+		this(urls, mimeTypes, false, 0, false);
+	}
+	
+	public ResourceDiscoveryAPI(String url, List<String> mimeTypes) {
+		this(Arrays.asList(new String[] {url}), mimeTypes, false, 0, false);
+	}
+	
+	public ResourceDiscoveryAPI(List<String> urls, List<String> mimeTypes, boolean ignoreRobotsTxt, int sleep_time, boolean getFirstOnly) {
+		this.mimeTypes = mimeTypes;
+		this.urls = urls;
+		this.ignoreRobotsTxt = ignoreRobotsTxt;
+		this.sleep_time = sleep_time;
+		this.getFirstOnly = getFirstOnly;
+	}
+	
+	public List<String> getResources() throws Exception {
+		List<String> resources = new ArrayList<String>();
+		Crawler crawler = setupCrawler();
+		boolean found = false;
+		
+		for (String url : this.urls) {
+			try {
+				url = StringEscapeUtils.unescapeHtml(url);
+				logger.debug("Looking for resources in " + url);
+				URL u = new URL(url);
+				
+				if (blacklistedHosts.contains(this.getHostURL(u))) {
+					logger.warn("Host " + u.getHost() + " is blacklisted. Ignoring");
+					
+					continue;
+				}
+				
+				url = UrlFilter.resolveRedirections(url);
+	
+				if (mimeTypes.contains(UrlFilter.getMimeType(url))) {
+					resources.add(url);
+				} else {
+					Collection<String> links = crawler.getLinks(url);
+		
+					for (String link : links) {
+						logger.debug("Looking for resources in " + link);
+						
+						u = new URL(link);
+						
+						if (blacklistedHosts.contains(this.getHostURL(u))) {
+							logger.warn("Host " + u.getHost() + " is blacklisted. Ignoring");
+							
+							break;
+						} else if (this.mimeTypes.contains(UrlFilter.getMimeType(link))) {
+							logger.debug("Resource found in " + link);
+							resources.add(link);
+							if(getFirstOnly) {
+								found = true;
+								break;
+							}
+						}
+					}
+				}
+			} catch (MalformedURLException mue) {
+				logger.error("Error using url", mue);
+			}
+			if(found)
+				break;
+		}
+		
+		return resources;
+	}
+	
+	private URL getHostURL(URL url) throws MalformedURLException {
+		return new URL(url.getProtocol() + "://" + url.getHost());
+	}
+
+	private Crawler setupCrawler() throws IOException, HttpException {
+		Crawler crawler = new Crawler();
+		WebRobot robot = crawler.getCrawler();
+		
+		if(sleep_time > 0) {
+//			robot.setSleep(true);
+			robot.setSleepTime(sleep_time);
+		}
+		
+		robot.setMaxDepth(this.maxDepth);
+		robot.setIgnoreRobotsTxt(this.ignoreRobotsTxt);
+
+		DownloadRuleSet rules = new DownloadRuleSet();
+		int minSize = 1, maxSize = 104857600;
+		rules.addRule("text", "html", minSize, maxSize, true);
+		rules.addRule("application", "xhtml+xml", minSize, maxSize, true);
+
+		for (String mimeType : this.mimeTypes) {
+			String[] parts = mimeType.split("/");
+			if (parts.length < 2) {
+				logger.error("WARNING: Requested mimetype "+mimeType+" seems to be malformed");
+				continue;
+			}
+			rules.addRule(parts[0], parts[1], minSize, maxSize, true);
+		}
+		rules.addRule("*", "*", minSize, maxSize, false);
+		robot.setDownloadRuleSet(rules);
+		return crawler;
+	}
+
+	public int getMaxDepth() {
+		return maxDepth;
+	}
+
+	public void setMaxDepth(int maxDepth) {
+		this.maxDepth = maxDepth;
+	}
+
+	public List<String> getMimeTypes() {
+		return mimeTypes;
+	}
+
+	public void setMimeTypes(List<String> mimeTypes) {
+		this.mimeTypes = mimeTypes;
+	}
+
+	public List<String> getUrls() {
+		return urls;
+	}
+
+	public void setUrls(List<String> urls) {
+		this.urls = urls;
+	}
+	
+	public boolean isIgnoreRobotsTxt() {
+		return ignoreRobotsTxt;
+	}
+
+	public void setIgnoreRobotsTxt(boolean ignoreRobotsTxt) {
+		this.ignoreRobotsTxt = ignoreRobotsTxt;
+	}
+	
+	public void setBlacklistedHosts(Collection<String> hosts) {
+		this.blacklistedHosts.clear();
+		
+		for (String host:hosts) {
+			try {
+				URL u = new URL(host);
+				
+				this.blacklistedHosts.add(this.getHostURL(u));
+			} catch (MalformedURLException e) {
+				logger.warn("Error using url " + host, e);
+			}
+		}
+	}
+
+	static public void main(String[] args) throws Exception {
+		BasicConfigurator.configure();
+	/*	ResourceDiscoveryAPI api = new ResourceDiscoveryAPI();
+		List<String> urls = new ArrayList<String>();
+		// the starting urls for the search
+//		urls.add("http://www.hindawi.com/journals/asp/2010/724087.html");
+		urls.add("http://www.di.uoa.gr/gr");
+		urls.add("http://www.di.uoa.gr/gr/apotimisi/ApotimisiStoxoi.pdf");
+		urls.add("http://orbit.dtu.dk/getResource?recordId=279356&amp;objectId=2&amp;versionId=1");
+//		urls.add("http://cdsweb.cern.ch/record/1210586");
+//		urls.add("http://www.ann-geophys.net/27/3791/2009/angeo-27-3791-2009.html");
+		
+		api.setUrls(urls);
+		api.setMaxDepth(1);
+		api.setIgnoreRobotsTxt(true);
+		api.setBlacklistedHosts(Arrays.asList("http://www.di.uoa.gr"));
+		
+		List<String> mimetypes = new ArrayList<String>();
+		// the mimetypes of the files that will be retrieved
+		//mimetypes.add("application/pdf");
+		//mimetypes.add("image/jpeg");
+		api.setMimeTypes(mimetypes);
+		
+		System.out.println(api.getResources());*/
+		
+		ResourceDiscoveryAPI fileDiscoverer = new ResourceDiscoveryAPI();
+		fileDiscoverer.setMimeTypes( Arrays.asList( new String[] { "application/pdf" } ) );
+		fileDiscoverer.setUrls( Arrays.asList( new String[] { "http://arxiv.org/pdf/0910.0377.pdf" } ) );
+		fileDiscoverer.setMaxDepth(1);
+		fileDiscoverer.setSleep_time(2);
+		fileDiscoverer.setGetFirstOnly(true);
+		fileDiscoverer.setIgnoreRobotsTxt(true);
+		
+		System.out.println(fileDiscoverer.getResources());
+	}
+
+	public Set<URL> getBlacklistedHosts() {
+		return blacklistedHosts;
+	}
+
+	public void setBlacklistedHosts(Set<URL> blacklistedHosts) {
+		this.blacklistedHosts = blacklistedHosts;
+	}
+
+	public int getSleep_time() {
+		return sleep_time;
+	}
+
+	public void setSleep_time(int sleep_time) {
+		this.sleep_time = sleep_time;
+	}
+
+	public boolean isGetFirstOnly() {
+		return getFirstOnly;
+	}
+
+	public void setGetFirstOnly(boolean getFirstOnly) {
+		this.getFirstOnly = getFirstOnly;
+	}
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/config/Configs.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/config/Configs.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/config/Configs.java	(revision 47463)
@@ -0,0 +1,220 @@
+package eu.dnetlib.data.utility.resource_discovery.crawler.config;
+
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.StringWriter;
+import java.util.Collection;
+import java.util.Vector;
+
+import net.matuschek.http.DownloadRuleSet;
+import net.matuschek.http.HttpException;
+import net.matuschek.http.URLLogger;
+import net.matuschek.spider.WebRobot;
+
+import org.apache.log4j.Logger;
+import org.dlese.dpc.xml.XMLDoc;
+import org.dlese.dpc.xml.XMLException;
+
+public class Configs {
+	private final static Logger logger = Logger.getLogger(Configs.class);
+	static public String agentName = "JoBo";
+	static public boolean ignoreRobotsTxt=false;
+	static public int sleepTime=3000;
+	static public int maxDepth=1;
+
+	static public boolean walkToOtherHosts=false;
+	static public boolean allowWholeHost=false;
+	static public boolean allowWholeDomain=false;
+	static public boolean flexibleHostCheck=true;
+	static public boolean localizeLinks=false;
+	static public boolean enableCookies=false;
+	
+	static public String startReferer=null;
+	static public int maxDocumentAge=-1;
+	static public String[] allowedUrl=null;
+	static public String[] visitMany=null;
+	static public String proxy=null;
+	static public int bandwidth=-1;
+	
+	private static String readXMLDoc(String filename) throws IOException {
+		BufferedReader br = new BufferedReader(new InputStreamReader(Configs.class.getResourceAsStream(filename)));
+		String strLine = null;
+		StringBuilder builder = new StringBuilder();
+		
+		try {
+			while ((strLine = br.readLine()) != null)
+				builder.append(strLine);
+
+		} finally {
+			br.close();
+		}		
+		
+		return builder.toString();
+	}
+
+	static {
+		try {
+//		XMLDoc xd = new XMLDoc("configs/robot.xml", true, true, true);
+		logger.debug("Reading configuration file for crawler");
+		XMLDoc xd = new XMLDoc();
+		//xd.useXmlString(readXMLDoc("/eu/dnetlib/functionality/validator/robot.xml"), true, true, true);
+		xd.useXmlString(readXMLDoc("/eu/dnetlib/data/utility/resource_discovery/robot.xml"), true, true, true);
+		
+		String[] ret1 = xd.getXmlFields(0,1,"AgentName");		
+		String [] ret2 = xd.getXmlFields(0,1,"IgnoreRobotsTxt");
+		String [] ret3 = xd.getXmlFields(0,1,"SleepTime");
+		String [] ret4 = xd.getXmlFields(0,1,"MaxDepth");
+		String [] ret5 = xd.getXmlFields(0,1,"WalkToOtherHosts");
+		String [] ret6 = xd.getXmlFields(0,1,"AllowWholeHost");
+		String [] ret7 = xd.getXmlFields(0,1,"AllowWholeDomain");
+		String [] ret8 = xd.getXmlFields(0,1,"FlexibleHostCheck");
+		String [] ret9 = xd.getXmlFields(0,1,"LocalizeLinks");
+		String [] ret10 = xd.getXmlFields(0,1,"EnableCookies");
+		String [] ret11 = xd.getXmlFields(0,1,"StartReferer");
+		String [] ret12 = xd.getXmlFields(0,1,"MaxDocumentAge");
+		String [] ret13 = xd.getXmlFields(0,0,"AllowedUrl");
+		String [] ret14 = xd.getXmlFields(0,0,"VisitMany");
+		String [] ret15 = xd.getXmlFields(0,1,"Proxy");
+		String [] ret16 = xd.getXmlFields(0,1,"Bandwidth");
+		if(ret1.length > 0)
+			agentName = ret1[0];
+		if(ret2.length > 0)
+			ignoreRobotsTxt = Boolean.parseBoolean(ret2[0]);
+		if(ret3.length > 0)
+			sleepTime = Integer.parseInt(ret3[0]) * 1000;
+		if(ret4.length > 0)
+			maxDepth = Integer.parseInt(ret4[0]);
+		if(ret5.length > 0)
+			walkToOtherHosts = Boolean.parseBoolean(ret5[0]);
+		if(ret6.length > 0)
+			allowWholeHost = Boolean.parseBoolean(ret6[0]);
+		if(ret7.length > 0)
+			allowWholeDomain = Boolean.parseBoolean(ret7[0]);
+		if(ret8.length > 0)
+			flexibleHostCheck = Boolean.parseBoolean(ret8[0]);
+		if(ret9.length > 0)
+			localizeLinks = Boolean.parseBoolean(ret9[0]);
+		if(ret10.length > 0)
+			enableCookies = Boolean.parseBoolean(ret10[0]);
+		if(ret11.length > 0)
+			startReferer = ret11[0];
+		if(ret12.length > 0)
+			maxDocumentAge = Integer.parseInt(ret12[0]);
+		if(ret13.length > 0)
+			allowedUrl = ret13;
+		if(ret14.length > 0)
+			visitMany = ret14;
+		if(ret15.length > 0)
+			proxy = ret15[0];
+		if(ret16.length > 0)
+			bandwidth = Integer.parseInt(ret16[0]);
+		} catch (IOException e) {
+			logger.debug("Error reading robots.txt", e);
+		} catch(XMLException e) {
+			logger.debug("WARNING: The file robot.xml seems to be malformed. The default settings will be used for the crawler.", e);
+		} catch(NumberFormatException e) {
+			logger.debug("WARNING: The file robot.xml seems to be malformed (an integer doesn't seem to be of type integer). The default settings will be used for the crawler.", e);
+		} catch (Exception e) {
+			logger.error("Error configuring", e);
+		}
+	}
+	
+	public static void configureCrawlerForValidation(WebRobot crawler) throws IOException {
+		logger.debug("Configuring crawler for validation");
+		crawler.setAgentName("Validator");
+		crawler.setIgnoreRobotsTxt(false);
+		crawler.setSleepTime(1);
+		crawler.setMaxDepth(1);
+		crawler.setWalkToOtherHosts(false);
+		crawler.setAllowWholeHost(true);
+		crawler.setAllowWholeDomain(true);
+		crawler.setFlexibleHostCheck(true);
+		crawler.setEnableCookies(true);
+			
+		DownloadRuleSet rules = new DownloadRuleSet();
+		int minSize = 1, maxSize = 104857600;
+		rules.addRule("text", "html", minSize, maxSize, true);
+		Collection<String> mimeTypes = UrlFilter.getRequestedMimeTypes();
+		for(String mimeType : mimeTypes) {
+			String[] parts = mimeType.split("/");
+			if(parts.length < 2) {
+				logger.debug("WARNING: Requested mimetype "+mimeType+" seems to be malformed");
+				throw new IOException();
+			}
+			rules.addRule(parts[0], parts[1], minSize, maxSize, true);
+		}
+		rules.addRule("*", "*", minSize, maxSize, false);
+		crawler.setDownloadRuleSet(rules);		
+		
+	}
+	
+	public static void configureCrawler(WebRobot crawler) throws IOException, HttpException {
+		logger.debug("Configuring crawler using configuration file parameters");
+		crawler.setAgentName(agentName);
+		crawler.setIgnoreRobotsTxt(ignoreRobotsTxt);
+		crawler.setSleepTime(sleepTime/1000);
+		crawler.setMaxDepth(maxDepth);
+		crawler.setWalkToOtherHosts(walkToOtherHosts);
+		crawler.setAllowWholeHost(allowWholeHost);
+		crawler.setAllowWholeDomain(allowWholeDomain);
+		crawler.setFlexibleHostCheck(flexibleHostCheck);
+		crawler.setEnableCookies(enableCookies);
+		
+		if(startReferer != null)
+			crawler.setStartReferer(startReferer);
+		if(maxDocumentAge > 0)
+			crawler.setMaxDocumentAge(maxDocumentAge);
+		if(allowedUrl != null) {
+			Vector<String> urls = new Vector<String>();
+			for(int i=0; i<allowedUrl.length; i++)
+				urls.add(allowedUrl[i]);
+			crawler.setAllowedURLs(urls);
+		}
+		if(visitMany != null) {
+			Vector<String> urls = new Vector<String>();
+			for(int i=0; i<visitMany.length; i++)
+				urls.add(visitMany[i]);
+			crawler.setVisitMany(urls);
+		}
+		if(proxy != null)
+			crawler.setProxy(proxy);
+		if(bandwidth > 0)
+			crawler.setBandwidth(bandwidth);
+			
+		DownloadRuleSet rules = new DownloadRuleSet();
+		int minSize = 1, maxSize = 104857600;
+		rules.addRule("text", "html", minSize, maxSize, true);
+		Collection<String> mimeTypes = UrlFilter.getRequestedMimeTypes();
+		for(String mimeType : mimeTypes) {
+			String[] parts = mimeType.split("/");
+			if(parts.length < 2) {
+				logger.debug("WARNING: Requested mimetype "+mimeType+" seems to be malformed");
+				throw new IOException();
+			}
+			rules.addRule(parts[0], parts[1], minSize, maxSize, true);
+		}
+		rules.addRule("*", "*", minSize, maxSize, false);
+		crawler.setDownloadRuleSet(rules);
+		
+	}
+	
+	public static void main(String[] args) {
+		WebRobot robby = new WebRobot();
+		try {
+		configureCrawler(robby);
+		StringWriter sw = new StringWriter();
+	    URLLogger log = new URLLogger(sw);
+	    robby.setDocManager(log);
+
+	    robby.run();
+	    
+	    logger.debug(sw.getBuffer().toString());
+		}
+		catch(Exception e) {
+			logger.debug(e.getLocalizedMessage());
+		}
+	}
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/Crawler.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/Crawler.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/Crawler.java	(revision 47463)
@@ -0,0 +1,101 @@
+package eu.dnetlib.data.utility.resource_discovery.crawler;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Vector;
+
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.data.utility.resource_discovery.crawler.config.Configs;
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+import net.matuschek.http.HttpException;
+import net.matuschek.http.URLLogger;
+import net.matuschek.spider.WebRobot;
+
+public class Crawler {
+	private final static Logger logger = Logger.getLogger(Crawler.class);
+	private WebRobot crawler;
+
+	public Crawler() throws IOException, HttpException {
+		crawler = new WebRobot();
+		Configs.configureCrawler(crawler);
+	}
+	
+	public Crawler(boolean isValidator) throws IOException, HttpException {
+		crawler = new WebRobot();
+		if(isValidator)
+			Configs.configureCrawlerForValidation(crawler);
+		else
+			Configs.configureCrawler(crawler);
+	}
+	
+	public void reconfigureForRetry() {
+		crawler.setMaxDepth(2);
+	}
+	
+	public Vector<String> getLinks(String url) throws MalformedURLException, IOException, InterruptedException {
+		logger.debug("Retrieving links from url "+url);
+		crawler.setStartURL(new URL(UrlFilter.resolveRedirections(url)));
+		StringWriter sw = new StringWriter();
+	    URLLogger log = new URLLogger(sw);
+	    crawler.setDocManager(log);
+
+	    crawler.run();
+	    
+	    String [] links = sw.getBuffer().toString().split("\n");
+	    Vector<String> linksV = new Vector<String>();
+	    for(int i=0; i<links.length; i++)
+	    	linksV.add(links[i]);
+	    return linksV;
+	}
+	
+	/*public Vector<String> getLinksFaster(String url) throws ParserException, IOException, InterruptedException {
+		Thread.sleep(Configs.sleepTime);
+		HttpURLConnection.setFollowRedirects(true);
+		URL URL = new URL(url);
+        HttpURLConnection conn = (HttpURLConnection) URL.openConnection();
+        Parser parser = new Parser(conn);
+        
+        NodeList list = parser.parse(new TagNameFilter("A"));
+        Vector<String> links = new Vector<String>();
+		for(int i=0; i<list.size(); i++) {
+			LinkTag n = (LinkTag) list.elementAt(i);
+			links.add(n.extractLink());
+		}
+		
+		return links;
+	}*/
+	
+	/**
+	 * Only for testing purposes, not supposed to be called
+	 */
+	public static void main(String[] args) {
+		Crawler c;
+		try {
+		c = new Crawler();
+		System.out.println(c.crawler.getAllowWholeHost()+" "+c.crawler.getAllowWholeDomain());
+		}
+		catch(Exception e) {
+			System.err.println("FATAL ERROR: Crawler could not be configured. Please check your robot.xml parameters and try again.");
+			System.err.println(e.getLocalizedMessage());
+			e.printStackTrace();
+			return;
+		}
+		String url = "http://www.di.uoa.gr/gr";
+		try {
+		System.out.println(c.getLinks(url));
+		}
+		catch(Exception e) {
+			System.err.println("ERROR: Crawler could not retrieve links from url "+url);
+			System.err.println(e.getLocalizedMessage());
+			e.printStackTrace();
+		}
+	}
+
+	public WebRobot getCrawler() {
+		return this.crawler;
+	}
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/ResourceExtractor.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/ResourceExtractor.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/crawler/ResourceExtractor.java	(revision 47463)
@@ -0,0 +1,43 @@
+package eu.dnetlib.data.utility.resource_discovery.crawler;
+
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+import java.io.IOException;
+import java.util.Vector;
+
+import org.apache.log4j.Logger;
+
+public class ResourceExtractor {
+	private final static Logger logger = Logger.getLogger(ResourceExtractor.class);
+	private Vector<String> filter;
+	private Vector<String> latest;
+	private int runned;
+	
+	public ResourceExtractor() {
+		runned = 0;
+		filter = new Vector<String>();
+		latest = new Vector<String>();
+	}
+	
+	public Vector<String> extractResource(Vector<String> urls) throws IOException, InterruptedException {
+		logger.debug("Extracting resources from links "+urls);
+		runned++;
+		Vector<String> ret = new Vector<String>();
+		if(runned == 1) {
+			filter.addAll(urls);
+			for(String url : urls) {
+				if(UrlFilter.checkExtension(url) || UrlFilter.checkMimeType(url))
+					ret.add(url);
+			}
+			return ret;
+		}
+		for(String url : urls) {
+			if(!latest.contains(url) && !filter.contains(url) && (UrlFilter.checkExtension(url) || UrlFilter.checkMimeType(url)))
+				ret.add(url);
+		}
+		latest.clear();
+		latest.addAll(ret);
+		return ret;
+	}
+	
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/robot.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/robot.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/robot.xml	(revision 47463)
@@ -0,0 +1,18 @@
+<config>
+	<AgentName>JoBo</AgentName>
+	<IgnoreRobotsTxt>false</IgnoreRobotsTxt>
+	<SleepTime>10</SleepTime>
+	<MaxDepth>1</MaxDepth>
+	<WalkToOtherHosts>true</WalkToOtherHosts>
+	<AllowWholeHost>true</AllowWholeHost>
+	<AllowWholeDomain>true</AllowWholeDomain>
+	<FlexibleHostCheck>true</FlexibleHostCheck>
+	<LocalizeLinks>false</LocalizeLinks>
+	<EnableCookies>false</EnableCookies>
+	<StartReferer></StartReferer>
+	<MaxDocumentAge></MaxDocumentAge>
+	<AllowedUrl></AllowedUrl>
+	<VisitMany></VisitMany>
+	<Proxy></Proxy>
+	<Bandwidth></Bandwidth>
+</config>
\ No newline at end of file
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/url_filter/UrlFilter.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/url_filter/UrlFilter.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/url_filter/UrlFilter.java	(revision 47463)
@@ -0,0 +1,222 @@
+package eu.dnetlib.data.utility.resource_discovery.url_filter;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Collection;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.data.utility.resource_discovery.crawler.config.Configs;
+
+public class UrlFilter {
+	private final static Logger logger = Logger.getLogger(UrlFilter.class);
+	static private HashMap<String, String> filetypes;
+	static private HashMap<String, String> knownTypes;
+	
+	static {
+		filetypes = null;
+		knownTypes = new HashMap<String, String>();
+		knownTypes.put("xls", "application/excel");
+		knownTypes.put("doc", "application/word"); 
+		knownTypes.put("dot", "application/word");
+		knownTypes.put("pdf", "application/pdf");
+		knownTypes.put("ai", "application/postscript");
+		knownTypes.put("eps", "application/postscript");
+		knownTypes.put("ps", "application/postscript");
+		knownTypes.put("ppt", "application/powerpoint");
+		knownTypes.put("pps", "application/powerpoint");
+		knownTypes.put("rtf", "application/rtf");
+		knownTypes.put("zip", "application/zip");
+		knownTypes.put("bmp", "image/bmp");
+		knownTypes.put("gif", "image/gif");
+		knownTypes.put("jpg", "image/jpeg");
+		knownTypes.put("jpeg", "image/jpeg");
+		knownTypes.put("jpe", "image/jpeg");
+		knownTypes.put("png", "image/png");
+		knownTypes.put("tif", "image/tiff");
+		knownTypes.put("tiff", "image/tiff");
+		
+		try {
+			BufferedReader rd = new BufferedReader(new InputStreamReader(UrlFilter.class.getResourceAsStream("/eu/dnetlib/data/utility/resource_discovery/filetypes.txt")));
+			String type;
+			filetypes = new HashMap<String, String>();
+			while((type = rd.readLine()) != null) {
+				int start;
+				if(type.trim().equals("") || type.trim().charAt(0) == '%')
+					continue;
+				if(type.indexOf('\t') == -1 && !knownTypes.containsKey(type.trim())) {
+					logger.debug("Warning: Type "+type.trim()+" was provided for harvesting and is not of any known type. The type will be used for harvesting, but you should provide its mime type manually");
+					filetypes.put(type, null);
+				}
+				else if((start=type.indexOf('\t')) != -1) {
+					String mimeType = type.substring(start).trim();
+					String ext = type.substring(0, start).trim();
+					filetypes.put(ext, mimeType);
+				}
+				else if(type.indexOf('\t') == -1 && knownTypes.containsKey(type.trim())) {
+					filetypes.put(type.trim(), knownTypes.get(type.trim()));
+				}
+			}
+			rd.close();
+		}
+		catch(Exception e) {
+			logger.error("File configs/filetypes could not be opened or parsed");
+		}
+	}
+	
+	static public Collection<String> getRequestedMimeTypes() {
+		return filetypes.values();
+	}
+	
+	static public boolean isUrl(String str) {
+		if(str.matches("http://(.)+") || str.matches("https://(.)+") || str.matches("www\\.(.)+"))
+			return true;
+		return false;
+	}
+	
+	static public String getMimeType(String resourceURL) throws InterruptedException, IOException {
+		URL url = null;
+		
+		try {
+			url = new URL(resourceURL);
+		} catch (MalformedURLException mue) {
+			logger.info("Error getting mime type", mue);
+			
+			return null;
+		}
+		
+		HttpURLConnection.setFollowRedirects(true);
+		
+		Thread.sleep(getSleepTime());
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+		conn.setRequestMethod("HEAD");
+		
+		conn = openConnectionCheckRedirects(conn);
+		
+        int statusCode = conn.getResponseCode();
+        if(statusCode == 503) {
+        	logger.debug("WARNING: Url "+resourceURL+" reported status code 503. Please increase the crawler's sleep time.");
+        	conn.disconnect();
+        	
+        	return null;
+        } else if (conn.getResponseCode() >= 400) {
+        	// Client or server error received
+        	logger.debug("WARNING: Url "+resourceURL+" seems to be unreachable. If this url is not of importance you can ignore this error.");
+        	conn.disconnect();
+
+        	return null;
+        } else {
+	        String mimeType = conn.getContentType();
+	        
+	        logger.debug("mime type for " + resourceURL + ": " + mimeType);
+	        logger.debug("response code was: " + statusCode);
+	        conn.disconnect();
+	        return mimeType;
+        }
+	}
+	
+	private static long getSleepTime() {
+		return Configs.sleepTime + (-3 + 6 * (int)Math.random())*1000;
+	}
+
+	static public boolean checkExtension(String url) {
+		for(String type : filetypes.keySet()) {
+			if(url.matches("(.)+\\."+type))
+				return true;
+		}
+		return false;		
+	}
+	
+	static public boolean checkMimeTypeProvided(String mimeType) {
+		if(filetypes.containsValue(mimeType))
+			return true;
+		return false;
+	}
+	
+	static public boolean checkMimeType(String url) throws IOException, InterruptedException {
+		String mimeType = getMimeType(url);
+		
+		return filetypes.containsValue(mimeType);
+	}
+	
+	static public String resolveRedirections(String resourceURL) throws IOException, InterruptedException {
+		HttpURLConnection.setFollowRedirects(true);
+		Thread.sleep(getSleepTime());
+		URL url = new URL(resourceURL);
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+
+        conn.setRequestMethod("HEAD");
+		conn = openConnectionCheckRedirects(conn);
+		
+        int statusCode = conn.getResponseCode();
+        if(statusCode == 503) {
+        	logger.warn("WARNING: Url "+resourceURL+" reported status code 503. Please increase the crawler's sleep time.");
+        	conn.disconnect();
+        	return null;
+        }
+        // Client or server error received
+        if(conn.getResponseCode() >= 400) {
+        	logger.warn("WARNING: Url "+resourceURL+" seems to be unreachable. If this url is not of importance you can ignore this error.");
+        	conn.disconnect();
+        	return null;
+        }
+        
+        String ret = conn.getURL().toExternalForm();
+        
+        logger.debug("resolved url: " + ret);
+        conn.disconnect();
+        return ret;
+	}
+	
+	private static HttpURLConnection openConnectionCheckRedirects(URLConnection c)
+			throws IOException {
+		boolean redir;
+		int redirects = 0;
+		
+		do {
+			redir = false;
+			if (c instanceof HttpURLConnection) {
+				HttpURLConnection http = (HttpURLConnection) c;
+				int stat = http.getResponseCode();
+				
+				if (stat >= 300 && stat <= 307 && stat != 306
+						&& stat != HttpURLConnection.HTTP_NOT_MODIFIED) {
+					URL base = http.getURL();
+					String loc = http.getHeaderField("Location");
+					URL target = null;
+					if (loc != null) {
+						target = new URL(base, loc);
+					}
+					http.disconnect();
+					// Redirection should be allowed only for HTTP and HTTPS
+					// and should be limited to 5 redirections at most.
+					if (target == null
+							|| !(target.getProtocol().equals("http") || target.getProtocol().equals("https"))
+							|| redirects >= 5) {
+						throw new IOException("Redirection should be allowed only for HTTP and HTTPS and should be limited to 5 redirections at most.");
+					}
+					redir = true;
+					c = target.openConnection();
+					redirects++;
+				}
+			}
+		} while (redir);
+		
+		return (HttpURLConnection) c;
+	}
+
+	
+	public static void main(String[] args) throws Exception {
+		System.out.println(isUrl(args[0]));
+		System.out.println(checkExtension(args[0]));
+		String type = getMimeType(args[0]);
+		System.out.println(type);
+		System.out.println(checkMimeType(type));
+	}
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/harvester/ResourceHarvester.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/harvester/ResourceHarvester.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/harvester/ResourceHarvester.java	(revision 47463)
@@ -0,0 +1,200 @@
+package eu.dnetlib.data.utility.resource_discovery.harvester;
+
+import eu.dnetlib.data.utility.resource_discovery.crawler.Crawler;
+import eu.dnetlib.data.utility.resource_discovery.crawler.ResourceExtractor;
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Vector;
+
+import org.apache.log4j.Logger;
+import org.dlese.dpc.oai.harvester.Harvester;
+import org.dlese.dpc.oai.harvester.Hexception;
+import org.dlese.dpc.oai.harvester.OAIErrorException;
+import org.dlese.dpc.xml.XMLDoc;
+import org.dlese.dpc.xml.XMLException;
+
+public class ResourceHarvester {
+
+	private final static Logger logger = Logger.getLogger(ResourceHarvester.class);
+	static public String[][] getRecordsFromRepository(String baseUrl) throws Hexception, OAIErrorException {
+		return Harvester.harvest(baseUrl, "oai_dc", null, null, null, null, true);
+	}
+	
+	static public String[][] getRecordsFromRepository(String baseUrl, String set, Date from, Date until) throws Hexception, OAIErrorException {
+		return Harvester.harvest(baseUrl, "oai_dc", set, from, until, null, true);
+	}
+	
+	static public Vector<Vector<String>> getResourceAndLinks(String header, String oaiDcRecord, Crawler crawler, ResourceExtractor extractor) throws IOException, InterruptedException {
+		Vector<Vector<String>> retrievedAndExtracted = new Vector<Vector<String>>();
+		String id = getDcIdentifier(oaiDcRecord, header);
+		if(id !=  null) {
+			String idUrl = UrlFilter.resolveRedirections(id);
+			Vector<String> urls;
+			urls = crawler.getLinks(idUrl);
+			retrievedAndExtracted.add(urls);
+			retrievedAndExtracted.add(extractor.extractResource(urls));
+			return retrievedAndExtracted;
+		}
+		return null;
+	}
+	
+	static public String getIdentifier(String oaiDcRecord, String identifier) throws IOException {
+		XMLDoc xd = new XMLDoc();
+		try {
+		xd.useXmlString(oaiDcRecord, true, true, true);
+		}
+		catch(XMLException e) {
+			logger.debug("WARNING: The record "+identifier+" seems to be malformed (deleted maybe?)");
+			return null;
+		}
+		try {
+		String [] fields = xd.getXmlFields(1,0,"dc:identifier");
+		Vector<String> urls = new Vector<String>();
+		for(String field : fields) {
+			if(UrlFilter.isUrl(field))
+				urls.add(field);
+		}
+		fields = xd.getXmlFields(0,0,"dc:source");
+		for(String field : fields) {
+			if(UrlFilter.isUrl(field))
+				urls.add(field);
+		}
+		fields = xd.getXmlFields(0,0,"dc:relation");
+		for(String field : fields) {
+			if(UrlFilter.isUrl(field))
+				urls.add(field);
+		}
+		if(urls.size() == 0) {
+			logger.debug("WARNING: The record "+identifier+" does not seem to have a field that is a url");
+			return null;
+		}
+		if(urls.size() == 1)
+			return urls.elementAt(0);
+		for(String url : urls) {
+			if(UrlFilter.checkExtension(url))
+				return url;
+		}
+		logger.debug("WARNING: The record "+identifier+" has multiple fields with valid urls and there is no way to choose one. The first one will be used");
+		return urls.elementAt(0);
+		}
+		catch(XMLException e) {
+			logger.debug("WARNING: The record "+identifier+" does not seem to have a dc:identifier field");
+			return null;
+		}
+	}
+	
+	static public String getDcIdentifier(String oaiDcRecord, String identifier) throws IOException {
+		XMLDoc xd = new XMLDoc();
+		try {
+		xd.useXmlString(oaiDcRecord, true, true, true);
+		}
+		catch(XMLException e) {
+			logger.debug("WARNING: The record "+identifier+" seems to be malformed (deleted maybe?)");
+			return null;
+		}
+		try {
+		String [] fields = xd.getXmlFields(1,0,"dc:identifier");
+		Vector<String> urls = new Vector<String>();
+		for(String field : fields) {
+			if(UrlFilter.isUrl(field))
+				urls.add(field);
+		}
+		if(urls.size() == 0) {
+			logger.debug("WARNING: The record "+identifier+" does not seem to have a field that is a url");
+			return null;
+		}
+		if(urls.size() == 1)
+			return urls.elementAt(0);
+		for(String url : urls) {
+			if(UrlFilter.checkExtension(url))
+				return url;
+		}
+		logger.debug("WARNING: The record "+identifier+" has multiple fields with valid urls and there is no way to choose one. The first one will be used");
+		return urls.elementAt(0);
+		}
+		catch(XMLException e) {
+			logger.debug("WARNING: The record "+identifier+" does not seem to have a dc:identifier field");
+			return null;
+		}
+	}
+	
+	@SuppressWarnings("deprecation")
+	public static void main(String[] args) {
+		if(args.length == 0) {
+			System.err.println("Usage:\t program_name baseUrl [-from day:month:year] [-until day:month:year] [-set setName]");
+			return;
+		}
+		String baseUrl = args[0];
+		String set = null;
+		Date from = null, until = null;
+		for(int i=1; i<args.length; i++) {
+			if(args[i].equals("-set") && i+1<args.length) {
+				set = args[i+1];
+				i++;
+			}
+			else if(args[i].equals("-from") && i+1<args.length) {
+				String[] fields = args[i+1].split(":");
+				i++;
+				if(fields.length != 3) {
+					System.err.println("The date format for the -from field provided is incorrect");
+					return;
+				}
+				from = new Date(Integer.parseInt(fields[2])-1900, Integer.parseInt(fields[1])-1, Integer.parseInt(fields[0]));
+			}
+			else if(args[i].equals("-until") && i+1<args.length) {
+				String[] fields = args[i+1].split(":");
+				i++;
+				if(fields.length != 3) {
+					System.err.println("The date format for the -until field provided is incorrect");
+					return;
+				}
+				until = new Date(Integer.parseInt(fields[2])-1900, Integer.parseInt(fields[1])-1, Integer.parseInt(fields[0]));
+			}
+			else {
+				System.err.println("The arguments you specified are incorrect");
+				return;
+			}
+		}
+		String[][] records;
+		try {
+			records = getRecordsFromRepository(baseUrl, set, from, until);
+		}
+		catch(Exception e) {
+			System.err.println(e.getLocalizedMessage());
+			e.printStackTrace();
+			return;
+		}
+		Crawler crawler;
+		try {
+		crawler = new Crawler();
+		}
+		catch(Exception e) {
+			System.err.println("FATAL ERROR: Crawler could not be configured. Please check your robot.xml parameters and try again.");
+			System.err.println(e.getLocalizedMessage());
+			e.printStackTrace();
+			return;			
+		}
+		ResourceExtractor extractor = new ResourceExtractor();
+		for(int i=0; i<records.length; i++) {
+			String id = "";
+			try {
+			id = getDcIdentifier(records[i][1], records[i][0]);
+			if(id !=  null) {
+				String idUrl = UrlFilter.resolveRedirections(id);
+				System.out.println("Now processing " + idUrl);
+				Vector<String> urls = crawler.getLinks(idUrl);
+				System.out.println("Retrieved links are: "+ urls);
+				System.out.println("Resources seem to be available in: " + extractor.extractResource(urls));
+				System.out.println();
+			}
+			}
+			catch(Exception e) {
+				System.err.println("WARNING: Record "+id+" couldn't be processed");
+				continue;				
+			}
+		}
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/HttpFilter.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/HttpFilter.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/HttpFilter.java	(revision 47463)
@@ -0,0 +1,60 @@
+package eu.dnetlib.data.utility.resource_discovery.plugin.crawler;
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+import net.matuschek.http.AbstractHttpDocManager;
+import net.matuschek.http.DocManagerException;
+import net.matuschek.http.HttpDoc;
+import net.matuschek.http.HttpHeader;
+
+public class HttpFilter extends AbstractHttpDocManager {
+	  private final static Logger logger = Logger.getLogger(HttpFilter.class);
+
+	  private ArrayList<ResourceObject> objects;
+
+	  public HttpFilter() {
+	    this.objects = new ArrayList<ResourceObject>();
+	  }
+
+	  public void processDocument(HttpDoc doc) 
+	    throws DocManagerException 
+	  {
+		  try {
+			  logger.debug("Getting mimeType from "+doc.getURL()+" which is "+doc.getHeaderValue(HttpHeader.CONTENT_TYPE));
+			  if(UrlFilter.checkMimeTypeProvided(doc.getHeaderValue(HttpHeader.CONTENT_TYPE))) {
+				  logger.debug("Adding to resourceUrls");
+				  ResourceObject obj = new ResourceObject();
+				  obj.setUrl(doc.getURL().toString());
+				  if(doc.getContent().length < 51200) {
+					  URL url  = new URL(doc.getURL().toString());
+					  InputStream is = url.openStream();
+					  ByteArrayOutputStream bos = new ByteArrayOutputStream();
+					  int nextChar;
+					  while ((nextChar=is.read()) != -1)
+						  bos.write(nextChar);
+					  obj.setContent(bos.toByteArray());
+				  }
+				  else
+					  obj.setContent(doc.getContent());
+				  objects.add(obj);
+			  }
+		  }
+		  catch(Exception e) {
+			  throw new DocManagerException(e.toString());
+		  }
+	  }
+	  
+	  public List<ResourceObject> getResources() {
+		  return this.objects;
+	  }
+		
+	
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/ResourceDiscoverer.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/ResourceDiscoverer.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/ResourceDiscoverer.java	(revision 47463)
@@ -0,0 +1,108 @@
+package eu.dnetlib.data.utility.resource_discovery.plugin.crawler;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.List;
+
+import net.matuschek.http.HttpException;
+import net.matuschek.spider.WebRobot;
+
+import org.apache.log4j.Logger;
+
+import eu.dnetlib.data.utility.resource_discovery.crawler.config.Configs;
+import eu.dnetlib.data.utility.resource_discovery.harvester.ResourceHarvester;
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+public class ResourceDiscoverer {
+	private final static Logger logger = Logger.getLogger(ResourceDiscoverer.class);
+	private WebRobot crawler;
+
+	public ResourceDiscoverer() throws IOException, HttpException {
+		crawler = new WebRobot();
+		Configs.configureCrawler(crawler);
+	}
+	
+	public List<ResourceObject> getResources(String jumpOffPageUrl) throws MalformedURLException, IOException, InterruptedException {
+		logger.debug("Retrieving links from url "+jumpOffPageUrl);
+		crawler.setStartURL(new URL(UrlFilter.resolveRedirections(jumpOffPageUrl)));
+		HttpFilter manager = new HttpFilter();
+	    crawler.setDocManager(manager);
+
+	    crawler.run();
+	    
+	    logger.debug("Resources are available in "+manager.getResources());
+	    return manager.getResources();
+	}
+	
+	public static void main(String[] args) {
+		String record = "";
+		if(args.length < 2) {
+			System.err.println("Arguments must be (one of them):");
+			System.err.println("1) path -f filename");
+			System.err.println("2) path xml_content");
+			System.exit(-1);
+		}
+		else if(args[1].equals("-f") && (args.length  >= 3)) {
+			File f = new File(args[2]);
+			try {
+				FileReader rd = new FileReader(f);
+				BufferedReader buf = new BufferedReader(rd);
+				String line;
+				while( (line=buf.readLine()) != null) {
+					record += line;
+				}
+			}
+			catch(Exception e) {
+				System.err.println("File "+args[1]+" could not be processed");
+				System.exit(-2);
+			}
+		}
+		else if(!args[1].equals("-f")) {
+			record = args[1];
+		}
+		else {
+			System.err.println("Arguments must be (one of them):");
+			System.err.println("1) -f filename");
+			System.err.println("2) xml_content");
+			System.exit(-1);
+		}
+		String unresolvedUrl=null;
+		String path = args[0];
+		try {
+			unresolvedUrl = ResourceHarvester.getIdentifier(record, "");
+		}
+		catch(Exception e) {
+			System.err.println("Error while trying to determine the dc:identifier field: "+e);
+			System.exit(-3);
+		}
+		if (unresolvedUrl != null) {
+			String jumpOffPageUrl="";
+			try {
+				jumpOffPageUrl = UrlFilter.resolveRedirections(unresolvedUrl);
+				ResourceDiscoverer discoverer = new ResourceDiscoverer();
+				List<ResourceObject> resources = discoverer.getResources(jumpOffPageUrl);
+			
+				int i=0;
+				for(ResourceObject obj : resources) {
+					i++;
+					System.out.println(obj.getUrl());
+					BufferedOutputStream os = 
+						  new BufferedOutputStream(new FileOutputStream(path+"/"+i+".pdf"));
+					os.write(obj.getContent());
+					os.flush();
+					os.close();
+				}
+			}
+			catch(Exception e) {
+				System.err.println("Error while trying to download resources from "+jumpOffPageUrl+" : "+e);
+				System.exit(-3);
+			}
+		}
+	}
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/ResourceObject.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/ResourceObject.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/crawler/ResourceObject.java	(revision 47463)
@@ -0,0 +1,23 @@
+package eu.dnetlib.data.utility.resource_discovery.plugin.crawler;
+
+public class ResourceObject {
+
+	private String url;
+	private byte[] content;
+	
+	public void setUrl(String url) {
+		this.url = url;
+	}
+	
+	public void setContent(byte[] content) {
+		this.content = content;
+	}
+	
+	public String getUrl() {
+		return this.url;
+	}
+	
+	public byte[] getContent() {
+		return this.content;
+	}
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceDescriptionRecord.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceDescriptionRecord.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceDescriptionRecord.java	(revision 47463)
@@ -0,0 +1,24 @@
+package eu.dnetlib.data.utility.resource_discovery.plugin;
+
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlAccessType;
+
+import eu.dnetlib.data.utility.featureextraction.record.DescriptionRecord;
+
+@XmlRootElement(namespace = "", name = "descriptionRecord")
+@XmlAccessorType(XmlAccessType.NONE)
+public class ResourceDescriptionRecord extends DescriptionRecord {
+
+	@XmlElement(name = "resourceUrls", type = ResourceUrls.class)
+	private ResourceUrls resourceUrls;
+	
+	public void setResourceUrls(ResourceUrls resourceUrls) {
+		this.resourceUrls = resourceUrls;
+	}
+	
+	public ResourceUrls getResourceUrls() {
+		return this.resourceUrls;
+	}
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceUrls.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceUrls.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceUrls.java	(revision 47463)
@@ -0,0 +1,23 @@
+package eu.dnetlib.data.utility.resource_discovery.plugin;
+
+import java.util.List;
+ 	
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+
+@XmlRootElement(name = "resourceUrls")
+public class ResourceUrls {
+
+	private List<String> urls;
+	
+	public void setUrls(List<String> urls) {
+		this.urls = urls;
+	}
+	
+	@XmlElement(name = "url")
+	public List<String> getUrls() {
+		return this.urls;
+	}
+	
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceDiscoveryPlugin.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceDiscoveryPlugin.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/plugin/ResourceDiscoveryPlugin.java	(revision 47463)
@@ -0,0 +1,82 @@
+package eu.dnetlib.data.utility.resource_discovery.plugin;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+import org.dlese.dpc.xml.XMLDoc;
+
+import eu.dnetlib.common.utils.XMLSerializer;
+import eu.dnetlib.data.utility.featureextraction.FeatureExtractionException;
+import eu.dnetlib.data.utility.featureextraction.plugin.AbstractPlugin;
+import eu.dnetlib.data.utility.resource_discovery.harvester.ResourceHarvester;
+import eu.dnetlib.data.utility.resource_discovery.plugin.crawler.ResourceDiscoverer;
+import eu.dnetlib.data.utility.resource_discovery.plugin.crawler.ResourceObject;
+import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;
+
+public class ResourceDiscoveryPlugin extends AbstractPlugin {
+	private final static Logger log = Logger
+			.getLogger(ResourceDiscoveryPlugin.class);
+	private static XMLSerializer<ResourceDescriptionRecord> handler = new XMLSerializer<ResourceDescriptionRecord>(ResourceDescriptionRecord.class);
+
+	@Override
+	public void execute() throws FeatureExtractionException {
+		String srcRecord = null;
+		try {
+			while ((srcRecord = this.nextRecord()) != null) {
+				XMLDoc xd = new XMLDoc();
+				xd.useXmlString(srcRecord, true, true, true);
+				String [] fields = xd.getXmlFields(1,0,"dri:objIdentifier");
+				String objIdentifier = fields[0];
+				// get the resource or jump-off-page from the DMF record
+				String unresolvedUrl = ResourceHarvester.getIdentifier(
+						srcRecord, objIdentifier);
+				// if the record has a dc:identifier that is a web-page (jump-off page or directly pointing to a resource)
+				if (unresolvedUrl != null) {
+					// resolve any redirections
+					String jumpOffPageUrl = UrlFilter
+							.resolveRedirections(unresolvedUrl);
+					log.debug("Now processing " + jumpOffPageUrl);
+					// ResourceDiscoverer will crawl the jump-off-page and return a list of ResourceObject
+					// Each ResourceObject contains the url of the resource (pdf) that was found inside the jump-off-page
+					// and the content of the actual resource as an array of byte[]
+					ResourceDiscoverer discoverer = new ResourceDiscoverer();
+					List<ResourceObject> resources = discoverer.getResources(jumpOffPageUrl);
+					
+					// prepare the description record that will be added to the resultDAO
+					ResourceDescriptionRecord descrRecord = new ResourceDescriptionRecord();
+					descrRecord.setObjectIdentifier(objIdentifier);
+					ArrayList<String> urlList = new ArrayList<String>();
+					// For now, nothing is done for the resource content, only the url is returned
+					for(ResourceObject obj : resources)
+						urlList.add(obj.getUrl());
+
+					ResourceUrls rUrls = new ResourceUrls();
+					rUrls.setUrls(urlList);
+					descrRecord.setResourceUrls(rUrls);
+
+					String[] record = {handler.getAsXml(descrRecord)};
+					log.debug("Returning "+Arrays.asList(record));
+					this.dao.addResults(Arrays.asList(record));
+				}
+				// if the record has no dc:identifier field, then return a place-holder record, containing only the obj-identifier
+				else {
+					ResourceDescriptionRecord descrRecord = new ResourceDescriptionRecord();
+					descrRecord.setObjectIdentifier(objIdentifier);
+					
+					String[] record = {handler.getAsXml(descrRecord)};
+					log.debug("Returning "+Arrays.asList(record));
+					this.dao.addResults(Arrays.asList(record));
+				}
+			}
+		} catch (Throwable e) {
+			log.error(e);
+			this.properties.setException(e);
+			throw new FeatureExtractionException(e);
+		} finally {
+			isFinished = true;
+		}
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/filetypes.txt
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/filetypes.txt	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/filetypes.txt	(revision 47463)
@@ -0,0 +1 @@
+pdf
\ No newline at end of file
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/testing/html_viewer/HtmlViewer.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/testing/html_viewer/HtmlViewer.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/testing/html_viewer/HtmlViewer.java	(revision 47463)
@@ -0,0 +1,34 @@
+package eu.dnetlib.data.utility.resource_discovery.testing.html_viewer;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * Used for testing purposes when unexpected behaviour is encountered on the JumpOffPageExpression.
+ * Unexpected behaviour is many times due to html defects.
+ * @author manos.karvounis
+ *
+ */
+
+public class HtmlViewer {
+
+	public static void main(String[] args) throws Exception {
+        HttpURLConnection.setFollowRedirects(false);
+		URL url;
+		if(args.length > 0)
+			url = new URL(args[0]);
+		else
+			url = new URL("http://docserv.uni-duesseldorf.de/servlets/DerivateServlet/Derivate-68");
+		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+        String line;
+        while((line = rd.readLine()) != null)
+        	System.out.println(line);
+        System.out.println(conn.getContentType());
+        rd.close();
+        conn.disconnect();
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/testing/validator/Validate.java
===================================================================
--- modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/testing/validator/Validate.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/src/main/eu/dnetlib/data/utility/resource_discovery/testing/validator/Validate.java	(revision 47463)
@@ -0,0 +1,280 @@
+package eu.dnetlib.data.utility.resource_discovery.testing.validator;
+/*
+import gr.uoa.di.crawler.Crawler;
+import gr.uoa.di.crawler.ResourceExtractor;
+import gr.uoa.di.oai.harvester.ResourceHarvester;
+
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Vector;
+
+import org.dlese.dpc.xml.XMLDoc;
+import org.dlese.dpc.xml.XMLException;
+*/
+
+
+public class Validate {
+/*	
+	static private int RECORDS_FOR_VALIDATION;
+	static private int RECORDS_BEFORE_RETRY;
+	
+	private boolean fatalErrorOccured;
+	private String[][] records;
+	private Vector<Integer> successful;
+	
+	static {
+		try {
+		XMLDoc xd = new XMLDoc("configs/validator.xml", true, true, true);
+		String[] ret1 = xd.getXmlFields(0,1,"RecordsForValidation");		
+		String [] ret2 = xd.getXmlFields(0,1,"RecordsBeforeRetry");
+		if(ret1.length > 0)
+			RECORDS_FOR_VALIDATION = Integer.parseInt(ret1[0]);
+		else
+			RECORDS_FOR_VALIDATION = 20;
+		if(ret2.length > 0)
+			RECORDS_BEFORE_RETRY = Integer.parseInt(ret2[0]);
+		else
+			RECORDS_BEFORE_RETRY = 4;
+		}
+		catch(XMLException e) {
+			System.err.println(e.getLocalizedMessage());
+			System.err.println("WARNING: The file validator.xml seems to be malformed. The default settings will be used for the validator.");
+			RECORDS_FOR_VALIDATION = 20;
+			RECORDS_BEFORE_RETRY = 4;
+		}
+		catch(NumberFormatException e) {
+			System.err.println("WARNING: The file validator.xml seems to be malformed (an integer doesn't seem to be of type integer). The default settings will be used for the validator.");
+			RECORDS_FOR_VALIDATION = 20;
+			RECORDS_BEFORE_RETRY = 4;
+		}
+	}
+	
+	public Validate() {
+		fatalErrorOccured = false;
+		successful = new Vector<Integer>(); 
+	}
+	
+	private void logError(String baseUrl, String msg, String type, boolean fatal) {
+		System.out.println("Error of type \""+type+"\" occured: "+msg);
+		if(fatal) {
+			this.fatalErrorOccured = true;
+			System.out.println("The process cannot continue. Please resolve the reported errors and try again.");
+		}
+		else {
+			System.out.println("The error is not fatal though, and the process can safely conntinue");
+		}
+	}
+	
+	public boolean validateRepository(String baseUrl, String set, Date from, Date until) {
+		int numOfRecords;
+		if(records.length < (RECORDS_FOR_VALIDATION/2)) {
+			logError(baseUrl, "The number of records retrieved are insufficient for an accurate validation of the repository", "too few records retrieved (2)", true);
+			return false;
+		}
+		else if(records.length < RECORDS_FOR_VALIDATION) {
+			logError(baseUrl, "The number of records retrieved are insufficient for an accurate validation of the repository", "too few records retrieved", false);
+			numOfRecords = records.length;
+		}
+		else
+			numOfRecords = RECORDS_FOR_VALIDATION;
+		Crawler crawler;
+		ResourceExtractor extractor;
+		try {
+		crawler = new Crawler(true);
+		extractor = new ResourceExtractor();
+		}
+		catch(Exception e) {
+			logError(baseUrl, "An internal error occured. Please notify the administrator and report the following error mesage:", "internal error", true);
+			System.out.println("Start of error message");
+			System.out.println(e.getLocalizedMessage());
+			e.printStackTrace();
+			System.out.println("End of error message\n");
+			return false;
+		}
+		int conEmpties = 0;
+		int exceptions = 0;
+		int successes = 0;
+		int i = 0;
+		// Ama teleiwseis eksatias tou i == records.length, diladi sou teleiwsan ta records, tote olh
+		// h diadikasia einai anepitixis
+		while(successes < numOfRecords && i < records.length) {
+			if(exceptions >= RECORDS_BEFORE_RETRY + 1) {
+				logError(baseUrl, "Too many unprocessable records. The oai_dc format used might be malformed", "many unprocessable records", true);
+				return false;
+			}
+			System.out.println("Processing record with id "+records[i][0]);
+			Vector<Vector<String>> retrievedAndExtracted = new Vector<Vector<String>>();
+			try {
+			retrievedAndExtracted = ResourceHarvester.getResourceAndLinks(records[i][0], records[i][1], crawler, extractor);
+			// Sou irthe ena record to opoio den eixe dc:identifier. Aposiopise to, mias kai
+			// pithanotata einai deleted record
+			if(retrievedAndExtracted == null) {
+				i++;
+				continue;
+			}
+			}
+			// To na sou erthei omws record pou den egine process einai provlima ama ginei sixna
+			catch(Exception e) {
+				logError(baseUrl, "Resource with id "+records[i][0]+" could not be processed", "record processing error", false);
+				exceptions++;
+				i++;
+				continue;
+			}
+			System.out.println("Retrieved: "+retrievedAndExtracted.elementAt(0));
+			System.out.println("Resources: "+retrievedAndExtracted.elementAt(1));
+			if(retrievedAndExtracted.elementAt(1).isEmpty()) {
+				conEmpties++;
+				// Tha ta metriseis gia successes, alla den tha ta valeis sto successful
+				// Afto gia na petixeis anoxh apenanti se pages xwris resource pou mas endiaferei,
+				// alla sto retry thes apo tetoia na ksekinas
+				successes++;
+			}
+			else {
+				conEmpties = 0;
+				successes++;
+			}
+			// edw
+			if(!retry && conEmpties >= RECORDS_BEFORE_RETRY) {
+				logError(baseUrl, "Resources could not be retrieved for "+RECORDS_BEFORE_RETRY+" consecutive records. The process will restart with different settings. If the problem persists, the validation process will fail.", "consecutive empty records", false);
+				return false;
+			}
+			if(retry && conEmpties >= RECORDS_BEFORE_RETRY) {
+				logError(baseUrl, "Resources could not be retrieved for "+RECORDS_BEFORE_RETRY+" consecutive records, despite the different settings used. The validation process failed.\n"+
+						"Possible reasons for this behaviour include (but are not limited to):\n"+
+						"1. Use of javascript for the retrieval of the resource. Javascript cannot be easily parsed by bots, so only html is supported (as well as all technologies that send html files to the client, ie php, jsp etc)\n"+
+						"2. Filetype mismatch. The mime types of the resources that your repository offers are either incorrect or not supported\n"+
+						"3. Bots are not allowed to crawl the resource page (due to the robots.txt file)\n"+
+						"4. The resource is present in more than two steps (clicks) away from the jump-off page included in the dc:identifier field. This might be due to the use of frames (they count as one step)\n", "consecutive empty records (2)", true);
+				return false;
+				
+			}
+		}
+		return true;
+	}
+	
+	public boolean validateRepositoryRetry(String baseUrl, String set, Date from, Date until) {
+		int numOfRecords;
+		if(records.length < RECORDS_FOR_VALIDATION) {
+			logError(baseUrl, "The number of records retrieved are insufficient for an accurate validation of the repository", "too few records retrieved", false);
+			numOfRecords = records.length;
+		}
+		else
+			numOfRecords = RECORDS_FOR_VALIDATION;
+		Crawler crawler;
+		ResourceExtractor extractor;
+		try {
+		crawler = new Crawler(true);
+		if(retry)
+			crawler.reconfigureForRetry();
+		extractor = new ResourceExtractor();
+		}
+		catch(Exception e) {
+			System.out.println("An internal error occured. Please notify the administrator and report the following error mesage:");
+			System.out.println("Start of error message");
+			System.out.println(e.getLocalizedMessage());
+			e.printStackTrace();
+			System.out.println("End of error message\n");
+			return false;
+		}
+		int empties = 0;
+		int exceptions = 0;
+		for(int i=0; i<numOfRecords; i++) {
+			System.out.println("Processing record with id "+records[i][0]);
+			Vector<Vector<String>> retrievedAndExtracted = new Vector<Vector<String>>();
+			try {
+			retrievedAndExtracted = ResourceHarvester.getResourceAndLinks(records[i][0], records[i][1], crawler, extractor);
+			if(retrievedAndExtracted == null)
+				continue;
+			}
+			catch(Exception e) {
+				logError(baseUrl, "Resource with id "+records[i][0]+" could not be processed", "record processing error", false);
+				exceptions++;
+				continue;
+			}
+			System.out.println("Retrieved: "+retrievedAndExtracted.elementAt(0));
+			System.out.println("Resources: "+retrievedAndExtracted.elementAt(1));
+			if(retrievedAndExtracted.elementAt(1).isEmpty()) {
+				empties++;
+			}
+			else
+				empties = 0;
+			if(!retry && empties >= RECORDS_BEFORE_RETRY) {
+				logError(baseUrl, "Resources could not be retrieved for "+RECORDS_BEFORE_RETRY+" consecutive records. The process will restart with different settings. If the problem persists, the validation process will fail.", "consecutive empty records", false);
+				return false;
+			}
+			if(retry && empties >= RECORDS_BEFORE_RETRY) {
+				logError(baseUrl, "Resources could not be retrieved for "+RECORDS_BEFORE_RETRY+" consecutive records, despite the different settings used. The validation process failed.\n"+
+						"Possible reasons for this behaviour include (but are not limited to):\n"+
+						"1. Use of javascript for the retrieval of the resource. Javascript cannot be easily parsed by bots, so only html is supported (as well as all technologies that send html files to the client, ie php, jsp etc)\n"+
+						"2. Filetype mismatch. The mime types of the resources that your repository offers are either incorrect or not supported\n"+
+						"3. Bots are not allowed to crawl the resource page (due to the robots.txt file)\n"+
+						"4. The resource is present in more than two steps (clicks) away from the jump-off page included in the dc:identifier field. This might be due to the use of frames (they count as one step)\n", "consecutive empty records (2)", true);
+				return false;
+				
+			}
+		}
+		return true;
+	}
+
+	@SuppressWarnings("deprecation")
+	public static void main(String[] args) {
+		if(args.length == 0) {
+			System.err.println("Usage:\t program_name baseUrl [-from day:month:year] [-until day:month:year] [-set setName]");
+			return;
+		}
+		String baseUrl = args[0];
+		String set = null;
+		Date from = null, until = null;
+		for(int i=1; i<args.length; i++) {
+			if(args[i].equals("-set") && i+1<args.length) {
+				set = args[i+1];
+				i++;
+			}
+			else if(args[i].equals("-from") && i+1<args.length) {
+				String[] fields = args[i+1].split(":");
+				i++;
+				if(fields.length != 3) {
+					System.err.println("The date format for the -from field provided is incorrect");
+					return;
+				}
+				from = new Date(Integer.parseInt(fields[2])-1900, Integer.parseInt(fields[1])-1, Integer.parseInt(fields[0]));
+			}
+			else if(args[i].equals("-until") && i+1<args.length) {
+				String[] fields = args[i+1].split(":");
+				i++;
+				if(fields.length != 3) {
+					System.err.println("The date format for the -until field provided is incorrect");
+					return;
+				}
+				until = new Date(Integer.parseInt(fields[2])-1900, Integer.parseInt(fields[1])-1, Integer.parseInt(fields[0]));
+			}
+			else {
+				System.err.println("The arguments you specified are incorrect");
+				return;
+			}
+		}
+		Validate validator = new Validate();
+		Calendar cal1 = Calendar.getInstance();
+		System.out.println(cal1.getTime());
+		try {
+		validator.records = ResourceHarvester.getRecordsFromRepository(baseUrl, set, from, until);
+		}
+		catch(Exception e) {
+		validator.logError(baseUrl, e.getLocalizedMessage(), "repository fatal error", true);
+		System.out.println(false);
+		Calendar cal2 = Calendar.getInstance();
+	    System.out.println(cal2.getTime());
+		return;
+		}
+		boolean success = validator.validateRepository(baseUrl, set, from, until, false);
+		System.out.println(success);
+		if(!success && !validator.fatalErrorOccured) {
+			success = validator.validateRepository(baseUrl, set, from, until, true);
+			System.out.println(success);
+		}
+		Calendar cal2 = Calendar.getInstance();
+	    System.out.println(cal2.getTime());
+	}
+	*/
+
+}
Index: modules/uoa-resource-discovery/trunk/build/ivy.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/build/ivy.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/build/ivy.xml	(revision 47463)
@@ -0,0 +1,31 @@
+<ivy-module version="1.0">
+	<info organisation="driver" module="uoa-resource-discovery"
+		status="integration" />
+	<configurations defaultconfmapping="*->default">
+		<conf name="default" />
+		<conf name="junit" visibility="private" extends="default" />
+	</configurations>
+	<publications>
+		<artifact name="uoa-resource-discovery" type="jar" />
+		<artifact name="uoa-resource-discovery" type="src" ext="zip" />
+	</publications>
+	<dependencies>
+		<dependency org="apache" name="commons-logging" rev="1.0.4" />
+		<dependency org="org.w3c" name="tidy" rev="+" />
+		<dependency name="log4j" org="log4j" rev="+"/>
+		<dependency org="junit" name="junit" rev="+" conf="junit"/>
+		
+		<dependency org="net.matuschek" name="jobo" rev="1.4+" />
+		<dependency org="DLS" name="jOAI" rev="2.0.9.3+" />
+		
+		<dependency org="driver" name="unibi-commons" rev="+" />
+		<dependency org="driver" name="unibi-data-utility-featureextraction-plugins" rev="+" />
+		
+		<dependency org="com.thoughtworks" name="xstream" rev="+" />
+		<dependency org="org.cyberneko" name="nekohtml" rev="+" />
+		<dependency org="com.jira" name="heritrix-commons" rev="+" />
+		<dependency org="com.jira" name="heritrix-modules" rev="+" />
+		<dependency org="org.kryo" name="kryo" rev="+" />
+		<dependency org="edu.indiana" name="xpp3" rev="+" />
+	</dependencies>
+</ivy-module>
Index: modules/uoa-resource-discovery/trunk/build/build.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/build/build.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/build/build.xml	(revision 47463)
@@ -0,0 +1,26 @@
+<project name="uoa-resource-discovery" default="build">
+	<!--
+	organisation:		driver
+	module: 		uoa-resource-discovery
+	-->
+
+	<property file="local.properties"/>
+
+	<!-- Layout paths properties -->
+	<property file="../../build/yvy-setup.properties"/>
+	<property file="../../../../build/yvy-trunk.properties"/>
+	<property file="../../../../../build/yvy-tag.properties"/>
+
+	<property file="build.properties"/>
+	<property name="yvy.build.common" value="${yvy.root.dir}/build/build-common.xml"/>
+
+	<!-- Common build definitions -->
+	<import file="${yvy.build.common}"/>
+	
+	<target name="build" depends="jar"/>
+
+	<target name="default">
+		<antcall target="${yvy.target.build}" inheritAll="true" inheritRefs="true"/>
+	</target>
+
+</project>
Index: modules/uoa-resource-discovery/trunk/build
===================================================================
--- modules/uoa-resource-discovery/trunk/build	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/build	(revision 47463)

Property changes on: modules/uoa-resource-discovery/trunk/build
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,2 ##
+out
+reports
Index: modules/uoa-resource-discovery/trunk/test/junit/TestXML.java
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/TestXML.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/TestXML.java	(revision 47463)
@@ -0,0 +1,28 @@
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.log4j.BasicConfigurator;
+import org.junit.Test;
+
+import eu.dnetlib.common.utils.*;
+import eu.dnetlib.data.utility.resource_discovery.plugin.ResourceDescriptionRecord;
+import eu.dnetlib.data.utility.resource_discovery.plugin.ResourceUrls;
+public class TestXML {
+	
+	@Test
+	public void TestCrawlingAndExtraction() throws Exception {
+		BasicConfigurator.configure();
+		
+		XMLSerializer<ResourceDescriptionRecord> handler = new XMLSerializer<ResourceDescriptionRecord>(ResourceDescriptionRecord.class);
+		ResourceDescriptionRecord descrRecord = new ResourceDescriptionRecord();
+		descrRecord.setObjectIdentifier("0239184018501");
+		List<String> urlList = Arrays.asList("res1", "res2");
+
+		ResourceUrls urls = new ResourceUrls();
+		urls.setUrls(urlList);
+		descrRecord.setResourceUrls(urls);
+		
+		System.out.println(handler.getAsXml(descrRecord));
+	}
+	
+}
Index: modules/uoa-resource-discovery/trunk/test/junit/MySourceDataProvider.java
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/MySourceDataProvider.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/MySourceDataProvider.java	(revision 47463)
@@ -0,0 +1,44 @@
+import java.util.ArrayList;
+import java.util.List;
+
+import eu.dnetlib.data.utility.featureextraction.FeatureExtractionException;
+import eu.dnetlib.data.utility.featureextraction.dataprovider.SourceDataProvider;
+
+
+public class MySourceDataProvider implements SourceDataProvider {
+	
+	ArrayList<String> DMFRecords;
+	
+	public MySourceDataProvider() {
+		this.DMFRecords = new ArrayList<String>();
+	}
+	
+	public void addDMFRecord(String record) {
+		DMFRecords.add(record);
+	}
+
+	@Override
+	public List<String> getRecords(int from, int to)
+			throws FeatureExtractionException {
+		try {
+			if(to >= this.DMFRecords.size())
+				to = this.DMFRecords.size();
+			return DMFRecords.subList(from-1, to);
+		}
+		catch(Exception e) {
+			throw new FeatureExtractionException(e);
+		}
+	}
+
+	@Override
+	public int getSize() throws FeatureExtractionException {
+		return this.DMFRecords.size();
+	}
+
+	@Override
+	public String getStatus() throws FeatureExtractionException {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+}
Index: modules/uoa-resource-discovery/trunk/test/junit/record4.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/record4.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/record4.xml	(revision 47463)
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>  
+  <record rank="0.45643544">
+
+    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">
+      <header>
+        <dri:objIdentifier>3297df8d-c100-44a2-8aa9-64729c406e05_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:www.tara.tcd.ie:2262/32968</dri:objIdentifier>
+        <dri:dateOfCollection>2010-01-05T18:27:02Z</dri:dateOfCollection>
+      </header>
+      <metadata>
+        <dr:CobjContentSynthesis/>
+        <dr:CobjTypology>Textual</dr:CobjTypology>
+
+        <dr:CobjIdentifier>Wilson, S., Flood, B., Goyal, S., Mosher, J., Bergin, S., O'Brien, J., Kennedy, R. ...Parameter estimation for a model with both imperfect test and repair... in Proceedings of the IEEE VLSI Test Symposium, Berkeley, CA, 6-10 May 2007, IEEE, 2007, pp 271-276</dr:CobjIdentifier>
+        <dr:CobjIdentifier>Y</dr:CobjIdentifier>
+        <dr:CobjIdentifier>Y</dr:CobjIdentifier>
+        <dr:CobjModel>OAI</dr:CobjModel>
+        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>
+        <dr:CobjDescriptionSynthesis/>
+
+        <dr:repositoryName>TARA</dr:repositoryName>
+        <dr:repositoryLink>http://www.tara.tcd.ie/</dr:repositoryLink>
+        <dr:repositoryCountry>IE</dr:repositoryCountry>
+        <dr:repositoryInstitution/>
+        <dc:creator>WILSON, SIMON PAUL</dc:creator>
+        <dc:title>Parameter estimation for a model with both imperfect test and repair</dc:title>
+
+        <dc:subject>Statistics</dc:subject>
+        <dr:CobjCategory>0004</dr:CobjCategory>
+        <dc:language>eng</dc:language>
+        <dc:dateAccepted>2009-09-18</dc:dateAccepted>
+        <dc:identifier>http://hdl.handle.net/2262/32968</dc:identifier>
+
+        <dc:publisher>IEEE</dc:publisher>
+        <dc:source/>
+        <dc:contributor/>
+        <dc:relation/>
+        <dc:description>The involvement of BLI researchers is supported by a grant from the Industrial Development Agency of Ireland. The involvement of CTVR researchers is supported by Science Foundation Ireland grant 03/CE3/I405.</dc:description>
+        <dc:description>We describe estimation of the parameters of a manufacturing test and repair model using data available from that test. The model allows imperfect testing and imperfect repair. The principal problem that we address is of parameter identification, given insufficient data, that we address by making conservative assumptions on the property being measured and the associated parameter values. Several cases of commonly occurring test types, in the manufacture of electronic products, are considered.</dc:description>
+        <dc:description>PUBLISHED</dc:description>
+
+      </metadata>
+    </result>
+  </record>
Index: modules/uoa-resource-discovery/trunk/test/junit/record7.xml
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/record7.xml	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/record7.xml	(revision 47463)
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>  
+  <record rank="0.42426404">
+    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">
+      <header>
+        <dri:objIdentifier>9dffbf71-6914-40fe-b110-8e41a977ba90_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dro.dur.ac.uk.OAI2:1691</dri:objIdentifier>
+        <dri:dateOfCollection>2009-12-29T22:13:17Z</dri:dateOfCollection>
+
+      </header>
+      <metadata>
+        <dr:CobjContentSynthesis/>
+        <dr:CobjTypology>Textual</dr:CobjTypology>
+        <dr:CobjIdentifier>Gott, R. and Roberts, R. (2004) 'A written test for procedural understanding : a way forward for assessment in the UK science curriculum ?', Research in science &amp;amp;amp; technological education., 22 (1). pp. 5-21.</dr:CobjIdentifier>
+        <dr:CobjModel>OAI</dr:CobjModel>
+        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>
+
+        <dr:CobjDescriptionSynthesis/>
+        <dr:repositoryName>Durham Research Online</dr:repositoryName>
+        <dr:repositoryLink>http://dro.dur.ac.uk</dr:repositoryLink>
+        <dr:repositoryCountry>UK</dr:repositoryCountry>
+        <dr:repositoryInstitution/>
+        <dc:creator>Gott, R.</dc:creator>
+        <dc:creator>Roberts, R.</dc:creator>
+
+        <dc:title>A written test for procedural understanding : a way forward for assessment in the UK science curriculum ?</dc:title>
+        <dc:subject/>
+        <dr:CobjCategory>0001</dr:CobjCategory>
+        <dc:language>eng</dc:language>
+        <dc:dateAccepted>2004-05-01</dc:dateAccepted>
+        <dc:identifier>http://dx.doi.org/10.1080/0263514042000187511</dc:identifier>
+        <dc:publisher>Routledge</dc:publisher>
+
+        <dc:source/>
+        <dc:contributor/>
+        <dc:relation>http://igitur-archive.library.uu.nl/lg/2008-0326-201100/preprint172.pdf</dc:relation>
+        <dc:relation>http://dx.doi.org/10.1080/0263514042000187511</dc:relation>
+        <dc:description>A recent UK House of Commons report on Science 14-19 identified problems with coursework and argued for a greater emphasis on teaching and assessment of scientific literacy. This paper describes a written test for procedural understanding, given to 15 year olds, that addresses both of these issues. Comparisons are made between the scores on a written test of procedural understanding with both assessments made of subject knowledge and pupil accounts of investigations. The potential advantages of assessing procedural understanding by written tests are discussed.</dc:description>
+      </metadata>
+    </result>
+
+  </record>
Index: modules/uoa-resource-discovery/trunk/test/junit/TestPlugin.java
===================================================================
--- modules/uoa-resource-discovery/trunk/test/junit/TestPlugin.java	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/test/junit/TestPlugin.java	(revision 47463)
@@ -0,0 +1,47 @@
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.apache.log4j.BasicConfigurator;
+import org.junit.*;
+
+import eu.dnetlib.data.utility.resource_discovery.plugin.ResourceDiscoveryPlugin;
+
+
+public class TestPlugin {
+	
+	MySourceDataProvider provider;
+	static private int N = 7;
+	
+	@BeforeClass
+	public static void Config() {
+		BasicConfigurator.configure();
+	}
+	
+	@Before
+	public void InitializeProvider() throws Exception {
+		provider = new MySourceDataProvider();
+		for(int i=7;i<=N;i++) {
+			BufferedReader br = new BufferedReader(new InputStreamReader(TestPlugin.class.getResourceAsStream("record"+i+".xml")));
+			
+			String line;
+			String record = "";
+			while((line = br.readLine()) != null)
+				record += line;
+			provider.addDMFRecord(record);
+		}
+	}
+	
+	@Test
+	public void TestResourcePlugin() throws Exception {
+		ResourceDiscoveryPlugin plugin = new ResourceDiscoveryPlugin();
+		plugin.setDao(new MyResultDao());
+		plugin.setSourceDataProvider(provider);
+		plugin.init();
+		
+		//System.out.println(provider.getRecords(1, 2));
+		//System.out.println(provider.getRecords(1, 2).size());
+		plugin.execute();
+		System.out.println("\n\n" + plugin.getDao().getResults(1, 1));
+	}
+	
+}
Index: modules/uoa-resource-discovery/trunk/.project
===================================================================
--- modules/uoa-resource-discovery/trunk/.project	(nonexistent)
+++ modules/uoa-resource-discovery/trunk/.project	(revision 47463)
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>uoa-resource-discovery</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>
Index: modules/uoa-resource-discovery/trunk
===================================================================
--- modules/uoa-resource-discovery/trunk	(nonexistent)
+++ modules/uoa-resource-discovery/trunk	(revision 47463)

Property changes on: modules/uoa-resource-discovery/trunk
___________________________________________________________________
Added: svn:ignore
## -0,0 +1 ##
+.classpath
