D-Net

<ivy-module version="1.0">

	<info organisation="driver" module="uoa-resource-discovery"

		status="integration" />

	<configurations defaultconfmapping="*->default">

		<conf name="default" />

		<conf name="junit" visibility="private" extends="default" />

	</configurations>

	<publications>

		<artifact name="uoa-resource-discovery" type="jar" />

		<artifact name="uoa-resource-discovery" type="src" ext="zip" />

	</publications>

	<dependencies>

		<dependency org="apache" name="commons-logging" rev="1.0.4" />

		<dependency org="org.w3c" name="tidy" rev="+" />

		<dependency name="log4j" org="log4j" rev="+"/>

		<dependency org="junit" name="junit" rev="+" conf="junit"/>

		<dependency org="net.matuschek" name="jobo" rev="1.4+" />

		<dependency org="DLS" name="jOAI" rev="2.0.9.3+" />

		<dependency org="driver" name="unibi-commons" rev="+" />

		<dependency org="driver" name="unibi-data-utility-featureextraction-plugins" rev="+" />

		<dependency org="com.thoughtworks" name="xstream" rev="+" />

		<dependency org="org.cyberneko" name="nekohtml" rev="+" />

		<dependency org="com.jira" name="heritrix-commons" rev="+" />

		<dependency org="com.jira" name="heritrix-modules" rev="+" />

		<dependency org="org.kryo" name="kryo" rev="+" />

		<dependency org="edu.indiana" name="xpp3" rev="+" />

	</dependencies>

</ivy-module>

<ivysettings>

  <include file="${yvy.build.dir}/ivysettings.xml"/>

  <settings defaultResolver="yvy-resolver" default="yvy.default.cache" defaultCacheDir="${yvy.cache.dir}"/>

</ivysettings>

source.dirs		= main

project.name		= uoa-resource-discovery

project.version		= 0.0.7

project.label		=

<project name="uoa-resource-discovery" default="build">

	<!--

	organisation:		driver

	module: 		uoa-resource-discovery

	-->

	<property file="local.properties"/>

	<!-- Layout paths properties -->

	<property file="../../build/yvy-setup.properties"/>

	<property file="../../../../build/yvy-trunk.properties"/>

	<property file="../../../../../build/yvy-tag.properties"/>

	<property file="build.properties"/>

	<property name="yvy.build.common" value="${yvy.root.dir}/build/build-common.xml"/>

	<!-- Common build definitions -->

	<import file="${yvy.build.common}"/>

	<target name="build" depends="jar"/>

	<target name="default">

		<antcall target="${yvy.target.build}" inheritAll="true" inheritRefs="true"/>

	</target>

</project>

<?xml version="1.0"?>  

  <record rank="0.4898979">

    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

      <header>

        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/29575</dri:objIdentifier>

        <dri:dateOfCollection>2009-12-30T00:16:01Z</dri:dateOfCollection>

      </header>

      <metadata>

        <dr:CobjContentSynthesis/>

        <dr:CobjTypology>Textual</dr:CobjTypology>

        <dr:CobjIdentifier>Journal of Personality and Social Psychology 89, 696-716 (2005)</dr:CobjIdentifier>

        <dr:CobjModel>OAI</dr:CobjModel>

        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>

        <dr:CobjDescriptionSynthesis/>

        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>

        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>

        <dr:repositoryCountry>NL</dr:repositoryCountry>

        <dr:repositoryInstitution/>

        <dc:creator>Wicherts, J.M.</dc:creator>

        <dc:creator>Dolan, C.V.</dc:creator>

        <dc:creator>Hessen, D.J.</dc:creator>

        <dc:title>Stereotype threat and group differences in test performance: A question of measurement invariance.</dc:title>

        <dc:subject>ethnic differences</dc:subject>

        <dc:subject>sex differences</dc:subject>

        <dc:subject>measurement invariance</dc:subject>

        <dc:subject>Sociale Wetenschappen</dc:subject>

        <dc:subject>stereotype threat</dc:subject>

        <dc:subject>test performance</dc:subject>

        <dr:CobjCategory>0001</dr:CobjCategory>

        <dc:language>eng</dc:language>

        <dc:dateAccepted>2005-01-01</dc:dateAccepted>

        <dc:identifier>http://igitur-archive.library.uu.nl/fss/2008-0807-201603/UUindex.html</dc:identifier>

        <dc:publisher>American Psychological Association</dc:publisher>

        <dc:source/>

        <dc:contributor/>

        <dc:relation>0022-3514</dc:relation>

        <dc:description>Studies into the effects of stereotype threat (ST) on test performance have shed new light on race and sex differences in achievement and intelligence test scores. In this article,the authors relate ST theory to the psychometric concept of measurement invariance and show that ST effects may be viewed as a source of measurement bias. As such,ST effects are detectable by means of multigroup confirmatory factor analysis. This enables research into the generalizability of ST effects to real-life or high-stakes testing. The modeling approach is described in detail and applied to 3 experiments in which the amount of ST for minorities and women was manipulated. Results indicate that ST results in measurement bias of intelligence and mathematics tests.</dc:description>

      </metadata>

    </result>

  </record>

import java.util.Arrays;

import java.util.List;

import org.apache.log4j.BasicConfigurator;

import org.junit.Test;

import eu.dnetlib.common.utils.*;

import eu.dnetlib.data.utility.resource_discovery.plugin.ResourceDescriptionRecord;

import eu.dnetlib.data.utility.resource_discovery.plugin.ResourceUrls;

public class TestXML {

	@Test

	public void TestCrawlingAndExtraction() throws Exception {

		BasicConfigurator.configure();

		XMLSerializer<ResourceDescriptionRecord> handler = new XMLSerializer<ResourceDescriptionRecord>(ResourceDescriptionRecord.class);

		ResourceDescriptionRecord descrRecord = new ResourceDescriptionRecord();

		descrRecord.setObjectIdentifier("0239184018501");

		List<String> urlList = Arrays.asList("res1", "res2");

		ResourceUrls urls = new ResourceUrls();

		urls.setUrls(urlList);

		descrRecord.setResourceUrls(urls);

		System.out.println(handler.getAsXml(descrRecord));

	}

}

<?xml version="1.0"?>

  <record rank="0.4618802">

    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

      <header>

        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/26815</dri:objIdentifier>

        <dri:dateOfCollection>2009-12-30T00:13:45Z</dri:dateOfCollection>

      </header>

      <metadata>

        <dr:CobjContentSynthesis/>

        <dr:CobjTypology>Textual</dr:CobjTypology>

        <dr:CobjIdentifier>Logic Group Preprint Series 172 (2008)</dr:CobjIdentifier>

        <dr:CobjModel>OAI</dr:CobjModel>

        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>

        <dr:CobjDescriptionSynthesis/>

        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>

        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>

        <dr:repositoryCountry>NL</dr:repositoryCountry>

        <dr:repositoryInstitution/>

        <dc:creator>Hollenberg, M.</dc:creator>

        <dc:title>Equational axioms of test algebra</dc:title>

        <dc:subject>Wijsbegeerte</dc:subject>

        <dr:CobjCategory>0000</dr:CobjCategory>

        <dc:language>eng</dc:language>

        <dc:dateAccepted>1996-12-09</dc:dateAccepted>

        <dc:identifier>http://igitur-archive.library.uu.nl/lg/2008-0326-201100/UUindex.html</dc:identifier>

        <dc:publisher/>

        <dc:source/>

        <dc:contributor/>

        <dc:relation/>

        <dc:description>We present a complete axiomatization of test algebra ([24,18,29]), the two-sorted algebraic variant of Propositional Dynamic Logic (PDL,[21,7]). The axiomatization consists of adding a finite number of equations to any axiomatization of Kleene algebra ([15,26,17,4]) and algebraic translations of the Segerberg ([27]) axioms for PDL. Kleene algebras are not finitely axiomatizable ([25,6]), so our result does not give us a finite axiomatization of test algebra: in fact, no finite equational axiomatization exists. We also present a single-sorted version of test algebra, using the notion of dynamic negation ([9,2,11]), to which the previous results carry over.</dc:description>

      </metadata>

    </result>

  </record>

<?xml version="1.0"?>  

  <record rank="0.45643544">

    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

      <header>

        <dri:objIdentifier>3297df8d-c100-44a2-8aa9-64729c406e05_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:www.tara.tcd.ie:2262/32968</dri:objIdentifier>

        <dri:dateOfCollection>2010-01-05T18:27:02Z</dri:dateOfCollection>

      </header>

      <metadata>

        <dr:CobjContentSynthesis/>

        <dr:CobjTypology>Textual</dr:CobjTypology>

        <dr:CobjIdentifier>Wilson, S., Flood, B., Goyal, S., Mosher, J., Bergin, S., O'Brien, J., Kennedy, R. ...Parameter estimation for a model with both imperfect test and repair... in Proceedings of the IEEE VLSI Test Symposium, Berkeley, CA, 6-10 May 2007, IEEE, 2007, pp 271-276</dr:CobjIdentifier>

        <dr:CobjIdentifier>Y</dr:CobjIdentifier>

        <dr:CobjIdentifier>Y</dr:CobjIdentifier>

        <dr:CobjModel>OAI</dr:CobjModel>

        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>

        <dr:CobjDescriptionSynthesis/>

        <dr:repositoryName>TARA</dr:repositoryName>

        <dr:repositoryLink>http://www.tara.tcd.ie/</dr:repositoryLink>

        <dr:repositoryCountry>IE</dr:repositoryCountry>

        <dr:repositoryInstitution/>

        <dc:creator>WILSON, SIMON PAUL</dc:creator>

        <dc:title>Parameter estimation for a model with both imperfect test and repair</dc:title>

        <dc:subject>Statistics</dc:subject>

        <dr:CobjCategory>0004</dr:CobjCategory>

        <dc:language>eng</dc:language>

        <dc:dateAccepted>2009-09-18</dc:dateAccepted>

        <dc:identifier>http://hdl.handle.net/2262/32968</dc:identifier>

        <dc:publisher>IEEE</dc:publisher>

        <dc:source/>

        <dc:contributor/>

        <dc:relation/>

        <dc:description>The involvement of BLI researchers is supported by a grant from the Industrial Development Agency of Ireland. The involvement of CTVR researchers is supported by Science Foundation Ireland grant 03/CE3/I405.</dc:description>

        <dc:description>We describe estimation of the parameters of a manufacturing test and repair model using data available from that test. The model allows imperfect testing and imperfect repair. The principal problem that we address is of parameter identification, given insufficient data, that we address by making conservative assumptions on the property being measured and the associated parameter values. Several cases of commonly occurring test types, in the manufacture of electronic products, are considered.</dc:description>

        <dc:description>PUBLISHED</dc:description>

      </metadata>

    </result>

  </record>

<?xml version="1.0"?>  

  <record rank="0.42426404">

    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

      <header>

        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/34935</dri:objIdentifier>

        <dri:dateOfCollection>2009-12-30T00:22:10Z</dri:dateOfCollection>

      </header>

      <metadata>

        <dr:CobjContentSynthesis/>

        <dr:CobjTypology>Textual</dr:CobjTypology>

        <dr:CobjIdentifier/>

        <dr:CobjModel>OAI</dr:CobjModel>

        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>

        <dr:CobjDescriptionSynthesis/>

        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>

        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>

        <dr:repositoryCountry>NL</dr:repositoryCountry>

        <dr:repositoryInstitution/>

        <dc:creator>Hoofd, M.G.V. van het</dc:creator>

        <dc:title>Exercise and depressionafter stroke, a systematic review Shuttle Walk Test in patients who suffered a stroke, a feasibility study</dc:title>

        <dc:subject>exercise</dc:subject>

        <dc:subject>Fysiotherapiewetenschap</dc:subject>

        <dc:subject>feasibility study</dc:subject>

        <dc:subject>systematic review Shuttle Walk Test in patients who suffered a stroke, a feasibility study: stroke</dc:subject>

        <dc:subject>Exercise and depressionafter stroke, a systematic review: stroke</dc:subject>

        <dc:subject>Shuttle Walk Test</dc:subject>

        <dc:subject>Geneeskunde</dc:subject>

        <dc:subject>depression</dc:subject>

        <dc:subject>aerobic capacity</dc:subject>

        <dr:CobjCategory>0007</dr:CobjCategory>

        <dc:language>eng</dc:language>

        <dc:dateAccepted>2009-06-30</dc:dateAccepted>

        <dc:identifier>http://igitur-archive.library.uu.nl/student-theses/2009-0807-200622/UUindex.html</dc:identifier>

        <dc:publisher/>

        <dc:source/>

        <dc:contributor>Brussel, M. van</dc:contributor>

        <dc:contributor>Port, I.G.L. van de</dc:contributor>

        <dc:contributor>Takken, T.</dc:contributor>

        <dc:relation/>

        <dc:description>Exercise and depression after stroke Purpose-Aim of this review is to summarize the evidence from (randomized) controlled trials regarding the effects of exercise on depression or depressive symptoms in patients who had suffered a stroke. Methods-Studies that included patients who suffered a stroke and measured an outcome concerning depression were systematically reviewed. After determining the methodological quality by the Pedro-scale, a best evidence synthesis was applied. Results-Two out of seven studies showed significant differences between both groups, in favor of the intervention group. Best evidence synthesis showed insufficient evidence for positive effects of exercise on depression in patients who suffered a stroke. Conclusion-From the studies included in the present review it cannot be concluded that exercise interventions had a positive effect on depression in patients who suffered a stroke. Key Words: stroke ... depression ... exercise ... systematic review Shuttle Walk Test in patients who suffered a stroke Objective-To evaluate the feasibility of the Shuttle Walk Test (SWT) and the Shuttle Run Test for children with cerebral palsy at GMFCS level II (SRT-II) in patients who suffered a stroke. Methods-Fifteen patients who suffered a stroke completed both the SWT and SRT-II to evaluate aerobic capacity. Results-Significant differences were found in maximum heart rate and test duration in favor of the SRT-II. No significant difference was found in perceived exertion. Conclusion-The SRT-II is more feasible to assess aerobic capacity in patients who suffered a stroke compared to the SWT. Key Words: stroke ... aerobic capacity ... feasibility study ... Shuttle Walk Test Nederlandse samenvatting De invloed van bewegen op depressie bij mensen met een CVA Doel-Dit review heeft als doel om te onderzoeken wat de invloed van bewegen is op depressie bij mensen met een CVA. Methode-Een systematische zoektocht is gedaan naar (gerandomiseerde), gecontroleerde studies waarin pati..nten met een CVA ge..ncludeerd waren en depressie als uitkomstmaat gemeten werd. Nadat de methodologische kwaliteit werd bepaald met de PEDro-schaal, is een best evidence synthese opgesteld. Resultaten-Twee van zeven studies lieten een significant, positief effect zien van bewegen op depressie bij mensen met een CVA. De best evidence synthese toonde onvoldoende bewijs voor positieve effecten van bewegen bij mensen met een CVA. Conclusie-Uit dit review kan niet geconcludeerd worden dat bewegen een positief effect heeft op depressie bij mensen met een CVA. Trefwoorden: CVA ... depressie ... bewegen ... systematisch review Een shuttle wandeltest bij mensen met een CVA Doel-In deze studie wordt de toepasbaarheid onderzocht van de Shuttle Wandel Test (SWT) en de Shuttle Run Test voor kinderen met een cerebrale parese met GMFCS-niveau II (SRT-II) bij mensen met een cerebrovasculair accident (CVA). Methode-Vijftien pati..nten met een CVA voerden de SWT en de SRT-II uit om het maximale inspanningsvermogen te meten. Resultaten-Significante verschillen werden gevonden wat betreft maximale hartslag en testduur tussen beide testen ten gunste van SRT-II. Er werd geen significant verschil gevonden in ervaren vermoeidheid (Borg-schaal). Conclusie-De SRT-II is beter toepasbaar om het maximale inspanningsvermogen te meten van mensen met een CVA dan de SWT. Trefwoorden: CVA ... maximaal inspanningsvermogen ... haalbaarheidsstudie ... Shuttle Walk Test</dc:description>

      </metadata>

    </result>

  </record>

<?xml version="1.0"?>  

  <record rank="0.42426404">

    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

      <header>

        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/36190</dri:objIdentifier>

        <dri:dateOfCollection>2009-12-30T00:23:45Z</dri:dateOfCollection>

      </header>

      <metadata>

        <dr:CobjContentSynthesis/>

        <dr:CobjTypology>Textual</dr:CobjTypology>

        <dr:CobjIdentifier/>

        <dr:CobjModel>OAI</dr:CobjModel>

        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>

        <dr:CobjDescriptionSynthesis/>

        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>

        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>

        <dr:repositoryCountry>NL</dr:repositoryCountry>

        <dr:repositoryInstitution/>

        <dc:creator>Junte, R.D.</dc:creator>

        <dc:title>Acaricide resistance in the blue cattle tick in South Africa; A comparison of three assays for determining tick resistance</dc:title>

        <dc:subject>Acaricide resistance, 3 South African Provinces, blue cattle tick, Riphicephalus (Boophilus) decoloratus, comparison, Adult Immersion Test, Shaw Larval Test, Larval Packet Test, cattle, resistant</dc:subject>

        <dc:subject>Diergeneeskunde</dc:subject>

        <dc:subject>Diergeneeskunde</dc:subject>

        <dr:CobjCategory>0007</dr:CobjCategory>

        <dc:language>eng</dc:language>

        <dc:dateAccepted>2007-08-21</dc:dateAccepted>

        <dc:identifier>http://igitur-archive.library.uu.nl/student-theses/2009-1015-200152/UUindex.html</dc:identifier>

        <dc:publisher/>

        <dc:source/>

        <dc:contributor>Dr. E. van Dalen, Prof. Dr. F. Jongejan</dc:contributor>

        <dc:relation/>

        <dc:description>The susceptibility of engorged females and larvae of Boophilus decoloratus ticks,- collected from cattle on breeding farms in Kwazula Natal, Eastern Cape and Limpopo provinces in South Africa-, for amitraz, cypermetrin and chlorfenvinphos. Was examined by means of the Adult Immersion test (AIT), the Shaw Larval Immersion Test (SLIT) and the Larval Packet Test (LPT). The results indicated resistance of Boophilus ticks to cypermetrin on all nine farms examined. On six farms ticks had developed resistance against amitraz demonstrated by all three tests procedures. On one farm ticks were still susceptible for amitraz in all tests, whereas in 2 remaining farms tests were not in agreement. Furthermore ticks were found susceptible for chlorfinvenphos on three farms in all three tests, whereas in 5 farms showed they appeared to be resistant. In general the results obtained with the different tests were in agreement.</dc:description>

      </metadata>

    </result>

  </record>

<?xml version="1.0"?>  

  <record rank="0.42426404">

    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

      <header>

        <dri:objIdentifier>9dffbf71-6914-40fe-b110-8e41a977ba90_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dro.dur.ac.uk.OAI2:1691</dri:objIdentifier>

        <dri:dateOfCollection>2009-12-29T22:13:17Z</dri:dateOfCollection>

      </header>

      <metadata>

        <dr:CobjContentSynthesis/>

        <dr:CobjTypology>Textual</dr:CobjTypology>

        <dr:CobjIdentifier>Gott, R. and Roberts, R. (2004) 'A written test for procedural understanding : a way forward for assessment in the UK science curriculum ?', Research in science &amp;amp;amp; technological education., 22 (1). pp. 5-21.</dr:CobjIdentifier>

        <dr:CobjModel>OAI</dr:CobjModel>

        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>

        <dr:CobjDescriptionSynthesis/>

        <dr:repositoryName>Durham Research Online</dr:repositoryName>

        <dr:repositoryLink>http://dro.dur.ac.uk</dr:repositoryLink>

        <dr:repositoryCountry>UK</dr:repositoryCountry>

        <dr:repositoryInstitution/>

        <dc:creator>Gott, R.</dc:creator>

        <dc:creator>Roberts, R.</dc:creator>

        <dc:title>A written test for procedural understanding : a way forward for assessment in the UK science curriculum ?</dc:title>

        <dc:subject/>

        <dr:CobjCategory>0001</dr:CobjCategory>

        <dc:language>eng</dc:language>

        <dc:dateAccepted>2004-05-01</dc:dateAccepted>

        <dc:identifier>http://dx.doi.org/10.1080/0263514042000187511</dc:identifier>

        <dc:publisher>Routledge</dc:publisher>

        <dc:source/>

        <dc:contributor/>

        <dc:relation>http://igitur-archive.library.uu.nl/lg/2008-0326-201100/preprint172.pdf</dc:relation>

        <dc:relation>http://dx.doi.org/10.1080/0263514042000187511</dc:relation>

        <dc:description>A recent UK House of Commons report on Science 14-19 identified problems with coursework and argued for a greater emphasis on teaching and assessment of scientific literacy. This paper describes a written test for procedural understanding, given to 15 year olds, that addresses both of these issues. Comparisons are made between the scores on a written test of procedural understanding with both assessments made of subject knowledge and pupil accounts of investigations. The potential advantages of assessing procedural understanding by written tests are discussed.</dc:description>

      </metadata>

    </result>

  </record>

import java.util.Vector;

import org.apache.log4j.BasicConfigurator;

import org.junit.Test;

import eu.dnetlib.data.utility.resource_discovery.crawler.Crawler;

import eu.dnetlib.data.utility.resource_discovery.crawler.ResourceExtractor;

import eu.dnetlib.data.utility.resource_discovery.url_filter.UrlFilter;

public class SuperTester {

	@Test

	public void TestCrawlingAndExtraction() throws Exception {

		BasicConfigurator.configure();

		Crawler crawler = new Crawler();

		ResourceExtractor extractor = new ResourceExtractor();

		String idUrl = UrlFilter.resolveRedirections("http://www.di.uoa.gr");

		System.out.println("Now processing " + idUrl);

		Vector<String> urls = crawler.getLinks(idUrl);

		System.out.println("Retrieved links are: "+ urls);

		System.out.println("Resources seem to be available in: " + extractor.extractResource(urls));

		System.out.println();

	}

}

<?xml version="1.0"?>    

  <record rank="0.42426404">

    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

      <header>

        <dri:objIdentifier>9dffbf71-6914-40fe-b110-8e41a977ba90_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dro.dur.ac.uk.OAI2:1856</dri:objIdentifier>

        <dri:dateOfCollection>2009-12-29T22:13:24Z</dri:dateOfCollection>

      </header>

      <metadata>

        <dr:CobjContentSynthesis/>

        <dr:CobjTypology>Textual</dr:CobjTypology>

        <dr:CobjIdentifier>Remedios, R. and Ritchie, K. and Lieberman, D. A. (2005) 'I used to like it but now I don't : the effect of the transfer test in Northern Ireland on pupils' intrinsic motivation.', British journal of educational psychology., 75 (3). pp. 435-452.</dr:CobjIdentifier>

        <dr:CobjModel>OAI</dr:CobjModel>

        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>

        <dr:CobjDescriptionSynthesis/>

        <dr:repositoryName>Durham Research Online</dr:repositoryName>

        <dr:repositoryLink>http://dro.dur.ac.uk</dr:repositoryLink>

        <dr:repositoryCountry>UK</dr:repositoryCountry>

        <dr:repositoryInstitution/>

        <dc:creator>Remedios, R.</dc:creator>

        <dc:creator>Lieberman, D. A.</dc:creator>

        <dc:creator>Ritchie, K.</dc:creator>

        <dc:title>I used to like it but now I don't : the effect of the transfer test in Northern Ireland on pupils' intrinsic motivation.</dc:title>

        <dc:subject/>

        <dr:CobjCategory>0001</dr:CobjCategory>

        <dc:language>eng</dc:language>

        <dc:dateAccepted>2005-09-01</dc:dateAccepted>

        <dc:identifier/>

        <dc:publisher>British Psychological Society</dc:publisher>

        <dc:source/>

        <dc:contributor/>

        <dc:relation>http://dro.dur.ac.uk/1856/</dc:relation>

        <dc:relation>http://dx.doi.org/10.1348/000709904X24771</dc:relation>

        <dc:description>Background. Research has suggested that the pressure of exams could undermine pupils' interest in their subjects, but almost all of this research has been conducted in laboratory settings. The Transfer Test in Northern Ireland provides an unusual opportunity to assess the effects of exam pressure in real life because some 10- and 11-year-olds sit a Transfer Test to be admitted to grammar school while others are not tested until they are 14. Aim. To assess the effect of exams on pupils' interest in their subjects both during the period before the exam and after the results are known. Sample. The sample comprised 66 pupils preparing to sit the Transfer Test and 55 not preparing for the test. Method. Pupils' interest in their school subjects was assessed by questionnaires administered 2 weeks before the Transfer Test and then again 2 weeks after the results were announced. Results. Surprisingly, prior to sitting the test, there was no significant difference in motivation between the test and no-test pupils. However, after sitting the test, the motivation of the test pupils decreased significantly relative to their no-test counterparts, despite the fact that most achieved the grades they needed for admission to grammar school. Conclusions. Exams provide a valuable tool for assessing academic progress, but under some circumstances they can reduce pupils' interest in the subjects they are studying. </dc:description>

      </metadata>

    </result>

  </record>

import java.util.ArrayList;

import java.util.List;

import eu.dnetlib.data.utility.featureextraction.dao.IResultDao;

public class MyResultDao implements IResultDao {

	ArrayList<String> results;

	public MyResultDao() {

		this.results = new ArrayList<String>();

	}

	@Override

	public void addResults(List<String> results) {

		for(String result : results)

			this.results.add(result);

	}

	@Override

	public void close() {

		// TODO Auto-generated method stub

	}

	@Override

	public int getNumberOfElements() {

		return results.size();

	}

	@Override

	public List<String> getResults(int from, int to) {

		return results.subList(from-1, to);

	}

}

package gr.uoa.di.resourcediscovery.test;

import gr.uoa.di.resourcediscovery.MethodProvider;

import gr.uoa.di.resourcediscovery.MethodProviderFileStorageImpl;

import gr.uoa.di.resourcediscovery.methods.ResourceDiscoveryMethod;

import gr.uoa.di.resourcediscovery.methods.XPathAndCrawl;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import org.apache.log4j.BasicConfigurator;

public class Sample {

	public static void main(String[] args) throws Exception {

		BasicConfigurator.configure();

		URL conUrl = new URL("http://rudie.di.uoa.gr:8080/files/");

		MethodProvider provider = new MethodProviderFileStorageImpl("method-map.xml");

		ResourceDiscoveryMethod method = provider.getMethod(conUrl);

		if(method == null) {

			List<String> mimeTypes = new ArrayList<String>();

			mimeTypes.add("application/pdf");

			XPathAndCrawl xpath = new XPathAndCrawl(mimeTypes, "http://rudie.di.uoa.gr:8080/robots.txt");

			method = xpath;

		}

		System.out.println("resources found in: " + method.getResources(conUrl, provider));

	}

}

import java.util.ArrayList;

import java.util.List;

import eu.dnetlib.data.utility.featureextraction.FeatureExtractionException;

import eu.dnetlib.data.utility.featureextraction.dataprovider.SourceDataProvider;

public class MySourceDataProvider implements SourceDataProvider {

	ArrayList<String> DMFRecords;

	public MySourceDataProvider() {

		this.DMFRecords = new ArrayList<String>();

	}

	public void addDMFRecord(String record) {

		DMFRecords.add(record);

	}

	@Override

	public List<String> getRecords(int from, int to)

			throws FeatureExtractionException {

		try {

			if(to >= this.DMFRecords.size())

				to = this.DMFRecords.size();

			return DMFRecords.subList(from-1, to);

		}

		catch(Exception e) {

			throw new FeatureExtractionException(e);

		}

	}

	@Override

	public int getSize() throws FeatureExtractionException {

		return this.DMFRecords.size();

	}

	@Override

	public String getStatus() throws FeatureExtractionException {

		// TODO Auto-generated method stub

		return null;

	}

}

import java.util.Calendar;

import org.junit.BeforeClass;

import org.junit.Test;

import eu.dnetlib.data.utility.resource_discovery.crawler.Crawler;

import eu.dnetlib.data.utility.resource_discovery.crawler.ResourceExtractor;

import eu.dnetlib.data.utility.resource_discovery.plugin.crawler.ResourceDiscoverer;

public class TestResourceDiscoverer {

	@BeforeClass

	public static void Config() {

		//BasicConfigurator.configure();

	}

	@Test

	public void test() throws Exception {

		String url = "http://elib.uni-stuttgart.de/opus/volltexte/1999/1/";

		Calendar cal = Calendar.getInstance();

		System.out.println(cal.getTime());

		ResourceDiscoverer discoverer = new ResourceDiscoverer();

		System.out.println(discoverer.getResources(url));

		cal = Calendar.getInstance();

		System.out.println(cal.getTime());

		Crawler crawler = new Crawler();

		ResourceExtractor extractor = new ResourceExtractor();

		System.out.println(extractor.extractResource(crawler.getLinks(url)));

		cal = Calendar.getInstance();

		System.out.println(cal.getTime());

	}

}

import java.io.BufferedReader;

import java.io.InputStreamReader;

import org.apache.log4j.BasicConfigurator;

import org.junit.*;

import eu.dnetlib.data.utility.resource_discovery.plugin.ResourceDiscoveryPlugin;

public class TestPlugin {

	MySourceDataProvider provider;

	static private int N = 7;

	@BeforeClass

	public static void Config() {

		BasicConfigurator.configure();

	}

	@Before

	public void InitializeProvider() throws Exception {

		provider = new MySourceDataProvider();

		for(int i=7;i<=N;i++) {

			BufferedReader br = new BufferedReader(new InputStreamReader(TestPlugin.class.getResourceAsStream("record"+i+".xml")));

			String line;

			String record = "";

			while((line = br.readLine()) != null)

				record += line;

			provider.addDMFRecord(record);

		}

	}

	@Test

	public void TestResourcePlugin() throws Exception {

		ResourceDiscoveryPlugin plugin = new ResourceDiscoveryPlugin();

		plugin.setDao(new MyResultDao());

		plugin.setSourceDataProvider(provider);

		plugin.init();

		//System.out.println(provider.getRecords(1, 2));

		//System.out.println(provider.getRecords(1, 2).size());

		plugin.execute();

		System.out.println("\n\n" + plugin.getDao().getResults(1, 1));

	}

}

<?xml version="1.0"?>

  <record rank="0.99999994">

    <result xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:dr="http://www.driver-repository.eu/namespace/dr">

      <header>

        <dri:objIdentifier>7a22e67a-364b-4a2d-bcc1-cd6cc8a4e9d0_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=::oai:dspace.library.uu.nl:1874/34916</dri:objIdentifier>

        <dri:dateOfCollection>2009-12-30T00:22:07Z</dri:dateOfCollection>

      </header>

      <metadata>

        <dr:CobjContentSynthesis/>

        <dr:CobjTypology>Textual</dr:CobjTypology>

        <dr:CobjIdentifier/>

        <dr:CobjModel>OAI</dr:CobjModel>

        <dr:CobjMDFormats>oai_dc</dr:CobjMDFormats>

        <dr:CobjDescriptionSynthesis/>

        <dr:repositoryName>DSpace at Utrecht University</dr:repositoryName>

        <dr:repositoryLink>http://www.igitur.nl/</dr:repositoryLink>

        <dr:repositoryCountry>NL</dr:repositoryCountry>

        <dr:repositoryInstitution/>

        <dc:creator>Westerlaken, J.</dc:creator>

        <dc:title>Test</dc:title>

        <dc:subject>test</dc:subject>

        <dc:subject>Geneeskunde</dc:subject>

        <dc:subject>Verplegingswetenschap</dc:subject>

        <dr:CobjCategory>0007</dr:CobjCategory>

        <dc:language>dut/nla</dc:language>

        <dc:dateAccepted>2009-07-31</dc:dateAccepted>

        <dc:identifier>http://igitur-archive.library.uu.nl/dissertations/2005-1018-200018/index.htm</dc:identifier>

        <dc:publisher/>

        <dc:source/>

        <dc:contributor>Teat, test</dc:contributor>

        <dc:contributor>Test, Test</dc:contributor>

        <dc:relation/>

        <dc:description>Test</dc:description>

      </metadata>

    </result>

  </record>

<?xml version="1.0" encoding="UTF-8"?>

<projectDescription>

	<name>uoa-resource-discovery</name>

	<comment></comment>

	<projects>

	</projects>

	<buildSpec>

		<buildCommand>

			<name>org.eclipse.jdt.core.javabuilder</name>

			<arguments>

			</arguments>

		</buildCommand>

	</buildSpec>

	<natures>

		<nature>org.eclipse.jdt.core.javanature</nature>

	</natures>

</projectDescription>

package gr.uoa.di.resourcediscovery;

public class UnknownMethodException extends Exception {

	private static final long serialVersionUID = 760327436365242998L;

}

package gr.uoa.di.resourcediscovery;

import gr.uoa.di.resourcediscovery.methods.ResourceDiscoveryMethod;

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.FileWriter;

import java.io.IOException;

import java.net.URL;

import java.util.HashMap;

import com.thoughtworks.xstream.XStream;

public class MethodProviderFileStorageImpl implements MethodProvider {

	private String pathToFile = null;

	HashMap<URL, ResourceDiscoveryMethod> map = new HashMap<URL, ResourceDiscoveryMethod>();

	public MethodProviderFileStorageImpl() {

	}

	@SuppressWarnings("unchecked")

	public MethodProviderFileStorageImpl(String pathToFile) throws FileNotFoundException {

		XStream xstream = new XStream();

		if(!(new File(pathToFile).exists()))

			map = new  HashMap<URL, ResourceDiscoveryMethod>();

		else

			map = (HashMap<URL, ResourceDiscoveryMethod>) xstream.fromXML(new FileReader(new File(pathToFile)));

		this.pathToFile = pathToFile;

	}

	@Override

	public ResourceDiscoveryMethod getMethod(URL baseUrl) throws MalformedConfigurationException, UnknownMethodException, IOException {

		baseUrl = new URL(Toolkit.getRedirectedUrl(baseUrl.toString(), 500));

		ResourceDiscoveryMethod ret = map.get(new URL(baseUrl.getProtocol()+"://"+baseUrl.getHost()));

		return ret;

	}

	@Override

	public void setMethod(URL baseUrl, ResourceDiscoveryMethod method) {

		map.put(baseUrl, method);

		try {

			store();

		} catch (IOException e) {

			e.printStackTrace();

		}

	}

	public String getPathToFile() {

		return pathToFile;

	}

	public void setPathToFile(String pathToFile) {

		this.pathToFile = pathToFile;

	}

	public void store() throws IOException {

		XStream xstream = new XStream();

		xstream.toXML(map, new FileWriter(new File(pathToFile)));

	}

}

package gr.uoa.di.resourcediscovery.methods;

import gr.uoa.di.resourcediscovery.MethodProvider;

import java.io.IOException;

import java.net.URL;

import java.util.List;

import org.xml.sax.SAXException;

public interface ResourceDiscoveryMethod {

	public List<String> getResources(URL upageUrl, MethodProvider provider) throws SAXException, IOException;

}

package gr.uoa.di.resourcediscovery.methods;

import gr.uoa.di.resourcediscovery.MalformedConfigurationException;

import gr.uoa.di.resourcediscovery.MethodProvider;

import gr.uoa.di.resourcediscovery.Toolkit;

import java.io.BufferedReader;

import java.io.FileNotFoundException;

import java.io.IOException;

import java.io.InputStreamReader;

import java.net.MalformedURLException;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import org.apache.log4j.Logger;

import org.archive.modules.net.RobotsDirectives;

import org.archive.modules.net.Robotstxt;

import org.cyberneko.html.parsers.DOMParser;

import org.w3c.dom.Document;

import org.w3c.dom.Node;

import org.w3c.dom.traversal.DocumentTraversal;

import org.w3c.dom.traversal.NodeFilter;

import org.w3c.dom.traversal.NodeIterator;

import org.xml.sax.SAXException;

public class XPathAndCrawl implements ResourceDiscoveryMethod {

	transient Logger logger = Logger.getLogger(XPathAndCrawl.class);

	private boolean resolveFrames = true;

	private boolean skipFirstPage = false;

	private long sleepMillis = 100;

	private boolean ignoreRobotsTxt = false;

	private String agentName = "OpenAIRE_Harvester";

	private List<String> mimeTypes = new ArrayList<String>();

	private boolean fallback = true;

	private String robotstxtUrl = null;

	transient private Robotstxt robot = null;

	transient private RobotsDirectives directives = null;

	private List<String> xpaths = new ArrayList<String>();

	public XPathAndCrawl() {

		this.ignoreRobotsTxt = true;

	}

	// you need one per repository!

	public XPathAndCrawl(List<String> mimeTypes, String robotstxtUrl) throws FileNotFoundException, IOException {

		this.mimeTypes.addAll(mimeTypes);

		if (robotstxtUrl != null) {

			URL url = new URL(robotstxtUrl);

			try {

				BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));

				this.robot = new Robotstxt(in);

				this.directives = this.robot.getDirectivesFor(agentName);

			} catch (FileNotFoundException ex) {

				logger.debug("Robots.txt was not found at " + robotstxtUrl);

				ignoreRobotsTxt = true;

			}

		} else {

			ignoreRobotsTxt = true;

		}

	}

	public void setRobotstxt(String robotstxtUrl) throws FileNotFoundException, IOException {

		this.robotstxtUrl = robotstxtUrl;

		if (robotstxtUrl != null) {

			URL url = new URL(robotstxtUrl);

			try {

				BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));

				this.robot = new Robotstxt(in);

				this.directives = this.robot.getDirectivesFor(agentName);

			} catch (FileNotFoundException ex) {

				logger.debug("Robots.txt was not found at " + robotstxtUrl);

				ignoreRobotsTxt = true;

			}

		} else {

			ignoreRobotsTxt = true;

		}

	}

	public String getRobotstxtUrl() {

		return robotstxtUrl;

	}

	@Override

	public List<String> getResources(URL upageUrl, MethodProvider provider) throws SAXException, IOException {

		String pageUrl = upageUrl.toString();

		logger.debug("Known xpaths: "+this.xpaths);

		pageUrl = Toolkit.getRedirectedUrl(pageUrl, this.sleepMillis);

		logger.debug("Resolved possible redirections. Url: "+pageUrl);

		List<String> ret = new ArrayList<String>();

		List<String> urls = new ArrayList<String>();

		urls.add(pageUrl);

		// check if url is a redirection

		if(this.mimeTypes.contains(Toolkit.getMimeType(pageUrl, this.sleepMillis))) {

			ret.add(Toolkit.makeAbsolute(pageUrl, new URL(pageUrl)));

			return ret;

		}

		if (this.resolveFrames) {

			DOMParser parser = new DOMParser();

			parser.parse(pageUrl);

			Document doc = parser.getDocument();

			urls.addAll(resolveFrames(doc, new URL(pageUrl)));

			logger.debug("urls after resolving frames: " + urls);

		}

		if (this.skipFirstPage) {

			List<String> addme = new ArrayList<String>();

			for (String url : urls) {

				DOMParser parser = new DOMParser();

				parser.parse(url);

				Document doc = parser.getDocument();

				addme.addAll(oneDepthDown(doc, new URL(url)));

			}

			urls.remove(pageUrl);

			if (this.resolveFrames) {

				for (String url : urls) {

					DOMParser parser = new DOMParser();

					parser.parse(url);

					Document doc = parser.getDocument();

					addme.addAll(resolveFrames(doc, new URL(url)));

				}

			}

			urls.addAll(addme);

			logger.debug("urls after skipping 1st page and resolving frames: " + urls);

		}

		for (String url : urls) {

			logger.debug("looking for resource in: " + url);

			try {

				url = Toolkit.makeAbsolute(url, new URL(pageUrl));

			} catch (Exception e) {

				e.printStackTrace();

				continue;

			}

			URL startingUrl = new URL(url);

			if (!this.ignoreRobotsTxt)

				if (!this.directives.allows(Toolkit.makeRelative(startingUrl))) {

					logger.debug("Skipping " + startingUrl + ". Disallowed by robots.txt directives.");

					continue;

				}

			if (this.xpaths.size() == 0) {

				logger.debug("No xpath information, crawling");

				// this for the first time

				DOMParser parser = new DOMParser();

				parser.parse(startingUrl.toString());

				Document doc = parser.getDocument();

				List<Node> resourceNodes = findNodesWithResource(doc, startingUrl);

				for (Node resourceNode : resourceNodes) {

					String xp = getXpathToRoot(resourceNode);

					xpaths.add(xp);

					logger.debug(xp);

				}

				try {

					URL methodUrl = new URL(pageUrl);

					provider.setMethod(new URL(methodUrl.getProtocol()+"://"+methodUrl.getHost()), this);

				} catch(MalformedConfigurationException e) {

					logger.error("Error updating xpath information", e);

				}

				for (String xp : xpaths) {

					String resourceUrl = getResourceUrl(xp, doc, startingUrl);

					if (resourceUrl != null) {

						logger.debug(resourceUrl);

						ret.add(resourceUrl);

					}

				}

			} else {

				// this is for the rest of the pages of the repo

				DOMParser parser = new DOMParser();

				parser.parse(startingUrl.toString());

				Document doc = parser.getDocument();

				for (String xp : xpaths) {

					String resourceUrl = getResourceUrl(xp, doc, startingUrl);

					if (resourceUrl != null) {

						logger.debug(resourceUrl);

						ret.add(resourceUrl);

					}

				}

			}

		}

		if (ret.size() == 0 && this.fallback) {

			// if no xpath contained the resource, try to find it and add

			// all the xpaths

			for (String url : urls) {

				logger.debug("looking for resource in (not found in xpath): " + url);

				try {

					url = Toolkit.makeAbsolute(url, new URL(pageUrl));

				} catch (Exception e) {

					e.printStackTrace();

					continue;

				}

				URL startingUrl = new URL(url);

				if (!this.ignoreRobotsTxt)

					if (!this.directives.allows(Toolkit.makeRelative(startingUrl))) {

						logger.debug("Skipping " + startingUrl + ". Disallowed by robots.txt directives.");

						continue;

					}

				DOMParser parser = new DOMParser();

				parser.parse(startingUrl.toString());

				Document doc = parser.getDocument();

				List<Node> resourceNodes = findNodesWithResource(doc, startingUrl);

				for (Node resourceNode : resourceNodes) {

					String xp = getXpathToRoot(resourceNode);

					xpaths.add(xp);

					logger.debug(xp);

				}

				try {

					URL methodUrl = new URL(pageUrl);

					provider.setMethod(new URL(methodUrl.getProtocol()+"://"+methodUrl.getHost()), this);

				} catch(MalformedConfigurationException e) {

					logger.error("Error updating xpath information", e);

				}

				for (String xp : xpaths) {

					String resourceUrl = getResourceUrl(xp, doc, startingUrl);

					if (resourceUrl != null) {

						logger.debug(resourceUrl);

						ret.add(resourceUrl);

					}

				}

			}

		}

		return ret;

	}

	private List<String> resolveFrames(Document doc, URL connectionUrl) {

		List<String> ret = new ArrayList<String>();

		DocumentTraversal traversal = (DocumentTraversal) doc;

		NodeIterator iterator = null;

		try {

			iterator = traversal.createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, null, true);

		} catch (Exception e) {

			e.printStackTrace();

			return ret;

		}

		for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) {

			if (n.getNodeName().equals("FRAME") || n.getNodeName().equals("IFRAME")) {

				String url = n.getAttributes().getNamedItem("src").getNodeValue();

				try {

					url = Toolkit.makeAbsolute(url, connectionUrl);

					ret.add(url);

				} catch (MalformedURLException ex) {

					continue;

				}

			}

		}

		return ret;

	}

	private List<String> oneDepthDown(Document doc, URL connectionUrl) throws IOException {

		List<String> ret = new ArrayList<String>();

		DocumentTraversal traversal = (DocumentTraversal) doc;

		NodeIterator iterator = null;

		try {

			iterator = traversal.createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, null, true);

		} catch (Exception e) {

			e.printStackTrace();

			return ret;

		}

		for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) {

			if (n.getNodeName().equals("A")) {

				String url = n.getAttributes().getNamedItem("href").getNodeValue();

				try {

					url = Toolkit.makeAbsolute(url, connectionUrl);

					if (Toolkit.getMimeType(url, this.sleepMillis).trim().contains("text/html"))

						ret.add(url);

				} catch (MalformedURLException ex) {

					continue;

				}

			}

		}

		return ret;

	}

	private String getXpathToRoot(Node node) {

		String xpath = "";

		do {

			if (node.getNodeName().equals("HTML")) {

				int before = 1;

				while ((node = node.getPreviousSibling()) != null)

					before++;

				return "/HTML["+before+"]" + xpath;

			}

			int before = 0;

			Node current = node;

			while ((current = current.getPreviousSibling()) != null)

				if (current.getNodeName().equals(node.getNodeName()))

					before++;

			xpath = "/" + node.getNodeName() + "[" + (before + 1) + "]" + xpath;

		} while ((node = node.getParentNode()) != null);

		return xpath;

	}

	private List<Node> findNodesWithResource(Document doc, URL connectionUrl) throws IOException {

		List<Node> ret = new ArrayList<Node>();

		DocumentTraversal traversal = (DocumentTraversal) doc;

		NodeIterator iterator = null;

		try {

			iterator = traversal.createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, null, true);

		} catch (Exception e) {

			e.printStackTrace();

			return ret;

		}

		for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) {

			if (n.getNodeName().equals("A")) {

				String url = null;

				try {

					url = n.getAttributes().getNamedItem("href").getNodeValue();

				} catch(NullPointerException e) {

					// anchor without href

					continue;

				}

				if (url == null)

					continue;

				try {

					url = Toolkit.makeAbsolute(url, connectionUrl);

					if (this.mimeTypes.contains(Toolkit.getMimeType(url, this.sleepMillis).trim()))

						ret.add(n);

				} catch (MalformedURLException ex) {

					continue;

				}

			}

		}

		return ret;

	}

	private String getResourceUrl(String xpath, Document doc, URL url) throws MalformedURLException {

		try {

			Node current = doc.getFirstChild();

			String[] elements = xpath.split("/");

			for (String element : elements) {

				if (element.trim().equals(""))

					continue;

				int position = Integer.parseInt(element.substring(element.indexOf('[')).replaceAll("\\[", "").replaceAll("\\]", ""));

				String name = element.substring(0, element.indexOf('['));

				int found = 0;

				do {

					if (current.getNodeName().equals(name)) {

						found++;

						if (found == position) {

							current = current.getFirstChild();

							break;

						}

					}

				} while ((current = current.getNextSibling()) != null);

			}

			String ret = current.getParentNode().getAttributes().getNamedItem("href").getNodeValue();

			return Toolkit.makeAbsolute(ret, url);

		} catch (Exception e) {

			return null;

		}

	}

	private Object readResolve() throws IOException {

		if (robotstxtUrl != null) {

			URL url = new URL(robotstxtUrl);

			BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));

			this.robot = new Robotstxt(in);

			this.directives = this.robot.getDirectivesFor(agentName);

		} else {

			ignoreRobotsTxt = true;

		}

		logger = Logger.getLogger(XPathAndCrawl.class);

		return this;

	}

	public boolean isResolveFrames() {

		return resolveFrames;

	}

	public void setResolveFrames(boolean resolveFrames) {

		this.resolveFrames = resolveFrames;

	}

	public boolean isSkipFirstPage() {

		return skipFirstPage;

	}

	public void setSkipFirstPage(boolean skipFirstPage) {

		this.skipFirstPage = skipFirstPage;

	}

	public long getSleepMillis() {

		return sleepMillis;

	}

	public void setSleepMillis(long sleepMillis) {

		this.sleepMillis = sleepMillis;

	}

	public List<String> getMimeTypes() {

		return mimeTypes;

	}

	public void setMimeTypes(List<String> mimeTypes) {

		this.mimeTypes = mimeTypes;

	}

	public List<String> getXpaths() {

		return xpaths;

	}

	public void setXpaths(List<String> xpaths) {

		this.xpaths = xpaths;

	}

	public void setIgnoreRobotsTxt(boolean ignoreRobotsTxt) {

		this.ignoreRobotsTxt = ignoreRobotsTxt;

	}

	public boolean isIgnoreRobotsTxt() {

		return ignoreRobotsTxt;

	}

	public void setAgentName(String agentName) {

		this.agentName = agentName;

		this.directives = this.robot.getDirectivesFor(agentName);

	}

	public String getAgentName() {

		return agentName;

	}

	public void setFallback(boolean fallback) {

		this.fallback = fallback;

	}

	public boolean isFallback() {

		return fallback;

	}

}

package gr.uoa.di.resourcediscovery.methods;

import gr.uoa.di.resourcediscovery.MethodProvider;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

public class URLTransformation implements ResourceDiscoveryMethod {

	private String regex = null, replacement = "";

	private String addToEnd = "";

	@Override

	public List<String> getResources(URL upageUrl, MethodProvider provider) {

		String pageUrl = upageUrl.toString();

		String trsf = pageUrl;

		if (regex != null && !regex.trim().equals(""))

			trsf = pageUrl.replaceAll(regex, replacement);

		trsf = trsf + addToEnd;

		List<String> ret = new ArrayList<String>();

		ret.add(trsf);

		return ret;

	}

	public String getRegex() {

		return regex;

	}

	public void setRegex(String regex) {

		this.regex = regex;

	}

	public String getAddToEnd() {

		return addToEnd;

	}

	public void setAddToEnd(String addToEnd) {

		this.addToEnd = addToEnd;

	}

	public String getReplacement() {

		return replacement;

	}

	public void setReplacement(String replacement) {

		this.replacement = replacement;

	}

}

package gr.uoa.di.resourcediscovery;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.MalformedURLException;

import java.net.URL;

import java.net.URLConnection;

import org.apache.log4j.Logger;

public class Toolkit {

	static transient Logger logger = Logger.getLogger(Toolkit.class);

	static int timeout = 10000;

	static public String makeAbsolute(String url, URL connectionUrl) throws MalformedURLException {

		return new URL(connectionUrl, url).toString();

	}

	static public String makeRelative(URL connectionUrl) throws MalformedURLException {

		return connectionUrl.getPath();

	}

	static public String getRedirectedUrl(String resourceURL, long sleepMillis) throws IOException, MalformedURLException {

		URL url = null;

		try {

			url = new URL(resourceURL);

		} catch (MalformedURLException mue) {

			logger.error("Error opening first url", mue);

			throw mue;

		}

		HttpURLConnection.setFollowRedirects(false);

		HttpURLConnection conn = null;

		try {

			Thread.sleep(sleepMillis);

			conn = (HttpURLConnection) url.openConnection();

			conn.setConnectTimeout(timeout);

			conn.setReadTimeout(timeout);

			conn.setAllowUserInteraction(false);         

			conn.setDoOutput(true);

		} catch (ClassCastException ex) {

			throw new MalformedURLException();

		} catch (InterruptedException e) {

			e.printStackTrace();

		}

		conn.setRequestMethod("HEAD");

		try {

			conn = openConnectionCheckRedirects(conn, sleepMillis);

		} catch (Exception ex) {

			throw new MalformedURLException();

		}

		try {

			Thread.sleep(sleepMillis);

		} catch (InterruptedException e) {

			e.printStackTrace();

		}

		int statusCode = conn.getResponseCode();

		if (statusCode == 503) {

			logger.error("Url " + conn.getURL() + " reported status code 503. Please increase the crawler's sleep time.");