D-Net

{"type_source": "SVN", "goal": "package -U -T 4C source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-download-plugins/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-download-plugins"}

import org.joda.time.DateTime;

import org.joda.time.Days;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

public class DateIntervalTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void test() {

		DateTime beforeDate = new DateTime(2010, 01, 1, 0, 0);

		DateTime now = new DateTime();

		Days day = Days.daysBetween(beforeDate, now);

		Assert.assertTrue(day.getDays() > 0);

	}

}

package eu.dnetlib.download.plugin;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.After;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

import org.junit.rules.TemporaryFolder;

import java.io.File;

import java.io.IOException;

import java.util.Arrays;

public class PathRetreiverTest {

    private static final Log log = LogFactory.getLog(PathRetreiverTest.class); // NOPMD by marko on 11/24/08 5:02 PM

    private TemporaryFolder t = new TemporaryFolder();

    private final PathRetreiver pt = new PathRetreiver();

    @Before

    public void setUp() throws IOException {

        t.create();

        IOUtils.readLines(getClass().getResourceAsStream("pmc_dirs.txt")).forEach(t::newFolder);

        for(final File dir : t.getRoot().listFiles(pathname -> pathname.isDirectory())) {

            int lower = Integer.parseInt(StringUtils.substringBefore(dir.getName(), "_").replaceAll("PMC", ""));

            int upper = Integer.parseInt(StringUtils.substringAfter(dir.getName(), "_").replaceAll("PMC", ""));

            for(int i=lower;i<lower+3 && i<upper;i++) {

               t.newFile(dir.getName() + "/" + i + ".xml");

            }

            for(int i=upper;i>upper-3 && i>lower;i--) {

                t.newFile(dir.getName() + "/" + i + ".xml");

            }

        }

        pt.setBase_path(t.getRoot().getPath());

    }

    @Test

    public void testPathRetriever() {

        String pathForPMCID = pt.getPathForPMCID(4676029);

        Assert.assertNotNull(pathForPMCID);

        log.info(pathForPMCID);

        pathForPMCID = pt.getPathForPMCID(4676028);

        Assert.assertNotNull(pathForPMCID);

        log.info(pathForPMCID);

        pathForPMCID = pt.getPathForPMCID(4676026);

        Assert.assertNull(pathForPMCID);

    }

    @After

    public void tearDown() {

        t.delete();

    }

}

package eu.dnetlib.download.plugin;

import junit.framework.Assert;

import org.junit.Before;

import org.junit.Test;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.data.download.rmi.DownloadItem;

public class ArxivImportFromFileTest {

	@Before

	public void setUp() throws Exception {}

	@Test

	public void testREgEx() {

		String regExp = "\\d{4}\\.\\d{4}";

		String input = "1308.0021";

		Assert.assertTrue(input.matches(regExp));

	}

	@Test

	public void testPlugin() {

		DownloadItem item = new DownloadItem();

		item.setOpenAccess("OPEN");

		String[] myList = new String[] { "http://arxiv.org/abs/1710.04937", "b" };

		Gson g = new GsonBuilder().disableHtmlEscaping().create();

		item.setUrl(g.toJson(myList));

		ArxivImportFromFile plugin = new ArxivImportFromFile();

		plugin.setBasePath("/tmp");

		plugin.retrieveUrl(item);

        System.out.println(item.getUrl());

		Assert.assertNotNull(item);

	}

}

package eu.dnetlib.download.plugin;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import org.junit.Assert;

import org.junit.Ignore;

import org.junit.Test;

import org.springframework.beans.factory.annotation.Autowired;

import java.util.ArrayList;

import java.util.List;

/**

 * Created by sandro on 5/15/17.

 */

@Ignore

public class DSpacePDFLinkPluginsTest {

    @Autowired

    DSpacePDFLinkPlugins dp;

    @Test

    public void testURL () {

        final DSpacePDFLinkPlugins plugin = new DSpacePDFLinkPlugins();

        final String s1 = plugin.extractURL("http://acikerisim.baskent.edu.tr/handle/11727/2340");

        Assert.assertEquals(s1, "http://acikerisim.baskent.edu.tr/bitstream/11727/2340/1/10008302.pdf");

    }

//    @Test

//    public void testDOI() {

//        final DSpacePDFLinkPlugins plugin = new DSpacePDFLinkPlugins();

//

//        final String s1 = plugin.extractURL("http://hdl.handle.net/10017/29866");

//       // Assert.assertEquals(s1, "http://acikerisim.baskent.edu.tr/bitstream/11727/2340/1/10008302.pdf");

//        System.out.println(s1);

//    }

    @Test

    public void testURLUniTO(){

        final DSpacePDFLinkPlugins plugin = new DSpacePDFLinkPlugins();

//        List<String> lst = new ArrayList<>();

//        lst.add("^http.*pdf$");

//        plugin.setRegularExpression(lst);

        final String s1 = plugin.extractURL("http://hdl.handle.net/2318/37517");

        Assert.assertEquals("https://iris.unito.it/bitstream/2318/37517/1/2002%20JBC%20-%20Integrin-induced%20epidermal%20growth%20factor%20%28EGF%29%20receptor%20activation%20requires%20c-Src%20and%20p130Cas.pdf",s1);

    }

    @Test

    public void downloadTest(){

    }

}

PMC2114664_PMC2135116

PMC2135117_PMC2173206

PMC2173207_PMC2193606

PMC2193607_PMC2228428

PMC2228429_PMC2289497

PMC2289498_PMC2376858

PMC2376859_PMC2453635

PMC2453636_PMC2562811

PMC2562812_PMC2600168

PMC2600169_PMC2639698

PMC2639699_PMC2679038

PMC2679039_PMC2709009

PMC2709010_PMC2739494

PMC2739495_PMC2774688

PMC2774689_PMC2807107

PMC2807108_PMC2837157

PMC2837158_PMC2872237

PMC2872246_PMC2900260

PMC2900261_PMC2931455

PMC2931456_PMC2959298

PMC2959299_PMC2977153

PMC2977154_PMC3000089

PMC3000090_PMC3018434

PMC3018435_PMC3043758

PMC3043759_PMC3066749

PMC3066750_PMC3094005

PMC3094006_PMC3115464

PMC3115476_PMC3139354

PMC3139355_PMC3161964

PMC3161966_PMC3184122

PMC3184123_PMC3205810

PMC3205811_PMC3226963

PMC3226964_PMC3246723

PMC3246724_PMC3267738

PMC3267739_PMC3287439

PMC3287440_PMC3305035

PMC3305036_PMC3324463

PMC3324464_PMC3342054

PMC3342055_PMC3359980

PMC3359981_PMC3379654

PMC3379655_PMC3400869

PMC3400870_PMC3420321

PMC3420322_PMC3440431

PMC3440432_PMC3465972

PMC3465973_PMC3485114

PMC3485115_PMC3504117

PMC3504118_PMC3521368

PMC3521369_PMC3540995

PMC3541002_PMC3559779

PMC3559780_PMC3579017

PMC3579018_PMC3597585

PMC3597586_PMC3616740

PMC3616741_PMC3637505

PMC3637506_PMC3656847

PMC3656848_PMC3674696

PMC3674697_PMC3694027

PMC3694028_PMC3713561

PMC3713562_PMC3734723

PMC3734724_PMC3756282

PMC3756283_PMC3775526

PMC3775527_PMC3794997

PMC3794998_PMC3817535

PMC3817536_PMC3834982

PMC3834984_PMC3851930

PMC3851934_PMC3870407

PMC3870408_PMC3888268

PMC3888269_PMC3905545

PMC3905546_PMC3923943

PMC3923951_PMC3942209

PMC3942210_PMC3962119

PMC3962121_PMC3980435

PMC3980436_PMC3997492

PMC3997493_PMC4016582

PMC4016583_PMC4034045

PMC4034053_PMC4052667

PMC4052668_PMC4069525

PMC4069526_PMC4087078

PMC4087079_PMC4104043

PMC4104070_PMC4123881

PMC4123882_PMC4143513

PMC4143514_PMC4161060

PMC4161143_PMC4179239

PMC4179240_PMC4196090

PMC4196091_PMC4213380

PMC4213381_PMC4229908

PMC4229909_PMC4246363

PMC4246371_PMC4265290

PMC4265291_PMC4283529

PMC4283530_PMC4301549

PMC4301550_PMC4320116

PMC4320117_PMC4336280

PMC4336281_PMC4354467

PMC4354468_PMC4372225

PMC4372226_PMC4389034

PMC4389035_PMC4406610

PMC4406611_PMC4423035

PMC4423048_PMC4439555

PMC4439556_PMC4456817

PMC4456818_PMC4474724

PMC4474725_PMC4491649

PMC4491650_PMC4508683

PMC4508684_PMC4525193

PMC4525194_PMC4543216

PMC4543217_PMC4560130

PMC4560131_PMC4579196

PMC4579197_PMC4595007

PMC4595008_PMC4609474

PMC4609475_PMC4626773

PMC4626774_PMC4643236

PMC4643237_PMC4660323

PMC4660324_PMC4676029

PMC4676030_PMC4693769

PMC4693770_PMC4710023

PMC4710024_PMC4728071

PMC4728075_PMC4743341

PMC4743342_PMC4758928

PMC4758929_PMC4775014

PMC4775015_PMC4793100

PMC4793101_PMC4809441

PMC4809442_PMC4826653

PMC4826654_PMC4844069

PMC4844073_PMC4859985

PMC4859986_PMC4877359

PMC4877365_PMC4894427

PMC4894428_PMC4910313

PMC4910314_PMC4928029

PMC4928030_PMC4943888

PMC4943889_PMC4963471

PMC4963472_PMC4979068

PMC4979069_PMC4996129

PMC4996130_PMC5011754

PMC5011755_PMC5027112

PMC5027113_PMC5041839

PMC5041840_PMC5055367

PMC5055368_PMC5070294

PMC5070295_PMC5084992

PMC5084998_PMC5098989

PMC5098990_PMC5113742

PMC5113743_PMC5128874

PMC5128875_PMC5142017

PMC5142018_PMC5155182

PMC5155183_PMC5168587

PMC5168588_PMC5181444

PMC5181445_PMC5195159

PMC5195160_PMC5207976

PMC5207977_PMC5220330

PMC5220331_PMC5232993

PMC5232994_PMC5245953

PMC5245954_PMC5258212

PMC5258213_PMC5269906

PMC5269907_PMC5281504

PMC5281505_PMC5294573

PMC5294574_PMC5307639

PMC5307640_PMC5320633

PMC5320634_PMC5334419

PMC5334420_PMC5348122

PMC5348123_PMC5362478

PMC5362479_PMC5379575

PMC5379578_PMC5393849

PMC5393850_PMC5409456

PMC5409462_PMC5424203

PMC5424204_PMC5437630

PMC5437631_PMC5452760

PMC5452761_PMC5467055

PMC5467056_PMC5483227

PMC5483229_PMC5497534

PMC549050_PMC1240576

PMC5497535_PMC5513250

PMC5513251_PMC5528017

PMC5528018_PMC5544983

PMC5544984_PMC5561184

PMC5561185_PMC5576750

PMC5576751_PMC5591643

PMC5591783_PMC5608430

PMC5608431_PMC5623045

PMC5623046_PMC5637514

PMC5637515_PMC5653330

PMC5653331_PMC5669206

PMC5669215_PMC5686621

PMC5686622_PMC5703149

PMC5703150_PMC5718512

PMC5718513_PMC5732977

PMC5732978_PMC5748310

PMC5748311_PMC5763634

PMC5763635_PMC5779436

PMC5779437_PMC5794819

PMC5794820_PMC5810839

PMC5810840_PMC5827115

PMC5827136_PMC5842751

PMC5842752_PMC5857142

PMC5857143_PMC5872445

PMC5872446_PMC5887736

PMC5887737_PMC5903694

PMC5903695_PMC5920276

PMC5920278_PMC5938386

PMC5938387_PMC5956503

PMC5956504_PMC5975655

PMC5975656_PMC5991267

PMC5991268_PMC6006820

PMC6006821_PMC6022687

PMC6022688_PMC6036291

PMC6036292_PMC6053207

PMC6053208_PMC6069616

PMC6069617_PMC6090019

PMC6090020_PMC6105291

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

// TODO: Auto-generated Javadoc

/**

 * The Class EuropePMC.

 */

public class EuropePMC extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(EuropePMC.class);

    /**

     * The base path.

     */

    private String basePath;

    // //*[local-name()='metadata']//*[local-name()='identifier' and ./@identifierType='pmc']/text()

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "europePMCDownloadPlugin";

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) throws DownloadPluginException {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) throws DownloadPluginException {

        try {

            String url = input.getOriginalUrl();

            if ((url == null) || (url.trim().length() == 0)) return input;

            @SuppressWarnings("unchecked")

            List<String> urls = new Gson().fromJson(url, ArrayList.class);

            log.debug(String.format("urls is %s", url));

            if ((urls == null) || (urls.size() == 0)) return input;

            input.setFileName(input.getIdItemMetadata());

            Boolean added = false;

            for (String s : urls) {

                if (s.startsWith("http")) {

                    log.debug(String.format("found url starting with http replace original URL with %s", s));

                    input.setOriginalUrl(s);

                } else if (s.startsWith("PMC")) {

                    String correctUrl = s.replace("PMC", "");

                    log.debug(String.format("found url starting with PMC %s", correctUrl));

                    added = true;

                    String path = PathRetreiver.getInstance(basePath).getPathForPMCID(Integer.parseInt(correctUrl));

                    if (path != null) {

                        input.setUrl("file://" + path);

                    } else {

                        input.setUrl(path);

                    }

                }

            }

            if (added == false) {

                input.setOriginalUrl(null);

                input.setUrl(null);

            }

            return input;

        } catch (Throwable e) {

            log.error("Exception on Download Plugin");

            log.error(e);

            throw new DownloadPluginException(e);

        }

    }

    /**

     * Gets the base path.

     *

     * @return the basePath

     */

    public String getBasePath() {

        return basePath;

    }

    /**

     * Sets the base path.

     *

     * @param basePath the basePath to set

     */

    @Override

    public void setBasePath(final String basePath) {

        this.basePath = basePath;

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * @Deprecated: use ArxivImportFromFile instead

 */

@Deprecated

public class ArxivDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        for (String s : urls) {

            if (s.startsWith("http")) {

                input.setOriginalUrl(s);

                String correctUrl = s.replace("abs", "pdf");

                correctUrl += ".pdf";

                input.setUrl(correctUrl);

            }

        }

        return input;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    @Override

    public String getPluginName() {

        return "ArxivDownloadPlugin";

    }

    @Override

    public void setBasePath(final String basePath) {

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * The Class EasyPDFDownloadPlugin.

 */

public class EasyPDFDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        if (input == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "easyPDFDownloadPlugin";

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return baseURL.endsWith(".pdf") ? baseURL : null;

    }

}

package eu.dnetlib.download.plugin;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

/**

 * The Class ELisDownloadPlugin.

 */

public class ELisDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(ELisDownloadPlugin.class);

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            Document doc = Jsoup.connect(url).get();

            Elements links = doc.select("a[href$=.pdf]");

            for (Element link : links) {

                String linkvalue = link.attr("abs:href");

                if (!linkvalue.toLowerCase().contains("thumbnailversion")) {

                    return linkvalue;

                }

            }

            return null;

        } catch (Exception e) {

            throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "ELisDownloadPlugin";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) {

            return null;

        }

        String url = input.getOriginalUrl();

        if (url == null || url.trim().length() == 0) {

            return input;

        }

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if (urls == null || urls.size() == 0) {

            return input;

        }

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

package eu.dnetlib.download.plugin;

import java.io.File;

import java.io.FileFilter;

import java.util.*;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * The Class PathRetreiver.

 */

public class PathRetreiver {

	private static final Log log = LogFactory.getLog(PathRetreiver.class);

	/** The instance. */

	private static PathRetreiver instance;

	/**

	 * Gets the single instance of PathRetreiver.

	 *

	 * @param base_path

	 *            the base_path

	 * @return single instance of PathRetreiver

	 */

	public static PathRetreiver getInstance(final String base_path) {

		if (instance == null) {

			instance = new PathRetreiver();

			instance.setBase_path(base_path);

		}

		return instance;

	}

	/** The base_path. */

	private String base_path;

	/** The values. */

	private TreeMap<Integer, InfoPath> values;

	/**

	 * Bootstrap.

	 */

	private void bootstrap() {

		values = new TreeMap<>();

		File basePath = new File(this.base_path);

		File[] selectedFiles = basePath.listFiles(pathname -> pathname.isDirectory());

		for (File f : selectedFiles) {

			String lower = StringUtils.substringAfter(StringUtils.substringBefore(f.getName(), "_"), "PMC");

			String upper = StringUtils.substringAfter(StringUtils.substringAfter(f.getName(), "_"), "PMC");

			String path = f.getPath();

			InfoPath i = new InfoPath();

			i.setLower(Integer.parseInt(lower));

			i.setUpper(Integer.parseInt(upper));

			i.setPath(path);

			values.put(i.getLower(), i);

		}

		if (log.isDebugEnabled()) {

			for (InfoPath p : values.values()) {

				log.debug(String.format("%s -- %s : %s", p.getLower(), p.getUpper(), p.getPath()));

			}

		}

	}

	/**

	 * Gets the path for pmcid.

	 *

	 * @param pmcID

	 *            the pmc id

	 * @return the path for pmcid

	 */

	public String getPathForPMCID(final int pmcID) {

		if (values == null) {

			bootstrap();

		}

		Map.Entry<Integer, InfoPath> infoPath = values.floorEntry(pmcID);

		if (infoPath != null) {

			final String currentPath = infoPath.getValue().getPath() + "/" + pmcID + ".xml";

			final File f = new File(currentPath);

			log.debug(String.format("try to search in path %s", currentPath));

			String s = null;

			if (f.exists()) {

				s = f.getPath();

				log.debug(String.format("found in %s", s));

			} else {

				log.debug(String.format("not found in %s", s));

			}

			return s;

		}

		log.debug(String.format("PMC with ID: %s not found", pmcID));

		return null;

	}

	/**

	 * Sets the base_path.

	 *

	 * @param base_path

	 *            the new base_path

	 */

	public void setBase_path(final String base_path) {

		this.base_path = base_path;

	}

	/**

	 * Gets the base_path.

	 *

	 * @return the base_path

	 */

	public String getBase_path() {

		return this.base_path;

	}

}

package eu.dnetlib.download.plugin;

import java.io.File;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.util.List;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class ArxivImportFromFile extends AbstractDownloadPlugin implements DownloadPlugin {

    private static final Log log = LogFactory.getLog(ArxivImportFromFile.class); // NOPMD by marko on 11/24/08 5:02 PM

    /**

     * The base path.

     */

    private String basePath;

    @Override

    public DownloadItem retrieveUrl(final DownloadItem item) {

        if (checkOpenAccess(item) == null) return null;

        final String baseURLs = item.getUrl();

        final List<String> urlsList = new Gson().fromJson(baseURLs, List.class);

        for (final String baseURL : urlsList) {

            if (baseURL.isEmpty() == false && baseURL.trim().startsWith("http://") == true) {

                final String name = StringUtils.substringAfter(baseURL, "abs/").trim();

                if (name == null) {

                    item.setUrl(null);

                    return item;

                }

                final String fileURL = createPath(name);

                if (StringUtils.isBlank(fileURL)) {

                    item.setUrl(null);

                    return item;

                }

                final File f = new File(fileURL);

                if (f.exists()) {

                    if (log.isDebugEnabled()) {

                        log.debug("found path associated to " + item.getIdItemMetadata() + " with path : " + fileURL);

                    }

                    item.setUrl("file://" + fileURL);

                } else {

                    if (log.isDebugEnabled()) {

                        log.debug("NOT found path associated to " + item.getIdItemMetadata()+": "+fileURL);

                    }

                    item.setUrl(null);

                }

                item.setOriginalUrl(baseURL);

                return item;

            } else {

                item.setUrl(null);

            }

        }

        return item;

    }

    public String createPath(final String name) {

        final String regExp = "\\d+\\.\\d+";

        if (name.matches(regExp)) {

            final String[] values = name.split("\\.");

            final Path bsPath = Paths.get(basePath);

            final Path filePath = Paths.get(String.format("%s/%s.pdf", values[0], name));

            final String fileURL = bsPath.resolve(filePath).toString();

            return fileURL;

        } else {

            if (name.contains("/")) {

                final String[] values = name.split("/");

                if (values.length != 2) return null;

                if (values[1].length() > 4) {

                    final String middle = values[1].substring(0, 4);

                    final Path bsPath = Paths.get(basePath);

                    final Path filePath = Paths.get(String.format("%s/%s.pdf", middle, name.replace("/", "")));

                    final String fileURL = bsPath.resolve(filePath).toString();

                    return fileURL;

                }

            }

        }

        return null;

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> items) {

        return Iterables.transform(items, input -> retrieveUrl(input));

    }

    @Override

    public String getPluginName() {

        return "ArxivImportFromFile";

    }

    /**

     * @return the basePath

     */

    public String getBasePath() {

        return basePath;

    }

    /**

     * @param basePath the basePath to set

     */

    @Override

    public void setBasePath(final String basePath) {

        this.basePath = basePath;

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return null;

    }

}

package eu.dnetlib.download.plugin;

/**

 * The Class InfoPath.

 */

public class InfoPath {

	/** The lower. */

	private int lower;

	/** The upper. */

	private int upper;

	/** The path. */

	private String path;

	/**

	 * Gets the lower.

	 *

	 * @return the lower

	 */

	public int getLower() {

		return lower;

	}

	/**

	 * Sets the lower.

	 *

	 * @param lower

	 *            the new lower

	 */

	public void setLower(final int lower) {

		this.lower = lower;

	}

	/**

	 * Gets the upper.

	 *

	 * @return the upper

	 */

	public int getUpper() {

		return upper;

	}

	/**

	 * Sets the upper.

	 *

	 * @param upper

	 *            the new upper

	 */

	public void setUpper(final int upper) {

		this.upper = upper;

	}

	/**

	 * Gets the path.

	 *

	 * @return the path

	 */

	public String getPath() {

		return path;

	}

	/**

	 * Sets the path.

	 *

	 * @param path

	 *            the new path

	 */

	public void setPath(final String path) {

		this.path = path;

	}

}

/**

 *

 */

package eu.dnetlib.download.plugin;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * The Class EasyPDFDownloadPlugin.

 */

public class DLibPlugin extends HALPdfDocumentPlugin {

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

            @Override

            public DownloadItem apply(final DownloadItem input) {

                return retrieveUrl(input);

            }

        });

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "DLIBDownloadPlugin";

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

    @Override

    public String extractURL(String baseURL) throws DownloadPluginException {

        return baseURL.trim().endsWith(".html") ? baseURL : null;

    }

}

package eu.dnetlib.download.plugin;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

public class DSpacePDFLinkPlugins extends AbstractDownloadPlugin implements DownloadPlugin {

    /**

     * The Constant log.

     */

    private static final Log log = LogFactory.getLog(DSpacePDFLinkPlugins.class);

    private final static int maxNumberJump = 10;

    private final static int maxNumberConnectRetries = 5;

    /**

     * Milliseconds used to backoff in case of connection errors.

     */

    private final static int BACKOFF_FACTOR = 100;

    private String getHTTPRedirectedURL(final String mainURL) throws Exception {

        URL startURL = new URL(mainURL);

        HttpURLConnection conn = (HttpURLConnection) startURL.openConnection();

        conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

        conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.

        HttpURLConnection.setFollowRedirects(true);

        String location = mainURL;

        int numJump = 1;

        int responseCode = conn.getResponseCode();

        while ((responseCode >= 300) && (responseCode < 400) && (numJump++ < maxNumberJump)) {

            location = conn.getHeaderFields().get("Location").get(0);

            conn.disconnect();

            startURL = new URL(location);

            conn = (HttpURLConnection) startURL.openConnection();

            conn.setConnectTimeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT);

            conn.setInstanceFollowRedirects(true);  // you still need to handle redirect manually.

            HttpURLConnection.setFollowRedirects(true);

            responseCode = conn.getResponseCode();

        }

        conn.disconnect();

        if (!((responseCode >= 200) && (responseCode < 300)))

            return null;

        return location;

    }

    /**

     * Extract url.

     *

     * @param url the url

     * @return the string

     */

    @Override

    public String extractURL(final String url) throws DownloadPluginException {

        try {

            final String location = getHTTPRedirectedURL(url);

            if (location == null) {

                return null;

            }

            Document doc = null;

            int retries = 0;

            boolean success = false;

            while(retries < maxNumberConnectRetries) {

                try {

                    doc = Jsoup.connect(location).timeout(AbstractDownloadPlugin.DEFAULT_TIMEOUT).get();

                    success = true;

                    break;

                } catch (IOException e) {

                    final int millis = BACKOFF_FACTOR * (retries + 1);

                    log.debug(String.format("backoff for %s ms before retrying on %s", millis, location));

                    Thread.sleep(millis);

                }

                retries++;

            }

            if (!success) {

                throw new DownloadPluginException("reached max number of connect retries for URL: " + location);

            }

            final Elements links = doc.select("meta[content][name=citation_pdf_url]");

            for (Element link : links) {

                String linkValue = link.attr("content");

                if (regularExpression != null) {

                    for (String regex : regularExpression) {

                        if (linkValue.matches(regex)) {

                            return linkValue;

                        }

                    }

                } else {

                    //if(linkValue.matches("^http.*pdf$")){

                        return linkValue;

                    //}

                }

            }

            return null;

        } catch (Throwable e) {

	        throw new DownloadPluginException("Error on extract URL", e);

        }

    }

    @Override

    public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) {

        return Iterables.transform(urls, input -> retrieveUrl(input));

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

    @Override

    public String getPluginName() {

        return "DSpacePDFLinkPlugins";

    }

    /*

     * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

     */

    @Override

    public DownloadItem retrieveUrl(final DownloadItem input) {

        if (checkOpenAccess(input) == null) return null;

        String url = input.getOriginalUrl();

        if ((url == null) || (url.trim().length() == 0)) return input;

        @SuppressWarnings("unchecked")

        List<String> urls = new Gson().fromJson(url, ArrayList.class);

        if ((urls == null) || (urls.size() == 0)) return input;

        if (checkUrlsNotNull(input, urls))

            return input;

        input.setOriginalUrl(null);

        input.setUrl(null);

        return input;

    }

    @Override

    public void setBasePath(final String basePath) {

        // TODO Auto-generated method stub

    }

}

package eu.dnetlib.download.plugin;

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

/**

 * Created by sandro on 3/1/16.

 */

public class HindawiDownloadPlugin extends AbstractDownloadPlugin implements DownloadPlugin {

	private String basePath;

	private Gson gson;

	public HindawiDownloadPlugin() {

		this.gson = new Gson();

	}

	@Override

	public String extractURL(final String baseURL) throws DownloadPluginException {

		return null;

	}

	@Override

	public void setBasePath(final String basePath) {

		this.basePath = basePath;

	}

	/*

	 * (non-Javadoc)

     *

     * @see eu.dnetlib.data.download.rmi.DownloadPlugin#getPluginName()

     */

	@Override

	public String getPluginName() {

		return "HindawiDownloadPlugin";

	}

	@Override

	public Iterable<DownloadItem> retrieveUrls(final Iterable<DownloadItem> urls) throws DownloadPluginException {

		return Iterables.transform(urls, new Function<DownloadItem, DownloadItem>() {

			@Override

			public DownloadItem apply(final DownloadItem input) {

				return retrieveUrl(input);

			}

		});

	}

	/*

	 * (non-Javadoc)

	 *

	 * @see eu.dnetlib.data.download.rmi.DownloadPlugin#retrieveUrl(eu.dnetlib.data.download.rmi.DownloadItem)

	 */

	@Override

	public DownloadItem retrieveUrl(final DownloadItem input) throws DownloadPluginException {

		List<String> inputList = this.gson.fromJson(input.getUrl(), ArrayList.class);

		if (inputList == null || inputList.size() == 0) {

			input.setUrl(null);

			return input;

		}

		for (String fileUrl : inputList) {

			Path inputhPath = FileSystems.getDefault().getPath(basePath).resolve("." + fileUrl);

			if (Files.exists(inputhPath)) {

				input.setOriginalUrl("file://" + inputhPath.toString());

				input.setUrl("file://" + inputhPath.toString());

				return input;

			}

		}

		input.setUrl(null);

		return input;

	}

}

package eu.dnetlib.download.plugin;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.ArrayList;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.gson.Gson;

import eu.dnetlib.data.download.rmi.AbstractDownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadItem;

import eu.dnetlib.data.download.rmi.DownloadPlugin;

import eu.dnetlib.data.download.rmi.DownloadPluginException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;