D-Net

package eu.dnetlib.data.collector.plugins.doiresolver;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.junit.MockitoJUnitRunner;

import static org.mockito.Mockito.when;

@RunWith(MockitoJUnitRunner.class)

public class DOIResolverIteratorTest {

    @Mock

    CrossrefResolver resolver;

    DOIResolverIterator it;

    @Before

    public void setup(){

        when(resolver.resolve("1")).thenReturn("RECORD1");

        when(resolver.resolve("2")).thenReturn(null);

        when(resolver.resolve("3")).thenReturn("RECORD3");

        String file = getClass().getResource("/eu/dnetlib/data/collector/plugins/doiresolver/doi_list.csv").getFile();

        it = new DOIResolverIterator(file, resolver);

    }

    @Test

    public void test(){

        while(it.hasNext()){

            System.out.println(it.next());

        }

    }

    @Test

    public void testCleanOk(){

        String doi = "10.1234/1234";

        Assert.assertEquals(doi, it.cleanDOI(doi));

    }

    @Test

    public void testCleanHttp(){

        String doi = "10.1234/1234";

        String doiURL = "http://dx.doi.org/"+doi;

        Assert.assertEquals(doi, it.cleanDOI(doiURL));

    }

    @Test

    public void testCleanHttps(){

        String doi = "10.1234/1234";

        String doiURL = "https://dx.doi.org/"+doi;

        Assert.assertEquals(doi, it.cleanDOI(doiURL));

    }

}

package eu.dnetlib.data.collector.plugins.doiresolver;

import eu.dnetlib.data.collector.plugins.HttpConnector;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

@Ignore

public class CrossrefResolverTest {

    HttpConnector httpConnector;

    CrossrefResolver resolver;

    @Before

    public void setup(){

        httpConnector = new HttpConnector();

        resolver = new CrossrefResolver();

        resolver.setHttpConnector(httpConnector);

    }

    @Test

    public void testUnexistingDOI(){

        String doi = "abcd";

        Assert.assertNull(resolver.resolve(doi));

    }

    @Test

    public void testResolveDOI(){

        String doi = "10.1016/j.carbpol.2020.115930";

        String record = resolver.resolve(doi);

        Assert.assertNotNull(record);

        System.out.println(record);

    }

}

1

2

3

package eu.dnetlib.data.collector.plugins.doiresolver;

import eu.dnetlib.data.collector.plugins.utils.JsonUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import java.io.IOException;

import java.nio.file.Files;

import java.nio.file.Paths;

import java.util.Iterator;

import java.util.concurrent.ArrayBlockingQueue;

public class DOIResolverIterator implements Iterator<String> {

    private static final Log log = LogFactory.getLog(DOIResolverIterator.class);

    private static final String STARTER = "FIRE";

    private static final String TERMINATOR = "ARNOLD";

    private static final String BAD_TERMINATOR = "BAD";

    private static final String UNRESOLVED = "UNRESOLVED";

    /** Path to the file that contains a list of DOIs, one per line. **/

    private String filePath;

    private ArrayBlockingQueue<String> queue;

    private CrossrefResolver crossrefResolver;

    public DOIResolverIterator(final String filePath, final CrossrefResolver crossrefResolver) {

        this.filePath = filePath;

        this.queue = new ArrayBlockingQueue<>(100);

        this.crossrefResolver = crossrefResolver;

        init();

    }

    private void init(){

        new Thread(() -> {

            // put first item in the queue

            if(queue.offer(STARTER)) {

                // read the file, ask the resolvers, put results in a shared queue

                //whatever exceptions, add terminator to the queue

                try{

                    Files.lines(Paths.get(filePath)).forEach(doi -> queue.offer(resolve(doi)));

                } catch (IOException e) {

                    log.error(e);

                    queue.offer(BAD_TERMINATOR);

                }

            }

            queue.offer(TERMINATOR);

        }

        ).start();

    }

    private String resolve(final String doi){

       log.debug("Resolving "+doi);

       log.debug("Crossref...");

       String record = crossrefResolver.resolve(cleanDOI(doi));

       if(StringUtils.isNotBlank(record)) return record;

       else {

           //try another resolver

       }

       return UNRESOLVED;

    }

    /**

     * Returns the identifier part of the DOI only.

     * @param doi

     * @return the DOI

     */

    protected String cleanDOI(final String doi){

       return doi.replace("http://dx.doi.org/", "").replace("https://dx.doi.org/", "");

    }

    @Override

    public boolean hasNext() {

        //If I get a null value, the queue is currently empty. so we wait for something

        if(queue.peek() == null){

            try {

                Thread.sleep(10);

            } catch (InterruptedException e) {

                e.printStackTrace();

            }

            return hasNext();

        }

       if(queue.peek().equals(TERMINATOR) || queue.peek().equals(BAD_TERMINATOR)){

           return false;

       }

       if(queue.peek().equals(UNRESOLVED) || queue.peek().equals(STARTER)){

           queue.poll();

           return hasNext();

       }

        return true;

    }

    @Override

    public String next() {

        return queue.poll();

    }

    public String getFilePath() {

        return filePath;

    }

    public void setFilePath(String filePath) {

        this.filePath = filePath;

    }

    public CrossrefResolver getCrossrefResolver() {

        return crossrefResolver;

    }

    public void setCrossrefResolver(CrossrefResolver crossrefResolver) {

        this.crossrefResolver = crossrefResolver;

    }

}

package eu.dnetlib.data.collector.plugins.doiresolver;

import eu.dnetlib.data.collector.plugins.HttpConnector;

import eu.dnetlib.data.collector.plugins.utils.JsonUtils;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import java.io.IOException;

import java.io.InputStream;

public class CrossrefResolver implements DOIResolver{

    private static final Log log = LogFactory.getLog(CrossrefResolver.class);

    private String baseURL = "https://api.crossref.org/works/";

    @Autowired

    private HttpConnector httpConnector;

    @Override

    public String resolve(String doi) {

        try {

            InputStream is = httpConnector.getInputSourceAsStream(getBaseURL()+doi);

            return asXml(IOUtils.toString(is));

        } catch (IOException | CollectorServiceException e) {

            log.error("Cannot resolve doi "+doi+" Exception: "+e);

            return null;

        }

    }

    protected String asXml(final String record){

       return new JsonUtils().convertToXML(record);

    }

    public String getBaseURL() {

        return baseURL;

    }

    public void setBaseURL(final String baseURL) {

        this.baseURL = baseURL;

    }

    public HttpConnector getHttpConnector() {

        return httpConnector;

    }

    public void setHttpConnector(HttpConnector httpConnector) {

        this.httpConnector = httpConnector;

    }

}

package eu.dnetlib.data.collector.plugins.doiresolver;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

public class DOIResolverPlugin extends AbstractCollectorPlugin {

    private CrossrefResolver crossrefResolver;

    @Override

    public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate) throws CollectorServiceException {

        //check baseurl not blank

        return () -> new DOIResolverIterator(interfaceDescriptor.getBaseUrl(), crossrefResolver);

    }

    public CrossrefResolver getCrossrefResolver() {

        return crossrefResolver;

    }

    public void setCrossrefResolver(CrossrefResolver crossrefResolver) {

        this.crossrefResolver = crossrefResolver;

    }

}

package eu.dnetlib.data.collector.plugins.doiresolver;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

public interface DOIResolver {

    String resolve(String doi);

    void setBaseURL(String baseURL);

}

	<bean id="doiResolverPlugin" class="eu.dnetlib.data.collector.plugins.doiresolver.DOIResolverPlugin" p:crossrefResolver-ref="crossrefResolver" >

		<property name="protocolDescriptor">

			<bean class="eu.dnetlib.data.collector.rmi.ProtocolDescriptor" p:name="doi_resolver"/>

		</property>

	</bean>

	<bean id="crossrefResolver" class="eu.dnetlib.data.collector.plugins.doiresolver.CrossrefResolver"/>

        <dependency>

            <groupId>org.mockito</groupId>

            <artifactId>mockito-core</artifactId>

            <version>3.3.3</version>

            <scope>test</scope>

        </dependency>

    </dependencies>