D-Net

package eu.dnetlib.data.collector.plugins.datacite;

import com.google.gson.Gson;

import com.google.gson.GsonBuilder;

import eu.dnetlib.data.collector.plugins.datacite.schema.DataciteSchema;

import eu.dnetlib.data.collector.plugins.datacite.schema.Result;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import java.io.ByteArrayOutputStream;

import java.io.IOException;

import java.net.URL;

import java.util.ArrayDeque;

import java.util.Iterator;

import java.util.Objects;

import java.util.Queue;

import java.util.zip.DataFormatException;

import java.util.zip.Inflater;

public class DataciteESIterator implements Iterator<String> {

    private final long timestamp;

    private String scrollId;

    private Queue<String> currentPage;

    private final Gson g =  new GsonBuilder().create();

    private static final String BASE_URL= "http://ip-90-147-167-25.ct1.garrservices.it:5000/new_scan";

    private static final String NEXT_URL= "http://ip-90-147-167-25.ct1.garrservices.it:5000/scan/%s";

    public DataciteESIterator(long timestamp) throws Exception {

        this.timestamp = timestamp;

        currentPage = new ArrayDeque<>();

        startRequest();

    }

    private static String decompression(final Result r) {

        try {

            byte[] byteArray = Base64.decodeBase64(r.getBody().getBytes());

            Inflater decompresser = new Inflater();

            decompresser.setInput(byteArray);

            ByteArrayOutputStream bos = new ByteArrayOutputStream(byteArray.length);

            byte[] buffer = new byte[8192];

            while (!decompresser.finished()) {

                int size = decompresser.inflate(buffer);

                bos.write(buffer, 0, size);

            }

            byte[] unzippeddata = bos.toByteArray();

            decompresser.end();

            return new String(unzippeddata);

        } catch (DataFormatException e) {

            return null;

        }

    }

    private void fillQueue(final String hits) {

        if (StringUtils.isBlank(hits) || "[]".equalsIgnoreCase(hits.trim()))

            return;

        try {

            DataciteSchema datacitepage = g.fromJson(hits, DataciteSchema.class);

            this.scrollId = datacitepage.getScrollId();

            datacitepage.getResult().stream().map(DataciteESIterator::decompression).filter(Objects::nonNull).forEach(this.currentPage::add);

        } catch (Throwable e) {

            System.out.println(hits);

            e.printStackTrace();

        }

    }

    private void startRequest() throws Exception {

        final URL startUrl = new URL(timestamp >0 ?BASE_URL + "?timestamp="+timestamp: BASE_URL);

        fillQueue(IOUtils.toString(startUrl.openStream()));

    }

    private void getNextPage() throws IOException {

        final URL startUrl = new URL(String.format(NEXT_URL,scrollId));

        fillQueue(IOUtils.toString(startUrl.openStream()));

    }

    @Override

    public boolean hasNext() {

        return  currentPage.size() >0;

    }

    @Override

    public String next() {

        if (currentPage.size() == 0) {

            return null;

        }

        String nextItem = currentPage.remove();

        if (currentPage.size() == 0) {

            try {

                getNextPage();

            } catch (Throwable e) {

                throw new RuntimeException(e);

            }

        }

        return nextItem;

    }

}

package eu.dnetlib.data.collector.plugins.datacite.schema;

import java.util.List;

import com.google.gson.annotations.Expose;

import com.google.gson.annotations.SerializedName;

public class DataciteSchema {

    @SerializedName("counter")

    @Expose

    private Integer counter;

    @SerializedName("result")

    @Expose

    private List<Result> result = null;

    @SerializedName("scroll_id")

    @Expose

    private String scrollId;

    @SerializedName("total")

    @Expose

    private Integer total;

    public Integer getCounter() {

        return counter;

    }

    public void setCounter(Integer counter) {

        this.counter = counter;

    }

    public List<Result> getResult() {

        return result;

    }

    public void setResult(List<Result> result) {

        this.result = result;

    }

    public String getScrollId() {

        return scrollId;

    }

    public void setScrollId(String scrollId) {

        this.scrollId = scrollId;

    }

    public Integer getTotal() {

        return total;

    }

    public void setTotal(Integer total) {

        this.total = total;

    }

}

package eu.dnetlib.data.collector.plugins.datacite.schema;

import com.google.gson.annotations.Expose;

import com.google.gson.annotations.SerializedName;

public class Result {

    @SerializedName("body")

    @Expose

    private String body;

    @SerializedName("id")

    @Expose

    private String id;

    @SerializedName("originalId")

    @Expose

    private String originalId;

    @SerializedName("timestamp")

    @Expose

    private Integer timestamp;

    public String getBody() {

        return body;

    }

    public void setBody(String body) {

        this.body = body;

    }

    public String getId() {

        return id;

    }

    public void setId(String id) {

        this.id = id;

    }

    public String getOriginalId() {

        return originalId;

    }

    public void setOriginalId(String originalId) {

        this.originalId = originalId;

    }

    public Integer getTimestamp() {

        return timestamp;

    }

    public void setTimestamp(Integer timestamp) {

        this.timestamp = timestamp;

    }

}

package eu.dnetlib.data.collector.plugins.datacite;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.plugin.CollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

import eu.dnetlib.miscutils.datetime.DateUtils;

import org.apache.commons.lang3.StringUtils;

import java.util.Date;

public class DataciteCollectorPlugin extends AbstractCollectorPlugin implements CollectorPlugin {

    @Override

    public Iterable<String> collect(InterfaceDescriptor interfaceDescriptor, String fromDate, String untilDate) throws CollectorServiceException {

        long timestamp= 0;

        if (StringUtils.isNotBlank(fromDate)) {

            Date parsed = new DateUtils().parse(fromDate);

            timestamp =parsed.getTime() /1000;

        }

        final long finalTimestamp = timestamp;

        return () -> {

            try {

                return new DataciteESIterator(finalTimestamp);

            } catch (Exception e) {

                throw new RuntimeException(e );

            }

        };

    }

}

	<bean id="dataciteESPlugin" class="eu.dnetlib.data.collector.plugins.datacite.DataciteCollectorPlugin">

		<property name="protocolDescriptor">

			<bean class="eu.dnetlib.data.collector.rmi.ProtocolDescriptor" p:name="dataciteESPlugins"/>

		</property>

	</bean>