D-Net

import org.apache.commons.lang3.StringUtils;

		try{

			final JSONArray arr = json.getJSONObject("replace").getJSONArray(elem);

			for(Object entry: arr)

		}catch(Throwable e){

			log.error("Problems filling the map for " + elem);

			throw(e);

		if (StringUtils.isNotEmpty(argument)){

			try{

				final JSONObject json = new JSONObject(argument);

				if(json.has("header"))

					fillMap(json, map_header,"header");

				if (json.has("body"))

					fillMap(json,map_body,"body");

				if(json.has("replace_currency"))

				{

					replace_currency = true	;

					from_currency = json.getJSONArray("replace_currency").getJSONObject(0).getString("from");

					to_currency = json.getJSONArray("replace_currency").getJSONObject(0).getString("to");

				}

				if (json.has("col_currency"))

					currency_column = json.getInt("col_currency");

			}catch(Throwable e){

				log.error("Problems while parsing the argument parameter.");

				throw (e);

			}

package eu.dnetlib.data.collector.plugins.HTTPWithFileName;

import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;

/**

 * Created by miriam on 04/05/2018.

 */

public class HTTPWithFileNameCollectorPlugin extends AbstractCollectorPlugin {

    @Override

    public Iterable<String> collect(InterfaceDescriptor interfaceDescriptor, String s, String s1) throws CollectorServiceException {

        return new HTTPWithFileNameCollectorIterable(interfaceDescriptor.getBaseUrl());

    }

}

package eu.dnetlib.data.collector.plugins.HTTPWithFileName;

import eu.dnetlib.data.collector.plugins.HttpConnector;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

/**

 * Created by miriam on 07/05/2018.

 */

public class Connector extends HttpConnector implements ConnectorInterface  {

    private String response;

    @Override

    public void get(final String requestUrl) throws CollectorServiceException {

        response = getInputSource(requestUrl);

    }

    @Override

    public String getResponse() {

        return response;

    }

    @Override

    public boolean isStatusOk() {

        return (response != null);

    }

    @Override

    public boolean responseTypeContains(String string) {

        String responseType = getResponseType();

        if (responseType != null)

            return responseType.contains(string);

        return false;

    }

}

package eu.dnetlib.data.collector.plugins.HTTPWithFileName;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

/**

 * Created by miriam on 07/05/2018.

 */

public interface ConnectorInterface {

    public void get(final String requestUrl) throws CollectorServiceException;

    public String getResponse();

    public boolean isStatusOk();

    public boolean responseTypeContains(String string);

}

package eu.dnetlib.data.collector.plugins.HTTPWithFileName;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.json.JSONObject;

import org.json.XML;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.function.Consumer;

/**

 * Created by miriam on 04/05/2018.

 */

public class HTTPWithFileNameCollectorIterable implements Iterable<String> {

    private static final Log log = LogFactory.getLog(HTTPWithFileNameCollectorIterable.class);

    private final ArrayList<String> urls = new ArrayList<>();

    private final ArrayList<String> jsons = new ArrayList<String>();

    public HTTPWithFileNameCollectorIterable(String startUrl){

        urls.add(startUrl);

    }

    private String addFilePath(String json,String url){

        String path = url.replace("metadata", "pdf");

        try {

            json = json.substring(0, json.length() - 1) + ",'downloadFileUrl':'" + path.substring(0, path.indexOf(".json")) + ".pdf'}";

        }catch(Exception ex){

            log.info("not file with extension .json");

        }

        JSONObject jsonobj = new JSONObject("{'metadata':"+json+"}");

        return XML.toString(jsonobj);

    }

    private void recurFolder(String text, String url){

        Document doc = Jsoup.parse(text);

        Elements links = doc.select("a");

        for(Element e:links){

            if (!e.text().equals("../")){

                String file = e.attr("href");

                if(file.endsWith(".json"))

                    jsons.add(url+file);

                else

                    urls.add(url+file);

            }

        }

    }

    @Override

    public Iterator<String> iterator() {

        final ArrayBlockingQueue<String> queue = new ArrayBlockingQueue<String>(100);

        return new Iterator<String>(){

            public void fillQueue() {

                Connector c = new Connector();

                String url;

                while((jsons.size()>0 || urls.size() > 0 ) && queue.size()<100){

                    if (jsons.size() > 0){

                        url = jsons.remove(0);

                        try {

                            c.get(url);

                        } catch (CollectorServiceException e) {

                            log.error("Impossible to collect url: " + url + " error: " + e.getMessage());

                        }

                        if(c.isStatusOk()){

                            try {

                                String ret = c.getResponse();

                                if (ret != null && ret.length()>0)

                                    queue.put(addFilePath(ret,url));

                            } catch (InterruptedException e) {

                                log.error("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );

                            }

                        }

                    }else{

                        url = urls.remove(0);

                        try {

                            c.get(url);

                        } catch (CollectorServiceException e) {

                            log.error("Impossible to collect url: " + url + " error: " + e.getMessage());

                        }

                        if(c.isStatusOk()){

                            if (c.responseTypeContains("text/html")){

                                recurFolder(c.getResponse(),url);

                            }

                            else if(c.responseTypeContains("application/json")){

                                try {

                                    queue.put(addFilePath(c.getResponse(),url));

                                } catch (InterruptedException e) {

                                    log.error("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );

                                }

                            }

                        }

                    }

                }

            }

            @Override

            public boolean hasNext() {

                if (queue.isEmpty()){

                    fillQueue();

                }

                return (!queue.isEmpty());

            }

            @Override

            public String next() {

                return queue.poll(); 

            }

            @Override

            public void remove() {

            }

            @Override

            public void forEachRemaining(Consumer<? super String> action) {

            }

        };

    }

}