D-Net

package eu.dnetlib.data.collector.plugins.httpfilename;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.NoSuchElementException;

import java.util.Objects;

import java.util.concurrent.ArrayBlockingQueue;

import java.util.concurrent.TimeUnit;

import eu.dnetlib.data.collector.rmi.CollectorServiceException;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.json.JSONObject;

import org.json.XML;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

/**

 * Created by miriam on 04/05/2018.

 */

public class HTTPWithFileNameCollectorIterable implements Iterable<String> {

    private static final Log log = LogFactory.getLog(HTTPWithFileNameCollectorIterable.class);

    private static final String TERMINATOR = "FINITO";

    private static final String JUNK = "<resource><url>%s</url><DOI>JUNK</DOI></resource>";

    public static final String APP_JSON = "application/json";

    public static final String APP_XML = "application/xml";

    public static final String TEXT_HTML = "text/html";

    private final ArrayBlockingQueue<String> queue = new ArrayBlockingQueue<String>(100);

    private long waitTime = 60L;

    private final ArrayList<String> urls = new ArrayList<>();

    private final ArrayList<String> metas = new ArrayList<String>();

    private String filterParam;

    int total = 0;

    int filtered = 0;

    public HTTPWithFileNameCollectorIterable(String startUrl, String filter){

        if (!startUrl.isEmpty())

            urls.add(startUrl);

        this.filterParam = filter;

        Thread ft = new Thread(new FillMetaQueue());

        ft.start();

    }

    @Override

    public Iterator<String> iterator() {

        return new Iterator<String>(){

            private String last = null;

            private boolean exec_next = true;

            @Override

            public boolean hasNext() {

                if(exec_next){

                    try {

                        last = queue.poll(waitTime, TimeUnit.SECONDS);

                        exec_next = false;

                    }catch(InterruptedException e){

                        log.warn(String.format("could not find elements to consume for more than %s%s", waitTime, TimeUnit.SECONDS));

                        throw new NoSuchElementException(e.getMessage());

                    }

                }

                return !(Objects.equals(last, TERMINATOR) || Objects.equals(last,null));

            }

            @Override

            public String next() {

                exec_next = true;

                return last;

            }

//            @Override

//            public boolean hasNext() {

//

//                return !Objects.equals(last, TERMINATOR);

//            }

//

//            @Override

//            public String next() {

//                try {

//                    last = queue.poll(waitTime, TimeUnit.SECONDS);

//                    if (Objects.equals(last, TERMINATOR)) {

//                        log.info("found terminator, omg!");

//                    }

//                } catch (InterruptedException e) {

//                    log.warn(String.format("could not find elements to consume for more than %s%s", waitTime, TimeUnit.SECONDS));

//                    throw new NoSuchElementException(e.getMessage());

//                }

//                return last;

//            }

        };

    }

    private class FillMetaQueue implements Runnable {

        final Connector c = new Connector();

        public void fillQueue() {

            String url;

            while((metas.size()>0 || urls.size() > 0 )) {

                log.debug("metas.size() = " + metas.size() + " urls.size() = " + urls.size() + " queue.size() = " +queue.size());

                if (metas.size() > 0) {

                    url = metas.remove(0);

                    try {

                        c.get(url);

                    } catch (CollectorServiceException e) {

                        log.info("Impossible to collect url: " + url + " error: " + e.getMessage());

                    }

                    if(c.isStatusOk()){

                        try {

                            String ret = c.getResponse();

                            if (ret != null && ret.length()>0) {

                                if (!containsFilter(ret))

                                    queue.offer(addFilePath(ret, url, url.endsWith(".json")), waitTime, TimeUnit.SECONDS);

                                else

                                    filtered++;

                                total++;

                            }

                        } catch (InterruptedException e) {

                            log.info("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );

                        }

                    }

                } else {

                    url = urls.remove(0);

                    try {

                        c.get(url);

                    } catch (CollectorServiceException e) {

                        log.info("Impossible to collect url: " + url + " error: " + e.getMessage());

                    }

                    if(c.isStatusOk()) {

                        if (c.responseTypeContains(TEXT_HTML)){

                            recurFolder(c.getResponse(), url);

                        } else if(c.responseTypeContains(APP_JSON) || c.responseTypeContains(APP_XML)){

                            try {

                                final String element = addFilePath(c.getResponse(), url, c.responseTypeContains(APP_JSON));

                                queue.offer(element, waitTime, TimeUnit.SECONDS);

                            } catch (InterruptedException e) {

                                log.info("not inserted in queue element associate to url " + url + " error: " + e.getMessage() );

                            }

                        }

                    }

                }

            }

            try {

                queue.offer(TERMINATOR, waitTime, TimeUnit.SECONDS);

            } catch (InterruptedException e) {

                throw new IllegalStateException(String.format("could not add element to queue for more than %s%s", waitTime, TimeUnit.SECONDS), e);

            }

        }

        private boolean containsFilter(String meta){

            if (filterParam == null || filterParam.isEmpty())

                return false;

            String[] filter = filterParam.split(";");

            for(String item:filter){

                if (meta.contains(item))

                    return true;

            }

            return false;

        }

        private String addFilePath(String meta, String url, boolean isJson){

            String path = url.replace("metadata", "pdf");

            try {

                if(isJson)

                    meta = meta.substring(0, meta.length() - 1) + ",'downloadFileUrl':'" + path.substring(0, path.indexOf(".json")) + ".pdf'}";

                else {

                    if (meta.contains("<!DOCTYPE")) {

                        meta = meta.substring(meta.indexOf("<!DOCTYPE"));

                        meta = meta.substring(meta.indexOf(">") + 1);

                    }

                    int index = meta.lastIndexOf("</");

                    meta = meta.substring(0, index) + "<downloadFileUrl>" + path.substring(0, path.indexOf(".xml")) + ".pdf</downloadFileUrl>" + meta.substring(index);

                }

            } catch(Exception ex) {

                log.info("not file with extension .json or .xml");

            }

            if(isJson) {

                try {

                    return XML.toString(new JSONObject("{'resource':" + meta + "}"));

                } catch(Exception e) {

                    log.fatal("Impossible to transform json object to xml \n" + meta + "\n " + e.getMessage() + "\n" + url);

                   // throw new RuntimeException();

                    final String junk = String.format(JUNK, url);

                    log.warn("returning " + junk);

                    return junk;

                }

            }

            return meta;

        }

        private void recurFolder(String text, String url){

            Document doc = Jsoup.parse(text);

            Elements links = doc.select("a");

            for(Element e:links){

                if (!e.text().equals("../")){

                    String file = e.attr("href");

                    if(file.endsWith(".json") || file.endsWith(".xml"))

                        metas.add(url+file);

                    else

                        urls.add(url+file);

                }

            }

        }

        @Override

        public void run() {

            fillQueue();

        }

    }

}