D-Net

package eu.dnetlib.data.claims.utils;

import org.apache.log4j.Logger;

import org.w3c.dom.Document;

import org.w3c.dom.NodeList;

import org.xml.sax.InputSource;

import javax.xml.parsers.DocumentBuilder;

import javax.xml.parsers.DocumentBuilderFactory;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathConstants;

import javax.xml.xpath.XPathFactory;

import java.io.*;

import java.net.HttpURLConnection;

import java.net.URL;

import java.net.URLEncoder;

import java.sql.Timestamp;

import java.util.Date;

/**

 * Created by argirok on 20/11/2015.

 */

/*

*Search and Parsing xmls from Search Service

*Search for DOIs

*

 */

public class SearchUtils {

    private static final Logger logger = Logger.getLogger(SearchUtils.class);

    private static String searchUrl="http://beta.services.openaire.eu:8480/search/search?action=search";

    private static String apiUrlForResults="http://beta.services.openaire.eu:8480/search/api";//"http://api.openaire.eu/search";

//    private static String apiUrlForDedupResults="http://rudie.di.uoa.gr:8080/dnet-functionality-services-1.2.0-SNAPSHOT/api/";

    private static String apiUrlForProjects="http://beta.services.openaire.eu:8480/search/api";//"http://rudie.di.uoa.gr:8080/dnet-functionality-services-1.2.0-SNAPSHOT/api/";

    private static String searchUrlProduction="http://services.openaire.eu:8480/search/search?action=search";

    private static String apiUrlForResultsProduction="http://services.openaire.eu:8480/search/api";//"http://api.openaire.eu/search";

    private static String apiUrlForProjectsProduction="http://services.openaire.eu:8480/search/api";//"http://rudie.di.uoa.gr:8080/dnet-functionality-services-1.2.0-SNAPSHOT/api/";

    private static String crossrefUrl="https://api.crossref.org/works?filter=doi:";

    private static String dataciteUrl="https://data.datacite.org/application/rdf+xml/";

    private static String dataciteNewAPIUrl="https://api.datacite.org/works/";

    private static String orcidUrlPrefix="https://pub.orcid.org/v2.1/";

    private static String orcidUrlSuffix="/works";

//    private static String contextsAPIUrl="https://dev-openaire.d4science.org/openaire/context";

    private static String contextsAPIUrl="http://beta.services.openaire.eu:8080/openaire/context";

    private static String contextsAPIUrlProduction="http://services.openaire.eu:8080/openaire/context";

    private boolean useApi=true;

    private ClaimValidation claimValidation = null;

    public SearchUtils(){

    }

    public static String fetchProjectXmlFromIndex(String id, boolean production) throws Exception{

            return getRequest(getProjectApiUrl(id, production));

    }

    /**

     *Look up in API, if there is no result,

     * consider that id is a dedup id and search for the objIdentifier id.

     * Then look up again in the API with the objIdentifier

     * @param id

     * @return xml or null

     * @throws Exception

     */

    public static String fetchPublicationXmlFromIndex(String id, boolean production) throws Exception{

        String xml=getRequest(getPublicationApiUrl(id, production));

        if(getNumberOfResultsInAPIXML(xml)==0){

            xml=null;

        }

        return xml;

    }

    public static String fetchDedupXmlFromIndex(String id, boolean production) throws Exception{

        String xml= null;

            String objId=getResultObjIdentifierFromSearch(id,production);

            if(objId!=null){

                xml=getRequest(getPublicationApiUrl(objId, production));

            }

        return xml;

    }

    public static String fetchSoftwareXmlFromIndex(String id, boolean production) throws Exception{

        String xml=getRequest(getSoftwareApiUrl(id,production));

         return xml;

    }

    public static String fetchOtherXmlFromIndex(String id, boolean production) throws Exception{

        String xml=getRequest(getOtherApiUrl(id,production));

        return xml;

    }

    /**

     *Look up in API, if there is no result,

     * consider that id is a dedup id and search for the objIdentifier id.

     * Then look up again in the API with the objIdentifier

     *

     * @param id

     * @return xml

     * @throws Exception

     */

    public static String fetchDatasetXmlFromIndex(String id, boolean production) throws Exception{

        String xml=getRequest(getDatasetApiUrl(id, production));

        if(getNumberOfResultsInAPIXML(xml)==0){

            xml=null;

        }

        return xml;

    }

    public static String fetchResultXMLFromDatacite(String id) throws Exception{

            return getRequest(getDataciteUrl(id));

    }

    public static String fetchResultXMLFromOrcid(String id) throws Exception{

            return getRequest(getOrcidUrl(id));

    }

    public static String fetchContext(String suffix, boolean production) throws Exception{

        return getRequest(getContextsAPIUrl(production)+suffix);

    }

     public static String getProjectApiUrl(String id, boolean production)  {

        return ((production)?apiUrlForProjectsProduction:apiUrlForProjects)+"/projects?openaireProjectID="+id;

    }

    public static String getDatasetApiUrl(String id, boolean production)  {

        return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/datasets?openaireDatasetID="+id;

    }

    public static String getPublicationApiUrl(String id, boolean production)  {

        return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/publications?openairePublicationID="+id;

    }

    public static String  getSoftwareApiUrl(String id, boolean production)  {

        return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/software?openaireSoftwareID="+id;

    }

    public static String  getOtherApiUrl(String id, boolean production)  {

        return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/other?openaireOtherID="+id;

    }

//    public static String getDedupPublicationApiUrl(String id)  {

//

//        return apiUrlForDedupResults+"/publications?openairePublicationID="+id;

//    }

    private static String getDataciteUrl(String id)  {

        return dataciteNewAPIUrl+id;

    }

    private static String getOrcidUrl(String id)  {

        return orcidUrlPrefix+id+orcidUrlSuffix;

    }

    private static String getContextsAPIUrl( boolean production)  {

        return ((production)?contextsAPIUrlProduction:contextsAPIUrl);

    }

    /**

     * Get result and objIdentifier  form openaire Search

     * @param resultdupid

     * @return

     */

    private static String getResultXMLByResultdupidSearchByUrl(String resultdupid, boolean production)  {

        String url= null;

        try {

             url = ((production)?searchUrlProduction:searchUrl)+"&sTransformer=results_openaire&query="+

                    URLEncoder.encode("(((deletedbyinference = false) AND (oaftype exact result)) )" +

                            " and (resultdupid exact " + resultdupid + ")", "UTF-8")

                    +"&size=10&locale=en_GB";

        } catch (UnsupportedEncodingException e) {

            logger.error("UnsupportedEncodingException",e);

        }

        return url;

    }

//    private static String getSoftwareXMLByIdSearchByUrl(String id, boolean production)  {

//        String url= null;

//        try {

//            url = ((production)?searchUrlProduction:searchUrl)+"&sTransformer=results_openaire&query="+

//                    URLEncoder.encode("(((deletedbyinference = false) AND (resulttypeid exact software) AND (oaftype exact result)) )" +

//                            " and (objIdentifier exact " + id + ")", "UTF-8")

//                    +"&size=1&locale=en_GB";

//        } catch (UnsupportedEncodingException e) {

//            logger.error("UnsupportedEncodingException",e);

//        }

//        return url;

//    }

    // HTTP GET request

    private static String getRequest(String url) throws Exception {

        URL obj = new URL(url);

        logger.debug(url);

        HttpURLConnection con = (HttpURLConnection) obj.openConnection();

        int responseCode = con.getResponseCode();

        if(responseCode != 200){

            return null;

        }

        BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));

        StringBuffer response = new StringBuffer();

        String inputLine;

        while ((inputLine = in.readLine()) != null) {

            response.append(inputLine).append("\n");

        }

        in.close();

        return response.toString();

    }

    public static String getCrossrefJsonRecord(String doi){

        String url=crossrefUrl+doi;

        URL obj = null;

        String responseStr=null;

        try {

            obj = new URL(url);

            HttpURLConnection con = (HttpURLConnection) obj.openConnection();

            BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));

            StringBuffer response = new StringBuffer();

            String inputLine;

            while ((inputLine = in.readLine()) != null) {

                response.append(inputLine).append("\n");

            }

            in.close();

            responseStr=response.toString();

            if(responseStr==null||(!responseStr.contains("\"status\":\"ok\"")||!(responseStr.contains("\"DOI\":\"")))){

                responseStr=null;

            }

        } catch (Exception e) {

            try{

                PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter("dois_malformed_urls.txt", true)));

                Date date= new java.util.Date();

                out.println(new Timestamp(date.getTime())+"Doi:"+doi+" Url:"+url);

                out.close();

            }catch (IOException e1) {

                logger.error("Couldn't write to file " + "dois_malformed_urls.txt",e);

            }

        }

        return responseStr;

    }

    /**

     *

     * @param doi

     * @return true  if the given doi request returns a valid answer otherwise false

     * @throws IOException

     */

     public static boolean isDoiValid(String doi) throws IOException {

            boolean found=false;

            String responseStr=getCrossrefJsonRecord(doi);

             if(responseStr!=null){

                found=true;

            }

            return found;

    }

    /**

     * Search in index for result with resultdupid

     * @param  resultdupid  Openaire Id

     * @return objIdentifier

     */

    private static String getResultObjIdentifierFromSearch(String resultdupid, boolean  production) throws Exception {

        String xml=getRequest(getResultXMLByResultdupidSearchByUrl(resultdupid,production));

        String objIdentifier=null;

            String size=null;

            DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();

            DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();

            InputSource inputSource= new InputSource(new StringReader(xml));

            Document document=dBuilder.parse(inputSource);

             XPathFactory xPathfactory= XPathFactory.newInstance();

            XPath xpath = xPathfactory.newXPath();

            NodeList nl = (NodeList) xpath.compile("//total/text()").evaluate(document, XPathConstants.NODESET);

            if (nl.getLength() > 0) {

                size= nl.item(0).getNodeValue();

            }

            if(size!=null && Integer.parseInt(size)>0){

                nl = (NodeList) xpath.compile("//field[@indexId='objIdentifier']/@value").evaluate(document, XPathConstants.NODESET);

                if (nl.getLength() > 0) {

                    objIdentifier=nl.item(0).getNodeValue();

                }

            }

        return objIdentifier;

    }

    /**

     *

     * @param xml : API result xml

     * @return number of results Found

     * @throws Exception

     */

    public static Integer getNumberOfResultsInAPIXML(String xml) throws Exception {

        if(xml==null){

            return 0;

        }

        String totalStr=null;

        Integer total=0;

        DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();

        DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();

        InputSource inputSource= new InputSource(new StringReader(xml));

        Document document=dBuilder.parse(inputSource);

        XPathFactory xPathfactory= XPathFactory.newInstance();

        XPath xpath = xPathfactory.newXPath();

        NodeList nl = (NodeList) xpath.compile("//total/text()").evaluate(document, XPathConstants.NODESET);

        if (nl.getLength() > 0) {

            totalStr= nl.item(0).getNodeValue();

        }

        if(totalStr!=null){

            total=Integer.parseInt(totalStr);

        }

        return total;

    }

    public ClaimValidation getClaimValidation() {

        return claimValidation;

    }

    public void setClaimValidation(ClaimValidation claimValidation) {

        this.claimValidation = claimValidation;

    }

}