Revision 49865
Added by Argiro Kokogiannaki over 6 years ago
| SearchUtils.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.claimsDemo; |
| 2 | 2 |
|
| 3 |
import eu.dnetlib.data.claims.migration.ClaimValidation; |
|
| 3 | 4 |
import org.apache.log4j.Logger; |
| 4 | 5 |
import org.w3c.dom.Document; |
| 5 | 6 |
import org.w3c.dom.NodeList; |
| ... | ... | |
| 14 | 15 |
import java.net.HttpURLConnection; |
| 15 | 16 |
import java.net.URL; |
| 16 | 17 |
import java.net.URLEncoder; |
| 18 |
import java.sql.Timestamp; |
|
| 19 |
import java.util.Date; |
|
| 17 | 20 |
|
| 18 | 21 |
/** |
| 19 | 22 |
* Created by argirok on 20/11/2015. |
| ... | ... | |
| 27 | 30 |
private static final Logger logger = Logger.getLogger(SearchUtils.class); |
| 28 | 31 |
|
| 29 | 32 |
private static String searchUrl="http://beta.services.openaire.eu:8480/search/search?action=search"; |
| 30 |
// private static String searchUrl="http://services.openaire.eu:8480/search/search?action=search"; |
|
| 31 |
private static String apiUrlForResults=" http://beta.services.openaire.eu:8480/search/api/";//"http://api.openaire.eu/search/"; |
|
| 32 |
private static String apiUrlForDedupResults="http://rudie.di.uoa.gr:8080/dnet-functionality-services-1.2.0-SNAPSHOT/api/"; |
|
| 33 |
private static String apiUrlForProjects="http://beta.services.openaire.eu:8480/search/api/";//"http://rudie.di.uoa.gr:8080/dnet-functionality-services-1.2.0-SNAPSHOT/api/"; |
|
| 33 |
private static String apiUrlForResults="http://beta.services.openaire.eu:8480/search/api";//"http://api.openaire.eu/search"; |
|
| 34 |
// private static String apiUrlForDedupResults="http://rudie.di.uoa.gr:8080/dnet-functionality-services-1.2.0-SNAPSHOT/api/"; |
|
| 35 |
private static String apiUrlForProjects="http://beta.services.openaire.eu:8480/search/api";//"http://rudie.di.uoa.gr:8080/dnet-functionality-services-1.2.0-SNAPSHOT/api/"; |
|
| 36 |
|
|
| 37 |
private static String searchUrlProduction="http://services.openaire.eu:8480/search/search?action=search"; |
|
| 38 |
private static String apiUrlForResultsProduction="http://services.openaire.eu:8480/search/api";//"http://api.openaire.eu/search"; |
|
| 39 |
private static String apiUrlForProjectsProduction="http://services.openaire.eu:8480/search/api";//"http://rudie.di.uoa.gr:8080/dnet-functionality-services-1.2.0-SNAPSHOT/api/"; |
|
| 40 |
|
|
| 41 |
|
|
| 34 | 42 |
private static String crossrefUrl="https://api.crossref.org/works?filter=doi:"; |
| 35 | 43 |
private static String dataciteUrl="https://data.datacite.org/application/rdf+xml/"; |
| 36 | 44 |
private static String dataciteNewAPIUrl="https://api.datacite.org/works/"; |
| 37 | 45 |
private static String orcidUrlPrefix="https://pub.orcid.org/"; |
| 38 | 46 |
private static String orcidUrlSuffix="/orcid-works"; |
| 39 | 47 |
private boolean useApi=true; |
| 48 |
private ClaimValidation claimValidation = null; |
|
| 40 | 49 |
|
| 41 | 50 |
public SearchUtils(){
|
| 42 | 51 |
|
| 43 | 52 |
} |
| 44 | 53 |
|
| 45 |
public static String fetchProjectXmlFromIndex(String id) throws Exception{
|
|
| 46 |
return getRequest(getProjectApiUrl(id)); |
|
| 54 |
public static String fetchProjectXmlFromIndex(String id, boolean production) throws Exception{
|
|
| 55 |
return getRequest(getProjectApiUrl(id, production));
|
|
| 47 | 56 |
} |
| 48 | 57 |
|
| 49 | 58 |
/** |
| ... | ... | |
| 54 | 63 |
* @return xml or null |
| 55 | 64 |
* @throws Exception |
| 56 | 65 |
*/ |
| 57 |
public static String fetchPublicationXmlFromIndex(String id) throws Exception{
|
|
| 66 |
public static String fetchPublicationXmlFromIndex(String id, boolean production) throws Exception{
|
|
| 58 | 67 |
|
| 59 |
String xml=getRequest(getPublicationApiUrl(id)); |
|
| 68 |
String xml=getRequest(getPublicationApiUrl(id, production));
|
|
| 60 | 69 |
if(getNumberOfResultsInAPIXML(xml)==0){
|
| 61 |
String objId=getResultObjIdentifierFromSearch(id); |
|
| 62 |
if(objId!=null){
|
|
| 63 |
xml=getRequest(getPublicationApiUrl(objId)); |
|
| 64 |
if(getNumberOfResultsInAPIXML(xml)==0){
|
|
| 65 |
xml=null; |
|
| 66 |
} |
|
| 67 |
}else{
|
|
| 68 |
xml=null; |
|
| 69 |
} |
|
| 70 |
xml=null; |
|
| 70 | 71 |
} |
| 71 | 72 |
return xml; |
| 72 | 73 |
} |
| 73 |
public static String fetchDedupPublicationXmlFromIndex(String id) throws Exception{
|
|
| 74 |
public static String fetchDedupXmlFromIndex(String id, boolean production) throws Exception{
|
|
| 74 | 75 |
|
| 75 |
String xml=getRequest(getDedupPublicationApiUrl(id)); |
|
| 76 |
if(getNumberOfResultsInAPIXML(xml)==0){
|
|
| 77 |
String objId=getResultObjIdentifierFromSearch(id); |
|
| 76 |
String xml= null; |
|
| 77 |
String objId=getResultObjIdentifierFromSearch(id,production); |
|
| 78 | 78 |
if(objId!=null){
|
| 79 |
xml=getRequest(getDedupPublicationApiUrl(objId)); |
|
| 80 |
if(getNumberOfResultsInAPIXML(xml)==0){
|
|
| 81 |
xml=null; |
|
| 82 |
} |
|
| 83 |
}else{
|
|
| 84 |
xml = null; |
|
| 79 |
xml=getRequest(getPublicationApiUrl(objId, production)); |
|
| 85 | 80 |
} |
| 86 |
} |
|
| 87 | 81 |
return xml; |
| 88 | 82 |
} |
| 89 | 83 |
|
| 84 |
public static String fetchSoftwareXmlFromIndex(String id, boolean production) throws Exception{
|
|
| 85 |
String xml=getRequest(getSoftwareApiUrl(id,production)); |
|
| 86 |
return xml; |
|
| 87 |
} |
|
| 88 |
|
|
| 89 |
|
|
| 90 | 90 |
/** |
| 91 | 91 |
*Look up in API, if there is no result, |
| 92 | 92 |
* consider that id is a dedup id and search for the objIdentifier id. |
| ... | ... | |
| 96 | 96 |
* @return xml |
| 97 | 97 |
* @throws Exception |
| 98 | 98 |
*/ |
| 99 |
public static String fetchDatasetXmlFromIndex(String id) throws Exception{
|
|
| 99 |
public static String fetchDatasetXmlFromIndex(String id, boolean production) throws Exception{
|
|
| 100 | 100 |
|
| 101 |
String xml=getRequest(getDatasetApiUrl(id)); |
|
| 101 |
String xml=getRequest(getDatasetApiUrl(id, production));
|
|
| 102 | 102 |
if(getNumberOfResultsInAPIXML(xml)==0){
|
| 103 |
String objId=getResultObjIdentifierFromSearch(id); |
|
| 104 |
if(objId!=null){
|
|
| 105 |
xml=getRequest(getDatasetApiUrl(objId)); |
|
| 106 |
if(getNumberOfResultsInAPIXML(xml)==0){
|
|
| 107 |
xml=null; |
|
| 108 |
} |
|
| 109 |
}else{
|
|
| 110 |
xml=null; |
|
| 111 |
} |
|
| 103 |
xml=null; |
|
| 112 | 104 |
} |
| 113 | 105 |
return xml; |
| 114 | 106 |
|
| 115 | 107 |
} |
| 116 |
public static String fetchDedupDatasetXmlFromIndex(String id) throws Exception{
|
|
| 117 | 108 |
|
| 118 |
String xml=getRequest(getDedupDatasetApiUrl(id)); |
|
| 119 |
if(getNumberOfResultsInAPIXML(xml)==0){
|
|
| 120 |
String objId=getResultObjIdentifierFromSearch(id); |
|
| 121 |
if(objId!=null){
|
|
| 122 |
xml=getRequest(getDedupDatasetApiUrl(objId)); |
|
| 123 |
if(getNumberOfResultsInAPIXML(xml)==0){
|
|
| 124 |
xml=null; |
|
| 125 |
} |
|
| 126 |
}else{
|
|
| 127 |
xml = null; |
|
| 128 |
} |
|
| 129 |
} |
|
| 130 |
return xml; |
|
| 131 |
|
|
| 132 |
} |
|
| 133 |
|
|
| 134 | 109 |
public static String fetchResultXMLFromDatacite(String id) throws Exception{
|
| 135 | 110 |
return getRequest(getDataciteUrl(id)); |
| 136 | 111 |
} |
| ... | ... | |
| 139 | 114 |
return getRequest(getOrcidUrl(id)); |
| 140 | 115 |
} |
| 141 | 116 |
|
| 142 |
public static String getProjectApiUrl(String id) {
|
|
| 117 |
public static String getProjectApiUrl(String id, boolean production) {
|
|
| 143 | 118 |
|
| 144 |
return apiUrlForProjects+"/projects?openaireProjectID="+id;
|
|
| 119 |
return ((production)?apiUrlForProjectsProduction:apiUrlForProjects)+"/projects?openaireProjectID="+id;
|
|
| 145 | 120 |
} |
| 146 | 121 |
|
| 147 |
public static String getDatasetApiUrl(String id) {
|
|
| 122 |
public static String getDatasetApiUrl(String id, boolean production) {
|
|
| 148 | 123 |
|
| 149 |
return apiUrlForResults+"/datasets?openaireDatasetID="+id;
|
|
| 124 |
return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/datasets?openaireDatasetID="+id;
|
|
| 150 | 125 |
} |
| 151 |
public static String getDedupDatasetApiUrl(String id) {
|
|
| 152 | 126 |
|
| 153 |
return apiUrlForDedupResults+"/datasets?openaireDatasetID="+id; |
|
| 154 |
} |
|
| 155 |
public static String getPublicationApiUrl(String id) {
|
|
| 127 |
public static String getPublicationApiUrl(String id, boolean production) {
|
|
| 156 | 128 |
|
| 157 |
return apiUrlForResults+"/publications?openairePublicationID="+id;
|
|
| 129 |
return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/publications?openairePublicationID="+id;
|
|
| 158 | 130 |
} |
| 159 |
public static String getDedupPublicationApiUrl(String id) {
|
|
| 160 |
|
|
| 161 |
return apiUrlForDedupResults+"/publications?openairePublicationID="+id; |
|
| 131 |
public static String getSoftwareApiUrl(String id, boolean production) {
|
|
| 132 |
//TODO change to beta when it's ready |
|
| 133 |
// return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/software/"+id; |
|
| 134 |
return "http://rudie.di.uoa.gr:8080/dnet-functionality-services-2.0.0-SNAPSHOT/api"+"/software?openaireSoftwareID="+id; |
|
| 162 | 135 |
} |
| 136 |
// public static String getDedupPublicationApiUrl(String id) {
|
|
| 137 |
// |
|
| 138 |
// return apiUrlForDedupResults+"/publications?openairePublicationID="+id; |
|
| 139 |
// } |
|
| 163 | 140 |
private static String getDataciteUrl(String id) {
|
| 164 | 141 |
|
| 165 | 142 |
return dataciteNewAPIUrl+id; |
| ... | ... | |
| 176 | 153 |
* @param resultdupid |
| 177 | 154 |
* @return |
| 178 | 155 |
*/ |
| 179 |
private static String getResultXMLByResultdupidSearchByUrl(String resultdupid) {
|
|
| 156 |
private static String getResultXMLByResultdupidSearchByUrl(String resultdupid, boolean production) {
|
|
| 180 | 157 |
String url= null; |
| 181 | 158 |
try {
|
| 182 |
url = searchUrl+"&sTransformer=results_openaire&query="+
|
|
| 159 |
url = ((production)?searchUrlProduction:searchUrl)+"&sTransformer=results_openaire&query="+
|
|
| 183 | 160 |
URLEncoder.encode("(((deletedbyinference = false) AND (oaftype exact result)) )" +
|
| 184 | 161 |
" and (resultdupid exact " + resultdupid + ")", "UTF-8") |
| 185 | 162 |
+"&size=10&locale=en_GB"; |
| ... | ... | |
| 188 | 165 |
} |
| 189 | 166 |
return url; |
| 190 | 167 |
} |
| 168 |
// private static String getSoftwareXMLByIdSearchByUrl(String id, boolean production) {
|
|
| 169 |
// String url= null; |
|
| 170 |
// try {
|
|
| 171 |
// url = ((production)?searchUrlProduction:searchUrl)+"&sTransformer=results_openaire&query="+ |
|
| 172 |
// URLEncoder.encode("(((deletedbyinference = false) AND (resulttypeid exact software) AND (oaftype exact result)) )" +
|
|
| 173 |
// " and (objIdentifier exact " + id + ")", "UTF-8") |
|
| 174 |
// +"&size=1&locale=en_GB"; |
|
| 175 |
// } catch (UnsupportedEncodingException e) {
|
|
| 176 |
// logger.error("UnsupportedEncodingException",e);
|
|
| 177 |
// } |
|
| 178 |
// return url; |
|
| 179 |
// } |
|
| 191 | 180 |
|
| 192 |
|
|
| 193 | 181 |
// HTTP GET request |
| 194 | 182 |
private static String getRequest(String url) throws Exception {
|
| 195 | 183 |
URL obj = new URL(url); |
| 184 |
// logger.debug(url); |
|
| 196 | 185 |
HttpURLConnection con = (HttpURLConnection) obj.openConnection(); |
| 197 | 186 |
int responseCode = con.getResponseCode(); |
| 198 | 187 |
if(responseCode != 200){
|
| ... | ... | |
| 229 | 218 |
} catch (Exception e) {
|
| 230 | 219 |
try{
|
| 231 | 220 |
PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter("dois_malformed_urls.txt", true)));
|
| 232 |
out.println("Doi:"+doi+" Url:"+url);
|
|
| 221 |
Date date= new java.util.Date(); |
|
| 222 |
out.println(new Timestamp(date.getTime())+"Doi:"+doi+" Url:"+url); |
|
| 233 | 223 |
out.close(); |
| 234 | 224 |
|
| 235 | 225 |
}catch (IOException e1) {
|
| ... | ... | |
| 260 | 250 |
* @param resultdupid Openaire Id |
| 261 | 251 |
* @return objIdentifier |
| 262 | 252 |
*/ |
| 263 |
private static String getResultObjIdentifierFromSearch(String resultdupid) throws Exception {
|
|
| 264 |
String xml=getRequest(getResultXMLByResultdupidSearchByUrl(resultdupid)); |
|
| 253 |
private static String getResultObjIdentifierFromSearch(String resultdupid, boolean production) throws Exception {
|
|
| 254 |
String xml=getRequest(getResultXMLByResultdupidSearchByUrl(resultdupid,production));
|
|
| 265 | 255 |
String objIdentifier=null; |
| 266 | 256 |
String size=null; |
| 267 | 257 |
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); |
| ... | ... | |
| 311 | 301 |
return total; |
| 312 | 302 |
} |
| 313 | 303 |
|
| 304 |
public ClaimValidation getClaimValidation() {
|
|
| 305 |
return claimValidation; |
|
| 306 |
} |
|
| 314 | 307 |
|
| 308 |
public void setClaimValidation(ClaimValidation claimValidation) {
|
|
| 309 |
this.claimValidation = claimValidation; |
|
| 310 |
} |
|
| 315 | 311 |
} |
| 316 | 312 |
|
| 317 | 313 |
|
Also available in: Unified diff
change migration procedure: search first in beta then in production| add software | change directIndexHandler check for response