Revision 52026
Added by Miriam Baglioni over 6 years ago
modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/HTTPWithFileName/HTTPWithFileNameCollectorIterable.java | ||
---|---|---|
23 | 23 |
private static final Log log = LogFactory.getLog(HTTPWithFileNameCollectorIterable.class); |
24 | 24 |
|
25 | 25 |
private final ArrayList<String> urls = new ArrayList<>(); |
26 |
private final ArrayList<String> jsons = new ArrayList<String>();
|
|
26 |
private final ArrayList<String> metas = new ArrayList<String>();
|
|
27 | 27 |
|
28 | 28 |
|
29 | 29 |
public HTTPWithFileNameCollectorIterable(String startUrl){ |
... | ... | |
31 | 31 |
urls.add(startUrl); |
32 | 32 |
} |
33 | 33 |
|
34 |
private String addFilePath(String json,String url){
|
|
34 |
private String addFilePath(String meta,String url, boolean isJson){
|
|
35 | 35 |
String path = url.replace("metadata", "pdf"); |
36 | 36 |
try { |
37 |
json = json.substring(0, json.length() - 1) + ",'downloadFileUrl':'" + path.substring(0, path.indexOf(".json")) + ".pdf'}"; |
|
37 |
if(isJson) |
|
38 |
meta = meta.substring(0, meta.length() - 1) + ",'downloadFileUrl':'" + path.substring(0, path.indexOf(".json")) + ".pdf'}"; |
|
39 |
else |
|
40 |
meta = meta.substring(0, meta.length() - 10) + "<downloadFileUrl>'" + path.substring(0, path.indexOf(".xml")) + ".pdf</downloadFileUrl></article>"; |
|
38 | 41 |
}catch(Exception ex){ |
39 |
log.info("not file with extension .json"); |
|
42 |
log.info("not file with extension .json or .xml");
|
|
40 | 43 |
} |
41 | 44 |
|
42 |
JSONObject jsonobj = new JSONObject("{'metadata':"+json+"}"); |
|
43 |
|
|
44 |
return XML.toString(jsonobj); |
|
45 |
//JSONObject jsonobj = new JSONObject("{'metadata':"+json+"}"); |
|
46 |
if(isJson){ |
|
47 |
JSONObject jsonobj = new JSONObject(meta); |
|
48 |
return XML.toString(jsonobj); |
|
49 |
} |
|
50 |
return meta; |
|
45 | 51 |
} |
46 | 52 |
|
47 | 53 |
private void recurFolder(String text, String url){ |
... | ... | |
50 | 56 |
for(Element e:links){ |
51 | 57 |
if (!e.text().equals("../")){ |
52 | 58 |
String file = e.attr("href"); |
53 |
if(file.endsWith(".json")) |
|
54 |
jsons.add(url+file);
|
|
59 |
if(file.endsWith(".json") || file.endsWith(".xml"))
|
|
60 |
metas.add(url+file);
|
|
55 | 61 |
else |
56 | 62 |
urls.add(url+file); |
57 | 63 |
} |
... | ... | |
69 | 75 |
public void fillQueue() { |
70 | 76 |
Connector c = new Connector(); |
71 | 77 |
String url; |
72 |
while((jsons.size()>0 || urls.size() > 0 ) && queue.size()<100){
|
|
73 |
if (jsons.size() > 0){
|
|
74 |
url = jsons.remove(0);
|
|
78 |
while((metas.size()>0 || urls.size() > 0 ) && queue.size()<100){
|
|
79 |
if (metas.size() > 0){
|
|
80 |
url = metas.remove(0);
|
|
75 | 81 |
try { |
76 | 82 |
c.get(url); |
77 | 83 |
} catch (CollectorServiceException e) { |
... | ... | |
81 | 87 |
try { |
82 | 88 |
String ret = c.getResponse(); |
83 | 89 |
if (ret != null && ret.length()>0) |
84 |
queue.put(addFilePath(ret,url)); |
|
90 |
queue.put(addFilePath(ret,url,url.endsWith(".json")));
|
|
85 | 91 |
} catch (InterruptedException e) { |
86 | 92 |
log.error("not inserted in queue element associate to url " + url + " error: " + e.getMessage() ); |
87 | 93 |
|
... | ... | |
98 | 104 |
if (c.responseTypeContains("text/html")){ |
99 | 105 |
recurFolder(c.getResponse(),url); |
100 | 106 |
} |
101 |
else if(c.responseTypeContains("application/json")){ |
|
107 |
else if(c.responseTypeContains("application/json") || c.responseTypeContains("application/xml")){
|
|
102 | 108 |
try { |
103 |
queue.put(addFilePath(c.getResponse(),url)); |
|
109 |
queue.put(addFilePath(c.getResponse(),url, c.responseTypeContains("application/json")));
|
|
104 | 110 |
} catch (InterruptedException e) { |
105 | 111 |
log.error("not inserted in queue element associate to url " + url + " error: " + e.getMessage() ); |
106 | 112 |
} |
Also available in: Unified diff
considered the case metadata are given in xml format instead of json