1
|
package eu.dnetlib.wds.collector.plugins;
|
2
|
|
3
|
import java.net.URL;
|
4
|
import java.util.ArrayList;
|
5
|
import java.util.Iterator;
|
6
|
import java.util.List;
|
7
|
|
8
|
import com.google.gson.JsonArray;
|
9
|
import com.google.gson.JsonElement;
|
10
|
import com.google.gson.JsonParser;
|
11
|
import org.apache.commons.io.IOUtils;
|
12
|
import org.apache.commons.logging.Log;
|
13
|
import org.apache.commons.logging.LogFactory;
|
14
|
|
15
|
/**
|
16
|
* Created by sandro on 5/27/16.
|
17
|
*/
|
18
|
public class CMRIterator implements Iterator<String> {
|
19
|
|
20
|
private final static String BASE_URL_PAGE = "https://cmr.earthdata.nasa.gov/search/collections.json?pretty=true&page_size=50&page_num=%d";
|
21
|
|
22
|
private static final Log log = LogFactory.getLog(CMRIterator.class);
|
23
|
|
24
|
private int current_page = 1;
|
25
|
|
26
|
private String nextElement = null;
|
27
|
|
28
|
private List<String> buffer = new ArrayList<>();
|
29
|
|
30
|
public CMRIterator() {
|
31
|
calculateNextElement();
|
32
|
}
|
33
|
|
34
|
private void calculateNextElement() {
|
35
|
try {
|
36
|
final URL myURL = new URL(String.format(BASE_URL_PAGE, current_page));
|
37
|
|
38
|
final String result = IOUtils.toString(myURL.openStream());
|
39
|
|
40
|
JsonElement element = new JsonParser().parse(result);
|
41
|
final JsonArray asJsonArray = element.getAsJsonObject().get("feed").getAsJsonObject().get("entry").getAsJsonArray();
|
42
|
|
43
|
if (asJsonArray.size() > 0) {
|
44
|
for (JsonElement el : asJsonArray) {
|
45
|
buffer.add(el.toString());
|
46
|
}
|
47
|
current_page++;
|
48
|
return;
|
49
|
}
|
50
|
|
51
|
} catch (Throwable e) {
|
52
|
log.error(String.format("Error on calculate next Element, The iterator will be shut down"), e);
|
53
|
|
54
|
}
|
55
|
|
56
|
this.nextElement = null;
|
57
|
|
58
|
}
|
59
|
|
60
|
@Override
|
61
|
public boolean hasNext() {
|
62
|
return this.buffer.size() > 0;
|
63
|
}
|
64
|
|
65
|
@Override
|
66
|
public String next() {
|
67
|
final String temp = buffer.remove(0);
|
68
|
if (buffer.size() < 10) {
|
69
|
calculateNextElement();
|
70
|
}
|
71
|
return temp;
|
72
|
}
|
73
|
}
|