Revision 41820
Added by Michele Artini about 8 years ago
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/ResultSetClientIterator.java | ||
---|---|---|
1 |
package eu.dnetlib.enabling.resultset.client; |
|
2 |
|
|
3 |
import java.util.Iterator; |
|
4 |
import java.util.LinkedList; |
|
5 |
import java.util.List; |
|
6 |
import java.util.NoSuchElementException; |
|
7 |
import java.util.Queue; |
|
8 |
import java.util.stream.Collectors; |
|
9 |
|
|
10 |
import org.apache.commons.httpclient.HttpClient; |
|
11 |
import org.apache.commons.httpclient.HttpMethod; |
|
12 |
import org.apache.commons.httpclient.HttpStatus; |
|
13 |
import org.apache.commons.httpclient.methods.GetMethod; |
|
14 |
import org.apache.commons.logging.Log; |
|
15 |
import org.apache.commons.logging.LogFactory; |
|
16 |
|
|
17 |
import com.google.gson.Gson; |
|
18 |
|
|
19 |
import eu.dnetlib.enabling.resultset.ResultSetResponse; |
|
20 |
import eu.dnetlib.rmi.common.ResultSetException; |
|
21 |
|
|
22 |
public class ResultSetClientIterator<T> implements Iterator<T> { |
|
23 |
|
|
24 |
private static final Log log = LogFactory.getLog(ResultSetClientIterator.class); |
|
25 |
private static final int PAGE_SIZE = 20; |
|
26 |
|
|
27 |
private final String id; |
|
28 |
private final String baseUrl; |
|
29 |
private final Class<T> clazz; |
|
30 |
private final Queue<T> buffer = new LinkedList<T>(); |
|
31 |
private final HttpClient client = new HttpClient(); |
|
32 |
private int total = -1; |
|
33 |
private int current = -1; |
|
34 |
|
|
35 |
public ResultSetClientIterator(final String id, final String baseUrl, final Class<T> clazz) { |
|
36 |
this.id = id; |
|
37 |
this.baseUrl = baseUrl; |
|
38 |
this.clazz = clazz; |
|
39 |
} |
|
40 |
|
|
41 |
@Override |
|
42 |
public boolean hasNext() { |
|
43 |
if (!this.buffer.isEmpty()) { return true; } |
|
44 |
try { |
|
45 |
return refillBuffer(); |
|
46 |
} catch (final ResultSetException e) { |
|
47 |
log.error("Error refilling buffer of rs: " + getId()); |
|
48 |
throw new RuntimeException("Error refilling buffer of rs: " + getId(), e); |
|
49 |
} |
|
50 |
} |
|
51 |
|
|
52 |
@Override |
|
53 |
public T next() { |
|
54 |
if (!hasNext()) { |
|
55 |
log.error("NoSuchElementException"); |
|
56 |
throw new NoSuchElementException(); |
|
57 |
} |
|
58 |
try { |
|
59 |
return this.buffer.poll(); |
|
60 |
} finally { |
|
61 |
this.current++; |
|
62 |
} |
|
63 |
} |
|
64 |
|
|
65 |
@Override |
|
66 |
public void remove() { |
|
67 |
throw new RuntimeException("Not implemented"); |
|
68 |
} |
|
69 |
|
|
70 |
private boolean refillBuffer() throws ResultSetException { |
|
71 |
final List<T> page = nextPage(); |
|
72 |
if (page == null || page.isEmpty()) { return false; } |
|
73 |
this.buffer.addAll(page); |
|
74 |
return true; |
|
75 |
} |
|
76 |
|
|
77 |
private List<T> nextPage() throws ResultSetException { |
|
78 |
final HttpMethod method = new GetMethod(getBaseUrl() + "/" + getId() + "/next/" + PAGE_SIZE); |
|
79 |
|
|
80 |
try { |
|
81 |
final int responseCode = this.client.executeMethod(method); |
|
82 |
|
|
83 |
if (HttpStatus.SC_OK != responseCode) { throw new ResultSetException("Error " + responseCode + " dowloading url: " + getBaseUrl()); } |
|
84 |
|
|
85 |
final String json = method.getResponseBodyAsString(); |
|
86 |
|
|
87 |
final Gson gson = new Gson(); |
|
88 |
|
|
89 |
final ResultSetResponse response = gson.fromJson(json, ResultSetResponse.class); |
|
90 |
|
|
91 |
this.total = response.getTotal(); |
|
92 |
|
|
93 |
return response.getElements() |
|
94 |
.stream() |
|
95 |
.map(s -> gson.fromJson(s, getClazz())) |
|
96 |
.collect(Collectors.toList()); |
|
97 |
} catch (final Throwable e) { |
|
98 |
throw new ResultSetException("Error dowloading url: " + getBaseUrl(), e); |
|
99 |
} |
|
100 |
} |
|
101 |
|
|
102 |
public String getId() { |
|
103 |
return this.id; |
|
104 |
} |
|
105 |
|
|
106 |
public String getBaseUrl() { |
|
107 |
return this.baseUrl; |
|
108 |
} |
|
109 |
|
|
110 |
public Class<T> getClazz() { |
|
111 |
return this.clazz; |
|
112 |
} |
|
113 |
|
|
114 |
public int getCurrent() { |
|
115 |
return this.current; |
|
116 |
} |
|
117 |
|
|
118 |
public int getTotal() { |
|
119 |
return this.total; |
|
120 |
} |
|
121 |
|
|
122 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/ResultSetInfo.java | ||
---|---|---|
1 |
package eu.dnetlib.enabling.resultset; |
|
2 |
|
|
3 |
import javax.xml.bind.annotation.XmlRootElement; |
|
4 |
|
|
5 |
@XmlRootElement |
|
6 |
public class ResultSetInfo { |
|
7 |
|
|
8 |
private String id; |
|
9 |
private int cursor = -1; |
|
10 |
private int total = -1; |
|
11 |
|
|
12 |
public ResultSetInfo() {} |
|
13 |
|
|
14 |
public ResultSetInfo(final String id, final int cursor, final int total) { |
|
15 |
this.id = id; |
|
16 |
this.cursor = cursor; |
|
17 |
this.total = total; |
|
18 |
} |
|
19 |
|
|
20 |
public String getId() { |
|
21 |
return this.id; |
|
22 |
} |
|
23 |
|
|
24 |
public void setId(final String id) { |
|
25 |
this.id = id; |
|
26 |
} |
|
27 |
|
|
28 |
public int getTotal() { |
|
29 |
return this.total; |
|
30 |
} |
|
31 |
|
|
32 |
public void setTotal(final int total) { |
|
33 |
this.total = total; |
|
34 |
} |
|
35 |
|
|
36 |
public int getCursor() { |
|
37 |
return this.cursor; |
|
38 |
} |
|
39 |
|
|
40 |
public void setCursor(final int cursor) { |
|
41 |
this.cursor = cursor; |
|
42 |
} |
|
43 |
|
|
44 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/factory/ResultSetFactory.java | ||
---|---|---|
19 | 19 |
public class ResultSetFactory { |
20 | 20 |
|
21 | 21 |
private ResultSetRegistry resultSetRegistry; |
22 |
|
|
22 |
private ResultSetClient resultSetClient; |
|
23 | 23 |
private String baseUrl; |
24 | 24 |
|
25 | 25 |
public <T> ResultSet<T> registerResultSet(final ResultSetListener<T> rs) { |
... | ... | |
36 | 36 |
} |
37 | 37 |
|
38 | 38 |
public <T, K> ResultSet<K> map(final ResultSet<?> rsIn, final Class<T> clazzIn, final Function<T, K> mapper) { |
39 |
final Iterable<T> iterIn = ResultSetClient.getClient(rsIn, clazzIn);
|
|
39 |
final Iterable<T> iterIn = this.resultSetClient.iter(rsIn, clazzIn);
|
|
40 | 40 |
return createResultSet(() -> StreamSupport.stream(iterIn.spliterator(), false).map(mapper).iterator()); |
41 | 41 |
} |
42 | 42 |
|
... | ... | |
78 | 78 |
this.baseUrl = baseUrl; |
79 | 79 |
} |
80 | 80 |
|
81 |
public ResultSetClient getResultSetClient() { |
|
82 |
return this.resultSetClient; |
|
83 |
} |
|
84 |
|
|
85 |
@Required |
|
86 |
public void setResultSetClient(final ResultSetClient resultSetClient) { |
|
87 |
this.resultSetClient = resultSetClient; |
|
88 |
} |
|
89 |
|
|
81 | 90 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/registry/ResultSetRegistry.java | ||
---|---|---|
29 | 29 |
* @throws ResultSetException |
30 | 30 |
*/ |
31 | 31 |
<T> ResultSetListener<T> getResultSetById(String rsId) throws ResultSetException; |
32 |
|
|
33 |
/** |
|
34 |
* |
|
35 |
* @param id |
|
36 |
* @return |
|
37 |
*/ |
|
38 |
boolean contains(String id); |
|
32 | 39 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/registry/ResultSetRegistryImpl.java | ||
---|---|---|
37 | 37 |
} |
38 | 38 |
} |
39 | 39 |
|
40 |
@Override |
|
41 |
public boolean contains(final String rsId) { |
|
42 |
return this.cache.get(rsId) != null; |
|
43 |
} |
|
44 |
|
|
40 | 45 |
public Cache getCache() { |
41 | 46 |
return this.cache; |
42 | 47 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/ResultSetController.java | ||
---|---|---|
45 | 45 |
return new ResultSetResponse(from, to, end, total, elements); |
46 | 46 |
} |
47 | 47 |
|
48 |
@RequestMapping(value = "/rs/{rsId}/info", method = RequestMethod.GET) |
|
49 |
public @ResponseBody ResultSetInfo getRsInfo(@PathVariable(value = "rsId") final String rsId) |
|
50 |
throws ResultSetException { |
|
51 |
final ResultSetListener<?> rs = this.resultSetRegistry.getResultSetById(rsId); |
|
52 |
|
|
53 |
return new ResultSetInfo(rsId, rs.getCount(), rs.getTotal()); |
|
54 |
} |
|
55 |
|
|
48 | 56 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/HttpResultSetClientIterator.java | ||
---|---|---|
1 |
package eu.dnetlib.enabling.resultset.client; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import java.util.stream.Collectors; |
|
5 |
|
|
6 |
import org.apache.commons.httpclient.HttpClient; |
|
7 |
import org.apache.commons.httpclient.HttpMethod; |
|
8 |
import org.apache.commons.httpclient.HttpStatus; |
|
9 |
import org.apache.commons.httpclient.methods.GetMethod; |
|
10 |
|
|
11 |
import com.google.gson.Gson; |
|
12 |
|
|
13 |
import eu.dnetlib.enabling.resultset.ResultSetResponse; |
|
14 |
import eu.dnetlib.rmi.common.ResultSetException; |
|
15 |
|
|
16 |
public class HttpResultSetClientIterator<T> extends AbstractResultSetClientIterator<T> { |
|
17 |
|
|
18 |
private static final int PAGE_SIZE = 20; |
|
19 |
|
|
20 |
private final String id; |
|
21 |
private final String baseUrl; |
|
22 |
private final Class<T> clazz; |
|
23 |
|
|
24 |
private final HttpClient client = new HttpClient(); |
|
25 |
|
|
26 |
public HttpResultSetClientIterator(final String id, final String baseUrl, final Class<T> clazz) { |
|
27 |
super(); |
|
28 |
this.id = id; |
|
29 |
this.baseUrl = baseUrl; |
|
30 |
this.clazz = clazz; |
|
31 |
} |
|
32 |
|
|
33 |
@Override |
|
34 |
protected List<T> nextPage() throws ResultSetException { |
|
35 |
final HttpMethod method = new GetMethod(getBaseUrl() + "/" + getId() + "/next/" + PAGE_SIZE); |
|
36 |
|
|
37 |
try { |
|
38 |
final int responseCode = this.client.executeMethod(method); |
|
39 |
|
|
40 |
if (HttpStatus.SC_OK != responseCode) { throw new ResultSetException("Error " + responseCode + " dowloading url: " + getBaseUrl()); } |
|
41 |
|
|
42 |
final String json = method.getResponseBodyAsString(); |
|
43 |
|
|
44 |
final Gson gson = new Gson(); |
|
45 |
|
|
46 |
final ResultSetResponse response = gson.fromJson(json, ResultSetResponse.class); |
|
47 |
|
|
48 |
return response.getElements() |
|
49 |
.stream() |
|
50 |
.map(s -> gson.fromJson(s, getClazz())) |
|
51 |
.collect(Collectors.toList()); |
|
52 |
} catch (final Throwable e) { |
|
53 |
throw new ResultSetException("Error dowloading url: " + getBaseUrl(), e); |
|
54 |
} |
|
55 |
} |
|
56 |
|
|
57 |
public String getId() { |
|
58 |
return this.id; |
|
59 |
} |
|
60 |
|
|
61 |
public String getBaseUrl() { |
|
62 |
return this.baseUrl; |
|
63 |
} |
|
64 |
|
|
65 |
public Class<T> getClazz() { |
|
66 |
return this.clazz; |
|
67 |
} |
|
68 |
|
|
69 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/AbstractResultSetClientIterator.java | ||
---|---|---|
1 |
package eu.dnetlib.enabling.resultset.client; |
|
2 |
|
|
3 |
import java.util.Iterator; |
|
4 |
import java.util.LinkedList; |
|
5 |
import java.util.List; |
|
6 |
import java.util.NoSuchElementException; |
|
7 |
import java.util.Queue; |
|
8 |
|
|
9 |
import org.apache.commons.logging.Log; |
|
10 |
import org.apache.commons.logging.LogFactory; |
|
11 |
|
|
12 |
import eu.dnetlib.rmi.common.ResultSetException; |
|
13 |
|
|
14 |
public abstract class AbstractResultSetClientIterator<T> implements Iterator<T> { |
|
15 |
|
|
16 |
private static final Log log = LogFactory.getLog(AbstractResultSetClientIterator.class); |
|
17 |
|
|
18 |
private final Queue<T> buffer = new LinkedList<T>(); |
|
19 |
|
|
20 |
@Override |
|
21 |
public boolean hasNext() { |
|
22 |
if (!this.buffer.isEmpty()) { return true; } |
|
23 |
try { |
|
24 |
return refillBuffer(); |
|
25 |
} catch (final ResultSetException e) { |
|
26 |
log.error("Error refilling resultSet buffer", e); |
|
27 |
throw new RuntimeException("Error refilling resultSet buffer", e); |
|
28 |
} |
|
29 |
} |
|
30 |
|
|
31 |
@Override |
|
32 |
public T next() { |
|
33 |
if (!hasNext()) { |
|
34 |
log.error("NoSuchElementException"); |
|
35 |
throw new NoSuchElementException(); |
|
36 |
} |
|
37 |
return this.buffer.poll(); |
|
38 |
} |
|
39 |
|
|
40 |
@Override |
|
41 |
public void remove() { |
|
42 |
throw new RuntimeException("Not implemented"); |
|
43 |
} |
|
44 |
|
|
45 |
private boolean refillBuffer() throws ResultSetException { |
|
46 |
final List<T> page = nextPage(); |
|
47 |
if (page == null || page.isEmpty()) { return false; } |
|
48 |
this.buffer.addAll(page); |
|
49 |
return true; |
|
50 |
} |
|
51 |
|
|
52 |
abstract protected List<T> nextPage() throws ResultSetException; |
|
53 |
|
|
54 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/ResultSetClient.java | ||
---|---|---|
1 | 1 |
package eu.dnetlib.enabling.resultset.client; |
2 | 2 |
|
3 |
import java.util.Iterator; |
|
3 |
import org.apache.commons.logging.Log; |
|
4 |
import org.apache.commons.logging.LogFactory; |
|
5 |
import org.springframework.beans.factory.annotation.Required; |
|
4 | 6 |
|
7 |
import eu.dnetlib.enabling.resultset.ResultSetInfo; |
|
8 |
import eu.dnetlib.enabling.resultset.listener.ResultSetListener; |
|
9 |
import eu.dnetlib.enabling.resultset.registry.ResultSetRegistry; |
|
5 | 10 |
import eu.dnetlib.rmi.common.ResultSet; |
6 | 11 |
|
7 |
public class ResultSetClient<T> implements Iterable<T> {
|
|
12 |
public class ResultSetClient { |
|
8 | 13 |
|
9 |
private final ResultSet<?> resultSet; |
|
10 |
private final Class<T> clazz; |
|
11 |
private ResultSetClientIterator<T> currentIter; |
|
14 |
private ResultSetRegistry resultSetRegistry; |
|
12 | 15 |
|
13 |
public static <T> ResultSetClient<T> getClient(final ResultSet<?> resultSet, final Class<T> clazz) { |
|
14 |
return new ResultSetClient<T>(resultSet, clazz); |
|
15 |
} |
|
16 |
private static final Log log = LogFactory.getLog(ResultSetClient.class); |
|
16 | 17 |
|
17 |
private ResultSetClient(final ResultSet<?> resultSet, final Class<T> clazz) { |
|
18 |
this.resultSet = resultSet; |
|
19 |
this.clazz = clazz; |
|
18 |
public <T> Iterable<T> iter(final ResultSet<?> resultSet, final Class<T> clazz) { |
|
19 |
try { |
|
20 |
if (this.resultSetRegistry.contains(resultSet.getId())) { |
|
21 |
final ResultSetListener<?> listener = this.resultSetRegistry.getResultSetById(resultSet.getId()); |
|
22 |
return () -> new LocalResultSetClientIterator<>(listener, clazz); |
|
23 |
} else { |
|
24 |
return () -> new HttpResultSetClientIterator<>(resultSet.getId(), resultSet.getBaseUrl(), clazz); |
|
25 |
} |
|
26 |
} catch (final Throwable e) { |
|
27 |
log.error("Error accessing resultset: " + resultSet.getId()); |
|
28 |
throw new RuntimeException(e); |
|
29 |
} |
|
20 | 30 |
} |
21 | 31 |
|
22 |
@Override |
|
23 |
public Iterator<T> iterator() { |
|
24 |
this.currentIter = new ResultSetClientIterator<>(this.resultSet.getId(), this.resultSet.getBaseUrl(), this.clazz); |
|
25 |
return this.currentIter; |
|
32 |
public ResultSetInfo info(final ResultSet<?> resultSet) { |
|
33 |
// TODO |
|
34 |
return null; |
|
26 | 35 |
} |
27 | 36 |
|
28 |
public int getTotal() {
|
|
29 |
return this.currentIter != null ? this.currentIter.getTotal() : -1;
|
|
37 |
public ResultSetRegistry getResultSetRegistry() {
|
|
38 |
return this.resultSetRegistry;
|
|
30 | 39 |
} |
31 | 40 |
|
32 |
public int getCurrent() { |
|
33 |
return this.currentIter != null ? this.currentIter.getCurrent() : -1; |
|
41 |
@Required |
|
42 |
public void setResultSetRegistry(final ResultSetRegistry resultSetRegistry) { |
|
43 |
this.resultSetRegistry = resultSetRegistry; |
|
34 | 44 |
} |
35 | 45 |
|
36 |
public ResultSet<?> getResultSet() { |
|
37 |
return this.resultSet; |
|
38 |
} |
|
39 |
|
|
40 |
public Class<T> getClazz() { |
|
41 |
return this.clazz; |
|
42 |
} |
|
43 |
|
|
44 | 46 |
} |
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/LocalResultSetClientIterator.java | ||
---|---|---|
1 |
package eu.dnetlib.enabling.resultset.client; |
|
2 |
|
|
3 |
import java.util.ArrayList; |
|
4 |
import java.util.List; |
|
5 |
|
|
6 |
import eu.dnetlib.enabling.resultset.listener.ResultSetListener; |
|
7 |
import eu.dnetlib.rmi.common.ResultSetException; |
|
8 |
|
|
9 |
public class LocalResultSetClientIterator<T> extends AbstractResultSetClientIterator<T> { |
|
10 |
|
|
11 |
private static final int PAGE_SIZE = 20; |
|
12 |
|
|
13 |
private final ResultSetListener<?> listener; |
|
14 |
private final Class<T> clazz; |
|
15 |
|
|
16 |
public LocalResultSetClientIterator(final ResultSetListener<?> listener, final Class<T> clazz) { |
|
17 |
super(); |
|
18 |
this.listener = listener; |
|
19 |
this.clazz = clazz; |
|
20 |
} |
|
21 |
|
|
22 |
@Override |
|
23 |
protected List<T> nextPage() throws ResultSetException { |
|
24 |
final List<T> res = new ArrayList<>(); |
|
25 |
for (int i = 0; i < PAGE_SIZE && this.listener.hasNext(); i++) { |
|
26 |
res.add(this.clazz.cast(this.listener.next())); |
|
27 |
} |
|
28 |
return res; |
|
29 |
} |
|
30 |
|
|
31 |
} |
modules/dnet-core-services/trunk/src/main/resources/eu/dnetlib/enabling/resultset/applicationContext-resultset.xml | ||
---|---|---|
16 | 16 |
p:cache-ref="resultSetRegistryCache" /> |
17 | 17 |
|
18 | 18 |
<bean id="resultSetFactory" class="eu.dnetlib.enabling.resultset.factory.ResultSetFactory" |
19 |
p:resultSetRegistry-ref="resultSetRegistry" p:baseUrl="${dnet.resultset.baseurl}" /> |
|
19 |
p:resultSetRegistry-ref="resultSetRegistry" |
|
20 |
p:resultSetClient-ref="resultSetClient" |
|
21 |
p:baseUrl="${dnet.resultset.baseurl}" /> |
|
22 |
|
|
23 |
<bean id="resultSetClient" class="eu.dnetlib.enabling.resultset.client.ResultSetClient" /> |
|
20 | 24 |
|
21 | 25 |
<!-- Cache --> |
22 | 26 |
<bean id="resultSetCacheManager" |
Also available in: Unified diff