Project

General

Profile

« Previous | Next » 

Revision 41820

View differences:

modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/ResultSetClientIterator.java
1
package eu.dnetlib.enabling.resultset.client;
2

  
3
import java.util.Iterator;
4
import java.util.LinkedList;
5
import java.util.List;
6
import java.util.NoSuchElementException;
7
import java.util.Queue;
8
import java.util.stream.Collectors;
9

  
10
import org.apache.commons.httpclient.HttpClient;
11
import org.apache.commons.httpclient.HttpMethod;
12
import org.apache.commons.httpclient.HttpStatus;
13
import org.apache.commons.httpclient.methods.GetMethod;
14
import org.apache.commons.logging.Log;
15
import org.apache.commons.logging.LogFactory;
16

  
17
import com.google.gson.Gson;
18

  
19
import eu.dnetlib.enabling.resultset.ResultSetResponse;
20
import eu.dnetlib.rmi.common.ResultSetException;
21

  
22
public class ResultSetClientIterator<T> implements Iterator<T> {
23

  
24
	private static final Log log = LogFactory.getLog(ResultSetClientIterator.class);
25
	private static final int PAGE_SIZE = 20;
26

  
27
	private final String id;
28
	private final String baseUrl;
29
	private final Class<T> clazz;
30
	private final Queue<T> buffer = new LinkedList<T>();
31
	private final HttpClient client = new HttpClient();
32
	private int total = -1;
33
	private int current = -1;
34

  
35
	public ResultSetClientIterator(final String id, final String baseUrl, final Class<T> clazz) {
36
		this.id = id;
37
		this.baseUrl = baseUrl;
38
		this.clazz = clazz;
39
	}
40

  
41
	@Override
42
	public boolean hasNext() {
43
		if (!this.buffer.isEmpty()) { return true; }
44
		try {
45
			return refillBuffer();
46
		} catch (final ResultSetException e) {
47
			log.error("Error refilling buffer of rs: " + getId());
48
			throw new RuntimeException("Error refilling buffer of rs: " + getId(), e);
49
		}
50
	}
51

  
52
	@Override
53
	public T next() {
54
		if (!hasNext()) {
55
			log.error("NoSuchElementException");
56
			throw new NoSuchElementException();
57
		}
58
		try {
59
			return this.buffer.poll();
60
		} finally {
61
			this.current++;
62
		}
63
	}
64

  
65
	@Override
66
	public void remove() {
67
		throw new RuntimeException("Not implemented");
68
	}
69

  
70
	private boolean refillBuffer() throws ResultSetException {
71
		final List<T> page = nextPage();
72
		if (page == null || page.isEmpty()) { return false; }
73
		this.buffer.addAll(page);
74
		return true;
75
	}
76

  
77
	private List<T> nextPage() throws ResultSetException {
78
		final HttpMethod method = new GetMethod(getBaseUrl() + "/" + getId() + "/next/" + PAGE_SIZE);
79

  
80
		try {
81
			final int responseCode = this.client.executeMethod(method);
82

  
83
			if (HttpStatus.SC_OK != responseCode) { throw new ResultSetException("Error " + responseCode + " dowloading url: " + getBaseUrl()); }
84

  
85
			final String json = method.getResponseBodyAsString();
86

  
87
			final Gson gson = new Gson();
88

  
89
			final ResultSetResponse response = gson.fromJson(json, ResultSetResponse.class);
90

  
91
			this.total = response.getTotal();
92

  
93
			return response.getElements()
94
					.stream()
95
					.map(s -> gson.fromJson(s, getClazz()))
96
					.collect(Collectors.toList());
97
		} catch (final Throwable e) {
98
			throw new ResultSetException("Error dowloading url: " + getBaseUrl(), e);
99
		}
100
	}
101

  
102
	public String getId() {
103
		return this.id;
104
	}
105

  
106
	public String getBaseUrl() {
107
		return this.baseUrl;
108
	}
109

  
110
	public Class<T> getClazz() {
111
		return this.clazz;
112
	}
113

  
114
	public int getCurrent() {
115
		return this.current;
116
	}
117

  
118
	public int getTotal() {
119
		return this.total;
120
	}
121

  
122
}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/ResultSetInfo.java
1
package eu.dnetlib.enabling.resultset;
2

  
3
import javax.xml.bind.annotation.XmlRootElement;
4

  
5
@XmlRootElement
6
public class ResultSetInfo {
7

  
8
	private String id;
9
	private int cursor = -1;
10
	private int total = -1;
11

  
12
	public ResultSetInfo() {}
13

  
14
	public ResultSetInfo(final String id, final int cursor, final int total) {
15
		this.id = id;
16
		this.cursor = cursor;
17
		this.total = total;
18
	}
19

  
20
	public String getId() {
21
		return this.id;
22
	}
23

  
24
	public void setId(final String id) {
25
		this.id = id;
26
	}
27

  
28
	public int getTotal() {
29
		return this.total;
30
	}
31

  
32
	public void setTotal(final int total) {
33
		this.total = total;
34
	}
35

  
36
	public int getCursor() {
37
		return this.cursor;
38
	}
39

  
40
	public void setCursor(final int cursor) {
41
		this.cursor = cursor;
42
	}
43

  
44
}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/factory/ResultSetFactory.java
19 19
public class ResultSetFactory {
20 20

  
21 21
	private ResultSetRegistry resultSetRegistry;
22

  
22
	private ResultSetClient resultSetClient;
23 23
	private String baseUrl;
24 24

  
25 25
	public <T> ResultSet<T> registerResultSet(final ResultSetListener<T> rs) {
......
36 36
	}
37 37

  
38 38
	public <T, K> ResultSet<K> map(final ResultSet<?> rsIn, final Class<T> clazzIn, final Function<T, K> mapper) {
39
		final Iterable<T> iterIn = ResultSetClient.getClient(rsIn, clazzIn);
39
		final Iterable<T> iterIn = this.resultSetClient.iter(rsIn, clazzIn);
40 40
		return createResultSet(() -> StreamSupport.stream(iterIn.spliterator(), false).map(mapper).iterator());
41 41
	}
42 42

  
......
78 78
		this.baseUrl = baseUrl;
79 79
	}
80 80

  
81
	public ResultSetClient getResultSetClient() {
82
		return this.resultSetClient;
83
	}
84

  
85
	@Required
86
	public void setResultSetClient(final ResultSetClient resultSetClient) {
87
		this.resultSetClient = resultSetClient;
88
	}
89

  
81 90
}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/registry/ResultSetRegistry.java
29 29
	 * @throws ResultSetException
30 30
	 */
31 31
	<T> ResultSetListener<T> getResultSetById(String rsId) throws ResultSetException;
32

  
33
	/**
34
	 *
35
	 * @param id
36
	 * @return
37
	 */
38
	boolean contains(String id);
32 39
}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/registry/ResultSetRegistryImpl.java
37 37
		}
38 38
	}
39 39

  
40
	@Override
41
	public boolean contains(final String rsId) {
42
		return this.cache.get(rsId) != null;
43
	}
44

  
40 45
	public Cache getCache() {
41 46
		return this.cache;
42 47
	}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/ResultSetController.java
45 45
		return new ResultSetResponse(from, to, end, total, elements);
46 46
	}
47 47

  
48
	@RequestMapping(value = "/rs/{rsId}/info", method = RequestMethod.GET)
49
	public @ResponseBody ResultSetInfo getRsInfo(@PathVariable(value = "rsId") final String rsId)
50
			throws ResultSetException {
51
		final ResultSetListener<?> rs = this.resultSetRegistry.getResultSetById(rsId);
52

  
53
		return new ResultSetInfo(rsId, rs.getCount(), rs.getTotal());
54
	}
55

  
48 56
}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/HttpResultSetClientIterator.java
1
package eu.dnetlib.enabling.resultset.client;
2

  
3
import java.util.List;
4
import java.util.stream.Collectors;
5

  
6
import org.apache.commons.httpclient.HttpClient;
7
import org.apache.commons.httpclient.HttpMethod;
8
import org.apache.commons.httpclient.HttpStatus;
9
import org.apache.commons.httpclient.methods.GetMethod;
10

  
11
import com.google.gson.Gson;
12

  
13
import eu.dnetlib.enabling.resultset.ResultSetResponse;
14
import eu.dnetlib.rmi.common.ResultSetException;
15

  
16
public class HttpResultSetClientIterator<T> extends AbstractResultSetClientIterator<T> {
17

  
18
	private static final int PAGE_SIZE = 20;
19

  
20
	private final String id;
21
	private final String baseUrl;
22
	private final Class<T> clazz;
23

  
24
	private final HttpClient client = new HttpClient();
25

  
26
	public HttpResultSetClientIterator(final String id, final String baseUrl, final Class<T> clazz) {
27
		super();
28
		this.id = id;
29
		this.baseUrl = baseUrl;
30
		this.clazz = clazz;
31
	}
32

  
33
	@Override
34
	protected List<T> nextPage() throws ResultSetException {
35
		final HttpMethod method = new GetMethod(getBaseUrl() + "/" + getId() + "/next/" + PAGE_SIZE);
36

  
37
		try {
38
			final int responseCode = this.client.executeMethod(method);
39

  
40
			if (HttpStatus.SC_OK != responseCode) { throw new ResultSetException("Error " + responseCode + " dowloading url: " + getBaseUrl()); }
41

  
42
			final String json = method.getResponseBodyAsString();
43

  
44
			final Gson gson = new Gson();
45

  
46
			final ResultSetResponse response = gson.fromJson(json, ResultSetResponse.class);
47

  
48
			return response.getElements()
49
					.stream()
50
					.map(s -> gson.fromJson(s, getClazz()))
51
					.collect(Collectors.toList());
52
		} catch (final Throwable e) {
53
			throw new ResultSetException("Error dowloading url: " + getBaseUrl(), e);
54
		}
55
	}
56

  
57
	public String getId() {
58
		return this.id;
59
	}
60

  
61
	public String getBaseUrl() {
62
		return this.baseUrl;
63
	}
64

  
65
	public Class<T> getClazz() {
66
		return this.clazz;
67
	}
68

  
69
}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/AbstractResultSetClientIterator.java
1
package eu.dnetlib.enabling.resultset.client;
2

  
3
import java.util.Iterator;
4
import java.util.LinkedList;
5
import java.util.List;
6
import java.util.NoSuchElementException;
7
import java.util.Queue;
8

  
9
import org.apache.commons.logging.Log;
10
import org.apache.commons.logging.LogFactory;
11

  
12
import eu.dnetlib.rmi.common.ResultSetException;
13

  
14
public abstract class AbstractResultSetClientIterator<T> implements Iterator<T> {
15

  
16
	private static final Log log = LogFactory.getLog(AbstractResultSetClientIterator.class);
17

  
18
	private final Queue<T> buffer = new LinkedList<T>();
19

  
20
	@Override
21
	public boolean hasNext() {
22
		if (!this.buffer.isEmpty()) { return true; }
23
		try {
24
			return refillBuffer();
25
		} catch (final ResultSetException e) {
26
			log.error("Error refilling resultSet buffer", e);
27
			throw new RuntimeException("Error refilling resultSet buffer", e);
28
		}
29
	}
30

  
31
	@Override
32
	public T next() {
33
		if (!hasNext()) {
34
			log.error("NoSuchElementException");
35
			throw new NoSuchElementException();
36
		}
37
		return this.buffer.poll();
38
	}
39

  
40
	@Override
41
	public void remove() {
42
		throw new RuntimeException("Not implemented");
43
	}
44

  
45
	private boolean refillBuffer() throws ResultSetException {
46
		final List<T> page = nextPage();
47
		if (page == null || page.isEmpty()) { return false; }
48
		this.buffer.addAll(page);
49
		return true;
50
	}
51

  
52
	abstract protected List<T> nextPage() throws ResultSetException;
53

  
54
}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/ResultSetClient.java
1 1
package eu.dnetlib.enabling.resultset.client;
2 2

  
3
import java.util.Iterator;
3
import org.apache.commons.logging.Log;
4
import org.apache.commons.logging.LogFactory;
5
import org.springframework.beans.factory.annotation.Required;
4 6

  
7
import eu.dnetlib.enabling.resultset.ResultSetInfo;
8
import eu.dnetlib.enabling.resultset.listener.ResultSetListener;
9
import eu.dnetlib.enabling.resultset.registry.ResultSetRegistry;
5 10
import eu.dnetlib.rmi.common.ResultSet;
6 11

  
7
public class ResultSetClient<T> implements Iterable<T> {
12
public class ResultSetClient {
8 13

  
9
	private final ResultSet<?> resultSet;
10
	private final Class<T> clazz;
11
	private ResultSetClientIterator<T> currentIter;
14
	private ResultSetRegistry resultSetRegistry;
12 15

  
13
	public static <T> ResultSetClient<T> getClient(final ResultSet<?> resultSet, final Class<T> clazz) {
14
		return new ResultSetClient<T>(resultSet, clazz);
15
	}
16
	private static final Log log = LogFactory.getLog(ResultSetClient.class);
16 17

  
17
	private ResultSetClient(final ResultSet<?> resultSet, final Class<T> clazz) {
18
		this.resultSet = resultSet;
19
		this.clazz = clazz;
18
	public <T> Iterable<T> iter(final ResultSet<?> resultSet, final Class<T> clazz) {
19
		try {
20
			if (this.resultSetRegistry.contains(resultSet.getId())) {
21
				final ResultSetListener<?> listener = this.resultSetRegistry.getResultSetById(resultSet.getId());
22
				return () -> new LocalResultSetClientIterator<>(listener, clazz);
23
			} else {
24
				return () -> new HttpResultSetClientIterator<>(resultSet.getId(), resultSet.getBaseUrl(), clazz);
25
			}
26
		} catch (final Throwable e) {
27
			log.error("Error accessing resultset: " + resultSet.getId());
28
			throw new RuntimeException(e);
29
		}
20 30
	}
21 31

  
22
	@Override
23
	public Iterator<T> iterator() {
24
		this.currentIter = new ResultSetClientIterator<>(this.resultSet.getId(), this.resultSet.getBaseUrl(), this.clazz);
25
		return this.currentIter;
32
	public ResultSetInfo info(final ResultSet<?> resultSet) {
33
		// TODO
34
		return null;
26 35
	}
27 36

  
28
	public int getTotal() {
29
		return this.currentIter != null ? this.currentIter.getTotal() : -1;
37
	public ResultSetRegistry getResultSetRegistry() {
38
		return this.resultSetRegistry;
30 39
	}
31 40

  
32
	public int getCurrent() {
33
		return this.currentIter != null ? this.currentIter.getCurrent() : -1;
41
	@Required
42
	public void setResultSetRegistry(final ResultSetRegistry resultSetRegistry) {
43
		this.resultSetRegistry = resultSetRegistry;
34 44
	}
35 45

  
36
	public ResultSet<?> getResultSet() {
37
		return this.resultSet;
38
	}
39

  
40
	public Class<T> getClazz() {
41
		return this.clazz;
42
	}
43

  
44 46
}
modules/dnet-core-services/trunk/src/main/java/eu/dnetlib/enabling/resultset/client/LocalResultSetClientIterator.java
1
package eu.dnetlib.enabling.resultset.client;
2

  
3
import java.util.ArrayList;
4
import java.util.List;
5

  
6
import eu.dnetlib.enabling.resultset.listener.ResultSetListener;
7
import eu.dnetlib.rmi.common.ResultSetException;
8

  
9
public class LocalResultSetClientIterator<T> extends AbstractResultSetClientIterator<T> {
10

  
11
	private static final int PAGE_SIZE = 20;
12

  
13
	private final ResultSetListener<?> listener;
14
	private final Class<T> clazz;
15

  
16
	public LocalResultSetClientIterator(final ResultSetListener<?> listener, final Class<T> clazz) {
17
		super();
18
		this.listener = listener;
19
		this.clazz = clazz;
20
	}
21

  
22
	@Override
23
	protected List<T> nextPage() throws ResultSetException {
24
		final List<T> res = new ArrayList<>();
25
		for (int i = 0; i < PAGE_SIZE && this.listener.hasNext(); i++) {
26
			res.add(this.clazz.cast(this.listener.next()));
27
		}
28
		return res;
29
	}
30

  
31
}
modules/dnet-core-services/trunk/src/main/resources/eu/dnetlib/enabling/resultset/applicationContext-resultset.xml
16 16
		p:cache-ref="resultSetRegistryCache" />
17 17
		
18 18
	<bean id="resultSetFactory" class="eu.dnetlib.enabling.resultset.factory.ResultSetFactory"
19
		p:resultSetRegistry-ref="resultSetRegistry"	p:baseUrl="${dnet.resultset.baseurl}" />
19
		p:resultSetRegistry-ref="resultSetRegistry"	
20
		p:resultSetClient-ref="resultSetClient"
21
		p:baseUrl="${dnet.resultset.baseurl}" />
22

  
23
	<bean id="resultSetClient" class="eu.dnetlib.enabling.resultset.client.ResultSetClient" />
20 24
		
21 25
	<!-- Cache -->
22 26
	<bean id="resultSetCacheManager"

Also available in: Unified diff