Project

General

Profile

« Previous | Next » 

Revision 61981

plugin to obtain services from fairsharing

View differences:

modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/fairsharing/FairSharingIterator.java
1
package eu.dnetlib.data.collector.plugins.fairsharing;
2

  
3
import java.util.Iterator;
4
import java.util.Queue;
5
import java.util.concurrent.PriorityBlockingQueue;
6

  
7
import org.apache.commons.io.IOUtils;
8
import org.apache.commons.logging.Log;
9
import org.apache.commons.logging.LogFactory;
10
import org.apache.http.client.methods.CloseableHttpResponse;
11
import org.apache.http.client.methods.HttpGet;
12
import org.apache.http.impl.client.CloseableHttpClient;
13
import org.apache.http.impl.client.HttpClients;
14
import org.json.JSONObject;
15
import org.json.XML;
16

  
17
import eu.dnetlib.data.collector.rmi.CollectorServiceException;
18

  
19
public class FairSharingIterator implements Iterator<String> {
20

  
21
	private static final Log log = LogFactory.getLog(FairSharingIterator.class); // NOPMD by marko on 11/24/08 5:02 PM
22

  
23
	private final Queue<String> queue = new PriorityBlockingQueue<>();
24

  
25
	private final String baseUrl;
26
	private final String authCode;
27
	private final int pageSize;
28

  
29
	private String nextUrl;
30
	private boolean started;
31

  
32
	public FairSharingIterator(final String baseUrl, final String authCode, final int pageSize) {
33
		this.baseUrl = baseUrl;
34
		this.authCode = authCode;
35
		this.pageSize = pageSize;
36
		this.started = false;
37
	}
38

  
39
	private void verifyStarted() {
40
		if (!this.started) {
41
			this.started = true;
42
			try {
43
				final String url = baseUrl + "/fairsharing_records/?page%5Bnumber%5D=1&page%5Bsize%5D=" + pageSize;
44
				this.nextUrl = downloadPage(url);
45
			} catch (final CollectorServiceException e) {
46
				throw new RuntimeException(e);
47
			}
48
		}
49
	}
50

  
51
	@Override
52
	public boolean hasNext() {
53
		synchronized (queue) {
54
			verifyStarted();
55
			return !queue.isEmpty();
56
		}
57
	}
58

  
59
	@Override
60
	public String next() {
61
		synchronized (queue) {
62
			verifyStarted();
63
			final String res = queue.poll();
64
			while (queue.isEmpty() && nextUrl != null && !nextUrl.isEmpty()) {
65
				try {
66
					nextUrl = downloadPage(nextUrl);
67
				} catch (final CollectorServiceException e) {
68
					throw new RuntimeException(e);
69
				}
70
			}
71
			return res;
72
		}
73
	}
74

  
75
	@Override
76
	public void remove() {}
77

  
78
	private String downloadPage(final String url) throws CollectorServiceException {
79
		log.debug("Fetching url: " + url);
80

  
81
		final HttpGet req = new HttpGet(url);
82
		req.addHeader("Accept", "application/json");
83
		req.addHeader("Content-Type", "application/json");
84
		req.addHeader("Authorization", "Bearer " + authCode);
85

  
86
		try (final CloseableHttpClient client = HttpClients.createDefault()) {
87
			try (final CloseableHttpResponse response = client.execute(req)) {
88
				final String content = IOUtils.toString(response.getEntity().getContent());
89
				final JSONObject obj = new JSONObject(content);
90

  
91
				obj.getJSONArray("data")
92
					.forEach(x -> queue.add(XML.toString(x, "record")));
93

  
94
				final JSONObject links = obj.getJSONObject("links");
95

  
96
				return links.isNull("next") ? null : links.getString("next");
97
			}
98
		} catch (final Exception e) {
99
			throw new CollectorServiceException("Error perfoming call fro login", e);
100
		}
101
	}
102

  
103
}
modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/fairsharing/FairSharingPlugin.java
1
package eu.dnetlib.data.collector.plugins.fairsharing;
2

  
3
import java.io.UnsupportedEncodingException;
4

  
5
import org.apache.commons.io.IOUtils;
6
import org.apache.commons.lang3.StringUtils;
7
import org.apache.commons.logging.Log;
8
import org.apache.commons.logging.LogFactory;
9
import org.apache.http.HttpEntity;
10
import org.apache.http.client.methods.CloseableHttpResponse;
11
import org.apache.http.client.methods.HttpPost;
12
import org.apache.http.entity.StringEntity;
13
import org.apache.http.impl.client.CloseableHttpClient;
14
import org.apache.http.impl.client.HttpClients;
15
import org.json.JSONObject;
16

  
17
import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;
18
import eu.dnetlib.data.collector.rmi.CollectorServiceException;
19
import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;
20

  
21
public class FairSharingPlugin extends AbstractCollectorPlugin {
22

  
23
	private static final int PAGE_SIZE = 100;
24

  
25
	private static final Log log = LogFactory.getLog(FairSharingPlugin.class); // NOPMD by marko on 11/24/08 5:02 PM
26

  
27
	@Override
28
	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)
29
		throws CollectorServiceException {
30

  
31
		// It should be https://api.fairsharing.org
32
		final String baseUrl = interfaceDescriptor.getBaseUrl();
33
		final String login = interfaceDescriptor.getParams().get("login");
34
		final String password = interfaceDescriptor.getParams().get("password");
35

  
36
		final String authCode = login(baseUrl, login, password);
37

  
38
		log.debug("authCode: " + authCode);
39

  
40
		if (StringUtils.isBlank(authCode)) { throw new CollectorServiceException("Authorization failed: authCode is empty"); }
41

  
42
		return () -> new FairSharingIterator(baseUrl, authCode, PAGE_SIZE);
43
	}
44

  
45
	private String login(final String baseUrl, final String login, final String password) throws CollectorServiceException {
46
		final HttpPost req = new HttpPost(baseUrl + "/users/sign_in");
47
		req.addHeader("Accept", "application/json");
48
		req.addHeader("Content-Type", "application/json");
49
		req.setEntity(prepareCredentials(login, password));
50

  
51
		try (final CloseableHttpClient client = HttpClients.createDefault()) {
52
			try (final CloseableHttpResponse response = client.execute(req)) {
53
				final String content = IOUtils.toString(response.getEntity().getContent());
54
				final JSONObject obj = new JSONObject(content);
55
				return obj.getString("jwt");
56
			}
57
		} catch (final Exception e) {
58
			throw new CollectorServiceException("Error perfoming call fro login", e);
59
		}
60
	}
61

  
62
	public HttpEntity prepareCredentials(final String login, final String password) throws CollectorServiceException {
63

  
64
		final JSONObject objUser = new JSONObject();
65
		objUser.put("login", login);
66
		objUser.put("password", password);
67

  
68
		final JSONObject objCredentials = new JSONObject();
69
		objCredentials.put("user", objUser);
70

  
71
		try {
72
			return new StringEntity(objCredentials.toString());
73
		} catch (final UnsupportedEncodingException e) {
74
			throw new CollectorServiceException("Error preparing http entity for login");
75
		}
76
	}
77

  
78
}
modules/dnet-collector-plugins/trunk/src/main/resources/eu/dnetlib/data/collector/plugins/applicationContext-dnet-modular-collector-plugins.xml
231 231
		</property>
232 232
	</bean>
233 233

  
234

  
234
	<bean id="fairSharingCollectorPlugin" class="eu.dnetlib.data.collector.plugins.fairsharing.FairSharingPlugin">
235
		<property name="protocolDescriptor">
236
			<bean class="eu.dnetlib.data.collector.rmi.ProtocolDescriptor" p:name="fairsharing">
237
				<property name="params">
238
					<list>
239
						<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter" 
240
							p:name="login"
241
							p:optional="false" />
242
						<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter" 
243
							p:name="password"
244
							p:optional="false" />
245
					</list>
246
				</property>
247
			</bean>
248
		</property>
249
	</bean>
235 250
</beans>
modules/dnet-collector-plugins/trunk/src/test/java/eu/dnetlib/data/collector/plugins/fairsharing/FairSharingPluginTest.java
1
package eu.dnetlib.data.collector.plugins.fairsharing;
2

  
3
import java.util.concurrent.atomic.AtomicLong;
4

  
5
import org.junit.Before;
6
import org.junit.Ignore;
7
import org.junit.Test;
8

  
9
import eu.dnetlib.data.collector.rmi.CollectorServiceException;
10
import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;
11

  
12
@Ignore
13
public class FairSharingPluginTest {
14

  
15
	private FairSharingPlugin plugin;
16
	private InterfaceDescriptor ifcDescr;
17

  
18
	@Before
19
	public void setUp() throws Exception {
20
		plugin = new FairSharingPlugin();
21
		ifcDescr = new InterfaceDescriptor();
22
		ifcDescr.setBaseUrl("https://api.fairsharing.org");
23
		ifcDescr.getParams().put("login", "");
24
		ifcDescr.getParams().put("password", "");
25
	}
26

  
27
	@Test
28
	public void testDownload() throws CollectorServiceException {
29
		final AtomicLong count = new AtomicLong(0);
30
		plugin.collect(ifcDescr, null, null).forEach(s -> {
31
			// System.out.println(s);
32
			final long x = count.incrementAndGet();
33
			if (x % 100 == 0) {
34
				System.out.println(x);
35
			}
36
		});
37

  
38
		System.out.println("TOTAL: " + count.get());
39
	}
40

  
41
}

Also available in: Unified diff