Project

General

Profile

« Previous | Next » 

Revision 62806

Added by Michele Artini about 1 year ago

reimplemented eosc plugin with new apis

View differences:

modules/dnet-collector-plugins/trunk/src/main/java/eu/dnetlib/data/collector/plugins/eosc/EoscServicesPlugin.java
1 1 
package eu.dnetlib.data.collector.plugins.eosc;
2 2 

  		




  
  3
  
    
import java.util.ArrayList;


  		




  
  4
  
    
import java.util.Iterator;


  		




  
  5
  
    
import java.util.List;


  		




  
  6
  
    

  		




  
  7
  
    
import org.apache.commons.io.IOUtils;


  		




  3
  8
  
    
import org.apache.commons.lang3.StringUtils;


  		




  4
  9
  
    
import org.apache.commons.lang3.math.NumberUtils;


  		




  
  10
  
    
import org.apache.commons.logging.Log;


  		




  
  11
  
    
import org.apache.commons.logging.LogFactory;


  		




  
  12
  
    
import org.apache.http.NameValuePair;


  		




  
  13
  
    
import org.apache.http.client.entity.UrlEncodedFormEntity;


  		




  5
  14
  
    
import org.apache.http.client.methods.CloseableHttpResponse;


  		




  6
  15
  
    
import org.apache.http.client.methods.HttpGet;


  		




  
  16
  
    
import org.apache.http.client.methods.HttpPost;


  		




  7
  17
  
    
import org.apache.http.impl.client.CloseableHttpClient;


  		




  8
  18
  
    
import org.apache.http.impl.client.HttpClients;


  		




  
  19
  
    
import org.apache.http.message.BasicNameValuePair;


  		




  9
  20
  
    
import org.dom4j.Document;


  		




  10
  
  
    
import org.dom4j.DocumentHelper;


  		




  11
  21
  
    
import org.dom4j.Element;


  		




  12
  
  
    
import org.dom4j.Node;


  		




  13
  22
  
    
import org.dom4j.io.SAXReader;


  		




  
  23
  
    
import org.json.JSONObject;


  		




  14
  24
  
    

  		




  
  25
  
    
import com.google.common.collect.Iterators;


  		




  
  26
  
    

  		




  15
  27
  
    
import eu.dnetlib.data.collector.plugin.AbstractCollectorPlugin;


  		




  16
  28
  
    
import eu.dnetlib.data.collector.rmi.CollectorServiceException;


  		




  17
  29
  
    
import eu.dnetlib.data.collector.rmi.InterfaceDescriptor;


  		




  ......




  19
  31
  
    
public class EoscServicesPlugin extends AbstractCollectorPlugin {


  		




  20
  32
  
    

  		




  21
  33
  
    
	// Suggested values:


  		




  22
  
  
    
	// baseUrl = https://api.eosc-portal.eu


  		




  23
  
  
    
	// maxProviders = 10000


  		




  24
  
  
    
	// XPATH_ID = /record/organization/id


  		




  
  34
  
    
	// baseUrl = https://api.eosc-portal.eu/public


  		




  
  35
  
    
	// XPATH_ID = /results/id


  		




  
  36
  
    
	// Go to https://aai.eosc-portal.eu/providers-api/ (user: d-net) for refreshToken and clientId


  		




  25
  37
  
    

  		




  26
  
  
    
	@SuppressWarnings("unchecked")


  		




  
  38
  
    
	private static final Log log = LogFactory.getLog(EoscServicesPlugin.class);


  		




  
  39
  
    

  		




  27
  40
  
    
	@Override


  		




  28
  41
  
    
	public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String untilDate)


  		




  29
  42
  
    
		throws CollectorServiceException {


  		




  30
  43
  
    

  		




  31
  44
  
    
		final String baseUrl = interfaceDescriptor.getBaseUrl();


  		




  32
  
  
    
		final String token = interfaceDescriptor.getParams().getOrDefault("token", null);


  		




  33
  
  
    
		final long maxProviders = NumberUtils.toLong(interfaceDescriptor.getParams().get("maxProviders"), 10000);


  		




  34
  45
  
    

  		




  35
  
  
    
		return () -> httpCall(baseUrl + "/provider/all?quantity=" + maxProviders, token)


  		




  36
  
  
    
			.selectNodes("/Paging/results/results")


  		




  37
  
  
    
			.stream()


  		




  38
  
  
    
			.map(o -> processProvider(baseUrl, token, (Node) o))


  		




  39
  
  
    
			.iterator();


  		




  40
  
  
    
	}


  		




  
  46
  
    
		final long quantity = NumberUtils.toLong(interfaceDescriptor.getParams().get("quantity"), 10000);


  		




  41
  47
  
    

  		




  42
  
  
    
	private String processProvider(final String baseUrl, final String token, final Node nodeProv) {


  		




  43
  
  
    
		final String orgId = nodeProv.valueOf("./id");


  		




  
  48
  
    
		final String authorizationUrl = interfaceDescriptor.getParams().get("authorizationUrl");


  		




  
  49
  
    
		final String clientId = interfaceDescriptor.getParams().get("clientId");


  		




  
  50
  
    
		final String refreshToken = interfaceDescriptor.getParams().get("refreshToken");


  		




  
  51
  
    
		final String accessToken = obtainAccessoToken(authorizationUrl, clientId, refreshToken);


  		




  44
  52
  
    

  		




  45
  
  
    
		final Document newDoc = DocumentHelper.createDocument();


  		




  46
  
  
    
		final Element newRoot = DocumentHelper.createElement("record");


  		




  
  53
  
    
		final Iterator<String> iter1 =


  		




  
  54
  
    
			iterCall(baseUrl + "/provider/bundle/all?quantity=" + quantity, accessToken, "organization");


  		




  47
  55
  
    

  		




  48
  
  
    
		newDoc.setRootElement(newRoot);


  		




  
  56
  
    
		final Iterator<String> iter2 =


  		




  
  57
  
    
			iterCall(baseUrl + "/service/adminPage/all?type=all&quantity=" + quantity, accessToken, "service");


  		




  49
  58
  
    

  		




  50
  
  
    
		nodeProv.setName("organization");


  		




  51
  
  
    
		newRoot.add(nodeProv.detach());


  		




  
  59
  
    
		return () -> Iterators.concat(iter1, iter2);


  		




  
  60
  
    
	}


  		




  52
  61
  
    

  		




  53
  
  
    
		final Document docSrvs = httpCall(baseUrl + "/provider/services/" + orgId, token);


  		




  54
  
  
    
		for (final Object o : docSrvs.selectNodes("/List/item")) {


  		




  55
  
  
    
			final Node nodeSrv = (Node) o;


  		




  56
  
  
    
			nodeSrv.setName("service");


  		




  57
  
  
    
			newRoot.add(nodeSrv.detach());


  		




  
  62
  
    
	private String obtainAccessoToken(final String url, final String clientId, final String refreshToken) throws CollectorServiceException {


  		




  
  63
  
    
		try (final CloseableHttpClient client = HttpClients.createDefault()) {


  		




  
  64
  
    
			final HttpPost req = new HttpPost(url);


  		




  
  65
  
    
			final List<NameValuePair> params = new ArrayList<>();


  		




  
  66
  
    
			params.add(new BasicNameValuePair("grant_type", "refresh_token"));


  		




  
  67
  
    
			params.add(new BasicNameValuePair("refresh_token", refreshToken));


  		




  
  68
  
    
			params.add(new BasicNameValuePair("client_id", clientId));


  		




  
  69
  
    
			params.add(new BasicNameValuePair("scope", "openid email profile"));


  		




  
  70
  
    

  		




  
  71
  
    
			req.setEntity(new UrlEncodedFormEntity(params, "UTF-8"));


  		




  
  72
  
    

  		




  
  73
  
    
			try (final CloseableHttpResponse response = client.execute(req)) {


  		




  
  74
  
    
				final String content = IOUtils.toString(response.getEntity().getContent());


  		




  
  75
  
    
				final JSONObject obj = new JSONObject(content);


  		




  
  76
  
    
				final String token = obj.getString("access_token");


  		




  
  77
  
    
				if (StringUtils.isNotBlank(token)) { return token; }


  		




  
  78
  
    
			}


  		




  
  79
  
    
		} catch (final Throwable e) {


  		




  
  80
  
    
			log.warn("Error obtaining access token", e);


  		




  
  81
  
    
			throw new CollectorServiceException("Error obtaining access token", e);


  		




  58
  82
  
    
		}


  		




  
  83
  
    
		throw new CollectorServiceException("Access token is missing");


  		




  
  84
  
    
	}


  		




  59
  85
  
    

  		




  60
  
  
    
		return newDoc.asXML();


  		




  
  86
  
    
	@SuppressWarnings("unchecked")


  		




  
  87
  
    
	private Iterator<String> iterCall(final String url, final String token, final String type) {


  		




  
  88
  
    
		return httpCall(url, token)


  		




  
  89
  
    
			.selectNodes("//results/results")


  		




  
  90
  
    
			.stream()


  		




  
  91
  
    
			.map(o -> {


  		




  
  92
  
    
				final Element el = (Element) o;


  		




  
  93
  
    
				el.addAttribute("type", type);


  		




  
  94
  
    
				return el.asXML();


  		




  
  95
  
    
			})


  		




  
  96
  
    
			.iterator();


  		




  61
  97
  
    
	}


  		




  62
  98
  
    

  		




  63
  99
  
    
	private Document httpCall(final String url, final String token) {


  		




  
  100
  
    
		log.info("EOSC API: " + url);


  		




  
  101
  
    

  		




  64
  102
  
    
		final SAXReader reader = new SAXReader();


  		




  65
  103
  
    

  		




  66
  104
  
    
		final HttpGet req = new HttpGet(url);


  		












  

    
      modules/dnet-collector-plugins/trunk/src/main/resources/eu/dnetlib/data/collector/plugins/applicationContext-dnet-modular-collector-plugins.xml
    
  





  222
  222
  
    
				<property name="params">


  		




  223
  223
  
    
					<list>


  		




  224
  224
  
    
						<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter" 


  		




  225
  
  
    
							p:name="maxProviders"


  		




  
  225
  
    
							p:name="quantity"


  		




  226
  226
  
    
							p:optional="true" 


  		




  227
  227
  
    
							p:type="NUMBER" />


  		




  228
  228
  
    
						<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter" 


  		




  229
  
  
    
							p:name="token"


  		




  230
  
  
    
							p:optional="true" /> 


  		




  
  229
  
    
							p:name="clientId" />


  		




  
  230
  
    
						<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter" 


  		




  
  231
  
    
							p:name="authorizationUrl" />


  		




  
  232
  
    
						<bean class="eu.dnetlib.data.collector.rmi.ProtocolParameter" 


  		




  
  233
  
    
							p:name="refreshToken" /> 


  		




  231
  234
  
    
					</list>


  		




  232
  235
  
    
				</property>


  		




  233
  236
  
    
			</bean>


  		












  

    
      modules/dnet-collector-plugins/trunk/src/test/java/eu/dnetlib/data/collector/plugins/eosc/EoscServicesPluginTest.java
    
  





  8
  8
  
    

  		




  9
  9
  
    
public class EoscServicesPluginTest {


  		




  10
  10
  
    

  		




  11
  
  
    
	public static final String BASE_URL = "https://api.eosc-portal.eu";


  		




  12
  
  
    
	public static final long MAX_PROVIDERS = 1;


  		




  
  11
  
    
	private static final String BASE_URL = "https://beta.providers.eosc-portal.eu/api/public";


  		




  
  12
  
    
	private static final long QUANTITY = 10000;


  		




  
  13
  
    
	private static final String AUTHORIZATION_URL = "https://aai.eosc-portal.eu/auth/realms/core/protocol/openid-connect/token";


  		




  
  14
  
    
	private static final String CLIENT_ID = "";


  		




  
  15
  
    
	private static final String REFRESH_TOKEN = "";


  		




  13
  16
  
    

  		




  14
  17
  
    
	@Test


  		




  15
  18
  
    
	@Ignore


  		




  ......




  18
  21
  
    

  		




  19
  22
  
    
		final InterfaceDescriptor desc = new InterfaceDescriptor();


  		




  20
  23
  
    
		desc.setBaseUrl(BASE_URL);


  		




  21
  
  
    
		desc.getParams().put("maxProviders", Long.toString(MAX_PROVIDERS));


  		




  
  24
  
    
		desc.getParams().put("quantity", Long.toString(QUANTITY));


  		




  
  25
  
    
		desc.getParams().put("authorizationUrl", AUTHORIZATION_URL);


  		




  
  26
  
    
		desc.getParams().put("clientId", CLIENT_ID);


  		




  
  27
  
    
		desc.getParams().put("refreshToken", REFRESH_TOKEN);


  		




  22
  28
  
    

  		




  23
  29
  
    
		int i = 1;


  		




  24
  30
  
    
		for (final String s : plugin.collect(desc, null, null)) {


  		












Also available in:  Unified diff