Project

General

Profile

« Previous | Next » 

Revision 47231

Added by Alessia Bardi almost 7 years ago

Let's use Iterators instead of Lists

View differences:

modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/data/collector/plugins/parthenos/ehri/EHRIIterator.java
2 2 

  		




  3
  3
  
    
import java.io.IOException;


  		




  4
  4
  
    
import java.net.URL;


  		




  5
  
  
    
import java.util.List;


  		




  
  5
  
    
import java.util.Iterator;


  		




  6
  6
  
    

  		




  7
  7
  
    
import eu.dnetlib.data.collector.ThreadSafeIterator;


  		




  8
  8
  
    
import eu.dnetlib.rmi.data.CollectorServiceRuntimeException;


  		




  ......




  18
  18
  
    
public class EHRIIterator extends ThreadSafeIterator {


  		




  19
  19
  
    

  		




  20
  20
  
    
	private static final Log log = LogFactory.getLog(EHRIIterator.class);


  		




  21
  
  
    
	private List<String> identifiers;


  		




  
  21
  
    
	private Iterator<String> identifiers;


  		




  22
  22
  
    
	private String baseUrl;


  		




  23
  23
  
    
	private String suffix;


  		




  24
  
  
    
	private int nextIdx = 0;


  		




  25
  24
  
    

  		




  26
  
  
    
	public EHRIIterator(final List<String> identifiers, final String baseUrl, final String suffix){


  		




  27
  
  
    
		this.identifiers = identifiers;


  		




  
  25
  
    
	public EHRIIterator(final Iterator<String> idIterator, final String baseUrl, final String suffix){


  		




  
  26
  
    
		this.identifiers = idIterator;


  		




  28
  27
  
    
		this.baseUrl = baseUrl;


  		




  29
  28
  
    
		this.suffix = suffix;


  		




  30
  29
  
    
	}


  		




  31
  30
  
    

  		




  32
  31
  
    
	@Override


  		




  33
  32
  
    
	public boolean doHasNext() {


  		




  34
  
  
    
		return nextIdx < identifiers.size();


  		




  
  33
  
    
		return identifiers.hasNext();


  		




  35
  34
  
    
	}


  		




  36
  35
  
    

  		




  37
  36
  
    
	@Override


  		




  38
  37
  
    
	public String doNext()  {


  		




  39
  
  
    
		String target = baseUrl+"/"+identifiers.get(nextIdx)+"/"+suffix;


  		




  40
  
  
    
		nextIdx++;


  		




  
  38
  
    
		String target = baseUrl+"/"+identifiers.next()+"/"+suffix;


  		




  41
  39
  
    
		log.debug("Getting "+target);


  		




  42
  40
  
    
		try {


  		




  43
  41
  
    
			URL url = new URL(target);


  		




  ......




  47
  45
  
    
		}


  		




  48
  46
  
    
	}


  		




  49
  47
  
    

  		




  50
  
  
    
	public List<String> getIdentifiers() {


  		




  
  48
  
    
	public Iterator<String> getIdentifiers() {


  		




  51
  49
  
    
		return identifiers;


  		




  52
  50
  
    
	}


  		




  53
  51
  
    

  		




  54
  
  
    
	public void setIdentifiers(final List<String> identifiers) {


  		




  
  52
  
    
	public void setIdentifiers(final Iterator<String> identifiers) {


  		




  55
  53
  
    
		this.identifiers = identifiers;


  		




  56
  54
  
    
	}


  		




  57
  55
  
    

  		












  

    
      modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/data/collector/plugins/parthenos/ehri/EHRIGraphQLClient.java
    
  





  4
  4
  
    
import java.io.InputStreamReader;


  		




  5
  5
  
    
import java.net.URI;


  		




  6
  6
  
    
import java.net.URISyntaxException;


  		




  7
  
  
    
import java.util.List;


  		




  
  7
  
    
import java.util.Iterator;


  		




  8
  8
  
    

  		




  9
  9
  
    
import com.google.common.collect.Iterables;


  		




  10
  10
  
    
import com.google.common.collect.Lists;


  		




  ......




  33
  33
  
    

  		




  34
  34
  
    
	private static final Log log = LogFactory.getLog(EHRIGraphQLClient.class);


  		




  35
  35
  
    

  		




  36
  
  
    
	public List<String> collect(final String baseURL, final String graphQLQuery) throws URISyntaxException, IOException, CollectorServiceException {


  		




  
  36
  
    
	public Iterator<String> collect(final String baseURL, final String graphQLQuery) throws URISyntaxException, IOException, CollectorServiceException {


  		




  37
  37
  
    
		/*


  		




  38
  38
  
    
		The curl equivalent:


  		




  39
  39
  
    
		curl --header X-Stream:true \


  		




  ......




  67
  67
  
    
		}


  		




  68
  68
  
    
	}


  		




  69
  69
  
    

  		




  70
  
  
    
	protected List<String> getIdentifiers(final InputStreamReader input){


  		




  
  70
  
    
	protected Iterator<String> getIdentifiers(final InputStreamReader input){


  		




  71
  71
  
    
		JsonObject jsonObject = new JsonParser().parse(input).getAsJsonObject();


  		




  72
  72
  
    
		JsonArray items = jsonObject.getAsJsonObject("data").getAsJsonObject("topLevelDocumentaryUnits").getAsJsonArray("items");


  		




  73
  73
  
    
		log.debug(items);


  		




  74
  
  
    
		return Lists.newArrayList(Iterables.transform(items, jelem -> jelem.getAsJsonObject().get("id").getAsString()));


  		




  
  74
  
    
		return Lists.newArrayList(Iterables.transform(items, jelem -> jelem.getAsJsonObject().get("id").getAsString())).iterator();


  		




  75
  75
  
    
	}


  		




  76
  76
  
    

  		




  77
  77
  
    
}


  		












  

    
      modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/data/collector/plugins/parthenos/ehri/EHRIIteratorFactory.java
    
  





  3
  3
  
    
import java.io.IOException;


  		




  4
  4
  
    
import java.net.URISyntaxException;


  		




  5
  5
  
    
import java.util.Iterator;


  		




  6
  
  
    
import java.util.List;


  		




  7
  6
  
    

  		




  8
  7
  
    
import eu.dnetlib.rmi.data.CollectorServiceException;


  		




  9
  8
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  ......




  22
  21
  
    
			final String baseUrl,


  		




  23
  22
  
    
			final String suffix) {


  		




  24
  23
  
    
		try {


  		




  25
  
  
    
			List<String> ids = ehriGraphQLClient.collect(baseURLIdentifiers, queryIdentifiers);


  		




  
  24
  
    
			Iterator<String> ids = ehriGraphQLClient.collect(baseURLIdentifiers, queryIdentifiers);


  		




  26
  25
  
    
			return new EHRIIterator(ids, baseUrl, suffix);


  		




  27
  26
  
    
		} catch (CollectorServiceException | IOException | URISyntaxException e) {


  		




  28
  27
  
    
			throw new RuntimeException(e);


  		












Also available in:  Unified diff