Project

General

Profile

« Previous | Next » 

Revision 27303

Added by Michele Artini over 10 years ago

from, until support

View differences:

modules/dnet-modular-collector-service/trunk/src/test/java/eu/dnetlib/data/collector/plugins/oai/OaiIteratorTest.java
14 14
15 15 
	@Before
16 16 
	public void setUp() {
17 
		oai = new OaiIterator(BASE_URL, FORMAT, SET);
17 
		oai = new OaiIterator(BASE_URL, FORMAT, SET, null, null);
18 18 
	}
19 19
20 20 
	@Test
modules/dnet-modular-collector-service/trunk/src/test/java/eu/dnetlib/data/collector/plugins/oai/OaiCollectorPluginTest.java
46 46 
	public void setUp() {
47 47 
		oai = new OaiCollectorPlugin();
48 48 
		oai.setOaiIteratorFactory(oaiIteratorFactory);
49 
		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_1)).thenReturn(oaiIterator1);
50 
		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_2)).thenReturn(oaiIterator2);
51 
		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_3)).thenReturn(oaiIterator3);
49 
		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_1, null, null)).thenReturn(oaiIterator1);
50 
		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_2, null, null)).thenReturn(oaiIterator2);
51 
		when(oaiIteratorFactory.newIterator(BASE_URL, FORMAT, SET_3, null, null)).thenReturn(oaiIterator3);
52 52 
	}
53 53
54 54 
	public void test() {
......
73 73 
		final Iterable<String> records = oai.collect(iface);
74 74
75 75 
		assertNotNull(records);
76 
		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_1);
77 
		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_2);
78 
		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_3);
76 
		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_1, null, null);
77 
		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_2, null, null);
78 
		verify(oaiIteratorFactory, new Times(0)).newIterator(BASE_URL, FORMAT, SET_3, null, null);
79 79
80 80 
		int count = 0;
81 81 
		for (String s : records) {
......
84 84 
			count++;
85 85 
		}
86 86 
		assertEquals(elements.size(), count);
87 
		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_1);
88 
		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_2);
89 
		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_3);
87 
		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_1, null, null);
88 
		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_2, null, null);
89 
		verify(oaiIteratorFactory).newIterator(BASE_URL, FORMAT, SET_3, null, null);
90 90 
	}
91 91 
}
modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/oai/OaiIterator.java
26 26 
	private String baseUrl;
27 27 
	private String set;
28 28 
	private String mdFormat;
29 
	private String fromDate;
30 
	private String untilDate;
29 31 
	private String token;
30 32 
	private boolean started;
31 33 

  		




  32
  
  
    
	public OaiIterator(final String baseUrl, final String mdFormat, final String set) {


  		




  
  34
  
    
	public OaiIterator(final String baseUrl, final String mdFormat, final String set, final String fromDate, final String untilDate) {


  		




  33
  35
  
    
		this.baseUrl = baseUrl;


  		




  34
  36
  
    
		this.mdFormat = mdFormat;


  		




  35
  37
  
    
		this.set = set;


  		




  
  38
  
    
		this.fromDate = fromDate;


  		




  
  39
  
    
		this.untilDate = untilDate;


  		




  36
  40
  
    
		this.started = false;


  		




  37
  41
  
    
	}


  		




  38
  42
  
    
	


  		




  ......




  73
  77
  
    
			url += "&set=" + set;


  		




  74
  78
  
    
		}


  		




  75
  79
  
    
		


  		




  
  80
  
    
		if ((fromDate != null) && fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) {


  		




  
  81
  
    
			url += "&from=" + fromDate;


  		




  
  82
  
    
		}


  		




  
  83
  
    
		


  		




  
  84
  
    
		if ((untilDate != null) && untilDate.matches("\\d{4}-\\d{2}-\\d{2}")) {


  		




  
  85
  
    
			url += "&until=" + untilDate;


  		




  
  86
  
    
		}


  		




  
  87
  
    
		


  		




  76
  88
  
    
		log.info("Downloading first page using url: " + url);


  		




  77
  89
  
    

  		




  78
  90
  
    
		return downloadPage(url);


  		












  

    
      modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/oai/OaiIteratorFactory.java
    
  





  3
  3
  
    
import java.util.Iterator;


  		




  4
  4
  
    

  		




  5
  5
  
    
public class OaiIteratorFactory {


  		




  6
  
  
    
	public Iterator<String> newIterator(final String baseUrl, final String mdFormat, final String set) {


  		




  7
  
  
    
		return new OaiIterator(baseUrl, mdFormat, set);


  		




  
  6
  
    
	public Iterator<String> newIterator(final String baseUrl, final String mdFormat, final String set, final String fromDate, final String untilDate) {


  		




  
  7
  
    
		return new OaiIterator(baseUrl, mdFormat, set, fromDate, untilDate);


  		




  8
  8
  
    
	}


  		




  9
  9
  
    
}


  		












  

    
      modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/oai/OaiCollectorPlugin.java
    
  





  17
  17
  
    

  		




  18
  18
  
    
	private static final String FORMAT_PARAM = "format";


  		




  19
  19
  
    
	private static final String OAI_SET_PARAM = "set";


  		




  
  20
  
    
	private static final String FROM_PARAM = "from";


  		




  
  21
  
    
	private static final String UNTIL_PARAM = "until";


  		




  20
  22
  
    
	


  		




  21
  23
  
    
	private OaiIteratorFactory oaiIteratorFactory;


  		




  22
  24
  
    

  		




  ......




  25
  27
  
    
		final String baseUrl = interfaceDescriptor.getBaseUrl();


  		




  26
  28
  
    
		final String mdFormat = interfaceDescriptor.getParams().get(FORMAT_PARAM);


  		




  27
  29
  
    
		final String setParam = interfaceDescriptor.getParams().get(OAI_SET_PARAM);


  		




  28
  
  
    

  		




  
  30
  
    
		final String fromDate = interfaceDescriptor.getParams().get(FROM_PARAM);


  		




  
  31
  
    
		final String untilDate = interfaceDescriptor.getParams().get(UNTIL_PARAM);


  		




  
  32
  
    
		


  		




  29
  33
  
    
		final Iterable<String> sets = Splitter.on(",").omitEmptyStrings().trimResults().split(setParam);


  		




  30
  34
  
    
		


  		




  31
  
  
    
		if ((baseUrl == null) || baseUrl.isEmpty()) { throw new CollectorServiceException("Param 'baseurl' is null or empty"); }


  		




  
  35
  
    
		if ((baseUrl == null) || baseUrl.isEmpty()) {


  		




  
  36
  
    
			throw new CollectorServiceException("Param 'baseurl' is null or empty");


  		




  
  37
  
    
		}


  		




  32
  38
  
    

  		




  33
  
  
    
		if ((mdFormat == null) || mdFormat.isEmpty()) { throw new CollectorServiceException("Param 'mdFormat' is null or empty"); }


  		




  
  39
  
    
		if ((mdFormat == null) || mdFormat.isEmpty()) {


  		




  
  40
  
    
			throw new CollectorServiceException("Param 'mdFormat' is null or empty");


  		




  
  41
  
    
		}


  		




  34
  42
  
    

  		




  
  43
  
    
		if ((fromDate != null) && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) {


  		




  
  44
  
    
			throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + fromDate);


  		




  
  45
  
    
		}


  		




  
  46
  
    
		


  		




  
  47
  
    
		if ((untilDate != null) && !untilDate.matches("\\d{4}-\\d{2}-\\d{2}")) {


  		




  
  48
  
    
			throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + untilDate);


  		




  
  49
  
    
		}


  		




  
  50
  
    
		


  		




  35
  51
  
    
		return new Iterable<String>() {


  		




  36
  52
  
    
			@SuppressWarnings("unchecked")


  		




  37
  53
  
    
			@Override


  		




  ......




  39
  55
  
    
				final Iterable<Iterator<String>> iter = Iterables.transform(sets, new Function<String, Iterator<String>>() {


  		




  40
  56
  
    
					@Override


  		




  41
  57
  
    
					public Iterator<String> apply(String set) {


  		




  42
  
  
    
						return oaiIteratorFactory.newIterator(baseUrl, mdFormat, set);


  		




  
  58
  
    
						return oaiIteratorFactory.newIterator(baseUrl, mdFormat, set, fromDate, untilDate);


  		




  43
  59
  
    
					}


  		




  44
  60
  
    
				});


  		




  45
  61
  
    
				return Iterators.concat(Iterables.toArray(iter, Iterator.class));


  		












Also available in:  Unified diff