Project

General

Profile

« Previous | Next » 

Revision 52889

SftpIterator can work incrementally with dates and datetimes

View differences:

modules/dnet-data-services/branches/saxonHE/src/test/java/eu/dnetlib/data/collector/plugins/sftp/SftpPubKeyCollectorPluginTest.java
27 27
	@Before
28 28
	public void prepare(){
29 29
		apiDescriptor = new InterfaceDescriptor();
30
		apiDescriptor.setBaseUrl("sftp://ariadne2.isti.cnr.it/../../data/transform/old/acdm_correct");
30
		apiDescriptor.setBaseUrl("sftp://ariadne2.isti.cnr.it/../../data/transform/old/acdm_correct/dir_021");
31 31
		apiDescriptor.setProtocol("sftp");
32 32
		apiDescriptor.setId("testSFTP");
33 33
		apiDescriptor.getParams().put("recursive", "true");
......
60 60
			Assert.assertNotNull(r);
61 61
			if(i > max_records) break;
62 62
		}
63
	}
63 64

  
65
	@Test
66
	public void testIncrementalWithAriadneServerFullDate() throws CollectorServiceException {
67

  
68
		Iterable<String> res = plugin.collect(apiDescriptor, "2016-02-21T10:58:53Z", null);
69
		Assert.assertNotNull(res);
70
		int i =0;
71
		for(String r : res){
72
			i++;
73
			Assert.assertNotNull(r);
74
			if(i > max_records) break;
75
		}
64 76
	}
77

  
78
	@Test(expected=CollectorServiceException.class)
79
	public void testIncrementalWithAriadneServerWrongDateFormat() throws CollectorServiceException {
80
		plugin.collect(apiDescriptor, "20160221", null);
81
	}
65 82
}
modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/AbstractSftpCollectorPlugin.java
31 31
		if ((extensions == null) || extensions.isEmpty()) {
32 32
			throw new CollectorServiceException("Param 'extensions' is null or empty");
33 33
		}
34
		if (fromDate != null && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) { throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + fromDate); }
34
		if (fromDate != null && (!fromDate.matches("\\d{4}-\\d{2}-\\d{2}(T\\d{2}:\\d{2}:\\d{2}Z)?+") && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}") )) {
35
			throw new CollectorServiceException("Invalid date (yyyy-MM-dd'T'HH:mm:ss'Z' or yyyy-MM-dd ): " + fromDate);
36
		}
35 37

  
36 38
		Set<String> extensionSet = parseSet(extensions);
37 39

  
modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/SftpIterator.java
8 8
import java.time.LocalDateTime;
9 9
import java.time.ZoneId;
10 10
import java.time.format.DateTimeFormatter;
11
import java.time.format.DateTimeParseException;
11 12
import java.util.LinkedList;
12 13
import java.util.Queue;
13 14
import java.util.Set;
......
57 58

  
58 59
	private LocalDateTime fromDate = null;
59 60
	private DateTimeFormatter simpleDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
61
	private DateTimeFormatter fullDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'");
60 62

  
61 63
	private void init(final String baseUrl, final boolean isRecursive, final Set<String> extensionsSet, final String fromDate) {
62 64
		this.baseUrl = baseUrl;
......
64 66
		this.extensionsSet = extensionsSet;
65 67
		this.incremental = StringUtils.isNotBlank(fromDate);
66 68
		if (incremental) {
67
			//I expect fromDate in the format 'yyyy-MM-dd'. See class eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode .
68
			//see https://stackoverflow.com/questions/27454025/unable-to-obtain-localdatetime-from-temporalaccessor-when-parsing-localdatetime
69
			this.fromDate = LocalDateTime.from(LocalDate.parse(fromDate, simpleDateTimeFormatter).atStartOfDay());
69
			try {
70
				this.fromDate = LocalDateTime.parse(fromDate, fullDateTimeFormatter);
71
			}catch(DateTimeParseException dtpe){
72
				//try with the simple formatter
73
				this.fromDate = LocalDateTime.from(LocalDate.parse(fromDate, simpleDateTimeFormatter).atStartOfDay());
74
			}
70 75
			log.debug("fromDate string: " + fromDate + " -- parsed: " + this.fromDate.toString());
71 76
		}
72 77
		try {

Also available in: Unified diff