Revision 52889
Added by Alessia Bardi over 5 years ago
modules/dnet-data-services/branches/saxonHE/src/test/java/eu/dnetlib/data/collector/plugins/sftp/SftpPubKeyCollectorPluginTest.java | ||
---|---|---|
27 | 27 |
@Before |
28 | 28 |
public void prepare(){ |
29 | 29 |
apiDescriptor = new InterfaceDescriptor(); |
30 |
apiDescriptor.setBaseUrl("sftp://ariadne2.isti.cnr.it/../../data/transform/old/acdm_correct"); |
|
30 |
apiDescriptor.setBaseUrl("sftp://ariadne2.isti.cnr.it/../../data/transform/old/acdm_correct/dir_021");
|
|
31 | 31 |
apiDescriptor.setProtocol("sftp"); |
32 | 32 |
apiDescriptor.setId("testSFTP"); |
33 | 33 |
apiDescriptor.getParams().put("recursive", "true"); |
... | ... | |
60 | 60 |
Assert.assertNotNull(r); |
61 | 61 |
if(i > max_records) break; |
62 | 62 |
} |
63 |
} |
|
63 | 64 |
|
65 |
@Test |
|
66 |
public void testIncrementalWithAriadneServerFullDate() throws CollectorServiceException { |
|
67 |
|
|
68 |
Iterable<String> res = plugin.collect(apiDescriptor, "2016-02-21T10:58:53Z", null); |
|
69 |
Assert.assertNotNull(res); |
|
70 |
int i =0; |
|
71 |
for(String r : res){ |
|
72 |
i++; |
|
73 |
Assert.assertNotNull(r); |
|
74 |
if(i > max_records) break; |
|
75 |
} |
|
64 | 76 |
} |
77 |
|
|
78 |
@Test(expected=CollectorServiceException.class) |
|
79 |
public void testIncrementalWithAriadneServerWrongDateFormat() throws CollectorServiceException { |
|
80 |
plugin.collect(apiDescriptor, "20160221", null); |
|
81 |
} |
|
65 | 82 |
} |
modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/AbstractSftpCollectorPlugin.java | ||
---|---|---|
31 | 31 |
if ((extensions == null) || extensions.isEmpty()) { |
32 | 32 |
throw new CollectorServiceException("Param 'extensions' is null or empty"); |
33 | 33 |
} |
34 |
if (fromDate != null && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) { throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + fromDate); } |
|
34 |
if (fromDate != null && (!fromDate.matches("\\d{4}-\\d{2}-\\d{2}(T\\d{2}:\\d{2}:\\d{2}Z)?+") && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}") )) { |
|
35 |
throw new CollectorServiceException("Invalid date (yyyy-MM-dd'T'HH:mm:ss'Z' or yyyy-MM-dd ): " + fromDate); |
|
36 |
} |
|
35 | 37 |
|
36 | 38 |
Set<String> extensionSet = parseSet(extensions); |
37 | 39 |
|
modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/SftpIterator.java | ||
---|---|---|
8 | 8 |
import java.time.LocalDateTime; |
9 | 9 |
import java.time.ZoneId; |
10 | 10 |
import java.time.format.DateTimeFormatter; |
11 |
import java.time.format.DateTimeParseException; |
|
11 | 12 |
import java.util.LinkedList; |
12 | 13 |
import java.util.Queue; |
13 | 14 |
import java.util.Set; |
... | ... | |
57 | 58 |
|
58 | 59 |
private LocalDateTime fromDate = null; |
59 | 60 |
private DateTimeFormatter simpleDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); |
61 |
private DateTimeFormatter fullDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); |
|
60 | 62 |
|
61 | 63 |
private void init(final String baseUrl, final boolean isRecursive, final Set<String> extensionsSet, final String fromDate) { |
62 | 64 |
this.baseUrl = baseUrl; |
... | ... | |
64 | 66 |
this.extensionsSet = extensionsSet; |
65 | 67 |
this.incremental = StringUtils.isNotBlank(fromDate); |
66 | 68 |
if (incremental) { |
67 |
//I expect fromDate in the format 'yyyy-MM-dd'. See class eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode . |
|
68 |
//see https://stackoverflow.com/questions/27454025/unable-to-obtain-localdatetime-from-temporalaccessor-when-parsing-localdatetime |
|
69 |
this.fromDate = LocalDateTime.from(LocalDate.parse(fromDate, simpleDateTimeFormatter).atStartOfDay()); |
|
69 |
try { |
|
70 |
this.fromDate = LocalDateTime.parse(fromDate, fullDateTimeFormatter); |
|
71 |
}catch(DateTimeParseException dtpe){ |
|
72 |
//try with the simple formatter |
|
73 |
this.fromDate = LocalDateTime.from(LocalDate.parse(fromDate, simpleDateTimeFormatter).atStartOfDay()); |
|
74 |
} |
|
70 | 75 |
log.debug("fromDate string: " + fromDate + " -- parsed: " + this.fromDate.toString()); |
71 | 76 |
} |
72 | 77 |
try { |
Also available in: Unified diff
SftpIterator can work incrementally with dates and datetimes