Project

General

Profile

« Previous | Next » 

Revision 48824

Added by Alessia Bardi over 6 years ago

Fixed incremental harvesting via sftp due to an issue with Java8 DateTime. Wish the time management would be easier with Java8?

View differences:

modules/dnet-data-services/branches/saxonHE/src/test/java/eu/dnetlib/data/collector/plugins/sftp/SftpPubKeyCollectorPluginTest.java
27 27 
	@Before
28 28 
	public void prepare(){
29 29 
		apiDescriptor = new InterfaceDescriptor();
30 
		apiDescriptor.setBaseUrl("sftp://ariadne2.isti.cnr.it/../../data/transform/acdm_correct");
30 
		apiDescriptor.setBaseUrl("sftp://ariadne2.isti.cnr.it/../../data/transform/old/acdm_correct");
31 31 
		apiDescriptor.setProtocol("sftp");
32 32 
		apiDescriptor.setId("testSFTP");
33 33 
		apiDescriptor.getParams().put("recursive", "true");
......
48 48 
		}
49 49 

  		




  50
  50
  
    
	}


  		




  
  51
  
    

  		




  
  52
  
    
	@Test


  		




  
  53
  
    
	public void testIncrementalWithAriadneServer() throws CollectorServiceException {


  		




  
  54
  
    

  		




  
  55
  
    
		Iterable<String> res = plugin.collect(apiDescriptor, "2017-06-06", null);


  		




  
  56
  
    
		Assert.assertNotNull(res);


  		




  
  57
  
    
		int i =0;


  		




  
  58
  
    
		for(String r : res){


  		




  
  59
  
    
			i++;


  		




  
  60
  
    
			Assert.assertNotNull(r);


  		




  
  61
  
    
			if(i > max_records) break;


  		




  
  62
  
    
		}


  		




  
  63
  
    

  		




  
  64
  
    
	}


  		




  51
  65
  
    
}


  		












  

    
      modules/dnet-data-services/branches/saxonHE/src/test/resources/eu/dnetlib/data/collector/plugins/sftp/SftpPubKeyCollectorPluginTest-context.xml
    
  





  28
  28
  
    
		<property name="properties">


  		




  29
  29
  
    
			<props>


  		




  30
  30
  
    
				<prop key="collector.sftp.auth.prvKeyPath">~/.ssh/id_rsa</prop>


  		




  31
  
  
    
				<prop key="collector.sftp.auth.passphrase"></prop>


  		




  
  31
  
    
				<prop key="collector.sftp.auth.passphrase">starnet82</prop>


  		




  32
  32
  
    
				<prop key="collector.sftp.auth.knownHostsPath">~/.ssh/known_hosts</prop>


  		




  33
  33
  
    
			</props>


  		




  34
  34
  
    
		</property>


  		












  

    
      modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/SftpIterator.java
    
  





  4
  4
  
    
import java.net.URI;


  		




  5
  5
  
    
import java.net.URISyntaxException;


  		




  6
  6
  
    
import java.time.Instant;


  		




  
  7
  
    
import java.time.LocalDate;


  		




  7
  8
  
    
import java.time.LocalDateTime;


  		




  8
  9
  
    
import java.time.ZoneId;


  		




  9
  10
  
    
import java.time.format.DateTimeFormatter;


  		




  10
  
  
    
import java.util.*;


  		




  
  11
  
    
import java.util.LinkedList;


  		




  
  12
  
    
import java.util.Queue;


  		




  
  13
  
    
import java.util.Set;


  		




  
  14
  
    
import java.util.Vector;


  		




  11
  15
  
    

  		




  12
  16
  
    
import com.jcraft.jsch.*;


  		




  13
  17
  
    
import eu.dnetlib.data.collector.ThreadSafeIterator;


  		




  ......




  61
  65
  
    
		this.incremental = StringUtils.isNotBlank(fromDate);


  		




  62
  66
  
    
		if (incremental) {


  		




  63
  67
  
    
			//I expect fromDate in the format 'yyyy-MM-dd'. See class eu.dnetlib.msro.workflows.nodes.collect.FindDateRangeForIncrementalHarvestingJobNode .


  		




  64
  
  
    
			this.fromDate = LocalDateTime.parse(fromDate, simpleDateTimeFormatter);


  		




  
  68
  
    
			//see https://stackoverflow.com/questions/27454025/unable-to-obtain-localdatetime-from-temporalaccessor-when-parsing-localdatetime


  		




  
  69
  
    
			this.fromDate = LocalDateTime.from(LocalDate.parse(fromDate, simpleDateTimeFormatter).atStartOfDay());


  		




  65
  70
  
    
			log.debug("fromDate string: " + fromDate + " -- parsed: " + this.fromDate.toString());


  		




  66
  71
  
    
		}


  		




  67
  72
  
    
		try {


  		












Also available in:  Unified diff