Revision 46300
Added by Alessia Bardi over 7 years ago
| modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/SftpCollectorPlugin.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.data.collector.plugins.sftp; |
| 2 | 2 |
|
| 3 |
import java.util.Iterator; |
|
| 4 | 3 |
import java.util.Set; |
| 5 | 4 |
|
| 6 |
import com.google.common.base.Splitter; |
|
| 7 |
import com.google.common.collect.Sets; |
|
| 8 | 5 |
import eu.dnetlib.rmi.data.CollectorServiceException; |
| 9 | 6 |
import eu.dnetlib.rmi.data.InterfaceDescriptor; |
| 10 |
import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin; |
|
| 11 | 7 |
|
| 12 | 8 |
/** |
| 13 | 9 |
* Created by andrea on 11/01/16. |
| 14 | 10 |
*/ |
| 15 |
public class SftpCollectorPlugin extends AbstractCollectorPlugin {
|
|
| 11 |
public class SftpCollectorPlugin extends AbstractSftpCollectorPlugin {
|
|
| 16 | 12 |
|
| 17 |
private SftpIteratorFactory sftpIteratorFactory; |
|
| 13 |
@Override |
|
| 14 |
protected Iterable<String> getIterable(final InterfaceDescriptor interfaceDescriptor, |
|
| 15 |
final String baseUrl, |
|
| 16 |
final boolean isRecursive, |
|
| 17 |
final Set<String> extensionSet, |
|
| 18 |
final String fromDate) throws CollectorServiceException {
|
|
| 18 | 19 |
|
| 19 |
@Override |
|
| 20 |
public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String toDate) |
|
| 21 |
throws CollectorServiceException {
|
|
| 22 |
final String baseUrl = interfaceDescriptor.getBaseUrl(); |
|
| 23 | 20 |
final String username = interfaceDescriptor.getParams().get("username");
|
| 24 | 21 |
final String password = interfaceDescriptor.getParams().get("password");
|
| 25 |
final String recursive = interfaceDescriptor.getParams().get("recursive");
|
|
| 26 |
final String extensions = interfaceDescriptor.getParams().get("extensions");
|
|
| 27 | 22 |
|
| 28 |
if ((baseUrl == null) || baseUrl.isEmpty()) {
|
|
| 29 |
throw new CollectorServiceException("Param 'baseurl' is null or empty");
|
|
| 30 |
} |
|
| 31 | 23 |
if ((username == null) || username.isEmpty()) {
|
| 32 | 24 |
throw new CollectorServiceException("Param 'username' is null or empty");
|
| 33 | 25 |
} |
| 34 | 26 |
if ((password == null) || password.isEmpty()) {
|
| 35 | 27 |
throw new CollectorServiceException("Param 'password' is null or empty");
|
| 36 | 28 |
} |
| 37 |
if ((recursive == null) || recursive.isEmpty()) {
|
|
| 38 |
throw new CollectorServiceException("Param 'recursive' is null or empty");
|
|
| 39 |
} |
|
| 40 |
if ((extensions == null) || extensions.isEmpty()) {
|
|
| 41 |
throw new CollectorServiceException("Param 'extensions' is null or empty");
|
|
| 42 |
} |
|
| 43 |
if (fromDate != null && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) { throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + fromDate); }
|
|
| 44 |
|
|
| 45 |
// final int fromDateIntSeconds = |
|
| 46 |
|
|
| 47 |
return new Iterable<String>() {
|
|
| 48 |
|
|
| 49 |
boolean isRecursive = "true".equals(recursive); |
|
| 50 |
|
|
| 51 |
Set<String> extensionsSet = parseSet(extensions); |
|
| 52 |
|
|
| 53 |
@Override |
|
| 54 |
public Iterator<String> iterator() {
|
|
| 55 |
return getSftpIteratorFactory().newIterator(baseUrl, username, password, isRecursive, extensionsSet, fromDate); |
|
| 56 |
} |
|
| 57 |
|
|
| 58 |
private Set<String> parseSet(final String extensions) {
|
|
| 59 |
return Sets.newHashSet(Splitter.on(",").omitEmptyStrings().trimResults().split(extensions));
|
|
| 60 |
} |
|
| 61 |
}; |
|
| 29 |
return () -> getSftpIteratorFactory().newIteratorSimplAuth(baseUrl, username, password, isRecursive, extensionSet, fromDate); |
|
| 62 | 30 |
} |
| 63 |
|
|
| 64 |
public SftpIteratorFactory getSftpIteratorFactory() {
|
|
| 65 |
return sftpIteratorFactory; |
|
| 66 |
} |
|
| 67 |
|
|
| 68 |
public void setSftpIteratorFactory(SftpIteratorFactory sftpIteratorFactory) {
|
|
| 69 |
this.sftpIteratorFactory = sftpIteratorFactory; |
|
| 70 |
} |
|
| 71 | 31 |
} |
| modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/SftpPubKeyCollectorPlugin.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.collector.plugins.sftp; |
|
| 2 |
|
|
| 3 |
import java.util.Set; |
|
| 4 |
|
|
| 5 |
import eu.dnetlib.rmi.data.CollectorServiceException; |
|
| 6 |
import eu.dnetlib.rmi.data.InterfaceDescriptor; |
|
| 7 |
import org.springframework.beans.factory.annotation.Value; |
|
| 8 |
|
|
| 9 |
/** |
|
| 10 |
* Created by alessia on 15/03/17. |
|
| 11 |
*/ |
|
| 12 |
public class SftpPubKeyCollectorPlugin extends AbstractSftpCollectorPlugin {
|
|
| 13 |
|
|
| 14 |
@Value("${collector.sftp.auth.prvKeyPath}")
|
|
| 15 |
private String prvKeyFilePath; |
|
| 16 |
@Value("${collector.sftp.auth.passphrase}")
|
|
| 17 |
private String passphrase; |
|
| 18 |
@Value("${collector.sftp.auth.knownHostsPath}")
|
|
| 19 |
private String knownHostsFile; |
|
| 20 |
|
|
| 21 |
@Override |
|
| 22 |
protected Iterable<String> getIterable(final InterfaceDescriptor interfaceDescriptor, |
|
| 23 |
final String baseUrl, |
|
| 24 |
final boolean isRecursive, |
|
| 25 |
final Set<String> extensionSet, |
|
| 26 |
final String fromDate) throws CollectorServiceException {
|
|
| 27 |
|
|
| 28 |
return () -> getSftpIteratorFactory().newIteratorPubKeyAuth(baseUrl, prvKeyFilePath, passphrase, knownHostsFile, isRecursive, extensionSet, fromDate); |
|
| 29 |
} |
|
| 30 |
|
|
| 31 |
} |
|
| modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/SftpIterator.java | ||
|---|---|---|
| 27 | 27 |
private static final int DEFAULT_TIMEOUT = 30000; |
| 28 | 28 |
private static final long BACKOFF_MILLIS = 10000; |
| 29 | 29 |
|
| 30 |
//params for simple authentication mode |
|
| 31 |
private String username; |
|
| 32 |
private String password; |
|
| 33 |
|
|
| 34 |
//params for pubkey authentication mode |
|
| 35 |
private String prvKeyFile; |
|
| 36 |
private String passPhrase; |
|
| 37 |
private String knownHostsFile; |
|
| 38 |
|
|
| 30 | 39 |
private String baseUrl; |
| 31 | 40 |
private String sftpURIScheme; |
| 32 | 41 |
private String sftpServerAddress; |
| 33 | 42 |
private String remoteSftpBasePath; |
| 34 |
private String username; |
|
| 35 |
private String password; |
|
| 43 |
|
|
| 36 | 44 |
private boolean isRecursive; |
| 37 | 45 |
private Set<String> extensionsSet; |
| 38 | 46 |
private boolean incremental; |
| ... | ... | |
| 45 | 53 |
private LocalDateTime fromDate = null; |
| 46 | 54 |
private DateTimeFormatter simpleDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
| 47 | 55 |
|
| 48 |
public SftpIterator(String baseUrl, String username, String password, boolean isRecursive, Set<String> extensionsSet, String fromDate) {
|
|
| 56 |
private void init(final String baseUrl, final boolean isRecursive, final Set<String> extensionsSet, final String fromDate) {
|
|
| 49 | 57 |
this.baseUrl = baseUrl; |
| 50 |
this.username = username; |
|
| 51 |
this.password = password; |
|
| 52 | 58 |
this.isRecursive = isRecursive; |
| 53 | 59 |
this.extensionsSet = extensionsSet; |
| 54 | 60 |
this.incremental = StringUtils.isNotBlank(fromDate); |
| ... | ... | |
| 58 | 64 |
log.debug("fromDate string: " + fromDate + " -- parsed: " + this.fromDate.toString());
|
| 59 | 65 |
} |
| 60 | 66 |
try {
|
| 61 |
URI sftpServer = new URI(baseUrl); |
|
| 67 |
URI sftpServer = new URI(this.baseUrl);
|
|
| 62 | 68 |
this.sftpURIScheme = sftpServer.getScheme(); |
| 63 | 69 |
this.sftpServerAddress = sftpServer.getHost(); |
| 64 | 70 |
this.remoteSftpBasePath = sftpServer.getPath(); |
| 65 | 71 |
} catch (URISyntaxException e) {
|
| 66 | 72 |
throw new CollectorServiceRuntimeException("Bad syntax in the URL " + baseUrl);
|
| 67 | 73 |
} |
| 74 |
} |
|
| 68 | 75 |
|
| 69 |
connectToSftpServer(); |
|
| 76 |
public SftpIterator(String baseUrl, String username, String password, boolean isRecursive, Set<String> extensionsSet, String fromDate) {
|
|
| 77 |
init(baseUrl, isRecursive, extensionsSet, fromDate); |
|
| 78 |
this.username = username; |
|
| 79 |
this.password = password; |
|
| 80 |
connectToSftpServerSimpleAuth(); |
|
| 70 | 81 |
initializeQueue(); |
| 71 | 82 |
} |
| 72 | 83 |
|
| 73 |
private void connectToSftpServer() {
|
|
| 84 |
public SftpIterator(final String baseUrl, final String prvKeyFilePath, final String passPhrase, final String knownHostsFile, final boolean isRecursive, final Set<String> extensionsSet, final String fromDate) {
|
|
| 85 |
init(baseUrl, isRecursive, extensionsSet, fromDate); |
|
| 86 |
this.passPhrase = passPhrase; |
|
| 87 |
this.prvKeyFile = prvKeyFilePath; |
|
| 88 |
this.knownHostsFile = knownHostsFile; |
|
| 89 |
connectToSftpServerPubKeyAuth(); |
|
| 90 |
initializeQueue(); |
|
| 91 |
} |
|
| 92 |
|
|
| 93 |
private void connectToSftpServerPubKeyAuth() {
|
|
| 74 | 94 |
JSch jsch = new JSch(); |
| 95 |
try {
|
|
| 96 |
jsch.setKnownHosts(this.knownHostsFile); |
|
| 97 |
jsch.addIdentity(this.prvKeyFile, this.passPhrase); |
|
| 98 |
sftpSession = jsch.getSession(sftpServerAddress); |
|
| 99 |
sftpSession.connect(); |
|
| 100 |
openChannelOnBasePath(); |
|
| 101 |
} catch (JSchException e) {
|
|
| 102 |
throw new CollectorServiceRuntimeException("Unable to create a session on remote SFTP server via Public key authentication.", e);
|
|
| 103 |
} |
|
| 75 | 104 |
|
| 105 |
} |
|
| 106 |
|
|
| 107 |
private void connectToSftpServerSimpleAuth() {
|
|
| 108 |
JSch jsch = new JSch(); |
|
| 76 | 109 |
try {
|
| 77 | 110 |
JSch.setConfig("StrictHostKeyChecking", "no");
|
| 78 | 111 |
sftpSession = jsch.getSession(username, sftpServerAddress); |
| 79 | 112 |
sftpSession.setPassword(password); |
| 80 | 113 |
sftpSession.connect(); |
| 114 |
openChannelOnBasePath(); |
|
| 115 |
} catch (JSchException e) {
|
|
| 116 |
throw new CollectorServiceRuntimeException("Unable to create a session on remote SFTP server via simple authentication.", e);
|
|
| 117 |
} |
|
| 118 |
} |
|
| 81 | 119 |
|
| 120 |
private void openChannelOnBasePath() {
|
|
| 121 |
try {
|
|
| 82 | 122 |
Channel channel = sftpSession.openChannel(sftpURIScheme); |
| 83 | 123 |
channel.connect(); |
| 84 | 124 |
sftpChannel = (ChannelSftp) channel; |
| ... | ... | |
| 89 | 129 |
log.debug("PWD from server 2 after 'cd " + fullPath + "' : " + sftpChannel.pwd());
|
| 90 | 130 |
log.info("Connected to SFTP server " + sftpServerAddress);
|
| 91 | 131 |
} catch (JSchException e) {
|
| 92 |
throw new CollectorServiceRuntimeException("Unable to connect to remote SFTP server.", e);
|
|
| 132 |
throw new CollectorServiceRuntimeException("Unable to open/connect SFTP channel.", e);
|
|
| 93 | 133 |
} catch (SftpException e) {
|
| 94 | 134 |
throw new CollectorServiceRuntimeException("Unable to access the base remote path on the SFTP server.", e);
|
| 95 | 135 |
} |
| modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/AbstractSftpCollectorPlugin.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.collector.plugins.sftp; |
|
| 2 |
|
|
| 3 |
import java.util.Set; |
|
| 4 |
|
|
| 5 |
import com.google.common.base.Splitter; |
|
| 6 |
import com.google.common.collect.Sets; |
|
| 7 |
import eu.dnetlib.rmi.data.CollectorServiceException; |
|
| 8 |
import eu.dnetlib.rmi.data.InterfaceDescriptor; |
|
| 9 |
import eu.dnetlib.rmi.data.plugin.AbstractCollectorPlugin; |
|
| 10 |
|
|
| 11 |
/** |
|
| 12 |
* Created by alessia on 15/03/17. |
|
| 13 |
*/ |
|
| 14 |
public abstract class AbstractSftpCollectorPlugin extends AbstractCollectorPlugin {
|
|
| 15 |
|
|
| 16 |
private SftpIteratorFactory sftpIteratorFactory; |
|
| 17 |
|
|
| 18 |
@Override |
|
| 19 |
public Iterable<String> collect(final InterfaceDescriptor interfaceDescriptor, final String fromDate, final String toDate) |
|
| 20 |
throws CollectorServiceException {
|
|
| 21 |
final String baseUrl = interfaceDescriptor.getBaseUrl(); |
|
| 22 |
final String recursive = interfaceDescriptor.getParams().get("recursive");
|
|
| 23 |
final String extensions = interfaceDescriptor.getParams().get("extensions");
|
|
| 24 |
|
|
| 25 |
if ((baseUrl == null) || baseUrl.isEmpty()) {
|
|
| 26 |
throw new CollectorServiceException("Param 'baseurl' is null or empty");
|
|
| 27 |
} |
|
| 28 |
if ((recursive == null) || recursive.isEmpty()) {
|
|
| 29 |
throw new CollectorServiceException("Param 'recursive' is null or empty");
|
|
| 30 |
} |
|
| 31 |
if ((extensions == null) || extensions.isEmpty()) {
|
|
| 32 |
throw new CollectorServiceException("Param 'extensions' is null or empty");
|
|
| 33 |
} |
|
| 34 |
if (fromDate != null && !fromDate.matches("\\d{4}-\\d{2}-\\d{2}")) { throw new CollectorServiceException("Invalid date (YYYY-MM-DD): " + fromDate); }
|
|
| 35 |
|
|
| 36 |
Set<String> extensionSet = parseSet(extensions); |
|
| 37 |
|
|
| 38 |
return getIterable(interfaceDescriptor, baseUrl, Boolean.parseBoolean(recursive), extensionSet, fromDate); |
|
| 39 |
} |
|
| 40 |
|
|
| 41 |
protected abstract Iterable<String> getIterable(final InterfaceDescriptor interfaceDescriptor, final String baseUrl, final boolean isRecursive, final Set<String> extensionSet, final String fromDate ) throws CollectorServiceException; |
|
| 42 |
|
|
| 43 |
private Set<String> parseSet(final String extensions) {
|
|
| 44 |
return Sets.newHashSet(Splitter.on(",").omitEmptyStrings().trimResults().split(extensions));
|
|
| 45 |
} |
|
| 46 |
|
|
| 47 |
public SftpIteratorFactory getSftpIteratorFactory() {
|
|
| 48 |
return sftpIteratorFactory; |
|
| 49 |
} |
|
| 50 |
|
|
| 51 |
public void setSftpIteratorFactory(SftpIteratorFactory sftpIteratorFactory) {
|
|
| 52 |
this.sftpIteratorFactory = sftpIteratorFactory; |
|
| 53 |
} |
|
| 54 |
|
|
| 55 |
} |
|
| modules/dnet-data-services/branches/saxonHE/src/main/java/eu/dnetlib/data/collector/plugins/sftp/SftpIteratorFactory.java | ||
|---|---|---|
| 8 | 8 |
*/ |
| 9 | 9 |
public class SftpIteratorFactory {
|
| 10 | 10 |
|
| 11 |
public Iterator<String> newIterator(final String baseUrl, |
|
| 11 |
public Iterator<String> newIteratorSimplAuth(final String baseUrl,
|
|
| 12 | 12 |
final String username, |
| 13 | 13 |
final String password, |
| 14 | 14 |
final boolean isRecursive, |
| 15 | 15 |
final Set<String> extensionsSet, final String fromDate) {
|
| 16 | 16 |
return new SftpIterator(baseUrl, username, password, isRecursive, extensionsSet, fromDate); |
| 17 | 17 |
} |
| 18 |
|
|
| 19 |
public Iterator<String> newIteratorPubKeyAuth(final String baseUrl, |
|
| 20 |
final String prvKeyFilePath, final String passPhrase, final String knownHostsFile, |
|
| 21 |
final boolean isRecursive, |
|
| 22 |
final Set<String> extensionsSet, final String fromDate) {
|
|
| 23 |
return new SftpIterator(baseUrl, prvKeyFilePath, passPhrase,knownHostsFile, isRecursive, extensionsSet, fromDate); |
|
| 24 |
} |
|
| 18 | 25 |
} |
| modules/dnet-data-services/branches/saxonHE/src/main/resources/eu/dnetlib/data/collector/plugins/applicationContext-dnet-modular-collector-plugins.properties | ||
|---|---|---|
| 2 | 2 |
collector.oai.http.readTimeOut=120 |
| 3 | 3 |
collector.oai.http.maxNumberOfRetry=6 |
| 4 | 4 |
services.objectstore.basePathList.xquery=collection('/db/DRIVER/ServiceResources/ObjectStoreServiceResourceType')//PROPERTY[@key='basePath']/@value/string()
|
| 5 |
collector.sftp.auth.prvKeyPath=~/.ssh/id_rsa |
|
| 6 |
collector.sftp.auth.passphrase=thePassPhrase |
|
| 7 |
collector.sftp.auth.knownHostsPath=~/.ssh/known_hosts |
|
| 5 | 8 |
|
| 9 |
|
|
| modules/dnet-data-services/branches/saxonHE/src/main/resources/eu/dnetlib/data/collector/plugins/applicationContext-dnet-modular-collector-plugins.xml | ||
|---|---|---|
| 146 | 146 |
</property> |
| 147 | 147 |
</bean> |
| 148 | 148 |
|
| 149 |
<bean id="sftpPubKeyCollectorPlugin" class="eu.dnetlib.data.collector.plugins.sftp.SftpPubKeyCollectorPlugin" |
|
| 150 |
p:sftpIteratorFactory-ref="sftpIteratorFactory"> |
|
| 151 |
<property name="protocolDescriptor"> |
|
| 152 |
<bean class="eu.dnetlib.rmi.data.ProtocolDescriptor" p:name="sftpPubKeyAuthentication"> |
|
| 153 |
<property name="params"> |
|
| 154 |
<list> |
|
| 155 |
<bean class="eu.dnetlib.rmi.data.ProtocolParameter" |
|
| 156 |
p:name="recursive" p:type="BOOLEAN"/> |
|
| 157 |
<bean class="eu.dnetlib.rmi.data.ProtocolParameter" |
|
| 158 |
p:name="extensions" p:type="LIST"/> |
|
| 159 |
</list> |
|
| 160 |
</property> |
|
| 161 |
</bean> |
|
| 162 |
</property> |
|
| 163 |
</bean> |
|
| 164 |
|
|
| 149 | 165 |
<bean id="filesystemCollectorPlugin" class="eu.dnetlib.data.collector.plugins.filesystem.FilesystemCollectorPlugin"> |
| 150 | 166 |
<property name="protocolDescriptor"> |
| 151 | 167 |
<bean class="eu.dnetlib.rmi.data.ProtocolDescriptor" p:name="filesystem"> |
Also available in: Unified diff
sftpPubKeyAuthentication API protocol for SFTP with pubKey authentication