1 |
26600
|
sandro.lab
|
package eu.dnetlib.msro.openaireplus.workflows.nodes.objectStore;
|
2 |
|
|
|
3 |
|
|
import java.io.File;
|
4 |
|
|
import java.util.List;
|
5 |
|
|
import javax.xml.ws.wsaddressing.W3CEndpointReference;
|
6 |
|
|
|
7 |
|
|
import com.googlecode.sarasvati.NodeToken;
|
8 |
32891
|
michele.ar
|
import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
|
9 |
26600
|
sandro.lab
|
import eu.dnetlib.data.objectstore.rmi.Protocols;
|
10 |
|
|
import eu.dnetlib.enabling.resultset.IterableResultSetFactory;
|
11 |
|
|
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
|
12 |
|
|
import eu.dnetlib.msro.workflows.nodes.BlackboardJobNode;
|
13 |
|
|
import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
|
14 |
|
|
import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory;
|
15 |
|
|
import eu.dnetlib.msro.workflows.util.ProgressProvider;
|
16 |
|
|
import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider;
|
17 |
48139
|
alessia.ba
|
import org.apache.commons.io.FileUtils;
|
18 |
|
|
import org.apache.commons.io.filefilter.TrueFileFilter;
|
19 |
26600
|
sandro.lab
|
|
20 |
|
|
public class ImportFilesJobNode extends BlackboardJobNode implements ProgressJobNode {
|
21 |
|
|
|
22 |
|
|
private String basePath;
|
23 |
|
|
private String repositoryPrefix;
|
24 |
|
|
private String oaiPrefix;
|
25 |
|
|
private String objectStoreId;
|
26 |
|
|
private String contentDescription;
|
27 |
|
|
|
28 |
|
|
private ResultsetProgressProvider progressProvider;
|
29 |
|
|
private ProcessCountingResultSetFactory processCountingResultSetFactory;
|
30 |
|
|
private IterableResultSetFactory iterableResultSetFactory;
|
31 |
|
|
|
32 |
|
|
public String getRepositoryPrefix() {
|
33 |
|
|
return repositoryPrefix;
|
34 |
|
|
}
|
35 |
|
|
|
36 |
|
|
public void setRepositoryPrefix(final String repositoryPrefix) {
|
37 |
|
|
this.repositoryPrefix = repositoryPrefix;
|
38 |
|
|
}
|
39 |
|
|
|
40 |
|
|
public String getBasePath() {
|
41 |
|
|
return basePath;
|
42 |
|
|
}
|
43 |
|
|
|
44 |
|
|
public void setBasePath(final String basePath) {
|
45 |
|
|
this.basePath = basePath;
|
46 |
|
|
}
|
47 |
|
|
|
48 |
|
|
@Override
|
49 |
|
|
public ProgressProvider getProgressProvider() {
|
50 |
|
|
return progressProvider;
|
51 |
|
|
}
|
52 |
|
|
|
53 |
|
|
@Override
|
54 |
32891
|
michele.ar
|
protected String obtainServiceId(final NodeToken token) {
|
55 |
|
|
return getServiceLocator().getServiceId(ObjectStoreService.class);
|
56 |
26600
|
sandro.lab
|
}
|
57 |
|
|
|
58 |
|
|
@Override
|
59 |
|
|
protected void prepareJob(final BlackboardJob job, final NodeToken token) throws Exception {
|
60 |
|
|
File dir = new File(basePath);
|
61 |
|
|
|
62 |
|
|
System.out.println("Getting all files in " + dir.getCanonicalPath() + " including those in subdirectories");
|
63 |
|
|
List<File> files = (List<File>) FileUtils.listFiles(dir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
|
64 |
|
|
IteratorRepository irepo;
|
65 |
|
|
if (repositoryPrefix.equals("od________18")) {
|
66 |
|
|
irepo = new ArxivIteratorRepository(files, repositoryPrefix, oaiPrefix);
|
67 |
|
|
} else if (repositoryPrefix.equals("od______2367")) {
|
68 |
|
|
irepo = new PUMAIteratorRepository(files, repositoryPrefix, oaiPrefix);
|
69 |
|
|
} else {
|
70 |
|
|
irepo = new RepositoryNoURLIterator(files, repositoryPrefix, oaiPrefix);
|
71 |
|
|
}
|
72 |
|
|
final W3CEndpointReference epr = iterableResultSetFactory.createIterableResultSet(irepo);
|
73 |
|
|
this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr);
|
74 |
|
|
job.setAction("FEED");
|
75 |
|
|
job.getParameters().put("obsID", getObjectStoreId());
|
76 |
|
|
job.getParameters().put("mime", getContentDescription());
|
77 |
|
|
job.getParameters().put("protocol", Protocols.File_System.toString());
|
78 |
|
|
job.getParameters().put("epr", progressProvider.getEpr().toString());
|
79 |
|
|
|
80 |
|
|
}
|
81 |
|
|
|
82 |
|
|
public IterableResultSetFactory getIterableResultSetFactory() {
|
83 |
|
|
return iterableResultSetFactory;
|
84 |
|
|
}
|
85 |
|
|
|
86 |
|
|
public void setIterableResultSetFactory(final IterableResultSetFactory iterableResultSetFactory) {
|
87 |
|
|
this.iterableResultSetFactory = iterableResultSetFactory;
|
88 |
|
|
}
|
89 |
|
|
|
90 |
|
|
public void setProgressProvider(final ResultsetProgressProvider progressProvider) {
|
91 |
|
|
this.progressProvider = progressProvider;
|
92 |
|
|
}
|
93 |
|
|
|
94 |
|
|
public ProcessCountingResultSetFactory getProcessCountingResultSetFactory() {
|
95 |
|
|
return processCountingResultSetFactory;
|
96 |
|
|
}
|
97 |
|
|
|
98 |
|
|
public void setProcessCountingResultSetFactory(final ProcessCountingResultSetFactory processCountingResultSetFactory) {
|
99 |
|
|
this.processCountingResultSetFactory = processCountingResultSetFactory;
|
100 |
|
|
}
|
101 |
|
|
|
102 |
|
|
public String getObjectStoreId() {
|
103 |
|
|
return objectStoreId;
|
104 |
|
|
}
|
105 |
|
|
|
106 |
|
|
public void setObjectStoreId(final String objectStoreId) {
|
107 |
|
|
this.objectStoreId = objectStoreId;
|
108 |
|
|
}
|
109 |
|
|
|
110 |
|
|
public String getContentDescription() {
|
111 |
|
|
return contentDescription;
|
112 |
|
|
}
|
113 |
|
|
|
114 |
|
|
public void setContentDescription(final String contentDescription) {
|
115 |
|
|
this.contentDescription = contentDescription;
|
116 |
|
|
}
|
117 |
|
|
|
118 |
|
|
public String getOaiPrefix() {
|
119 |
|
|
return oaiPrefix;
|
120 |
|
|
}
|
121 |
|
|
|
122 |
|
|
public void setOaiPrefix(final String oaiPrefix) {
|
123 |
|
|
this.oaiPrefix = oaiPrefix;
|
124 |
|
|
}
|
125 |
|
|
}
|