Revision 62135
Added by Michele Artini over 2 years ago
PrepareEnvCollectHadoopJobNode.java | ||
---|---|---|
1 | 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes.dhp; |
2 | 2 |
|
3 |
import static eu.dnetlib.dhp.common.Constants.COLLECTION_MODE; |
|
4 |
import static eu.dnetlib.dhp.common.Constants.CONNECT_TIMEOUT; |
|
5 |
import static eu.dnetlib.dhp.common.Constants.DNET_MESSAGE_MGR_URL; |
|
6 |
import static eu.dnetlib.dhp.common.Constants.MAX_NUMBER_OF_RETRY; |
|
7 |
import static eu.dnetlib.dhp.common.Constants.METADATA_ENCODING; |
|
8 |
import static eu.dnetlib.dhp.common.Constants.OOZIE_WF_PATH; |
|
9 |
import static eu.dnetlib.dhp.common.Constants.READ_TIMEOUT; |
|
10 |
import static eu.dnetlib.dhp.common.Constants.REQUEST_DELAY; |
|
11 |
import static eu.dnetlib.dhp.common.Constants.RETRY_DELAY; |
|
12 |
|
|
3 | 13 |
import java.text.SimpleDateFormat; |
4 | 14 |
import java.util.Date; |
5 | 15 |
import java.util.Iterator; |
... | ... | |
7 | 17 |
import java.util.Optional; |
8 | 18 |
import java.util.stream.Collectors; |
9 | 19 |
|
10 |
import eu.dnetlib.common.logging.DnetLogger; |
|
11 |
import eu.dnetlib.dhp.collection.ApiDescriptor; |
|
12 |
import eu.dnetlib.dhp.model.mdstore.Provenance; |
|
13 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
|
14 |
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; |
|
15 |
import eu.dnetlib.msro.rmi.MSROException; |
|
20 |
import javax.annotation.Resource; |
|
21 |
|
|
16 | 22 |
import org.apache.commons.lang.math.NumberUtils; |
17 | 23 |
import org.apache.commons.lang3.StringUtils; |
18 | 24 |
import org.apache.commons.logging.Log; |
19 | 25 |
import org.apache.commons.logging.LogFactory; |
20 | 26 |
import org.springframework.beans.factory.annotation.Autowired; |
27 |
import org.springframework.beans.factory.annotation.Value; |
|
21 | 28 |
|
22 | 29 |
import com.google.gson.Gson; |
23 | 30 |
import com.googlecode.sarasvati.Arc; |
24 | 31 |
import com.googlecode.sarasvati.NodeToken; |
25 | 32 |
|
33 |
import eu.dnetlib.common.logging.DnetLogger; |
|
34 |
import eu.dnetlib.dhp.collection.ApiDescriptor; |
|
35 |
import eu.dnetlib.dhp.model.mdstore.Provenance; |
|
26 | 36 |
import eu.dnetlib.enabling.datasources.common.ApiParam; |
37 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
|
38 |
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; |
|
27 | 39 |
import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager; |
40 |
import eu.dnetlib.msro.rmi.MSROException; |
|
28 | 41 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
29 | 42 |
import eu.dnetlib.msro.workflows.util.WorkflowsConstants; |
30 |
import org.springframework.beans.factory.annotation.Value; |
|
31 | 43 |
|
32 |
import javax.annotation.Resource; |
|
33 |
|
|
34 |
import static eu.dnetlib.dhp.common.Constants.*; |
|
35 |
|
|
36 | 44 |
public class PrepareEnvCollectHadoopJobNode extends SimpleJobNode { |
37 | 45 |
|
38 |
private static final Log log = LogFactory.getLog(PrepareEnvCollectHadoopJobNode.class);
|
|
46 |
private static final Log log = LogFactory.getLog(PrepareEnvCollectHadoopJobNode.class);
|
|
39 | 47 |
|
40 |
public static final String METADATA_IDENTIFIER_PATH = "metadata_identifier_path";
|
|
41 |
public static final String DATE_FORMAT = "yyyy-MM-dd";
|
|
48 |
public static final String METADATA_IDENTIFIER_PATH = "metadata_identifier_path";
|
|
49 |
public static final String DATE_FORMAT = "yyyy-MM-dd";
|
|
42 | 50 |
|
43 |
private long ONE_DAY = 1000 * 60 * 60 * 24;
|
|
51 |
private final long ONE_DAY = 1000 * 60 * 60 * 24;
|
|
44 | 52 |
|
45 |
@Resource(name = "msroWorkflowLogger")
|
|
46 |
private DnetLogger dnetLogger;
|
|
53 |
@Resource(name = "msroWorkflowLogger")
|
|
54 |
private DnetLogger dnetLogger;
|
|
47 | 55 |
|
48 |
@Autowired
|
|
49 |
private LocalDatasourceManager<?, ?> dsManager;
|
|
56 |
@Autowired
|
|
57 |
private LocalDatasourceManager<?, ?> dsManager;
|
|
50 | 58 |
|
51 |
/**
|
|
52 |
* MDStore identifier
|
|
53 |
*/
|
|
54 |
private String mdId;
|
|
59 |
/**
|
|
60 |
* MDStore identifier
|
|
61 |
*/
|
|
62 |
private String mdId;
|
|
55 | 63 |
|
56 |
/**
|
|
57 |
* REFRESH | INCREMENTAL
|
|
58 |
*/
|
|
59 |
private String collectionMode;
|
|
64 |
/**
|
|
65 |
* REFRESH | INCREMENTAL
|
|
66 |
*/
|
|
67 |
private String collectionMode;
|
|
60 | 68 |
|
61 |
/**
|
|
62 |
* used in INCREMENTAL mode
|
|
63 |
*/
|
|
64 |
private String fromDateOverride;
|
|
69 |
/**
|
|
70 |
* used in INCREMENTAL mode
|
|
71 |
*/
|
|
72 |
private String fromDateOverride;
|
|
65 | 73 |
|
66 |
/**
|
|
67 |
* used in to set the untilDate
|
|
68 |
*/
|
|
69 |
private String untilDateOverride;
|
|
74 |
/**
|
|
75 |
* used in to set the untilDate
|
|
76 |
*/
|
|
77 |
private String untilDateOverride;
|
|
70 | 78 |
|
71 |
/**
|
|
72 |
* XML | JSON, default = XML
|
|
73 |
*/
|
|
74 |
private String metadataEncoding = "XML";
|
|
79 |
/**
|
|
80 |
* XML | JSON, default = XML
|
|
81 |
*/
|
|
82 |
private String metadataEncoding = "XML";
|
|
75 | 83 |
|
76 |
/**
|
|
77 |
* Maximum number of allowed retires before failing
|
|
78 |
*/
|
|
79 |
private int maxNumberOfRetry = 5;
|
|
84 |
/**
|
|
85 |
* Maximum number of allowed retires before failing
|
|
86 |
*/
|
|
87 |
private int maxNumberOfRetry = 5;
|
|
80 | 88 |
|
81 |
/**
|
|
82 |
* Delay between request (Milliseconds)
|
|
83 |
*/
|
|
84 |
private int requestDelay = 0;
|
|
89 |
/**
|
|
90 |
* Delay between request (Milliseconds)
|
|
91 |
*/
|
|
92 |
private int requestDelay = 0;
|
|
85 | 93 |
|
86 |
/**
|
|
87 |
* Time to wait after a failure before retrying (Seconds)
|
|
88 |
*/
|
|
89 |
private int retryDelay = 60;
|
|
94 |
/**
|
|
95 |
* Time to wait after a failure before retrying (Seconds)
|
|
96 |
*/
|
|
97 |
private int retryDelay = 60;
|
|
90 | 98 |
|
91 |
/**
|
|
92 |
* Connect timeout (Seconds)
|
|
93 |
*/
|
|
94 |
private int connectTimeOut = 30;
|
|
99 |
/**
|
|
100 |
* Connect timeout (Seconds)
|
|
101 |
*/
|
|
102 |
private int connectTimeOut = 30;
|
|
95 | 103 |
|
96 |
/**
|
|
97 |
* Read timeout (Seconds)
|
|
98 |
*/
|
|
99 |
private int readTimeOut = 60;
|
|
104 |
/**
|
|
105 |
* Read timeout (Seconds)
|
|
106 |
*/
|
|
107 |
private int readTimeOut = 60;
|
|
100 | 108 |
|
101 |
@Value("${dnet.openaire.dhp.collection.app.path}")
|
|
102 |
private String oozieWfPath;
|
|
109 |
@Value("${dnet.openaire.dhp.collection.app.path}")
|
|
110 |
private String oozieWfPath;
|
|
103 | 111 |
|
104 |
@Value("${dnet.openaire.dhp.dnetMessageManagerURL}")
|
|
105 |
private String dnetMessageManagerURL;
|
|
112 |
@Value("${dnet.openaire.dhp.dnetMessageManagerURL}")
|
|
113 |
private String dnetMessageManagerURL;
|
|
106 | 114 |
|
107 |
@Override
|
|
108 |
protected String execute(final NodeToken token) throws Exception {
|
|
115 |
@Override
|
|
116 |
protected String execute(final NodeToken token) throws Exception {
|
|
109 | 117 |
|
110 |
final String dsId = token.getEnv().getAttribute("parentDatasourceId");
|
|
111 |
log.info("dsId: " + dsId);
|
|
112 |
final String apiId = token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE);
|
|
113 |
log.info("apiId: " + apiId);
|
|
118 |
final String dsId = token.getEnv().getAttribute("parentDatasourceId");
|
|
119 |
log.info("dsId: " + dsId);
|
|
120 |
final String apiId = token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE);
|
|
121 |
log.info("apiId: " + apiId);
|
|
114 | 122 |
|
115 |
final Optional<ApiDescriptor> opt = dsManager.getApis(dsId) |
|
116 |
.stream() |
|
117 |
.filter(a -> a.getId().equals(apiId)) |
|
118 |
.map(a -> { |
|
119 |
final ApiDescriptor res = new ApiDescriptor(); |
|
120 |
res.setBaseUrl(a.getBaseurl()); |
|
121 |
res.setId(a.getId()); |
|
122 |
res.setProtocol(a.getProtocol()); |
|
123 |
res.getParams().put(METADATA_IDENTIFIER_PATH, a.getMetadataIdentifierPath()); |
|
124 |
res.getParams().putAll(a.getApiParams() |
|
125 |
.stream() |
|
126 |
.map(o -> (ApiParam) o) |
|
127 |
.collect(Collectors.toMap(ApiParam::getParam, ApiParam::getValue))); |
|
128 |
return res; |
|
129 |
}) |
|
130 |
.findFirst(); |
|
123 |
final Optional<ApiDescriptor> opt = dsManager.getApis(dsId) |
|
124 |
.stream() |
|
125 |
.filter(a -> a.getId().equals(apiId)) |
|
126 |
.map(a -> { |
|
127 |
final ApiDescriptor res = new ApiDescriptor(); |
|
128 |
res.setBaseUrl(a.getBaseurl()); |
|
129 |
res.setId(a.getId()); |
|
130 |
res.setProtocol(a.getProtocol()); |
|
131 |
res.getParams().put(METADATA_IDENTIFIER_PATH, a.getMetadataIdentifierPath()); |
|
132 |
res.getParams() |
|
133 |
.putAll(a.getApiParams() |
|
134 |
.stream() |
|
135 |
.map(o -> (ApiParam) o) |
|
136 |
.collect(Collectors.toMap(ApiParam::getParam, ApiParam::getValue))); |
|
137 |
return res; |
|
138 |
}) |
|
139 |
.findFirst(); |
|
131 | 140 |
|
132 |
if (opt.isPresent()) {
|
|
133 |
token.getEnv().setAttribute("mdId", getMdId());
|
|
134 |
token.getEnv().setAttribute(COLLECTION_MODE, getCollectionMode());
|
|
135 |
token.getEnv().setAttribute(METADATA_ENCODING, getMetadataEncoding());
|
|
136 |
token.getEnv().setAttribute(OOZIE_WF_PATH, getOozieWfPath());
|
|
137 |
token.getEnv().setAttribute(DNET_MESSAGE_MGR_URL, getDnetMessageManagerURL());
|
|
141 |
if (opt.isPresent()) {
|
|
142 |
token.getEnv().setAttribute("mdId", getMdId());
|
|
143 |
token.getEnv().setAttribute(COLLECTION_MODE, getCollectionMode());
|
|
144 |
token.getEnv().setAttribute(METADATA_ENCODING, getMetadataEncoding());
|
|
145 |
token.getEnv().setAttribute(OOZIE_WF_PATH, getOozieWfPath());
|
|
146 |
token.getEnv().setAttribute(DNET_MESSAGE_MGR_URL, getDnetMessageManagerURL());
|
|
138 | 147 |
|
139 |
token.getEnv().setAttribute(MAX_NUMBER_OF_RETRY, getMaxNumberOfRetry());
|
|
140 |
token.getEnv().setAttribute(REQUEST_DELAY, getRequestDelay());
|
|
141 |
token.getEnv().setAttribute(RETRY_DELAY, getRetryDelay());
|
|
142 |
token.getEnv().setAttribute(CONNECT_TIMEOUT, getConnectTimeOut());
|
|
143 |
token.getEnv().setAttribute(READ_TIMEOUT, getReadTimeOut());
|
|
148 |
token.getEnv().setAttribute(MAX_NUMBER_OF_RETRY, getMaxNumberOfRetry());
|
|
149 |
token.getEnv().setAttribute(REQUEST_DELAY, getRequestDelay());
|
|
150 |
token.getEnv().setAttribute(RETRY_DELAY, getRetryDelay());
|
|
151 |
token.getEnv().setAttribute(CONNECT_TIMEOUT, getConnectTimeOut());
|
|
152 |
token.getEnv().setAttribute(READ_TIMEOUT, getReadTimeOut());
|
|
144 | 153 |
|
145 |
final ApiDescriptor api = opt.get();
|
|
146 |
if ("INCREMENTAL".equals(getCollectionMode())) {
|
|
147 |
final String fromDate = calculateFromDate(token);
|
|
154 |
final ApiDescriptor api = opt.get();
|
|
155 |
if ("INCREMENTAL".equals(getCollectionMode())) {
|
|
156 |
final String fromDate = calculateFromDate(token);
|
|
148 | 157 |
|
149 |
log.info("Incremental Harvesting from: " + fromDate);
|
|
158 |
log.info("Incremental Harvesting from: " + fromDate);
|
|
150 | 159 |
|
151 |
if (StringUtils.isNotBlank(fromDate)) {
|
|
152 |
api.getParams().put("fromDate", fromDate);
|
|
153 |
}
|
|
154 |
}
|
|
160 |
if (StringUtils.isNotBlank(fromDate)) {
|
|
161 |
api.getParams().put("fromDate", fromDate);
|
|
162 |
}
|
|
163 |
}
|
|
155 | 164 |
|
156 |
if (StringUtils.isNotBlank(getUntilDateOverride())) {
|
|
157 |
api.getParams().put("untilDate", getUntilDateOverride());
|
|
158 |
}
|
|
165 |
if (StringUtils.isNotBlank(getUntilDateOverride())) {
|
|
166 |
api.getParams().put("untilDate", getUntilDateOverride());
|
|
167 |
}
|
|
159 | 168 |
|
160 |
token.getEnv().setAttribute("apiDescription", new Gson().toJson(api));
|
|
169 |
token.getEnv().setAttribute("apiDescription", new Gson().toJson(api));
|
|
161 | 170 |
|
162 |
final Provenance provenance = new Provenance();
|
|
163 |
provenance.setDatasourceId(dsId);
|
|
164 |
final Datasource<?, ?> ds = dsManager.getDs(dsId);
|
|
165 |
provenance.setDatasourceName(ds.getOfficialname());
|
|
166 |
provenance.setNsPrefix(ds.getNamespaceprefix());
|
|
167 |
final String dsProvenance = new Gson().toJson(provenance);
|
|
168 |
log.info("datasource provenance: " + dsProvenance);
|
|
171 |
final Provenance provenance = new Provenance();
|
|
172 |
provenance.setDatasourceId(dsId);
|
|
173 |
final Datasource<?, ?, ?> ds = dsManager.getDs(dsId);
|
|
174 |
provenance.setDatasourceName(ds.getOfficialname());
|
|
175 |
provenance.setNsPrefix(ds.getNamespaceprefix());
|
|
176 |
final String dsProvenance = new Gson().toJson(provenance);
|
|
177 |
log.info("datasource provenance: " + dsProvenance);
|
|
169 | 178 |
|
170 |
token.getEnv().setAttribute("dataSourceInfo", dsProvenance);
|
|
171 |
token.getEnv().setAttribute("timestamp", ""+System.currentTimeMillis());
|
|
172 |
token.getEnv().setAttribute("identifierPath",api.getParams().get(METADATA_IDENTIFIER_PATH));
|
|
173 |
token.getEnv().setAttribute("workflowId",token.getProcess().getEnv().getAttribute("system:processId"));
|
|
179 |
token.getEnv().setAttribute("dataSourceInfo", dsProvenance);
|
|
180 |
token.getEnv().setAttribute("timestamp", "" + System.currentTimeMillis());
|
|
181 |
token.getEnv().setAttribute("identifierPath", api.getParams().get(METADATA_IDENTIFIER_PATH));
|
|
182 |
token.getEnv().setAttribute("workflowId", token.getProcess().getEnv().getAttribute("system:processId"));
|
|
174 | 183 |
|
175 |
token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE_BASEURL, api.getBaseUrl());
|
|
176 |
token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_PREFIX + "protocol", api.getProtocol());
|
|
177 |
final Map<String, String> params = api.getParams();
|
|
178 |
if (params != null) {
|
|
179 |
for(Map.Entry<String, String> e : params.entrySet()) {
|
|
180 |
token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_PREFIX + e.getKey(), e.getValue());
|
|
181 |
}
|
|
182 |
}
|
|
184 |
token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE_BASEURL, api.getBaseUrl());
|
|
185 |
token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_PREFIX + "protocol", api.getProtocol());
|
|
186 |
final Map<String, String> params = api.getParams();
|
|
187 |
if (params != null) {
|
|
188 |
for (final Map.Entry<String, String> e : params.entrySet()) {
|
|
189 |
token.getEnv().setAttribute(WorkflowsConstants.DATAPROVIDER_PREFIX + e.getKey(), e.getValue());
|
|
190 |
}
|
|
191 |
}
|
|
183 | 192 |
|
184 |
return Arc.DEFAULT_ARC;
|
|
185 |
} else {
|
|
186 |
throw new DsmNotFoundException("cannot find ds interface: " + apiId);
|
|
187 |
}
|
|
188 |
}
|
|
193 |
return Arc.DEFAULT_ARC;
|
|
194 |
} else {
|
|
195 |
throw new DsmNotFoundException("cannot find ds interface: " + apiId);
|
|
196 |
}
|
|
197 |
}
|
|
189 | 198 |
|
190 |
private String findCurrentWfProfileId(NodeToken token) throws MSROException { |
|
191 |
final String p1 = token.getEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID); |
|
192 |
if (p1 != null && !p1.isEmpty()) { |
|
193 |
return p1; |
|
194 |
} |
|
195 |
final String p2 = token.getFullEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID); |
|
196 |
if (p2 != null && !p2.isEmpty()) { |
|
197 |
return p2; |
|
198 |
} |
|
199 |
final String p3 = token.getProcess().getEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID); |
|
200 |
if (p3 != null && !p3.isEmpty()) { |
|
201 |
return p3; |
|
202 |
} |
|
203 |
throw new MSROException("Missing property in env: " + WorkflowsConstants.SYSTEM_WF_PROFILE_ID); |
|
204 |
} |
|
199 |
private String findCurrentWfProfileId(final NodeToken token) throws MSROException { |
|
200 |
final String p1 = token.getEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID); |
|
201 |
if (p1 != null && !p1.isEmpty()) { return p1; } |
|
202 |
final String p2 = token.getFullEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID); |
|
203 |
if (p2 != null && !p2.isEmpty()) { return p2; } |
|
204 |
final String p3 = token.getProcess().getEnv().getAttribute(WorkflowsConstants.SYSTEM_WF_PROFILE_ID); |
|
205 |
if (p3 != null && !p3.isEmpty()) { return p3; } |
|
206 |
throw new MSROException("Missing property in env: " + WorkflowsConstants.SYSTEM_WF_PROFILE_ID); |
|
207 |
} |
|
205 | 208 |
|
206 |
private String calculateFromDate(final NodeToken token) throws MSROException {
|
|
209 |
private String calculateFromDate(final NodeToken token) throws MSROException {
|
|
207 | 210 |
|
208 |
if (StringUtils.isNotBlank(getFromDateOverride())) {
|
|
209 |
log.info("using override FROM_DATE for incremental harvesting: " + getFromDateOverride());
|
|
210 |
return getFromDateOverride();
|
|
211 |
}
|
|
212 |
final String profId = findCurrentWfProfileId(token);
|
|
211 |
if (StringUtils.isNotBlank(getFromDateOverride())) {
|
|
212 |
log.info("using override FROM_DATE for incremental harvesting: " + getFromDateOverride());
|
|
213 |
return getFromDateOverride();
|
|
214 |
}
|
|
215 |
final String profId = findCurrentWfProfileId(token);
|
|
213 | 216 |
|
214 |
final long d = findLastSuccessStartDate(profId);
|
|
215 |
return (d > 0) ? (new SimpleDateFormat(DATE_FORMAT)).format(new Date(d - ONE_DAY)) : null;
|
|
216 |
}
|
|
217 |
final long d = findLastSuccessStartDate(profId);
|
|
218 |
return d > 0 ? new SimpleDateFormat(DATE_FORMAT).format(new Date(d - ONE_DAY)) : null;
|
|
219 |
}
|
|
217 | 220 |
|
218 |
private long findLastSuccessStartDate(String profId) {
|
|
219 |
long res = -1;
|
|
221 |
private long findLastSuccessStartDate(final String profId) {
|
|
222 |
long res = -1;
|
|
220 | 223 |
|
221 |
final Iterator<Map<String, String>> iter = dnetLogger.find(WorkflowsConstants.SYSTEM_WF_PROFILE_ID, profId);
|
|
222 |
while (iter.hasNext()) {
|
|
223 |
final Map<String, String> map = iter.next();
|
|
224 |
if ("true".equalsIgnoreCase(map.get(WorkflowsConstants.SYSTEM_COMPLETED_SUCCESSFULLY))) {
|
|
225 |
final long curr = NumberUtils.toLong(map.get(WorkflowsConstants.SYSTEM_START_DATE), -1);
|
|
226 |
if (curr > res) {
|
|
227 |
res = curr;
|
|
228 |
}
|
|
229 |
}
|
|
230 |
}
|
|
231 |
return res;
|
|
232 |
}
|
|
224 |
final Iterator<Map<String, String>> iter = dnetLogger.find(WorkflowsConstants.SYSTEM_WF_PROFILE_ID, profId);
|
|
225 |
while (iter.hasNext()) {
|
|
226 |
final Map<String, String> map = iter.next();
|
|
227 |
if ("true".equalsIgnoreCase(map.get(WorkflowsConstants.SYSTEM_COMPLETED_SUCCESSFULLY))) {
|
|
228 |
final long curr = NumberUtils.toLong(map.get(WorkflowsConstants.SYSTEM_START_DATE), -1);
|
|
229 |
if (curr > res) {
|
|
230 |
res = curr;
|
|
231 |
}
|
|
232 |
}
|
|
233 |
}
|
|
234 |
return res;
|
|
235 |
}
|
|
233 | 236 |
|
237 |
public String getMdId() { |
|
238 |
return mdId; |
|
239 |
} |
|
234 | 240 |
|
235 |
public String getMdId() {
|
|
236 |
return mdId;
|
|
237 |
}
|
|
241 |
public void setMdId(final String mdId) {
|
|
242 |
this.mdId = mdId;
|
|
243 |
}
|
|
238 | 244 |
|
239 |
public void setMdId(String mdId) {
|
|
240 |
this.mdId = mdId;
|
|
241 |
}
|
|
245 |
public String getCollectionMode() {
|
|
246 |
return collectionMode;
|
|
247 |
}
|
|
242 | 248 |
|
243 |
public String getCollectionMode() {
|
|
244 |
return collectionMode;
|
|
245 |
}
|
|
249 |
public void setCollectionMode(final String collectionMode) {
|
|
250 |
this.collectionMode = collectionMode;
|
|
251 |
}
|
|
246 | 252 |
|
247 |
public void setCollectionMode(String collectionMode) {
|
|
248 |
this.collectionMode = collectionMode;
|
|
249 |
}
|
|
253 |
public String getOozieWfPath() {
|
|
254 |
return oozieWfPath;
|
|
255 |
}
|
|
250 | 256 |
|
251 |
public String getOozieWfPath() {
|
|
252 |
return oozieWfPath;
|
|
253 |
}
|
|
257 |
public void setOozieWfPath(final String oozieWfPath) {
|
|
258 |
this.oozieWfPath = oozieWfPath;
|
|
259 |
}
|
|
254 | 260 |
|
255 |
public void setOozieWfPath(String oozieWfPath) {
|
|
256 |
this.oozieWfPath = oozieWfPath;
|
|
257 |
}
|
|
261 |
public String getMetadataEncoding() {
|
|
262 |
return metadataEncoding;
|
|
263 |
}
|
|
258 | 264 |
|
259 |
public String getMetadataEncoding() {
|
|
260 |
return metadataEncoding;
|
|
261 |
}
|
|
265 |
public void setMetadataEncoding(final String metadataEncoding) {
|
|
266 |
this.metadataEncoding = metadataEncoding;
|
|
267 |
}
|
|
262 | 268 |
|
263 |
public void setMetadataEncoding(String metadataEncoding) {
|
|
264 |
this.metadataEncoding = metadataEncoding;
|
|
265 |
}
|
|
269 |
public String getFromDateOverride() {
|
|
270 |
return fromDateOverride;
|
|
271 |
}
|
|
266 | 272 |
|
267 |
public String getFromDateOverride() {
|
|
268 |
return fromDateOverride;
|
|
269 |
}
|
|
273 |
public void setFromDateOverride(final String fromDateOverride) {
|
|
274 |
this.fromDateOverride = fromDateOverride;
|
|
275 |
}
|
|
270 | 276 |
|
271 |
public void setFromDateOverride(String fromDateOverride) {
|
|
272 |
this.fromDateOverride = fromDateOverride;
|
|
273 |
}
|
|
277 |
public String getUntilDateOverride() {
|
|
278 |
return untilDateOverride;
|
|
279 |
}
|
|
274 | 280 |
|
275 |
public String getUntilDateOverride() {
|
|
276 |
return untilDateOverride;
|
|
277 |
}
|
|
281 |
public void setUntilDateOverride(final String untilDateOverride) {
|
|
282 |
this.untilDateOverride = untilDateOverride;
|
|
283 |
}
|
|
278 | 284 |
|
279 |
public void setUntilDateOverride(String untilDateOverride) {
|
|
280 |
this.untilDateOverride = untilDateOverride;
|
|
281 |
}
|
|
285 |
public int getMaxNumberOfRetry() {
|
|
286 |
return maxNumberOfRetry;
|
|
287 |
}
|
|
282 | 288 |
|
283 |
public int getMaxNumberOfRetry() {
|
|
284 |
return maxNumberOfRetry;
|
|
285 |
}
|
|
289 |
public void setMaxNumberOfRetry(final int maxNumberOfRetry) {
|
|
290 |
this.maxNumberOfRetry = maxNumberOfRetry;
|
|
291 |
}
|
|
286 | 292 |
|
287 |
public void setMaxNumberOfRetry(int maxNumberOfRetry) {
|
|
288 |
this.maxNumberOfRetry = maxNumberOfRetry;
|
|
289 |
}
|
|
293 |
public int getRequestDelay() {
|
|
294 |
return requestDelay;
|
|
295 |
}
|
|
290 | 296 |
|
291 |
public int getRequestDelay() {
|
|
292 |
return requestDelay;
|
|
293 |
}
|
|
297 |
public void setRequestDelay(final int requestDelay) {
|
|
298 |
this.requestDelay = requestDelay;
|
|
299 |
}
|
|
294 | 300 |
|
295 |
public void setRequestDelay(int requestDelay) {
|
|
296 |
this.requestDelay = requestDelay;
|
|
297 |
}
|
|
301 |
public int getRetryDelay() {
|
|
302 |
return retryDelay;
|
|
303 |
}
|
|
298 | 304 |
|
299 |
public int getRetryDelay() {
|
|
300 |
return retryDelay;
|
|
301 |
}
|
|
305 |
public void setRetryDelay(final int retryDelay) {
|
|
306 |
this.retryDelay = retryDelay;
|
|
307 |
}
|
|
302 | 308 |
|
303 |
public void setRetryDelay(int retryDelay) {
|
|
304 |
this.retryDelay = retryDelay;
|
|
305 |
}
|
|
309 |
public int getConnectTimeOut() {
|
|
310 |
return connectTimeOut;
|
|
311 |
}
|
|
306 | 312 |
|
307 |
public int getConnectTimeOut() {
|
|
308 |
return connectTimeOut;
|
|
309 |
}
|
|
313 |
public void setConnectTimeOut(final int connectTimeOut) {
|
|
314 |
this.connectTimeOut = connectTimeOut;
|
|
315 |
}
|
|
310 | 316 |
|
311 |
public void setConnectTimeOut(int connectTimeOut) {
|
|
312 |
this.connectTimeOut = connectTimeOut;
|
|
313 |
}
|
|
317 |
public int getReadTimeOut() {
|
|
318 |
return readTimeOut;
|
|
319 |
}
|
|
314 | 320 |
|
315 |
public int getReadTimeOut() {
|
|
316 |
return readTimeOut;
|
|
317 |
}
|
|
321 |
public void setReadTimeOut(final int readTimeOut) {
|
|
322 |
this.readTimeOut = readTimeOut;
|
|
323 |
}
|
|
318 | 324 |
|
319 |
public void setReadTimeOut(int readTimeOut) {
|
|
320 |
this.readTimeOut = readTimeOut;
|
|
321 |
}
|
|
325 |
public String getDnetMessageManagerURL() {
|
|
326 |
return dnetMessageManagerURL;
|
|
327 |
}
|
|
322 | 328 |
|
323 |
public String getDnetMessageManagerURL() {
|
|
324 |
return dnetMessageManagerURL;
|
|
325 |
}
|
|
329 |
public void setDnetMessageManagerURL(final String dnetMessageManagerURL) {
|
|
330 |
this.dnetMessageManagerURL = dnetMessageManagerURL;
|
|
331 |
}
|
|
326 | 332 |
|
327 |
public void setDnetMessageManagerURL(String dnetMessageManagerURL) { |
|
328 |
this.dnetMessageManagerURL = dnetMessageManagerURL; |
|
329 |
} |
|
330 |
|
|
331 | 333 |
} |
Also available in: Unified diff
merge from eosc_services