1
|
package eu.dnetlib.enabling.datasources;
|
2
|
|
3
|
import java.io.StringReader;
|
4
|
import java.text.ParseException;
|
5
|
import java.util.Date;
|
6
|
import java.util.List;
|
7
|
import java.util.Map;
|
8
|
import java.util.regex.Pattern;
|
9
|
|
10
|
import javax.xml.ws.wsaddressing.W3CEndpointReference;
|
11
|
|
12
|
import org.antlr.stringtemplate.StringTemplate;
|
13
|
import org.apache.commons.io.IOUtils;
|
14
|
import org.apache.commons.logging.Log;
|
15
|
import org.apache.commons.logging.LogFactory;
|
16
|
import org.dom4j.Document;
|
17
|
import org.dom4j.io.SAXReader;
|
18
|
import org.quartz.CronExpression;
|
19
|
import org.springframework.beans.factory.annotation.Required;
|
20
|
import org.springframework.core.io.ClassPathResource;
|
21
|
import org.springframework.core.io.Resource;
|
22
|
|
23
|
import com.google.common.collect.Lists;
|
24
|
import com.google.common.collect.Maps;
|
25
|
|
26
|
import eu.dnetlib.enabling.database.rmi.DatabaseException;
|
27
|
import eu.dnetlib.enabling.database.rmi.DatabaseService;
|
28
|
import eu.dnetlib.enabling.datasources.common.DatasourceDesc;
|
29
|
import eu.dnetlib.enabling.datasources.common.DatasourceManagerServiceException;
|
30
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
|
31
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
32
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
33
|
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
|
34
|
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
|
35
|
import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
|
36
|
import eu.dnetlib.miscutils.functional.UnaryFunction;
|
37
|
import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
|
38
|
|
39
|
public class DatasourceManagerClients {
|
40
|
|
41
|
private static final Resource xslt = new ClassPathResource("/eu/dnetlib/enabling/datasources/repo_2_is.xslt");
|
42
|
private static final String REPOSITORY_SERVICE_RESOURCE_TYPE = "RepositoryServiceResourceType";
|
43
|
private static final Log log = LogFactory.getLog(DatasourceManagerClients.class);
|
44
|
private static final String TMPLS_BASEDIR = "/eu/dnetlib/enabling/datasources/";
|
45
|
private String db;
|
46
|
private UniqueServiceLocator serviceLocator;
|
47
|
private ResultSetClientFactory resultSetClientFactory;
|
48
|
|
49
|
public String findDatasourceId(final String profileId) throws DatasourceManagerServiceException {
|
50
|
try {
|
51
|
return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
|
52
|
"/*[.//RESOURCE_IDENTIFIER/@value='" + profileId + "']//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value/text()");
|
53
|
} catch (final Exception e) {
|
54
|
log.error("Error finding dsId of profile " + profileId, e);
|
55
|
throw new DatasourceManagerServiceException("Error finding dsId of profile " + profileId, e);
|
56
|
}
|
57
|
}
|
58
|
|
59
|
public String getDatasourceProfile(final String dsId) throws DatasourceManagerServiceException {
|
60
|
try {
|
61
|
return serviceLocator.getService(ISLookUpService.class)
|
62
|
.getResourceProfileByQuery(
|
63
|
"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')/*[.//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value/text() = '"
|
64
|
+ dsId + "']");
|
65
|
} catch (final Exception e) {
|
66
|
return null;
|
67
|
}
|
68
|
}
|
69
|
|
70
|
public boolean deleteProfile(final String dsId) throws DatasourceManagerServiceException {
|
71
|
try {
|
72
|
final SAXReader reader = new SAXReader();
|
73
|
|
74
|
final String profile = getDatasourceProfile(dsId);
|
75
|
|
76
|
if (profile != null) {
|
77
|
final Document docOld = reader.read(new StringReader(profile));
|
78
|
final String profId = docOld.valueOf("//RESOURCE_IDENTIFIER/@value");
|
79
|
serviceLocator.getService(ISRegistryService.class).deleteProfile(profId);
|
80
|
}
|
81
|
return true;
|
82
|
} catch (final Exception e) {
|
83
|
log.error("Error deleting profile", e);
|
84
|
throw new DatasourceManagerServiceException("Error deleting profile", e);
|
85
|
}
|
86
|
}
|
87
|
|
88
|
public boolean regenerateProfile(final String dsId) throws DatasourceManagerServiceException {
|
89
|
|
90
|
try {
|
91
|
final SAXReader reader = new SAXReader();
|
92
|
|
93
|
final List<String> list = getTransformedDatasourcesByCondition("ds.id= '" + dsId + "'", new ApplyXslt(xslt));
|
94
|
|
95
|
if (list.size() != 1) { throw new DatasourceManagerServiceException("Illegal number of datasource with id " + dsId + ", size: " + list.size()); }
|
96
|
|
97
|
final Document doc = reader.read(new StringReader(list.get(0)));
|
98
|
|
99
|
final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class);
|
100
|
|
101
|
final String profile = getDatasourceProfile(dsId);
|
102
|
|
103
|
if (profile != null) {
|
104
|
final Document docOld = reader.read(new StringReader(profile));
|
105
|
final String profId = docOld.valueOf("//RESOURCE_IDENTIFIER/@value");
|
106
|
|
107
|
doc.selectSingleNode("//RESOURCE_IDENTIFIER/@value").setText(profId);
|
108
|
|
109
|
registry.updateProfile(profId, doc.asXML(), REPOSITORY_SERVICE_RESOURCE_TYPE);
|
110
|
log.info("Profile " + profId + " UPDATED for ds " + dsId);
|
111
|
} else {
|
112
|
final String profId = registry.registerProfile(doc.asXML());
|
113
|
log.info("Valid Profile " + profId + " REGISTERED for ds " + dsId);
|
114
|
}
|
115
|
return true;
|
116
|
} catch (final Exception e) {
|
117
|
log.error("Error saving profile, id: " + dsId, e);
|
118
|
throw new DatasourceManagerServiceException("Error regenerating profile", e);
|
119
|
}
|
120
|
}
|
121
|
|
122
|
public Iterable<String> searchSQL(final String sql) throws DatabaseException {
|
123
|
final W3CEndpointReference epr = serviceLocator.getService(DatabaseService.class).searchSQL(getDb(), sql);
|
124
|
return resultSetClientFactory.getClient(epr);
|
125
|
}
|
126
|
|
127
|
public boolean updateSQL(final String dsId, final String sql, final boolean delete, final boolean updateprofile) throws DatasourceManagerServiceException {
|
128
|
log.debug("Executing query SQL: " + sql);
|
129
|
|
130
|
try {
|
131
|
serviceLocator.getService(DatabaseService.class).updateSQL(getDb(), sql);
|
132
|
} catch (final DatabaseException e) {
|
133
|
throw new DatasourceManagerServiceException(e);
|
134
|
}
|
135
|
|
136
|
if (updateprofile) {
|
137
|
if (delete) {
|
138
|
return deleteProfile(dsId);
|
139
|
} else {
|
140
|
return regenerateProfile(dsId);
|
141
|
}
|
142
|
}
|
143
|
return false;
|
144
|
}
|
145
|
|
146
|
public boolean updateSQL(final String dsId, final String sqlTemplate, final Map<String, Object> params, final boolean delete, final boolean updateProfile)
|
147
|
throws DatasourceManagerServiceException {
|
148
|
|
149
|
verifyParams(params.values());
|
150
|
verifyParams(params.keySet().toArray());
|
151
|
|
152
|
try {
|
153
|
final Resource resource = new ClassPathResource(TMPLS_BASEDIR + sqlTemplate);
|
154
|
final StringTemplate st = new StringTemplate(IOUtils.toString(resource.getInputStream()));
|
155
|
st.setAttributes(params);
|
156
|
return updateSQL(dsId, st.toString(), delete, updateProfile);
|
157
|
} catch (final Exception e) {
|
158
|
log.error("Error in updateSQL", e);
|
159
|
throw new DatasourceManagerServiceException("Error in updateSQL", e);
|
160
|
}
|
161
|
}
|
162
|
|
163
|
public List<DatasourceDesc> getDatasourcesByCondition(final String condition) throws DatasourceManagerServiceException {
|
164
|
final SAXReader reader = new SAXReader();
|
165
|
final List<DatasourceDesc> list = Lists.newArrayList();
|
166
|
try {
|
167
|
for (final String s : getXmlDatasourcesByCondition(condition)) {
|
168
|
final Document doc = reader.read(new StringReader(s));
|
169
|
list.add(DatasourceFunctions.xmlToDatasourceDesc(doc));
|
170
|
}
|
171
|
} catch (final Exception e) {
|
172
|
log.error("Error obtaining datasources from db", e);
|
173
|
throw new DatasourceManagerServiceException("Error obtaining datasources from db", e);
|
174
|
}
|
175
|
return list;
|
176
|
|
177
|
}
|
178
|
|
179
|
private void verifyParams(final Object... params) throws DatasourceManagerServiceException {
|
180
|
|
181
|
final Pattern pattern = Pattern.compile("\\n");
|
182
|
|
183
|
for (final Object p : params) {
|
184
|
log.debug("TESTING SQL PARAM:" + p);
|
185
|
if ((p == null) || p.toString().isEmpty()) {
|
186
|
log.error("Parameter null or empty");
|
187
|
throw new DatasourceManagerServiceException("Parameter null or empty");
|
188
|
} else if (pattern.matcher(p.toString()).matches()) {
|
189
|
log.error("Parameter [" + p + "] contains an invalid character");
|
190
|
throw new DatasourceManagerServiceException("Parameter [" + p + "] contains an invalid character");
|
191
|
} else {
|
192
|
log.debug("TEST OK");
|
193
|
}
|
194
|
}
|
195
|
}
|
196
|
|
197
|
private List<String> getTransformedDatasourcesByCondition(final String condition, final UnaryFunction<String, String> function)
|
198
|
throws DatasourceManagerServiceException {
|
199
|
final List<String> list = Lists.newArrayList();
|
200
|
try {
|
201
|
for (final String s : getXmlDatasourcesByCondition(condition)) {
|
202
|
list.add(function.evaluate(s));
|
203
|
}
|
204
|
} catch (final Exception e) {
|
205
|
log.error("Error obtaining datasources from db", e);
|
206
|
throw new DatasourceManagerServiceException("Error obtaining datasources from db", e);
|
207
|
}
|
208
|
return list;
|
209
|
}
|
210
|
|
211
|
private Iterable<String> getXmlDatasourcesByCondition(final String condition) throws DatasourceManagerServiceException {
|
212
|
try {
|
213
|
final Map<String, Object> params = Maps.newHashMap();
|
214
|
|
215
|
if ((condition != null) && !condition.trim().isEmpty()) {
|
216
|
params.put("condition", condition);
|
217
|
}
|
218
|
return searchSQL("getDatasources.sql.st", params);
|
219
|
} catch (final Exception e) {
|
220
|
log.error("Error obtaining datasources from db", e);
|
221
|
throw new DatasourceManagerServiceException("Error obtaining datasources from db", e);
|
222
|
}
|
223
|
}
|
224
|
|
225
|
public Iterable<String> searchSQL(final String sqlTemplate, final Map<String, Object> params) throws DatasourceManagerServiceException {
|
226
|
try {
|
227
|
final Resource resource = new ClassPathResource(TMPLS_BASEDIR + sqlTemplate);
|
228
|
final StringTemplate st = new StringTemplate(IOUtils.toString(resource.getInputStream()));
|
229
|
if (params != null) {
|
230
|
st.setAttributes(params);
|
231
|
}
|
232
|
|
233
|
final String sql = st.toString();
|
234
|
|
235
|
log.debug("Executing SQL: " + sql);
|
236
|
|
237
|
return searchSQL(sql);
|
238
|
} catch (final Exception e) {
|
239
|
log.error("Error executing sql", e);
|
240
|
|
241
|
throw new DatasourceManagerServiceException("Error obtaining datasources from db", e);
|
242
|
}
|
243
|
}
|
244
|
|
245
|
public boolean isDefinedParam(final String ifaceId, final String field) throws DatasourceManagerServiceException {
|
246
|
final Map<String, Object> params = Maps.newHashMap();
|
247
|
params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));
|
248
|
params.put("field", DatasourceFunctions.asSqlValue(field));
|
249
|
|
250
|
final List<String> list = Lists.newArrayList(searchSQL("searchApiCollectionParam.sql.st", params));
|
251
|
|
252
|
return !list.isEmpty();
|
253
|
}
|
254
|
|
255
|
public Date findNextScheduledExecution(final String dsId, final String ifaceId) throws DatasourceManagerServiceException {
|
256
|
final String xquery = "/*[.//DATAPROVIDER/@interface='" + ifaceId + "' and .//SCHEDULING/@enabled='true']//CRON/text()";
|
257
|
try {
|
258
|
final String cronExpression = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
|
259
|
final CronExpression cron = new CronExpression(cronExpression);
|
260
|
return cron.getNextValidTimeAfter(new Date());
|
261
|
} catch (final ISLookUpDocumentNotFoundException e) {
|
262
|
// When the value is not found a null value must be returned
|
263
|
return null;
|
264
|
} catch (final ISLookUpException e) {
|
265
|
log.error("Error in xquery: " + xquery, e);
|
266
|
throw new DatasourceManagerServiceException("Error in xquery: " + xquery, e);
|
267
|
} catch (final ParseException e) {
|
268
|
log.error("Error parsing cron expression", e);
|
269
|
throw new DatasourceManagerServiceException("Error parsing cron expression", e);
|
270
|
}
|
271
|
}
|
272
|
|
273
|
public String getDb() {
|
274
|
return db;
|
275
|
}
|
276
|
|
277
|
@Required
|
278
|
public void setDb(final String db) {
|
279
|
this.db = db;
|
280
|
}
|
281
|
|
282
|
public ResultSetClientFactory getResultSetClientFactory() {
|
283
|
return resultSetClientFactory;
|
284
|
}
|
285
|
|
286
|
@Required
|
287
|
public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
|
288
|
this.resultSetClientFactory = resultSetClientFactory;
|
289
|
}
|
290
|
|
291
|
public UniqueServiceLocator getServiceLocator() {
|
292
|
return serviceLocator;
|
293
|
}
|
294
|
|
295
|
@Required
|
296
|
public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
|
297
|
this.serviceLocator = serviceLocator;
|
298
|
}
|
299
|
|
300
|
}
|