Revision 62109
Added by Michele Artini over 2 years ago
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/templates/datasource_base.xml | ||
|---|---|---|
| 1 |
<RESOURCE_PROFILE> |
|
| 2 |
<HEADER> |
|
| 3 |
<RESOURCE_IDENTIFIER value="" /> |
|
| 4 |
<RESOURCE_TYPE value="RepositoryServiceResourceType" /> |
|
| 5 |
<RESOURCE_KIND value="RepositoryServiceResources" /> |
|
| 6 |
<RESOURCE_URI value="" /> |
|
| 7 |
<DATE_OF_CREATION value="" /> |
|
| 8 |
<PROTOCOL /> |
|
| 9 |
</HEADER> |
|
| 10 |
<BODY> |
|
| 11 |
<CONFIGURATION> |
|
| 12 |
<DATASOURCE_TYPE /> |
|
| 13 |
<DATASOURCE_ORIGINAL_ID provenance="OPENAIRE" /> |
|
| 14 |
<DATASOURCE_AGGREGATED>false</DATASOURCE_AGGREGATED> |
|
| 15 |
<ENVIRONMENTS> |
|
| 16 |
<ENVIRONMENT>OPENAIRE</ENVIRONMENT> |
|
| 17 |
</ENVIRONMENTS> |
|
| 18 |
<TYPOLOGY /> |
|
| 19 |
<MAX_SIZE_OF_DATASTRUCTURE>0</MAX_SIZE_OF_DATASTRUCTURE> |
|
| 20 |
<AVAILABLE_DISKSPACE>0</AVAILABLE_DISKSPACE> |
|
| 21 |
<MAX_NUMBER_OF_DATASTRUCTURE>0</MAX_NUMBER_OF_DATASTRUCTURE> |
|
| 22 |
<OFFICIAL_NAME /> |
|
| 23 |
<ENGLISH_NAME /> |
|
| 24 |
<ICON_URI /> |
|
| 25 |
<COUNTRY /> |
|
| 26 |
<LOCATION> |
|
| 27 |
<LONGITUDE>0.0</LONGITUDE> |
|
| 28 |
<LATITUDE>0.0</LATITUDE> |
|
| 29 |
<TIMEZONE>0.0</TIMEZONE> |
|
| 30 |
</LOCATION> |
|
| 31 |
<REPOSITORY_WEBPAGE /> |
|
| 32 |
<REPOSITORY_INSTITUTION /> |
|
| 33 |
<ADMIN_INFO /> |
|
| 34 |
<INTERFACES /> |
|
| 35 |
<EXTRA_FIELDS /> |
|
| 36 |
<REGISTERED_BY /> |
|
| 37 |
</CONFIGURATION> |
|
| 38 |
<STATUS> |
|
| 39 |
<NUMBER_OF_OBJECTS>0</NUMBER_OF_OBJECTS> |
|
| 40 |
<LAST_UPDATE value="" /> |
|
| 41 |
</STATUS> |
|
| 42 |
<QOS> |
|
| 43 |
<AVAILABILITY>0</AVAILABILITY> |
|
| 44 |
<CAPACITY /> |
|
| 45 |
<THROUGHPUT>0.0</THROUGHPUT> |
|
| 46 |
</QOS> |
|
| 47 |
<SECURITY_PARAMETERS /> |
|
| 48 |
<BLACKBOARD /> |
|
| 49 |
</BODY> |
|
| 50 |
</RESOURCE_PROFILE> |
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/setLastCollectionInfo.sql | ||
|---|---|---|
| 1 | 1 |
UPDATE dsm_api |
| 2 | 2 |
SET (last_collection_mdid, last_collection_total, last_collection_date) = (:mdId, :total, :date) |
| 3 |
WHERE id = :apiId AND datasource = :dsId |
|
| 3 |
WHERE id = :apiId AND service = :dsId |
|
| modules/dnet-openaire-datasource-manager/trunk/pom.xml | ||
|---|---|---|
| 9 | 9 |
<groupId>eu.dnetlib</groupId> |
| 10 | 10 |
<artifactId>dnet-openaire-datasource-manager</artifactId> |
| 11 | 11 |
<packaging>jar</packaging> |
| 12 |
<version>1.0.9-SNAPSHOT</version>
|
|
| 12 |
<version>2.0.0-SNAPSHOT</version>
|
|
| 13 | 13 |
<scm> |
| 14 | 14 |
<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaire-datasource-manager/trunk</developerConnection> |
| 15 | 15 |
</scm> |
| ... | ... | |
| 22 | 22 |
<dependency> |
| 23 | 23 |
<groupId>eu.dnetlib</groupId> |
| 24 | 24 |
<artifactId>dnet-datasource-manager-common</artifactId> |
| 25 |
<version>[1.0.0-SNAPSHOT,2.0.0)</version>
|
|
| 25 |
<version>[2.0.0,3.0.0)</version>
|
|
| 26 | 26 |
</dependency> |
| 27 | 27 |
<dependency> |
| 28 | 28 |
<groupId>org.springframework</groupId> |
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/setLastValidationJob.sql | ||
|---|---|---|
| 1 | 1 |
UPDATE dsm_api |
| 2 | 2 |
SET (last_validation_job) = (:jobId) |
| 3 |
WHERE id = :apiId AND datasource = :dsId |
|
| 3 |
WHERE id = :apiId AND service = :dsId |
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/setRemovable.sql | ||
|---|---|---|
| 1 |
UPDATE dsm_api SET removable = :removable WHERE datasource = :dsId |
|
| 1 |
UPDATE dsm_api SET removable = :removable WHERE service = :dsId |
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/java/eu/dnetlib/enabling/datasources/DatasourceFunctions.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.enabling.datasources; |
| 2 | 2 |
|
| 3 |
import java.io.IOException; |
|
| 4 | 3 |
import java.sql.Array; |
| 5 | 4 |
import java.sql.Date; |
| 6 | 5 |
import java.sql.SQLException; |
| 7 | 6 |
import java.sql.Timestamp; |
| 8 | 7 |
import java.util.Arrays; |
| 9 | 8 |
import java.util.HashMap; |
| 10 |
import java.util.List; |
|
| 11 | 9 |
import java.util.Map; |
| 12 | 10 |
import java.util.Objects; |
| 13 | 11 |
import java.util.stream.Collectors; |
| 14 | 12 |
|
| 15 |
import org.apache.commons.lang3.BooleanUtils; |
|
| 16 | 13 |
import org.apache.commons.lang3.StringUtils; |
| 17 | 14 |
import org.apache.commons.logging.Log; |
| 18 | 15 |
import org.apache.commons.logging.LogFactory; |
| 19 |
import org.dom4j.Document; |
|
| 20 |
import org.dom4j.DocumentException; |
|
| 21 |
import org.dom4j.Element; |
|
| 22 |
import org.dom4j.io.SAXReader; |
|
| 23 |
import org.springframework.core.io.ClassPathResource; |
|
| 24 |
import org.springframework.core.io.Resource; |
|
| 25 | 16 |
|
| 26 | 17 |
import eu.dnetlib.enabling.datasources.common.Api; |
| 27 | 18 |
import eu.dnetlib.enabling.datasources.common.ApiParam; |
| ... | ... | |
| 29 | 20 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
| 30 | 21 |
import eu.dnetlib.enabling.datasources.common.Identity; |
| 31 | 22 |
import eu.dnetlib.enabling.datasources.common.Organization; |
| 23 |
import eu.dnetlib.enabling.datasources.common.PidSystem; |
|
| 32 | 24 |
import eu.dnetlib.enabling.datasources.common.SearchApisEntry; |
| 33 | 25 |
import eu.dnetlib.enabling.datasources.common.SimpleDatasource; |
| 34 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
| 35 | 26 |
|
| 36 | 27 |
public class DatasourceFunctions {
|
| 37 | 28 |
|
| 29 |
private static final String EOSC_DATA_SOURCE = "Data Source"; |
|
| 30 |
|
|
| 38 | 31 |
private static final Log log = LogFactory.getLog(DatasourceFunctions.class); |
| 39 | 32 |
|
| 40 |
private static final Resource baseDsProfile = new ClassPathResource("/eu/dnetlib/enabling/datasources/templates/datasource_base.xml");
|
|
| 41 |
|
|
| 42 | 33 |
public static SimpleDatasource mapToSimpleDs(final Map<String, Object> map) {
|
| 43 | 34 |
final SimpleDatasource ds = new SimpleDatasource(); |
| 44 | 35 |
ds.setId(castObject(map.get("id"), String.class));
|
| 45 | 36 |
ds.setName(castObject(map.get("name"), String.class));
|
| 46 | 37 |
ds.setOrigId(castObject(map.get("id"), String.class));
|
| 47 |
ds.setTypology(castObject(map.get("typology"), String.class));
|
|
| 38 |
ds.setEoscDatasourceType(castObject(map.get("eosc_datasource_type"), String.class));
|
|
| 48 | 39 |
ds.setValid(true); |
| 49 | 40 |
try {
|
| 50 | 41 |
final Array arr = castObject(map.get("apis"), Array.class);
|
| 51 | 42 |
|
| 52 | 43 |
if (arr != null && ((Object[]) arr.getArray()).length > 0) {
|
| 53 | 44 |
ds.setApis(Arrays.stream((Object[]) arr.getArray()) |
| 54 |
.filter(Objects::nonNull)
|
|
| 55 |
.map(Object::toString)
|
|
| 56 |
.collect(Collectors.toSet()));
|
|
| 45 |
.filter(Objects::nonNull) |
|
| 46 |
.map(Object::toString) |
|
| 47 |
.collect(Collectors.toSet())); |
|
| 57 | 48 |
} |
| 58 | 49 |
} catch (final SQLException e) {
|
| 59 | 50 |
log.error("Error parsing array (apis)", e);
|
| ... | ... | |
| 63 | 54 |
return ds; |
| 64 | 55 |
} |
| 65 | 56 |
|
| 66 |
public static Datasource<Organization<?>, Identity> mapToDatasource(final Map<String, Object> map) {
|
|
| 67 |
final Datasource<Organization<?>, Identity> ds = new Datasource<>(); |
|
| 57 |
public static Datasource<Organization<?>, Identity, PidSystem> mapToDatasource(final Map<String, Object> map) {
|
|
| 58 |
final Datasource<Organization<?>, Identity, PidSystem> ds = new Datasource<>();
|
|
| 68 | 59 |
ds.setId(castObject(map.get("id"), String.class));
|
| 69 | 60 |
ds.setOfficialname(castObject(map.get("officialname"), String.class));
|
| 70 | 61 |
ds.setEnglishname(castObject(map.get("englishname"), String.class));
|
| ... | ... | |
| 76 | 67 |
ds.setTimezone(castObject(map.get("timezone"), String.class));
|
| 77 | 68 |
ds.setNamespaceprefix(castObject(map.get("namespaceprefix"), String.class));
|
| 78 | 69 |
ds.setLanguages(castObject(map.get("languages"), String.class));
|
| 79 |
ds.setOd_contenttypes(castObject(map.get("od_contenttypes"), String.class));
|
|
| 80 | 70 |
ds.setCollectedfrom(castObject(map.get("collectedfrom"), String.class));
|
| 81 | 71 |
ds.setDateofvalidation(castObject(map.get("dateofvalidation"), Date.class));
|
| 82 |
ds.setTypology(castObject(map.get("typology"), String.class));
|
|
| 72 |
ds.setEoscDatasourceType(castObject(map.get("eosc_datasource_type"), String.class));
|
|
| 83 | 73 |
ds.setProvenanceaction(castObject(map.get("provenanceaction"), String.class));
|
| 84 | 74 |
ds.setDateofcollection(castObject(map.get("dateofcollection"), Date.class));
|
| 85 | 75 |
ds.setPlatform(castObject(map.get("platform"), String.class));
|
| ... | ... | |
| 88 | 78 |
ds.setReleasestartdate(castObject(map.get("releasestartdate"), Date.class));
|
| 89 | 79 |
ds.setReleaseenddate(castObject(map.get("releaseenddate"), Date.class));
|
| 90 | 80 |
ds.setMissionstatementurl(castObject(map.get("missionstatementurl"), String.class));
|
| 91 |
ds.setDataprovider(castObject(map.get("dataprovider"), Boolean.class));
|
|
| 92 |
ds.setServiceprovider(castObject(map.get("serviceprovider"), Boolean.class));
|
|
| 93 | 81 |
ds.setDatabaseaccesstype(castObject(map.get("databaseaccesstype"), String.class));
|
| 94 | 82 |
ds.setDatauploadtype(castObject(map.get("datauploadtype"), String.class));
|
| 95 | 83 |
ds.setDatabaseaccessrestriction(castObject(map.get("databaseaccessrestriction"), String.class));
|
| 96 | 84 |
ds.setDatauploadrestriction(castObject(map.get("datauploadrestriction"), String.class));
|
| 97 |
ds.setVersioning(castObject(map.get("versioning"), Boolean.class));
|
|
| 98 | 85 |
ds.setCitationguidelineurl(castObject(map.get("citationguidelineurl"), String.class));
|
| 99 |
ds.setQualitymanagementkind(castObject(map.get("qualitymanagementkind"), String.class));
|
|
| 100 |
ds.setPidsystems(castObject(map.get("pidsystems"), String.class));
|
|
| 101 | 86 |
ds.setCertificates(castObject(map.get("certificates"), String.class));
|
| 102 | 87 |
ds.setAggregator(castObject(map.get("aggregator"), String.class));
|
| 103 | 88 |
ds.setIssn(castObject(map.get("issn"), String.class));
|
| ... | ... | |
| 106 | 91 |
ds.setRegisteredby(castObject(map.get("registeredby"), String.class));
|
| 107 | 92 |
ds.setSubjects(castObject(map.get("subjects"), String.class));
|
| 108 | 93 |
ds.setManaged(castObject(map.get("managed"), Boolean.class));
|
| 94 |
ds.setConsentTermsOfUse(castObject(map.get("consenttermsofuse"), Boolean.class));
|
|
| 95 |
ds.setConsentTermsOfUseDate(castObject(map.get("consenttermsofusedate"), Date.class));
|
|
| 96 |
ds.setFullTextDownload(castObject(map.get("fulltextdownload"), Boolean.class));
|
|
| 97 |
ds.setStatus(castObject(map.get("status"), String.class));
|
|
| 98 |
|
|
| 109 | 99 |
return ds; |
| 110 | 100 |
} |
| 111 | 101 |
|
| ... | ... | |
| 116 | 106 |
return id; |
| 117 | 107 |
} |
| 118 | 108 |
|
| 119 |
public static Organization<Datasource<?, ?>> mapToDsOrganization(final Map<String, Object> map) {
|
|
| 120 |
final Organization<Datasource<?, ?>> org = new Organization<>(); |
|
| 109 |
public static Organization<Datasource<?, ?, ?>> mapToDsOrganization(final Map<String, Object> map) {
|
|
| 110 |
final Organization<Datasource<?, ?, ?>> org = new Organization<>();
|
|
| 121 | 111 |
org.setId(castObject(map.get("id"), String.class));
|
| 122 | 112 |
org.setLegalshortname(castObject(map.get("legalshortname"), String.class));
|
| 123 | 113 |
org.setLegalname(castObject(map.get("legalname"), String.class));
|
| ... | ... | |
| 155 | 145 |
a.setContentdescription(castObject(map.get("contentdescription"), String.class));
|
| 156 | 146 |
a.setActive(castObject(map.get("active"), Boolean.class));
|
| 157 | 147 |
a.setRemovable(castObject(map.get("removable"), Boolean.class));
|
| 158 |
a.setTypology(castObject(map.get("typology"), String.class));
|
|
| 159 | 148 |
a.setCompatibility(castObject(map.get("compatibility"), String.class));
|
| 160 | 149 |
a.setCompatibilityOverrided(castObject(map.get("isCompatibilityOverrided"), Boolean.class));
|
| 161 | 150 |
a.setMetadataIdentifierPath(castObject(map.get("metadataIdentifierPath"), String.class));
|
| ... | ... | |
| 174 | 163 |
final Array arr = castObject(map.get("params"), Array.class);
|
| 175 | 164 |
if (arr != null) {
|
| 176 | 165 |
a.setApiParams(Arrays.stream((Object[]) arr.getArray()) |
| 177 |
.filter(Objects::nonNull)
|
|
| 178 |
.map(Object::toString)
|
|
| 179 |
.map(s -> {
|
|
| 180 |
final ApiParam p = new ApiParamImpl();
|
|
| 181 |
p.setParam(StringUtils.substringBefore(s, "="));
|
|
| 182 |
p.setValue(StringUtils.substringAfter(s, "="));
|
|
| 183 |
return p;
|
|
| 184 |
})
|
|
| 185 |
.collect(Collectors.toSet()));
|
|
| 166 |
.filter(Objects::nonNull) |
|
| 167 |
.map(Object::toString) |
|
| 168 |
.map(s -> {
|
|
| 169 |
final ApiParam p = new ApiParamImpl(); |
|
| 170 |
p.setParam(StringUtils.substringBefore(s, "=")); |
|
| 171 |
p.setValue(StringUtils.substringAfter(s, "=")); |
|
| 172 |
return p; |
|
| 173 |
}) |
|
| 174 |
.collect(Collectors.toSet())); |
|
| 186 | 175 |
} |
| 187 | 176 |
} catch (final SQLException e) {
|
| 188 | 177 |
log.error("Error parsing array params", e);
|
| ... | ... | |
| 192 | 181 |
return a; |
| 193 | 182 |
} |
| 194 | 183 |
|
| 195 |
public static String dsToProfile(final Datasource<Organization<?>, Identity> ds, final List<Api<ApiParam>> apis, final String profId) |
|
| 196 |
throws DocumentException, IOException {
|
|
| 197 |
final Document doc = new SAXReader().read(baseDsProfile.getInputStream()); |
|
| 198 |
|
|
| 199 |
setValue(doc, "//DATASOURCE_TYPE", ds.getTypology()); |
|
| 200 |
setValue(doc, "//DATASOURCE_ORIGINAL_ID", ds.getId()); |
|
| 201 |
setValue(doc, "//TYPOLOGY", ds.getPlatform()); |
|
| 202 |
setValue(doc, "//OFFICIAL_NAME", ds.getOfficialname()); |
|
| 203 |
setValue(doc, "//ENGLISH_NAME", ds.getEnglishname()); |
|
| 204 |
setValue(doc, "//ICON_URI", ds.getLogourl()); |
|
| 205 |
setValue(doc, "//COUNTRY", ds.getOrganizations().stream().map(Organization::getCountry).findFirst().orElse(""));
|
|
| 206 |
setValue(doc, "//LONGITUDE", ds.getLongitude()); |
|
| 207 |
setValue(doc, "//LATITUDE", ds.getLatitude()); |
|
| 208 |
setValue(doc, "//TIMEZONE", ds.getTimezone()); |
|
| 209 |
setValue(doc, "//REPOSITORY_WEBPAGE", ds.getWebsiteurl()); |
|
| 210 |
setValue(doc, "//REPOSITORY_INSTITUTION", ds.getOrganizations().stream().map(Organization::getLegalname).findFirst().orElse(""));
|
|
| 211 |
setValue(doc, "//ADMIN_INFO", ds.getContactemail()); |
|
| 212 |
setValue(doc, "//REGISTERED_BY", ds.getRegisteredby()); |
|
| 213 |
setValue(doc, "//LAST_UPDATE/@value", DateUtils.now_ISO8601()); |
|
| 214 |
|
|
| 215 |
final Element ef = (Element) doc.selectSingleNode("//EXTRA_FIELDS");
|
|
| 216 |
addExtraField(ef, "OpenAireDataSourceId", ds.getId()); |
|
| 217 |
addExtraField(ef, "NamespacePrefix", ds.getNamespaceprefix()); |
|
| 218 |
addExtraField(ef, "VERIFIED", "NO"); |
|
| 219 |
addExtraField(ef, "aggregatorName", ds.getAggregator()); |
|
| 220 |
addExtraField(ef, "dateOfValidation", ds.getDateofvalidation()); |
|
| 221 |
addExtraField(ef, "dateOfCollection", ds.getDateofcollection()); |
|
| 222 |
addExtraField(ef, "mergeHomonyms", "NO"); |
|
| 223 |
addExtraField(ef, "ACTID", ds.getActivationId()); |
|
| 224 |
|
|
| 225 |
if (apis != null) {
|
|
| 226 |
final Element ifaces = (Element) doc.selectSingleNode("//INTERFACES");
|
|
| 227 |
apis.forEach(api -> addInterface(ifaces, api)); |
|
| 228 |
} |
|
| 229 |
return doc.asXML(); |
|
| 230 |
} |
|
| 231 |
|
|
| 232 |
private static void setValue(final Document doc, final String xpath, final Object value) {
|
|
| 233 |
if (value != null) {
|
|
| 234 |
doc.selectSingleNode(xpath).setText(value.toString()); |
|
| 235 |
} |
|
| 236 |
} |
|
| 237 |
|
|
| 238 |
private static void addInterface(final Element ifaces, final Api<?> api) {
|
|
| 239 |
|
|
| 240 |
final Element ifc = ifaces.addElement("INTERFACE");
|
|
| 241 |
ifc.addAttribute("id", api.getId());
|
|
| 242 |
ifc.addAttribute("label", String.format("%s (%s)",
|
|
| 243 |
StringUtils.defaultIfBlank(api.getTypology(), "-"), |
|
| 244 |
StringUtils.defaultIfBlank(api.getCompatibility(), "-"))); |
|
| 245 |
ifc.addAttribute("typology", StringUtils.defaultIfBlank(api.getTypology(), ""));
|
|
| 246 |
ifc.addAttribute("active", "" + BooleanUtils.toBooleanDefaultIfNull(api.getActive(), false));
|
|
| 247 |
ifc.addAttribute("compliance", StringUtils.defaultIfBlank(api.getCompatibility(), ""));
|
|
| 248 |
ifc.addAttribute("contentDescription", StringUtils.defaultIfBlank(api.getContentdescription(), ""));
|
|
| 249 |
ifc.addAttribute("removable", "" + BooleanUtils.toBooleanDefaultIfNull(api.getRemovable(), false));
|
|
| 250 |
|
|
| 251 |
final Element ap = ifc.addElement("ACCESS_PROTOCOL");
|
|
| 252 |
api.getApiParams().forEach(p -> ap.addAttribute(p.getParam(), p.getValue())); |
|
| 253 |
ap.setText(StringUtils.defaultIfBlank(api.getProtocol(), "")); |
|
| 254 |
ifc.addElement("BASE_URL").setText(StringUtils.defaultIfBlank(api.getBaseurl(), ""));
|
|
| 255 |
|
|
| 256 |
if (api.isCompatibilityOverrided()) {
|
|
| 257 |
addInterfaceExtraField(ifc, "overriding_compliance", true); |
|
| 258 |
} |
|
| 259 |
|
|
| 260 |
addInterfaceExtraField(ifc, "metadata_identifier_path", api.getMetadataIdentifierPath()); |
|
| 261 |
addInterfaceExtraField(ifc, "last_collection_date", api.getLastCollectionDate()); |
|
| 262 |
addInterfaceExtraField(ifc, "last_collection_mdId", api.getLastCollectionMdid()); |
|
| 263 |
addInterfaceExtraField(ifc, "last_collection_total", api.getLastCollectionTotal()); |
|
| 264 |
addInterfaceExtraField(ifc, "last_aggregation_date", api.getLastAggregationDate()); |
|
| 265 |
addInterfaceExtraField(ifc, "last_aggregation_mdId", api.getLastAggregationMdid()); |
|
| 266 |
addInterfaceExtraField(ifc, "last_aggregation_total", api.getLastAggregationTotal()); |
|
| 267 |
addInterfaceExtraField(ifc, "last_download_date", api.getLastDownloadDate()); |
|
| 268 |
addInterfaceExtraField(ifc, "last_download_objId", api.getLastDownloadObjid()); |
|
| 269 |
addInterfaceExtraField(ifc, "last_download_total", api.getLastDownloadTotal()); |
|
| 270 |
addInterfaceExtraField(ifc, "last_validation_job", api.getLastValidationJob()); |
|
| 271 |
} |
|
| 272 |
|
|
| 273 |
private static void addExtraField(final Element parent, final String field, final Object value) {
|
|
| 274 |
if (value != null && StringUtils.isNotBlank(value.toString())) {
|
|
| 275 |
final Element f = parent.addElement("FIELD");
|
|
| 276 |
f.addElement("key").setText(field);
|
|
| 277 |
f.addElement("value").setText(value.toString());
|
|
| 278 |
} |
|
| 279 |
} |
|
| 280 |
|
|
| 281 |
private static void addInterfaceExtraField(final Element parent, final String field, final Object value) {
|
|
| 282 |
if (value != null && StringUtils.isNotBlank(value.toString())) {
|
|
| 283 |
final Element f = parent.addElement("INTERFACE_EXTRA_FIELD");
|
|
| 284 |
f.addAttribute("name", field);
|
|
| 285 |
f.setText(value.toString()); |
|
| 286 |
} |
|
| 287 |
} |
|
| 288 |
|
|
| 289 |
public static Map<String, Object> dsToMap(final Datasource<Organization<?>, Identity> ds) {
|
|
| 184 |
public static Map<String, Object> dsToMap(final Datasource<Organization<?>, Identity, PidSystem> ds) {
|
|
| 290 | 185 |
final Map<String, Object> map = new HashMap<>(); |
| 291 | 186 |
|
| 292 | 187 |
map.put("id", ds.getId());
|
| ... | ... | |
| 300 | 195 |
map.put("timezone", ds.getTimezone());
|
| 301 | 196 |
map.put("namespaceprefix", ds.getNamespaceprefix());
|
| 302 | 197 |
map.put("languages", ds.getLanguages());
|
| 303 |
map.put("od_contenttypes", ds.getOd_contenttypes());
|
|
| 304 | 198 |
map.put("collectedfrom", ds.getCollectedfrom());
|
| 305 | 199 |
map.put("dateofvalidation", ds.getDateofvalidation());
|
| 306 |
map.put("typology", ds.getTypology());
|
|
| 200 |
map.put("eosctype", EOSC_DATA_SOURCE);
|
|
| 201 |
map.put("eoscdatasourcetype", ds.getEoscDatasourceType());
|
|
| 307 | 202 |
map.put("provenanceaction", ds.getProvenanceaction());
|
| 308 | 203 |
map.put("platform", ds.getPlatform());
|
| 309 | 204 |
map.put("activationid", ds.getActivationId());
|
| ... | ... | |
| 311 | 206 |
map.put("releasestartdate", ds.getReleasestartdate());
|
| 312 | 207 |
map.put("releaseenddate", ds.getReleaseenddate());
|
| 313 | 208 |
map.put("missionstatementurl", ds.getMissionstatementurl());
|
| 314 |
map.put("dataprovider", ds.getDataprovider());
|
|
| 315 |
map.put("serviceprovider", ds.getServiceprovider());
|
|
| 316 | 209 |
map.put("databaseaccesstype", ds.getDatabaseaccesstype());
|
| 317 | 210 |
map.put("datauploadtype", ds.getDatauploadtype());
|
| 318 | 211 |
map.put("databaseaccessrestriction", ds.getDatabaseaccessrestriction());
|
| 319 | 212 |
map.put("datauploadrestriction", ds.getDatauploadrestriction());
|
| 320 |
map.put("versioning", ds.getVersioning());
|
|
| 321 | 213 |
map.put("citationguidelineurl", ds.getCitationguidelineurl());
|
| 322 |
map.put("qualitymanagementkind", ds.getQualitymanagementkind());
|
|
| 323 | 214 |
map.put("pidsystems", ds.getPidsystems());
|
| 324 | 215 |
map.put("certificates", ds.getCertificates());
|
| 325 | 216 |
map.put("aggregator", ds.getAggregator());
|
| ... | ... | |
| 329 | 220 |
map.put("registeredby", ds.getRegisteredby());
|
| 330 | 221 |
map.put("subjects", ds.getSubjects());
|
| 331 | 222 |
map.put("managed", ds.getManaged());
|
| 223 |
map.put("status", ds.getStatus());
|
|
| 332 | 224 |
|
| 333 | 225 |
return map; |
| 334 | 226 |
} |
| ... | ... | |
| 353 | 245 |
map.put("baseUrl", api.getBaseurl());
|
| 354 | 246 |
map.put("dsId", api.getDatasource());
|
| 355 | 247 |
map.put("contentDescription", api.getContentdescription());
|
| 356 |
map.put("typology", api.getTypology());
|
|
| 357 | 248 |
map.put("compatibility", api.getCompatibility());
|
| 358 | 249 |
map.put("metadataIdentifierPath", api.getMetadataIdentifierPath());
|
| 359 | 250 |
map.put("removable", api.getRemovable());
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/java/eu/dnetlib/enabling/datasources/DatasourceManagerClients.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.enabling.datasources; |
| 2 | 2 |
|
| 3 |
import java.io.StringReader; |
|
| 4 | 3 |
import java.text.ParseException; |
| 5 | 4 |
import java.util.Date; |
| 6 |
import java.util.HashMap; |
|
| 7 | 5 |
import java.util.List; |
| 8 | 6 |
import java.util.Map; |
| 9 | 7 |
import java.util.stream.Collectors; |
| ... | ... | |
| 11 | 9 |
import org.apache.commons.io.IOUtils; |
| 12 | 10 |
import org.apache.commons.logging.Log; |
| 13 | 11 |
import org.apache.commons.logging.LogFactory; |
| 14 |
import org.dom4j.Document; |
|
| 15 |
import org.dom4j.io.SAXReader; |
|
| 16 | 12 |
import org.quartz.CronExpression; |
| 17 | 13 |
import org.springframework.beans.factory.annotation.Autowired; |
| 18 | 14 |
import org.springframework.beans.factory.annotation.Required; |
| 19 | 15 |
import org.springframework.core.io.ClassPathResource; |
| 20 | 16 |
import org.springframework.core.io.Resource; |
| 21 | 17 |
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; |
| 18 |
import org.springframework.transaction.annotation.Transactional; |
|
| 22 | 19 |
|
| 23 | 20 |
import com.google.common.collect.ImmutableMap; |
| 24 | 21 |
|
| ... | ... | |
| 28 | 25 |
import eu.dnetlib.enabling.datasources.common.DsmException; |
| 29 | 26 |
import eu.dnetlib.enabling.datasources.common.Identity; |
| 30 | 27 |
import eu.dnetlib.enabling.datasources.common.Organization; |
| 28 |
import eu.dnetlib.enabling.datasources.common.PidSystem; |
|
| 31 | 29 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException; |
| 32 | 30 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; |
| 33 | 31 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
| 34 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; |
|
| 35 | 32 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
| 36 |
import org.springframework.transaction.annotation.Transactional; |
|
| 37 | 33 |
|
| 38 | 34 |
public class DatasourceManagerClients {
|
| 39 | 35 |
|
| 40 |
private static final String REPOSITORY_SERVICE_RESOURCE_TYPE = "RepositoryServiceResourceType"; |
|
| 41 | 36 |
private static final Log log = LogFactory.getLog(DatasourceManagerClients.class); |
| 42 | 37 |
|
| 43 | 38 |
private static final Resource dsQuery = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "getDatasource.sql"); |
| 39 |
private static final Resource dsQueryByPrefix = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "getDatasourceByPrefix.sql"); |
|
| 44 | 40 |
private static final Resource dsIdentitiesQuery = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "dsIdentitiesQuery.sql"); |
| 45 | 41 |
private static final Resource dsOrganizationsQuery = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "dsOrganizationsQuery.sql"); |
| 46 | 42 |
private static final Resource listApisByDsId = new ClassPathResource(LocalOpenaireDatasourceManagerImpl.QUERY_BASEDIR + "listApisByDsId.sql"); |
| ... | ... | |
| 48 | 44 |
|
| 49 | 45 |
private NamedParameterJdbcTemplate jdbcTemplate; |
| 50 | 46 |
|
| 51 |
public enum AfterSqlUpdate {
|
|
| 52 |
DELETE_DS_PROFILE, UPDATE_DS_PROFILE, NONE |
|
| 53 |
} |
|
| 54 |
|
|
| 55 | 47 |
@Autowired |
| 56 | 48 |
private UniqueServiceLocator serviceLocator; |
| 57 | 49 |
|
| 58 |
public String findDatasourceId(final String profileId) throws DsmException {
|
|
| 59 |
try {
|
|
| 60 |
return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery( |
|
| 61 |
"/*[.//RESOURCE_IDENTIFIER/@value='" + profileId + "']//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value/text()"); |
|
| 62 |
} catch (final Exception e) {
|
|
| 63 |
log.error("Error finding dsId of profile " + profileId, e);
|
|
| 64 |
throw new DsmException(-1, "Error finding dsId of profile " + profileId, e); |
|
| 65 |
} |
|
| 66 |
} |
|
| 67 |
|
|
| 68 |
public String getDatasourceProfile(final String dsId) throws DsmException {
|
|
| 69 |
try {
|
|
| 70 |
return serviceLocator.getService(ISLookUpService.class) |
|
| 71 |
.getResourceProfileByQuery( |
|
| 72 |
"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')/*[.//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value/text() = '"
|
|
| 73 |
+ dsId + "']"); |
|
| 74 |
} catch (final Exception e) {
|
|
| 75 |
return null; |
|
| 76 |
} |
|
| 77 |
} |
|
| 78 |
|
|
| 79 |
public boolean deleteProfile(final String dsId) throws DsmException {
|
|
| 80 |
try {
|
|
| 81 |
final SAXReader reader = new SAXReader(); |
|
| 82 |
|
|
| 83 |
final String profile = getDatasourceProfile(dsId); |
|
| 84 |
|
|
| 85 |
if (profile != null) {
|
|
| 86 |
final Document docOld = reader.read(new StringReader(profile)); |
|
| 87 |
final String profId = docOld.valueOf("//RESOURCE_IDENTIFIER/@value");
|
|
| 88 |
serviceLocator.getService(ISRegistryService.class).deleteProfile(profId); |
|
| 89 |
} |
|
| 90 |
return true; |
|
| 91 |
} catch (final Exception e) {
|
|
| 92 |
log.error("Error deleting profile", e);
|
|
| 93 |
throw new DsmException(-1, "Error deleting profile", e); |
|
| 94 |
} |
|
| 95 |
} |
|
| 96 |
|
|
| 97 |
public boolean regenerateProfile(final String dsId) throws DsmException {
|
|
| 98 |
return regenerateProfile(dsId, serviceLocator.getService(ISRegistryService.class)); |
|
| 99 |
} |
|
| 100 |
|
|
| 101 |
public boolean regenerateProfile(final String dsId, final ISRegistryService registry) throws DsmException {
|
|
| 102 |
|
|
| 103 |
final Datasource<Organization<?>, Identity> ds = getDatasourceById(dsId); |
|
| 104 |
final List<Api<ApiParam>> apis = getApis(dsId); |
|
| 105 |
|
|
| 106 |
try {
|
|
| 107 |
|
|
| 108 |
final String oldProfile = getDatasourceProfile(dsId); |
|
| 109 |
|
|
| 110 |
if (oldProfile != null) {
|
|
| 111 |
final Document docOld = new SAXReader().read(new StringReader(oldProfile)); |
|
| 112 |
final String profId = docOld.valueOf("//RESOURCE_IDENTIFIER/@value");
|
|
| 113 |
final String profile = DatasourceFunctions.dsToProfile(ds, apis, profId); |
|
| 114 |
registry.updateProfile(profId, profile, REPOSITORY_SERVICE_RESOURCE_TYPE); |
|
| 115 |
log.info("Profile " + profId + " UPDATED for ds " + dsId);
|
|
| 116 |
} else {
|
|
| 117 |
final String profile = DatasourceFunctions.dsToProfile(ds, apis, ""); |
|
| 118 |
final String profId = registry.registerProfile(profile); |
|
| 119 |
log.info("Valid Profile " + profId + " REGISTERED for ds " + dsId);
|
|
| 120 |
} |
|
| 121 |
return true; |
|
| 122 |
} catch (final Exception e) {
|
|
| 123 |
log.error("Error saving profile, id: " + dsId, e);
|
|
| 124 |
throw new DsmException(-1, "Error regenerating profile", e); |
|
| 125 |
} |
|
| 126 |
} |
|
| 127 |
|
|
| 128 |
|
|
| 129 | 50 |
@Transactional(readOnly = true) |
| 130 | 51 |
public List<Map<String, Object>> searchSQL(final String sql, final Map<String, Object> sqlParams) throws DsmException {
|
| 131 | 52 |
try {
|
| ... | ... | |
| 138 | 59 |
} |
| 139 | 60 |
} |
| 140 | 61 |
|
| 141 |
|
|
| 142 | 62 |
@Transactional(readOnly = true) |
| 143 | 63 |
public List<Map<String, Object>> searchSQL(final Resource sqlResource, final Map<String, Object> sqlParams) throws DsmException {
|
| 144 | 64 |
try {
|
| ... | ... | |
| 149 | 69 |
} |
| 150 | 70 |
} |
| 151 | 71 |
|
| 152 |
|
|
| 153 | 72 |
@Transactional |
| 154 |
public void updateSQL(final String dsId, final String sql, final AfterSqlUpdate op, final Map<String, Object> sqlparams)
|
|
| 155 |
throws DsmException {
|
|
| 73 |
public void updateSQL(final String dsId, final String sql, final Map<String, Object> sqlparams) |
|
| 74 |
throws DsmException {
|
|
| 156 | 75 |
log.debug("Executing query SQL: " + sql);
|
| 157 | 76 |
|
| 158 | 77 |
jdbcTemplate.update(sql, sqlparams); |
| 78 |
} |
|
| 159 | 79 |
|
| 160 |
switch (op) {
|
|
| 161 |
case DELETE_DS_PROFILE: |
|
| 162 |
deleteProfile(dsId); |
|
| 163 |
break; |
|
| 164 |
case UPDATE_DS_PROFILE: |
|
| 165 |
|
|
| 166 |
regenerateProfile(dsId, serviceLocator.getService(ISRegistryService.class)); |
|
| 167 |
break; |
|
| 168 |
default: |
|
| 169 |
break; |
|
| 170 |
} |
|
| 171 |
|
|
| 172 |
} |
|
| 173 | 80 |
@Transactional |
| 174 |
public void updateSQL(final String dsId, final Resource sqlResource, final AfterSqlUpdate op, final Map<String, Object> sqlparams)
|
|
| 175 |
throws DsmException {
|
|
| 81 |
public void updateSQL(final String dsId, final Resource sqlResource, final Map<String, Object> sqlparams) |
|
| 82 |
throws DsmException {
|
|
| 176 | 83 |
try {
|
| 177 |
updateSQL(dsId, IOUtils.toString(sqlResource.getInputStream()), op, sqlparams);
|
|
| 84 |
updateSQL(dsId, IOUtils.toString(sqlResource.getInputStream()), sqlparams); |
|
| 178 | 85 |
} catch (final Exception e) {
|
| 179 | 86 |
log.error("Error in updateSQL", e);
|
| 180 | 87 |
throw new DsmException(-1, "Error in updateSQL", e); |
| ... | ... | |
| 182 | 89 |
} |
| 183 | 90 |
|
| 184 | 91 |
@Transactional(readOnly = true) |
| 185 |
public Datasource<Organization<?>, Identity> getDatasourceById(final String id) throws DsmException {
|
|
| 92 |
public Datasource<Organization<?>, Identity, PidSystem> getDatasourceById(final String id) throws DsmException {
|
|
| 186 | 93 |
final List<Map<String, Object>> list = searchSQL(dsQuery, ImmutableMap.of("dsId", id));
|
| 187 | 94 |
|
| 188 | 95 |
if (list.size() != 1) { throw new DsmException("Invalid number of ds with id: " + id); }
|
| 189 | 96 |
|
| 190 |
final Datasource<Organization<?>, Identity> ds = DatasourceFunctions.mapToDatasource(list.get(0)); |
|
| 97 |
final Datasource<Organization<?>, Identity, PidSystem> ds = DatasourceFunctions.mapToDatasource(list.get(0));
|
|
| 191 | 98 |
ds.setIdentities(searchSQL(dsIdentitiesQuery, ImmutableMap.of("dsId", id))
|
| 192 |
.stream()
|
|
| 193 |
.map(DatasourceFunctions::mapToDsIdentity)
|
|
| 194 |
.collect(Collectors.toSet()));
|
|
| 99 |
.stream() |
|
| 100 |
.map(DatasourceFunctions::mapToDsIdentity) |
|
| 101 |
.collect(Collectors.toSet())); |
|
| 195 | 102 |
ds.setOrganizations(searchSQL(dsOrganizationsQuery, ImmutableMap.of("dsId", id))
|
| 196 |
.stream()
|
|
| 197 |
.map(DatasourceFunctions::mapToDsOrganization)
|
|
| 198 |
.collect(Collectors.toSet()));
|
|
| 103 |
.stream() |
|
| 104 |
.map(DatasourceFunctions::mapToDsOrganization) |
|
| 105 |
.collect(Collectors.toSet())); |
|
| 199 | 106 |
|
| 200 | 107 |
return ds; |
| 201 | 108 |
} |
| 202 | 109 |
|
| 110 |
public Datasource<Organization<?>, Identity, PidSystem> getDatasourceByPrefix(final String prefix) throws DsmException {
|
|
| 111 |
final List<Map<String, Object>> list = searchSQL(dsQueryByPrefix, ImmutableMap.of("prefix", prefix));
|
|
| 203 | 112 |
|
| 113 |
if (list.size() != 1) { throw new DsmException("Invalid number of ds with prefix: " + prefix); }
|
|
| 114 |
|
|
| 115 |
final Datasource<Organization<?>, Identity, PidSystem> ds = DatasourceFunctions.mapToDatasource(list.get(0)); |
|
| 116 |
ds.setIdentities(searchSQL(dsIdentitiesQuery, ImmutableMap.of("dsId", ds.getId()))
|
|
| 117 |
.stream() |
|
| 118 |
.map(DatasourceFunctions::mapToDsIdentity) |
|
| 119 |
.collect(Collectors.toSet())); |
|
| 120 |
ds.setOrganizations(searchSQL(dsOrganizationsQuery, ImmutableMap.of("dsId", ds.getId()))
|
|
| 121 |
.stream() |
|
| 122 |
.map(DatasourceFunctions::mapToDsOrganization) |
|
| 123 |
.collect(Collectors.toSet())); |
|
| 124 |
|
|
| 125 |
return ds; |
|
| 126 |
} |
|
| 127 |
|
|
| 204 | 128 |
@Transactional(readOnly = true) |
| 205 | 129 |
public List<Api<ApiParam>> getApis(final String dsId) throws DsmException {
|
| 206 | 130 |
|
| 207 | 131 |
return searchSQL(listApisByDsId, ImmutableMap.of("dsId", dsId))
|
| 208 |
.stream()
|
|
| 209 |
.map(DatasourceFunctions::mapToApi)
|
|
| 210 |
.collect(Collectors.toList());
|
|
| 132 |
.stream() |
|
| 133 |
.map(DatasourceFunctions::mapToApi) |
|
| 134 |
.collect(Collectors.toList()); |
|
| 211 | 135 |
} |
| 212 | 136 |
|
| 213 | 137 |
@Transactional(readOnly = true) |
| ... | ... | |
| 233 | 157 |
} |
| 234 | 158 |
} |
| 235 | 159 |
|
| 236 |
@Transactional(readOnly = true) |
|
| 237 |
public void regenerateProfiles() throws DsmException {
|
|
| 238 |
final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class); |
|
| 239 |
searchSQL("SELECT id FROM dsm_datasources", new HashMap<>()).stream()
|
|
| 240 |
.map(m -> m.get("id").toString())
|
|
| 241 |
.forEach(id -> {
|
|
| 242 |
try {
|
|
| 243 |
regenerateProfile(id, registry); |
|
| 244 |
} catch (final DsmException e) {
|
|
| 245 |
log.error("Error regeneating profile: " + id, e);
|
|
| 246 |
} |
|
| 247 |
}); |
|
| 248 |
} |
|
| 249 |
|
|
| 250 | 160 |
public NamedParameterJdbcTemplate getJdbcTemplate() {
|
| 251 | 161 |
return jdbcTemplate; |
| 252 | 162 |
} |
| modules/dnet-openaire-datasource-manager/trunk/src/main/java/eu/dnetlib/enabling/datasources/LocalOpenaireDatasourceManager.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.enabling.datasources; |
| 2 | 2 |
|
| 3 |
import eu.dnetlib.enabling.datasources.common.Api; |
|
| 4 |
import eu.dnetlib.enabling.datasources.common.ApiParam; |
|
| 3 | 5 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
| 4 |
import eu.dnetlib.enabling.datasources.common.Organization; |
|
| 5 | 6 |
import eu.dnetlib.enabling.datasources.common.Identity; |
| 6 |
import eu.dnetlib.enabling.datasources.common.ApiParam; |
|
| 7 |
import eu.dnetlib.enabling.datasources.common.Api; |
|
| 8 | 7 |
import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager; |
| 8 |
import eu.dnetlib.enabling.datasources.common.Organization; |
|
| 9 |
import eu.dnetlib.enabling.datasources.common.PidSystem; |
|
| 9 | 10 |
|
| 10 |
public interface LocalOpenaireDatasourceManager extends LocalDatasourceManager<Datasource<Organization<?>, Identity>, Api<ApiParam>> {
|
|
| 11 |
public interface LocalOpenaireDatasourceManager extends LocalDatasourceManager<Datasource<Organization<?>, Identity, PidSystem>, Api<ApiParam>> {
|
|
| 11 | 12 |
|
| 12 | 13 |
} |
| modules/dnet-openaire-datasource-manager/trunk/src/main/java/eu/dnetlib/enabling/datasources/LocalOpenaireDatasourceManagerImpl.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.enabling.datasources; |
| 2 | 2 |
|
| 3 | 3 |
import java.io.StringWriter; |
| 4 |
import java.util.*; |
|
| 4 |
import java.util.ArrayList; |
|
| 5 |
import java.util.Date; |
|
| 6 |
import java.util.HashMap; |
|
| 7 |
import java.util.HashSet; |
|
| 8 |
import java.util.List; |
|
| 9 |
import java.util.Map; |
|
| 10 |
import java.util.Optional; |
|
| 11 |
import java.util.Set; |
|
| 5 | 12 |
import java.util.stream.Collectors; |
| 6 | 13 |
|
| 7 |
import com.google.common.collect.ImmutableMap; |
|
| 8 |
import eu.dnetlib.enabling.datasources.DatasourceManagerClients.AfterSqlUpdate; |
|
| 9 |
import eu.dnetlib.enabling.datasources.common.*; |
|
| 10 | 14 |
import org.apache.commons.io.IOUtils; |
| 11 | 15 |
import org.apache.commons.lang3.StringUtils; |
| 12 | 16 |
import org.apache.commons.lang3.math.NumberUtils; |
| ... | ... | |
| 17 | 21 |
import org.springframework.core.io.Resource; |
| 18 | 22 |
import org.springframework.transaction.annotation.Transactional; |
| 19 | 23 |
|
| 24 |
import com.google.common.collect.ImmutableMap; |
|
| 25 |
|
|
| 26 |
import eu.dnetlib.enabling.datasources.common.Api; |
|
| 27 |
import eu.dnetlib.enabling.datasources.common.ApiParam; |
|
| 28 |
import eu.dnetlib.enabling.datasources.common.BrowsableField; |
|
| 29 |
import eu.dnetlib.enabling.datasources.common.BrowseTerm; |
|
| 30 |
import eu.dnetlib.enabling.datasources.common.BrowseTermImpl; |
|
| 31 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
|
| 32 |
import eu.dnetlib.enabling.datasources.common.DsmException; |
|
| 33 |
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; |
|
| 34 |
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; |
|
| 35 |
import eu.dnetlib.enabling.datasources.common.DsmRuntimeException; |
|
| 36 |
import eu.dnetlib.enabling.datasources.common.Identity; |
|
| 37 |
import eu.dnetlib.enabling.datasources.common.Organization; |
|
| 38 |
import eu.dnetlib.enabling.datasources.common.PidSystem; |
|
| 39 |
import eu.dnetlib.enabling.datasources.common.SearchApisEntry; |
|
| 40 |
import eu.dnetlib.enabling.datasources.common.SimpleDatasource; |
|
| 41 |
|
|
| 20 | 42 |
public class LocalOpenaireDatasourceManagerImpl implements LocalOpenaireDatasourceManager {
|
| 21 | 43 |
|
| 22 | 44 |
private DatasourceManagerClients datasourceManagerClients; |
| ... | ... | |
| 47 | 69 |
|
| 48 | 70 |
private static final Log log = LogFactory.getLog(LocalOpenaireDatasourceManagerImpl.class); |
| 49 | 71 |
|
| 50 |
private static final String REPO_PROFILEID_SUFFIX = "_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU="; |
|
| 51 |
|
|
| 52 | 72 |
@Override |
| 53 | 73 |
@Transactional(readOnly = true) |
| 54 | 74 |
public Set<String> listManagedDatasourceIds() throws DsmRuntimeException {
|
| 55 | 75 |
try {
|
| 56 |
return datasourceManagerClients.searchSQL("SELECT id FROM dsm_datasources WHERE managed = true", new HashMap<>())
|
|
| 57 |
.stream()
|
|
| 58 |
.map(m -> (String) m.get("id"))
|
|
| 59 |
.collect(Collectors.toCollection(HashSet::new));
|
|
| 76 |
return datasourceManagerClients.searchSQL("SELECT id FROM dsm_services WHERE managed = true", new HashMap<>())
|
|
| 77 |
.stream() |
|
| 78 |
.map(m -> (String) m.get("id"))
|
|
| 79 |
.collect(Collectors.toCollection(HashSet::new)); |
|
| 60 | 80 |
} catch (final DsmException e) {
|
| 61 | 81 |
throw new DsmRuntimeException(e); |
| 62 | 82 |
} |
| ... | ... | |
| 67 | 87 |
public List<SimpleDatasource> searchDatasourcesByType(final String type) throws DsmException {
|
| 68 | 88 |
|
| 69 | 89 |
return datasourceManagerClients.searchSQL(searchDsByType, ImmutableMap.of("type", type))
|
| 70 |
.stream()
|
|
| 71 |
.map(DatasourceFunctions::mapToSimpleDs)
|
|
| 72 |
.collect(Collectors.toList());
|
|
| 90 |
.stream() |
|
| 91 |
.map(DatasourceFunctions::mapToSimpleDs) |
|
| 92 |
.collect(Collectors.toList()); |
|
| 73 | 93 |
} |
| 74 | 94 |
|
| 75 | 95 |
@Override |
| ... | ... | |
| 88 | 108 |
} |
| 89 | 109 |
|
| 90 | 110 |
return datasourceManagerClients |
| 91 |
.searchSQL(sql.toString(), ImmutableMap.of("value", field.equalsIgnoreCase("__search__") ? "%" + value + "%" : value))
|
|
| 92 |
.stream()
|
|
| 93 |
.map(DatasourceFunctions::mapToSearchApisEntry)
|
|
| 94 |
.collect(Collectors.toList());
|
|
| 111 |
.searchSQL(sql.toString(), ImmutableMap.of("value", field.equalsIgnoreCase("__search__") ? "%" + value + "%" : value))
|
|
| 112 |
.stream() |
|
| 113 |
.map(DatasourceFunctions::mapToSearchApisEntry) |
|
| 114 |
.collect(Collectors.toList()); |
|
| 95 | 115 |
|
| 96 | 116 |
} catch (final Exception e) {
|
| 97 | 117 |
log.error("Error searching field " + field + " - value: " + value, e);
|
| ... | ... | |
| 101 | 121 |
|
| 102 | 122 |
@Override |
| 103 | 123 |
@Transactional |
| 104 |
public void saveDs(final Datasource<Organization<?>, Identity> ds) throws DsmException {
|
|
| 124 |
public void saveDs(final Datasource<Organization<?>, Identity, PidSystem> ds) throws DsmException {
|
|
| 105 | 125 |
|
| 106 | 126 |
if (StringUtils.isBlank(ds.getAggregator())) {
|
| 107 | 127 |
ds.setAggregator("OPENAIRE");
|
| ... | ... | |
| 109 | 129 |
|
| 110 | 130 |
ds.setManaged(true); |
| 111 | 131 |
|
| 112 |
datasourceManagerClients.updateSQL(ds.getId(), addDs, AfterSqlUpdate.NONE, DatasourceFunctions.dsToMap(ds));
|
|
| 132 |
datasourceManagerClients.updateSQL(ds.getId(), addDs, DatasourceFunctions.dsToMap(ds)); |
|
| 113 | 133 |
|
| 114 | 134 |
if (ds.getOrganizations() != null) {
|
| 115 | 135 |
for (final Organization<?> org : ds.getOrganizations()) {
|
| 116 | 136 |
final Map<String, Object> orgParams = DatasourceFunctions.orgToMap(ds.getId(), org); |
| 117 |
datasourceManagerClients.updateSQL(ds.getId(), addOrg, AfterSqlUpdate.NONE, orgParams);
|
|
| 137 |
datasourceManagerClients.updateSQL(ds.getId(), addOrg, orgParams); |
|
| 118 | 138 |
} |
| 119 | 139 |
} |
| 120 | 140 |
final Map<String, Object> params = new HashMap<>(); |
| 121 | 141 |
params.put("removable", true);
|
| 122 | 142 |
params.put("dsId", ds.getId());
|
| 123 |
datasourceManagerClients.updateSQL(ds.getId(), setRemovable, AfterSqlUpdate.NONE, params); |
|
| 124 |
|
|
| 125 |
datasourceManagerClients.regenerateProfile(ds.getId()); |
|
| 143 |
datasourceManagerClients.updateSQL(ds.getId(), setRemovable, params); |
|
| 126 | 144 |
} |
| 127 | 145 |
|
| 128 | 146 |
@Override |
| 129 | 147 |
@Transactional |
| 130 | 148 |
public void deleteDs(final String dsId) throws DsmException {
|
| 131 |
datasourceManagerClients.updateSQL(fixDsId(dsId), deleteDs, AfterSqlUpdate.DELETE_DS_PROFILE, ImmutableMap.of("dsId", dsId));
|
|
| 149 |
datasourceManagerClients.updateSQL(dsId, deleteDs, ImmutableMap.of("dsId", dsId));
|
|
| 132 | 150 |
} |
| 133 | 151 |
|
| 134 | 152 |
@Override |
| 135 |
public Datasource<Organization<?>, Identity> getDs(final String dsId) throws DsmException {
|
|
| 136 |
return datasourceManagerClients.getDatasourceById(fixDsId(dsId));
|
|
| 153 |
public Datasource<Organization<?>, Identity, PidSystem> getDs(final String dsId) throws DsmException {
|
|
| 154 |
return datasourceManagerClients.getDatasourceById(dsId);
|
|
| 137 | 155 |
} |
| 138 | 156 |
|
| 139 | 157 |
@Override |
| 140 |
public void regenerateProfiles() throws DsmException {
|
|
| 141 |
datasourceManagerClients.regenerateProfiles();
|
|
| 158 |
public Datasource<Organization<?>, Identity, PidSystem> getDsByNsPrefix(final String prefix) throws DsmException {
|
|
| 159 |
return datasourceManagerClients.getDatasourceByPrefix(prefix);
|
|
| 142 | 160 |
} |
| 143 | 161 |
|
| 144 | 162 |
@Override |
| 145 | 163 |
public List<Api<ApiParam>> getApis(final String dsId) throws DsmException {
|
| 146 |
return datasourceManagerClients.getApis(fixDsId(dsId));
|
|
| 164 |
return datasourceManagerClients.getApis(dsId);
|
|
| 147 | 165 |
} |
| 148 | 166 |
|
| 149 | 167 |
@Override |
| 150 | 168 |
public void setManaged(final String dsId, final boolean managed) throws DsmException {
|
| 151 |
final String id = fixDsId(dsId);
|
|
| 169 |
final String id = dsId;
|
|
| 152 | 170 |
|
| 153 | 171 |
final Map<String, Object> params = new HashMap<>(); |
| 154 | 172 |
params.put("managed", managed);
|
| 155 | 173 |
params.put("dsId", id);
|
| 156 | 174 |
|
| 157 |
datasourceManagerClients.updateSQL(id, setManaged, AfterSqlUpdate.NONE, params);
|
|
| 175 |
datasourceManagerClients.updateSQL(id, setManaged, params); |
|
| 158 | 176 |
|
| 159 | 177 |
params.clear(); |
| 160 | 178 |
params.put("removable", true);
|
| 161 | 179 |
params.put("dsId", id);
|
| 162 | 180 |
|
| 163 |
datasourceManagerClients.updateSQL(id, setRemovable, AfterSqlUpdate.UPDATE_DS_PROFILE, params);
|
|
| 181 |
datasourceManagerClients.updateSQL(id, setRemovable, params); |
|
| 164 | 182 |
|
| 165 | 183 |
} |
| 166 | 184 |
|
| 167 | 185 |
@Override |
| 168 | 186 |
public boolean isManaged(final String dsId) throws DsmException {
|
| 169 |
final String q = "SELECT * from dsm_datasources WHERE id = :dsId AND managed = true";
|
|
| 170 |
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId))).isEmpty();
|
|
| 187 |
final String q = "SELECT id from dsm_services WHERE id = :dsId AND managed = true";
|
|
| 188 |
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", dsId)).isEmpty();
|
|
| 171 | 189 |
} |
| 172 | 190 |
|
| 173 | 191 |
@Override |
| 174 | 192 |
@Transactional |
| 175 | 193 |
public void setActive(final String dsId, final String apiId, final boolean active) throws DsmException {
|
| 176 |
final String id = fixDsId(dsId);
|
|
| 194 |
final String id = dsId;
|
|
| 177 | 195 |
|
| 178 | 196 |
final Map<String, Object> params = new HashMap<>(); |
| 179 | 197 |
params.put("active", active);
|
| 180 | 198 |
params.put("apiId", apiId);
|
| 181 | 199 |
params.put("dsId", id);
|
| 182 | 200 |
|
| 183 |
datasourceManagerClients.updateSQL(id, setActive, AfterSqlUpdate.NONE, params);
|
|
| 201 |
datasourceManagerClients.updateSQL(id, setActive, params); |
|
| 184 | 202 |
if (!active) {
|
| 185 |
datasourceManagerClients.updateSQL(dsId, resetLastOperationsInfo, AfterSqlUpdate.NONE, params);
|
|
| 203 |
datasourceManagerClients.updateSQL(dsId, resetLastOperationsInfo, params); |
|
| 186 | 204 |
} |
| 187 | 205 |
|
| 188 |
setManaged(id, true); // It also update the IS profile
|
|
| 206 |
setManaged(id, true); |
|
| 189 | 207 |
} |
| 190 | 208 |
|
| 191 | 209 |
@Override |
| 192 | 210 |
@Transactional(readOnly = true) |
| 193 | 211 |
public boolean isActive(final String dsId, final String apiId) throws DsmException {
|
| 194 |
final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active = true";
|
|
| 195 |
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
|
|
| 212 |
final String q = "SELECT * from dsm_api WHERE id = :apiId AND service = :dsId AND active = true";
|
|
| 213 |
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", dsId, "apiId", apiId)).isEmpty();
|
|
| 196 | 214 |
} |
| 197 | 215 |
|
| 198 | 216 |
@Override |
| 199 | 217 |
@Transactional(readOnly = true) |
| 200 | 218 |
public boolean isRemovable(final String dsId, final String apiId) throws DsmException {
|
| 201 |
final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active != true AND removable = true";
|
|
| 202 |
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();
|
|
| 219 |
final String q = "SELECT * from dsm_api WHERE id = :apiId AND service = :dsId AND active != true AND removable = true";
|
|
| 220 |
return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", dsId, "apiId", apiId)).isEmpty();
|
|
| 203 | 221 |
} |
| 204 | 222 |
|
| 205 | 223 |
@Override |
| 206 | 224 |
@Transactional |
| 207 | 225 |
public void updateCompliance(final String dsId, final String apiId, final String level, final boolean override) throws DsmException {
|
| 208 |
final String id = fixDsId(dsId);
|
|
| 226 |
final String id = dsId;
|
|
| 209 | 227 |
|
| 210 | 228 |
if (!override) {
|
| 211 |
datasourceManagerClients.updateSQL(id, setCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
|
|
| 229 |
datasourceManagerClients.updateSQL(id, setCompliance, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
|
|
| 212 | 230 |
} else if (level != null) {
|
| 213 |
datasourceManagerClients.updateSQL(id, overrideCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
|
|
| 231 |
datasourceManagerClients.updateSQL(id, overrideCompliance, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));
|
|
| 214 | 232 |
} else {
|
| 215 |
datasourceManagerClients.updateSQL(id, resetCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("apiId", apiId, "dsId", id));
|
|
| 233 |
datasourceManagerClients.updateSQL(id, resetCompliance, ImmutableMap.of("apiId", apiId, "dsId", id));
|
|
| 216 | 234 |
} |
| 217 | 235 |
|
| 218 |
setManaged(id, true); // It also update the IS profile
|
|
| 236 |
setManaged(id, true); |
|
| 219 | 237 |
} |
| 220 | 238 |
|
| 221 | 239 |
@Override |
| 222 |
|
|
| 223 | 240 |
public void setLastCollectionInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date) |
| 224 |
throws DsmException {
|
|
| 225 |
setLastOperationInfo(setLastCollectionInfo, fixDsId(dsId), apiId, mdId, size, date);
|
|
| 241 |
throws DsmException {
|
|
| 242 |
setLastOperationInfo(setLastCollectionInfo, dsId, apiId, mdId, size, date);
|
|
| 226 | 243 |
} |
| 227 | 244 |
|
| 228 | 245 |
@Override |
| 229 | 246 |
public void setLastAggregationInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date) |
| 230 |
throws DsmException {
|
|
| 231 |
setLastOperationInfo(setLastAggregationInfo, fixDsId(dsId), apiId, mdId, size, date);
|
|
| 247 |
throws DsmException {
|
|
| 248 |
setLastOperationInfo(setLastAggregationInfo, dsId, apiId, mdId, size, date);
|
|
| 232 | 249 |
} |
| 233 | 250 |
|
| 234 | 251 |
@Override |
| 235 | 252 |
public void setLastDownloadInfo(final String dsId, final String apiId, final String objId, final Integer size, final Date date) |
| 236 |
throws DsmException {
|
|
| 237 |
setLastOperationInfo(setLastDownloadInfo, fixDsId(dsId), apiId, objId, size, date);
|
|
| 253 |
throws DsmException {
|
|
| 254 |
setLastOperationInfo(setLastDownloadInfo, dsId, apiId, objId, size, date);
|
|
| 238 | 255 |
} |
| 239 | 256 |
|
| 240 | 257 |
@Override |
| 241 |
public void setLastValidationJob(String dsId, String apiId, String jobId) throws DsmException {
|
|
| 258 |
public void setLastValidationJob(final String dsId, final String apiId, final String jobId) throws DsmException {
|
|
| 242 | 259 |
final Map<String, Object> params = new HashMap<>(); |
| 243 | 260 |
params.put("dsId", dsId);
|
| 244 | 261 |
params.put("apiId", apiId);
|
| 245 | 262 |
params.put("jobId", jobId);
|
| 246 | 263 |
|
| 247 |
datasourceManagerClients.updateSQL(dsId, setLastValidationJob, AfterSqlUpdate.NONE, params);
|
|
| 248 |
setManaged(dsId, true); // It also update the IS profile
|
|
| 264 |
datasourceManagerClients.updateSQL(dsId, setLastValidationJob, params); |
|
| 265 |
setManaged(dsId, true); |
|
| 249 | 266 |
} |
| 250 | 267 |
|
| 251 | 268 |
@Transactional |
| 252 |
protected void setLastOperationInfo(final Resource sqlResource, final String dsId, final String apiId, final String mdId, final Integer size, final Date date) |
|
| 253 |
throws DsmException {
|
|
| 269 |
protected void setLastOperationInfo(final Resource sqlResource, |
|
| 270 |
final String dsId, |
|
| 271 |
final String apiId, |
|
| 272 |
final String mdId, |
|
| 273 |
final Integer size, |
|
| 274 |
final Date date) |
|
| 275 |
throws DsmException {
|
|
| 254 | 276 |
final Map<String, Object> params = new HashMap<>(); |
| 255 | 277 |
params.put("dsId", dsId);
|
| 256 | 278 |
params.put("apiId", apiId);
|
| ... | ... | |
| 260 | 282 |
params.put("date", new java.sql.Timestamp(date.getTime()));
|
| 261 | 283 |
} |
| 262 | 284 |
|
| 263 |
datasourceManagerClients.updateSQL(dsId, sqlResource, AfterSqlUpdate.NONE, params);
|
|
| 285 |
datasourceManagerClients.updateSQL(dsId, sqlResource, params); |
|
| 264 | 286 |
|
| 265 |
setManaged(dsId, true); // It also update the IS profile
|
|
| 287 |
setManaged(dsId, true); |
|
| 266 | 288 |
} |
| 267 | 289 |
|
| 268 | 290 |
@Override |
| 269 | 291 |
@Transactional |
| 270 | 292 |
public void updateApiDetails(final String dsId, |
| 271 |
final String apiId,
|
|
| 272 |
final String metadataIdentifierPath,
|
|
| 273 |
final String baseUrl,
|
|
| 274 |
final Map<String, String> params)
|
|
| 275 |
throws DsmException {
|
|
| 293 |
final String apiId, |
|
| 294 |
final String metadataIdentifierPath, |
|
| 295 |
final String baseUrl, |
|
| 296 |
final Map<String, String> params) |
|
| 297 |
throws DsmException {
|
|
| 276 | 298 |
|
| 277 |
final String id = fixDsId(dsId);
|
|
| 299 |
final String id = dsId;
|
|
| 278 | 300 |
|
| 279 | 301 |
// Delete old params |
| 280 |
datasourceManagerClients.updateSQL(id, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
|
|
| 302 |
datasourceManagerClients.updateSQL(id, "DELETE FROM dsm_apiparams WHERE api = :api", ImmutableMap.of("api", apiId));
|
|
| 281 | 303 |
|
| 282 | 304 |
// Insert new params |
| 283 | 305 |
for (final Map.Entry<String, String> e : params.entrySet()) {
|
| 284 | 306 |
final Map<String, Object> sqlParams = ImmutableMap.of("param", e.getKey(), "value", e.getValue(), "api", apiId);
|
| 285 |
datasourceManagerClients.updateSQL(id, insertApiParam, AfterSqlUpdate.NONE, sqlParams);
|
|
| 307 |
datasourceManagerClients.updateSQL(id, insertApiParam, sqlParams); |
|
| 286 | 308 |
} |
| 287 | 309 |
|
| 288 | 310 |
// Update the BaseURL |
| 289 |
datasourceManagerClients.updateSQL(id, |
|
| 290 |
"UPDATE dsm_api SET baseurl = :baseurl WHERE id = :api", |
|
| 291 |
AfterSqlUpdate.NONE, |
|
| 292 |
ImmutableMap.of("baseurl", baseUrl, "api", apiId));
|
|
| 311 |
datasourceManagerClients |
|
| 312 |
.updateSQL(id, "UPDATE dsm_api SET baseurl = :baseurl WHERE id = :api", ImmutableMap.of("baseurl", baseUrl, "api", apiId));
|
|
| 293 | 313 |
|
| 294 | 314 |
// Update the metadata_identifier_path |
| 295 |
datasourceManagerClients.updateSQL(id, |
|
| 296 |
"UPDATE dsm_api SET metadata_identifier_path = :path WHERE id = :api", |
|
| 297 |
AfterSqlUpdate.NONE, |
|
| 298 |
ImmutableMap.of("path", metadataIdentifierPath, "api", apiId));
|
|
| 315 |
datasourceManagerClients.updateSQL(id, "UPDATE dsm_api SET metadata_identifier_path = :path WHERE id = :api", ImmutableMap |
|
| 316 |
.of("path", metadataIdentifierPath, "api", apiId));
|
|
| 299 | 317 |
|
| 300 |
setManaged(id, true); // It also update the IS profile
|
|
| 318 |
setManaged(id, true); |
|
| 301 | 319 |
} |
| 302 | 320 |
|
| 303 | 321 |
@Override |
| ... | ... | |
| 308 | 326 |
@Override |
| 309 | 327 |
public List<BrowseTerm> browseField(final String field) throws DsmException {
|
| 310 | 328 |
final Optional<DbBrowsableField> bf = getBrowsableFields() |
| 311 |
.stream()
|
|
| 312 |
.filter(f -> f.getId().equals(field))
|
|
| 313 |
.findFirst();
|
|
| 329 |
.stream() |
|
| 330 |
.filter(f -> f.getId().equals(field)) |
|
| 331 |
.findFirst(); |
|
| 314 | 332 |
|
| 315 | 333 |
if (bf.isPresent()) {
|
| 316 | 334 |
return datasourceManagerClients.searchSQL(bf.get().getSql(), new HashMap<>()) |
| 317 |
.stream()
|
|
| 318 |
.filter(m -> m.get("term") != null)
|
|
| 319 |
.filter(m -> m.get("count") != null)
|
|
| 320 |
.filter(m -> StringUtils.isNotBlank(m.get("term").toString()))
|
|
| 321 |
.map(m -> new BrowseTermImpl(m.get("term").toString(), NumberUtils.toInt(m.get("count").toString(), 0)))
|
|
| 322 |
.collect(Collectors.toList());
|
|
| 335 |
.stream() |
|
| 336 |
.filter(m -> m.get("term") != null)
|
|
| 337 |
.filter(m -> m.get("count") != null)
|
|
| 338 |
.filter(m -> StringUtils.isNotBlank(m.get("term").toString()))
|
|
| 339 |
.map(m -> new BrowseTermImpl(m.get("term").toString(), NumberUtils.toInt(m.get("count").toString(), 0)))
|
|
| 340 |
.collect(Collectors.toList()); |
|
| 323 | 341 |
} else {
|
| 324 | 342 |
log.error("Not browsable field:" + field);
|
| 325 | 343 |
throw new DsmException("Not browsable field:" + field);
|
| ... | ... | |
| 330 | 348 |
@Transactional |
| 331 | 349 |
public void addApi(final Api<ApiParam> api) throws DsmException {
|
| 332 | 350 |
|
| 333 |
datasourceManagerClients.updateSQL(api.getDatasource(), insertApi, AfterSqlUpdate.NONE, DatasourceFunctions.apiToMap(api));
|
|
| 351 |
datasourceManagerClients.updateSQL(api.getDatasource(), insertApi, DatasourceFunctions.apiToMap(api)); |
|
| 334 | 352 |
|
| 335 | 353 |
if (api.getApiParams() != null) {
|
| 336 | 354 |
api.getApiParams().forEach(p -> {
|
| 337 | 355 |
final ImmutableMap<String, Object> sqlParams = ImmutableMap.of("param", p.getParam(), "value", p.getValue(), "api", api.getId());
|
| 338 | 356 |
try {
|
| 339 |
datasourceManagerClients.updateSQL(api.getDatasource(), insertApiParam, AfterSqlUpdate.NONE, sqlParams);
|
|
| 357 |
datasourceManagerClients.updateSQL(api.getDatasource(), insertApiParam, sqlParams); |
|
| 340 | 358 |
} catch (final DsmException e) {
|
| 341 | 359 |
throw new RuntimeException(e); |
| 342 | 360 |
} |
| 343 | 361 |
}); |
| 344 | 362 |
} |
| 345 | 363 |
|
| 346 |
setManaged(api.getDatasource(), true); // It also update the IS profile
|
|
| 364 |
setManaged(api.getDatasource(), true); |
|
| 347 | 365 |
} |
| 348 | 366 |
|
| 349 | 367 |
@Override |
| 350 | 368 |
@Transactional |
| 351 |
public void deleteApi(final String dsId, final String apiId) throws DsmException {
|
|
| 352 |
if (!isRemovable(dsId, apiId)) { throw new DsmException("The api " + apiId + " can't be deleted"); }
|
|
| 369 |
public void deleteApi(final String dsId, final String apiId) throws DsmForbiddenException, DsmNotFoundException {
|
|
| 370 |
try {
|
|
| 371 |
if (!isRemovable(dsId, apiId)) { throw new DsmForbiddenException("The api " + apiId + " can't be deleted"); }
|
|
| 353 | 372 |
|
| 354 |
datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
|
|
| 355 |
datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_api WHERE id = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));
|
|
| 373 |
datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_apiparams WHERE api = :api", ImmutableMap.of("api", apiId));
|
|
| 374 |
datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_api WHERE id = :api", ImmutableMap.of("api", apiId));
|
|
| 356 | 375 |
|
| 357 |
setManaged(dsId, true); // It also update the IS profile |
|
| 376 |
setManaged(dsId, true); |
|
| 377 |
} catch (final Exception e) {
|
|
| 378 |
throw new DsmForbiddenException("The api " + apiId + " can't be deleted");
|
|
| 379 |
} |
|
| 358 | 380 |
} |
| 359 | 381 |
|
| 360 | 382 |
public DatasourceManagerClients getDatasourceManagerClients() {
|
| 361 | 383 |
return datasourceManagerClients; |
| 362 | 384 |
} |
| 363 | 385 |
|
| 364 |
private String fixDsId(final String dsId) throws DsmException {
|
|
| 365 |
return dsId.endsWith(REPO_PROFILEID_SUFFIX) ? datasourceManagerClients.findDatasourceId(dsId) : dsId; |
|
| 366 |
} |
|
| 367 |
|
|
| 368 | 386 |
@Required |
| 369 | 387 |
public void setDatasourceManagerClients(final DatasourceManagerClients datasourceManagerClients) {
|
| 370 | 388 |
this.datasourceManagerClients = datasourceManagerClients; |
| ... | ... | |
| 379 | 397 |
this.browsableFields = browsableFields; |
| 380 | 398 |
} |
| 381 | 399 |
|
| 400 |
@Override |
|
| 401 |
@Deprecated |
|
| 402 |
public void regenerateProfiles() throws DsmException {}
|
|
| 403 |
|
|
| 382 | 404 |
} |
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/db_fixes/script.sql | ||
|---|---|---|
| 1 |
alter table dsm_api add column compatibility_override varchar(255) default NULL;
|
|
| 1 |
alter table dsm_api add column compatibility_override text default NULL;
|
|
| 2 | 2 |
|
| 3 |
update dsm_api set removable = true where id in (select api.id from dsm_api api left join dsm_datasources d on d.id = api.datasource where d.managed = true and api.removable=false); |
|
| 3 |
update dsm_api set removable = true where id in (select api.id from dsm_api api left join dsm_services d on d.id = api.service where d.managed = true and api.removable=false); |
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/applicationContext-dnet-openaire-datasource-manager.xml | ||
|---|---|---|
| 20 | 20 |
<list> |
| 21 | 21 |
<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField" |
| 22 | 22 |
p:id="country" p:label="Datasource countries" |
| 23 |
p:sql="select o.country as term, count(*) as count from dsm_api a left outer join dsm_datasources d on (a.datasource = d.id) left outer join dsm_datasource_organization dao on (d.id = dao.datasource) left outer join dsm_organizations o on (dao.organization = o.id) group by o.country order by count desc" />
|
|
| 23 |
p:sql="select o.country as term, count(*) as count from dsm_api a left outer join dsm_services d on (a.service = d.id) left outer join dsm_service_organization dao on (d.id = dao.service) left outer join dsm_organizations o on (dao.organization = o.id) group by o.country order by count desc" />
|
|
| 24 | 24 |
<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField" |
| 25 | 25 |
p:id="type" p:label="API typologies" |
| 26 |
p:sql="select a.typology as term, count(*) as count from dsm_api a group by a.typology order by count desc" />
|
|
| 26 |
p:sql="select d.eosc_datasource_type as term, count(*) as count from dsm_api a left outer join dsm_services d on (a.service = d.id) group by d.eosc_datasource_type order by count desc" />
|
|
| 27 | 27 |
<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField" |
| 28 | 28 |
p:id="protocol" p:label="API protocols" |
| 29 | 29 |
p:sql="select a.protocol as term, count(*) as count from dsm_api a group by a.protocol order by count desc" /> |
| ... | ... | |
| 33 | 33 |
<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField" |
| 34 | 34 |
p:id="active" p:label="API activation" |
| 35 | 35 |
p:sql="select a.active as term, count(*) as count from dsm_api a group by a.active order by count desc" /> |
| 36 |
<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField" |
|
| 37 |
p:id="consenttermsofuse" p:label="Consent terms of use" |
|
| 38 |
p:sql="select d.consenttermsofuse as term, count(*) as count from dsm_api a left outer join dsm_services d on (a.service = d.id) group by d.consenttermsofuse order by count desc" /> |
|
| 39 |
<bean class="eu.dnetlib.enabling.datasources.DbBrowsableField" |
|
| 40 |
p:id="fulltextdownload" p:label="Fulltext download" |
|
| 41 |
p:sql="select d.fulltextdownload as term, count(*) as count from dsm_api a left outer join dsm_services d on (a.service = d.id) group by d.fulltextdownload order by count desc" /> |
|
| 36 | 42 |
</list> |
| 37 | 43 |
</property> |
| 38 | 44 |
</bean> |
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/addDatasource.sql | ||
|---|---|---|
| 1 |
INSERT INTO dsm_datasources(
|
|
| 1 |
INSERT INTO dsm_services(
|
|
| 2 | 2 |
id, |
| 3 | 3 |
officialname, |
| 4 | 4 |
englishname, |
| ... | ... | |
| 10 | 10 |
timezone, |
| 11 | 11 |
namespaceprefix, |
| 12 | 12 |
languages,--array |
| 13 |
od_contenttypes, |
|
| 14 | 13 |
collectedfrom, |
| 15 | 14 |
dateofvalidation, |
| 16 |
typology, |
|
| 17 | 15 |
provenanceaction, |
| 18 | 16 |
platform, |
| 19 | 17 |
activationid, |
| ... | ... | |
| 21 | 19 |
releasestartdate, |
| 22 | 20 |
releaseenddate, |
| 23 | 21 |
missionstatementurl, |
| 24 |
dataprovider, |
|
| 25 |
serviceprovider, |
|
| 26 | 22 |
databaseaccesstype, |
| 27 | 23 |
datauploadtype, |
| 28 | 24 |
databaseaccessrestriction, |
| 29 | 25 |
datauploadrestriction, |
| 30 |
versioning, |
|
| 31 | 26 |
citationguidelineurl, |
| 32 |
qualitymanagementkind, |
|
| 33 |
pidsystems, |
|
| 34 | 27 |
certificates, |
| 35 | 28 |
aggregator, |
| 36 | 29 |
issn, |
| ... | ... | |
| 38 | 31 |
lissn, |
| 39 | 32 |
registeredby, |
| 40 | 33 |
subjects,--array |
| 41 |
managed |
|
| 34 |
managed, |
|
| 35 |
eosc_datasource_type, |
|
| 36 |
eosc_type |
|
| 42 | 37 |
) VALUES ( |
| 43 | 38 |
:id, |
| 44 | 39 |
:officialname, |
| ... | ... | |
| 51 | 46 |
:timezone, |
| 52 | 47 |
:namespaceprefix, |
| 53 | 48 |
:languages,--array |
| 54 |
:od_contenttypes, |
|
| 55 | 49 |
:collectedfrom, |
| 56 | 50 |
:dateofvalidation, |
| 57 |
:typology, |
|
| 58 | 51 |
:provenanceaction, |
| 59 | 52 |
:platform, |
| 60 | 53 |
:activationid, |
| ... | ... | |
| 62 | 55 |
:releasestartdate, |
| 63 | 56 |
:releaseenddate, |
| 64 | 57 |
:missionstatementurl, |
| 65 |
:dataprovider, |
|
| 66 |
:serviceprovider, |
|
| 67 | 58 |
:databaseaccesstype, |
| 68 | 59 |
:datauploadtype, |
| 69 | 60 |
:databaseaccessrestriction, |
| 70 | 61 |
:datauploadrestriction, |
| 71 |
:versioning, |
|
| 72 | 62 |
:citationguidelineurl, |
| 73 |
:qualitymanagementkind, |
|
| 74 |
:pidsystems, |
|
| 75 | 63 |
:certificates, |
| 76 | 64 |
:aggregator, |
| 77 | 65 |
:issn, |
| ... | ... | |
| 79 | 67 |
:lissn, |
| 80 | 68 |
:registeredby, |
| 81 | 69 |
:subjects,--array |
| 82 |
:managed |
|
| 70 |
:managed, |
|
| 71 |
:eoscdatasourcetype, |
|
| 72 |
:eosctype |
|
| 83 | 73 |
) |
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/addOrganization.sql | ||
|---|---|---|
| 18 | 18 |
:collectedfrom |
| 19 | 19 |
); |
| 20 | 20 |
|
| 21 |
INSERT INTO dsm_datasource_organization(
|
|
| 22 |
datasource,
|
|
| 21 |
INSERT INTO dsm_service_organization(
|
|
| 22 |
service,
|
|
| 23 | 23 |
organization |
| 24 | 24 |
) VALUES ( |
| 25 | 25 |
:dsId, |
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/deleteDatasource.sql | ||
|---|---|---|
| 1 | 1 |
BEGIN; |
| 2 |
DELETE FROM dsm_apiparams USING dsm_api WHERE api=dsm_api.id AND api.datasource= :dsId;
|
|
| 3 |
DELETE FROM api WHERE datasource = :dsId;
|
|
| 4 |
DELETE FROM dsm_datasource_organization WHERE datasource = :dsId;
|
|
| 5 |
DELETE FROM dsm_datasourcepids WHERE datasource = :dsId;
|
|
| 6 |
DELETE FROM datasources WHERE id = :dsId;
|
|
| 2 |
DELETE FROM dsm_apiparams USING dsm_api WHERE api = dsm_api.id AND api.service = :dsId;
|
|
| 3 |
DELETE FROM api WHERE service = :dsId;
|
|
| 4 |
DELETE FROM dsm_service_organization WHERE service = :dsId;
|
|
| 5 |
DELETE FROM dsm_servicepids WHERE service = :dsId;
|
|
| 6 |
DELETE FROM dsm_services WHERE id = :dsId;
|
|
| 7 | 7 |
COMMIT; |
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/dsIdentitiesQuery.sql | ||
|---|---|---|
| 3 | 3 |
id.issuertype |
| 4 | 4 |
FROM |
| 5 | 5 |
dsm_identities id |
| 6 |
LEFT OUTER JOIN dsm_datasourcepids dp ON (id.pid = dp.pid)
|
|
| 6 |
LEFT OUTER JOIN dsm_servicepids dp ON (id.pid = dp.pid)
|
|
| 7 | 7 |
WHERE |
| 8 |
dp.datasource = :dsId |
|
| 8 |
dp.service = :dsId |
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/dsOrganizationsQuery.sql | ||
|---|---|---|
| 10 | 10 |
org.provenanceaction AS provenanceaction |
| 11 | 11 |
FROM |
| 12 | 12 |
dsm_organizations org |
| 13 |
LEFT OUTER JOIN dsm_datasource_organization dso ON (org.id = dso.organization)
|
|
| 13 |
LEFT OUTER JOIN dsm_service_organization dso ON (org.id = dso.service)
|
|
| 14 | 14 |
WHERE |
| 15 |
dso.datasource = :dsId |
|
| 15 |
dso.service = :dsId |
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/getDatasource.sql | ||
|---|---|---|
| 10 | 10 |
ds.timezone AS "timezone", |
| 11 | 11 |
ds.namespaceprefix AS "namespaceprefix", |
| 12 | 12 |
ds.languages AS "languages", |
| 13 |
ds.od_contenttypes AS "od_contenttypes", |
|
| 14 | 13 |
ds.collectedfrom AS "collectedfrom", |
| 15 | 14 |
ds.dateofvalidation AS "dateofvalidation", |
| 16 |
ds.typology AS "typology", |
|
| 17 | 15 |
ds.provenanceaction AS "provenanceaction", |
| 18 | 16 |
ds.dateofcollection AS "dateofcollection", |
| 19 | 17 |
ds.platform AS "platform", |
| ... | ... | |
| 22 | 20 |
ds.releasestartdate AS "releasestartdate", |
| 23 | 21 |
ds.releaseenddate AS "releaseenddate", |
| 24 | 22 |
ds.missionstatementurl AS "missionstatementurl", |
| 25 |
ds.dataprovider AS "dataprovider", |
|
| 26 |
ds.serviceprovider AS "serviceprovider", |
|
| 27 | 23 |
ds.databaseaccesstype AS "databaseaccesstype", |
| 28 | 24 |
ds.datauploadtype AS "datauploadtype", |
| 29 | 25 |
ds.databaseaccessrestriction AS "databaseaccessrestriction", |
| 30 | 26 |
ds.datauploadrestriction AS "datauploadrestriction", |
| 31 |
ds.versioning AS "versioning", |
|
| 32 | 27 |
ds.citationguidelineurl AS "citationguidelineurl", |
| 33 |
ds.qualitymanagementkind AS "qualitymanagementkind", |
|
| 34 |
ds.pidsystems AS "pidsystems", |
|
| 35 | 28 |
ds.certificates AS "certificates", |
| 36 | 29 |
ds.aggregator AS "aggregator", |
| 37 | 30 |
ds.issn AS "issn", |
| ... | ... | |
| 39 | 32 |
ds.lissn AS "lissn", |
| 40 | 33 |
ds.registeredby AS "registeredby", |
| 41 | 34 |
ds.subjects AS "subjects", |
| 42 |
ds.managed AS "managed" |
|
| 35 |
ds.managed AS "managed", |
|
| 36 |
ds.consenttermsofuse AS "consenttermsofuse", |
|
| 37 |
ds.consenttermsofusedate AS "consenttermsofusedate", |
|
| 38 |
ds.fulltextdownload AS "fulltextdownload", |
|
| 39 |
ds.eosc_datasource_type AS "eosc_datasource_type", |
|
| 40 |
ds.status AS "status" |
|
| 43 | 41 |
FROM |
| 44 |
dsm_datasources ds
|
|
| 42 |
dsm_services ds
|
|
| 45 | 43 |
WHERE |
| 46 |
ds.id = :dsId |
|
| 47 |
|
|
| 44 |
ds.id = :dsId |
|
| 45 |
AND eosc_datasource_type IS NOT NULL |
|
| 46 |
AND dedup_main_service = true |
|
| 48 | 47 |
|
| 49 | 48 |
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/getDatasourceByPrefix.sql | ||
|---|---|---|
| 1 |
SELECT |
|
| 2 |
ds.id AS "id", |
|
| 3 |
ds.officialname AS "officialname", |
|
| 4 |
ds.englishname AS "englishname", |
|
| 5 |
ds.websiteurl AS "websiteurl", |
|
| 6 |
ds.logourl AS "logourl", |
|
| 7 |
ds.contactemail AS "contactemail", |
|
| 8 |
ds.latitude AS "latitude", |
|
| 9 |
ds.longitude AS "longitude", |
|
| 10 |
ds.timezone AS "timezone", |
|
| 11 |
ds.namespaceprefix AS "namespaceprefix", |
|
| 12 |
ds.languages AS "languages", |
|
| 13 |
ds.collectedfrom AS "collectedfrom", |
|
| 14 |
ds.dateofvalidation AS "dateofvalidation", |
|
| 15 |
ds.provenanceaction AS "provenanceaction", |
|
| 16 |
ds.dateofcollection AS "dateofcollection", |
|
| 17 |
ds.platform AS "platform", |
|
| 18 |
ds.activationid AS "activationId", |
|
| 19 |
ds.description AS "description", |
|
| 20 |
ds.releasestartdate AS "releasestartdate", |
|
| 21 |
ds.releaseenddate AS "releaseenddate", |
|
| 22 |
ds.missionstatementurl AS "missionstatementurl", |
|
| 23 |
ds.databaseaccesstype AS "databaseaccesstype", |
|
| 24 |
ds.datauploadtype AS "datauploadtype", |
|
| 25 |
ds.databaseaccessrestriction AS "databaseaccessrestriction", |
|
| 26 |
ds.datauploadrestriction AS "datauploadrestriction", |
|
| 27 |
ds.citationguidelineurl AS "citationguidelineurl", |
|
| 28 |
ds.certificates AS "certificates", |
|
| 29 |
ds.aggregator AS "aggregator", |
|
| 30 |
ds.issn AS "issn", |
|
| 31 |
ds.eissn AS "eissn", |
|
| 32 |
ds.lissn AS "lissn", |
|
| 33 |
ds.registeredby AS "registeredby", |
|
| 34 |
ds.subjects AS "subjects", |
|
| 35 |
ds.managed AS "managed", |
|
| 36 |
ds.consenttermsofuse AS "consenttermsofuse", |
|
| 37 |
ds.consenttermsofusedate AS "consenttermsofusedate", |
|
| 38 |
ds.fulltextdownload AS "fulltextdownload", |
|
| 39 |
ds.eosc_datasource_type AS "eosc_datasource_type", |
|
| 40 |
ds.status AS "status" |
|
| 41 |
FROM |
|
| 42 |
dsm_services ds |
|
| 43 |
WHERE |
|
| 44 |
ds.namespaceprefix = :prefix |
|
| 45 |
AND eosc_datasource_type IS NOT NULL |
|
| 46 |
AND dedup_main_service = true |
|
| 47 |
|
|
| 48 |
|
|
| 49 |
|
|
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/insertApi.sql | ||
|---|---|---|
| 2 | 2 |
id, |
| 3 | 3 |
protocol, |
| 4 | 4 |
baseurl, |
| 5 |
datasource,
|
|
| 5 |
service,
|
|
| 6 | 6 |
contentdescription, |
| 7 |
typology, |
|
| 8 | 7 |
compatibility, |
| 9 | 8 |
metadata_identifier_path, |
| 10 | 9 |
removable |
| ... | ... | |
| 14 | 13 |
:baseUrl, |
| 15 | 14 |
:dsId, |
| 16 | 15 |
:contentDescription, |
| 17 |
:typology, |
|
| 18 | 16 |
:compatibility, |
| 19 | 17 |
:metadataIdentifierPath, |
| 20 | 18 |
:removable |
| modules/dnet-openaire-datasource-manager/trunk/src/main/resources/eu/dnetlib/enabling/datasources/queries/listApisByDsId.sql | ||
|---|---|---|
| 2 | 2 |
a.id AS "id", |
| 3 | 3 |
a.protocol AS "protocol", |
| 4 | 4 |
a.baseurl AS "baseUrl", |
| 5 |
a.datasource AS "datasource",
|
|
| 5 |
a.service AS "datasource",
|
|
| 6 | 6 |
a.contentdescription AS "contentdescription", |
| 7 | 7 |
a.active AS "active", |
| 8 | 8 |
a.removable AS "removable", |
| 9 |
a.typology AS "typology", |
|
Also available in: Unified diff
merge from eosc_services