Revision 62129
Added by Michele Artini about 2 years ago
| RepoUIUtils.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.functionality.modular.ui.repositories; |
| 2 | 2 |
|
| 3 | 3 |
import java.io.StringReader; |
| 4 |
import java.util.ArrayList; |
|
| 4 | 5 |
import java.util.Collections; |
| 5 | 6 |
import java.util.List; |
| 6 | 7 |
import java.util.Map; |
| 7 | 8 |
import java.util.Set; |
| 8 | 9 |
|
| 9 |
import javax.annotation.Resource; |
|
| 10 |
|
|
| 11 | 10 |
import org.antlr.stringtemplate.StringTemplate; |
| 12 | 11 |
import org.apache.commons.io.IOUtils; |
| 13 |
import org.apache.commons.lang.StringEscapeUtils;
|
|
| 14 |
import org.apache.commons.lang.math.NumberUtils;
|
|
| 12 |
import org.apache.commons.lang3.BooleanUtils;
|
|
| 13 |
import org.apache.commons.lang3.StringUtils;
|
|
| 15 | 14 |
import org.apache.commons.logging.Log; |
| 16 | 15 |
import org.apache.commons.logging.LogFactory; |
| 17 | 16 |
import org.dom4j.Document; |
| 18 | 17 |
import org.dom4j.Node; |
| 19 | 18 |
import org.dom4j.io.SAXReader; |
| 20 |
import org.springframework.core.io.ClassPathResource;
|
|
| 19 |
import org.springframework.beans.factory.annotation.Autowired;
|
|
| 21 | 20 |
|
| 22 | 21 |
import com.google.common.collect.Lists; |
| 23 | 22 |
import com.google.common.collect.Maps; |
| ... | ... | |
| 26 | 25 |
import eu.dnetlib.data.collector.rmi.CollectorService; |
| 27 | 26 |
import eu.dnetlib.data.collector.rmi.ProtocolDescriptor; |
| 28 | 27 |
import eu.dnetlib.data.collector.rmi.ProtocolParameter; |
| 28 |
import eu.dnetlib.enabling.datasources.common.Api; |
|
| 29 |
import eu.dnetlib.enabling.datasources.common.ApiParam; |
|
| 30 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
|
| 31 |
import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager; |
|
| 29 | 32 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; |
| 30 | 33 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
| 31 | 34 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
| ... | ... | |
| 33 | 36 |
import eu.dnetlib.functionality.modular.ui.repositories.objects.RepoMetaWfEntry; |
| 34 | 37 |
import eu.dnetlib.functionality.modular.ui.repositories.objects.SimpleParamEntry; |
| 35 | 38 |
import eu.dnetlib.functionality.modular.ui.repositories.objects.VocabularyEntry; |
| 39 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
| 36 | 40 |
import eu.dnetlib.msro.workflows.util.WorkflowsConstants.WorkflowStatus; |
| 37 | 41 |
|
| 38 | 42 |
public class RepoUIUtils {
|
| 39 | 43 |
|
| 40 |
@Resource
|
|
| 44 |
@Autowired
|
|
| 41 | 45 |
private UniqueServiceLocator serviceLocator; |
| 46 |
@Autowired |
|
| 47 |
private LocalDatasourceManager<Datasource<?, ?, ?>, Api<?>> dsManager; |
|
| 42 | 48 |
|
| 43 |
private final ClassPathResource getRepoApiQueryTmpl = |
|
| 44 |
new ClassPathResource("/eu/dnetlib/functionality/modular/ui/repositories/templates/getRepoApi.xquery.st");
|
|
| 45 |
|
|
| 46 | 49 |
private static final Log log = LogFactory.getLog(RepoUIUtils.class); |
| 47 | 50 |
|
| 48 | 51 |
private final Map<String, List<ProtocolParameter>> parametersMap = Maps.newHashMap(); |
| 49 | 52 |
|
| 50 |
public RepoInterfaceEntry getApi(final String repoId, final String ifaceId) throws Exception {
|
|
| 53 |
public RepoInterfaceEntry getApi(final String dsId, final String apiId) throws Exception {
|
|
| 51 | 54 |
final RepoInterfaceEntry ifc = new RepoInterfaceEntry(); |
| 52 | 55 |
|
| 53 |
final StringTemplate st = new StringTemplate(IOUtils.toString(getRepoApiQueryTmpl.getInputStream())); |
|
| 54 |
st.setAttribute("dsId", repoId);
|
|
| 55 |
st.setAttribute("ifaceId", ifaceId);
|
|
| 56 |
final Datasource<?, ?, ?> ds = dsManager.getDs(dsId); |
|
| 56 | 57 |
|
| 57 |
final String query = st.toString(); |
|
| 58 |
final Api<?> api = dsManager.getApis(dsId) |
|
| 59 |
.stream() |
|
| 60 |
.filter(a -> a.getId().equals(apiId)) |
|
| 61 |
.findFirst() |
|
| 62 |
.orElseThrow(() -> new Exception("Api not found"));
|
|
| 58 | 63 |
|
| 59 |
final SAXReader reader = new SAXReader();
|
|
| 64 |
final List<?> metaWfs = new ArrayList<>(); // TODO
|
|
| 60 | 65 |
|
| 61 |
final String s = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(query); |
|
| 66 |
ifc.setId(api.getId()); |
|
| 67 |
ifc.setLabel(String |
|
| 68 |
.format("%s (%s)", StringUtils.defaultIfBlank(ds.getEoscDatasourceType(), "-"), StringUtils.defaultIfBlank(api.getCompatibility(), "-")));
|
|
| 69 |
ifc.setRemovable(api.getRemovable() && metaWfs.isEmpty()); |
|
| 70 |
ifc.setRepoId(ds.getId()); |
|
| 71 |
ifc.setRepoName(ds.getOfficialname()); |
|
| 72 |
ifc.setRepoCountry(ds.getOrganizations() |
|
| 73 |
.stream() |
|
| 74 |
.map(o -> o.getCountry()) |
|
| 75 |
.filter(o -> o != null) |
|
| 76 |
.findFirst() |
|
| 77 |
.orElse("UNKNOWN"));
|
|
| 78 |
ifc.setRepoPrefix(ds.getNamespaceprefix()); |
|
| 79 |
ifc.setRepoType(ds.getEoscDatasourceType()); |
|
| 80 |
ifc.setEmail(ds.getContactemail()); |
|
| 81 |
ifc.setActive(api.getActive()); |
|
| 82 |
ifc.setProtocol(api.getProtocol()); |
|
| 83 |
ifc.setConsentTermsOfUseLabel(BooleanUtils.toString(ds.getConsentTermsOfUse(), "YES", "NO", "UNKNOWN")); |
|
| 84 |
ifc.setFullTextDownloadLabel(BooleanUtils.toString(ds.getFullTextDownload(), "YES", "NO", "UNKNOWN")); |
|
| 62 | 85 |
|
| 63 |
final Document doc = reader.read(new StringReader(s)); |
|
| 64 |
|
|
| 65 |
ifc.setId(doc.valueOf("/api/id"));
|
|
| 66 |
ifc.setLabel(doc.valueOf("/api/label"));
|
|
| 67 |
ifc.setRemovable(doc.valueOf("/api/removable").equalsIgnoreCase("true") && doc.selectNodes("//metaWF").isEmpty());
|
|
| 68 |
ifc.setRepoId(doc.valueOf("/api/repo/@id"));
|
|
| 69 |
ifc.setRepoName(StringEscapeUtils.unescapeHtml(doc.valueOf("/api/repo")));
|
|
| 70 |
ifc.setRepoCountry(doc.valueOf("/api/repo/@country"));
|
|
| 71 |
ifc.setRepoPrefix(doc.valueOf("/api/repo/@prefix"));
|
|
| 72 |
ifc.setRepoType(doc.valueOf("/api/repo/@type"));
|
|
| 73 |
ifc.setEmail(doc.valueOf("/api/repo/@email"));
|
|
| 74 |
final String protocol = doc.valueOf("/api/protocol");
|
|
| 75 |
ifc.setProtocol(protocol); |
|
| 76 |
|
|
| 77 | 86 |
final Set<String> toVerifyParams = getParameterNamesForProtocol(ifc.getProtocol()); |
| 78 |
for (final Object o : doc.selectNodes("/api/commonParams/param")) {
|
|
| 79 |
final Node n = (Node) o; |
|
| 80 |
ifc.getCommonParams().add(new SimpleParamEntry(n.valueOf("@name"), n.getText()));
|
|
| 81 |
} |
|
| 87 |
ifc.getAccessParams().add(new SimpleParamEntry("baseUrl", api.getBaseurl()));
|
|
| 82 | 88 |
|
| 83 |
log.debug("****** " + toVerifyParams);
|
|
| 84 |
for (final Object o : doc.selectNodes("/api/accessParams/param")) {
|
|
| 85 |
final Node n = (Node) o; |
|
| 86 |
final String pname = n.valueOf("@name");
|
|
| 87 |
if (toVerifyParams.contains(pname)) {
|
|
| 88 |
ifc.getAccessParams().add(new SimpleParamEntry(pname, n.getText())); |
|
| 89 |
toVerifyParams.remove(pname); |
|
| 89 |
for (final ApiParam p : api.getApiParams()) {
|
|
| 90 |
if (toVerifyParams.contains(p.getParam())) {
|
|
| 91 |
ifc.getAccessParams().add(new SimpleParamEntry(p.getParam(), p.getValue())); |
|
| 92 |
toVerifyParams.remove(p.getParam()); |
|
| 90 | 93 |
} else {
|
| 91 |
log.warn("Invalid param [" + pname + "] for protocol " + protocol + " in repo " + repoId);
|
|
| 94 |
log.warn("Invalid param [" + p.getParam() + "] for protocol " + p.getValue() + " in ds " + dsId);
|
|
| 92 | 95 |
} |
| 93 | 96 |
} |
| 97 |
|
|
| 94 | 98 |
for (final String pname : toVerifyParams) {
|
| 95 | 99 |
ifc.getAccessParams().add(new SimpleParamEntry(pname, "")); |
| 96 |
log.info("Adding missing param [" + pname + "] for protocol " + protocol + " in repo " + repoId);
|
|
| 100 |
log.info("Adding missing param [" + pname + "] for protocol " + api.getProtocol() + " in ds " + dsId);
|
|
| 97 | 101 |
} |
| 98 | 102 |
|
| 99 |
for (final Object o : doc.selectNodes("/api/extraFields/field")) {
|
|
| 100 |
final Node n = (Node) o; |
|
| 101 |
final String name = n.valueOf("@name");
|
|
| 102 |
final String value = n.getText(); |
|
| 103 |
if (name.equalsIgnoreCase("overriding_compliance")) {
|
|
| 104 |
for (final SimpleParamEntry e : ifc.getCommonParams()) {
|
|
| 105 |
if (e.getName().equals("compliance")) {
|
|
| 106 |
// The original compliance (assigned by the validator) is stored in otherValue |
|
| 107 |
e.setOtherValue(value); |
|
| 108 |
} |
|
| 109 |
} |
|
| 110 |
} else if (name.equalsIgnoreCase("last_aggregation_date")) {
|
|
| 111 |
ifc.setAggrDate(value); |
|
| 112 |
} else if (name.equalsIgnoreCase("last_aggregation_total")) {
|
|
| 113 |
ifc.setAggrTotal(NumberUtils.toInt(value, 0)); |
|
| 114 |
} else if (name.equalsIgnoreCase("last_aggregation_mdId")) {
|
|
| 115 |
ifc.setAggrMdId(value); |
|
| 116 |
} else if (name.equalsIgnoreCase("last_collection_date")) {
|
|
| 117 |
ifc.setCollDate(value); |
|
| 118 |
} else if (name.equalsIgnoreCase("last_collection_total")) {
|
|
| 119 |
ifc.setCollTotal(NumberUtils.toInt(value, 0)); |
|
| 120 |
} else if (name.equalsIgnoreCase("last_collection_mdId")) {
|
|
| 121 |
ifc.setCollMdId(value); |
|
| 122 |
} else if (name.equalsIgnoreCase("last_download_date")) {
|
|
| 123 |
ifc.setDownloadDate(value); |
|
| 124 |
} else if (name.equalsIgnoreCase("last_download_total")) {
|
|
| 125 |
ifc.setDownloadTotal(NumberUtils.toInt(value, 0)); |
|
| 126 |
} else if (name.equalsIgnoreCase("last_download_objId")) {
|
|
| 127 |
ifc.setDownloadObjId(value); |
|
| 128 |
} else {
|
|
| 129 |
ifc.getOtherParams().add(new SimpleParamEntry(name, value)); |
|
| 130 |
} |
|
| 131 |
} |
|
| 103 |
ifc.setMetadataIdentifierPath(api.getMetadataIdentifierPath()); |
|
| 104 |
ifc.setCompliance(api.getCompatibility()); |
|
| 105 |
ifc.setComplianceOverrided(api.isCompatibilityOverrided()); |
|
| 132 | 106 |
|
| 133 |
if (doc.selectNodes("/api/extraFields/field[@name='metadata_identifier_path']").isEmpty()) {
|
|
| 134 |
ifc.getOtherParams().add(new SimpleParamEntry("metadata_identifier_path", ""));
|
|
| 107 |
if (api.getLastAggregationDate() != null) {
|
|
| 108 |
ifc.setAggrDate(DateUtils.calculate_ISO8601(api.getLastAggregationDate().getTime())); |
|
| 109 |
ifc.setAggrTotal(api.getLastAggregationTotal()); |
|
| 110 |
ifc.setAggrMdId(api.getLastAggregationMdid()); |
|
| 135 | 111 |
} |
| 112 |
if (api.getLastCollectionDate() != null) {
|
|
| 113 |
ifc.setCollDate(DateUtils.calculate_ISO8601(api.getLastCollectionDate().getTime())); |
|
| 114 |
ifc.setCollTotal(api.getLastCollectionTotal()); |
|
| 115 |
ifc.setCollMdId(api.getLastCollectionMdid()); |
|
| 116 |
} |
|
| 117 |
if (api.getLastDownloadDate() != null) {
|
|
| 118 |
ifc.setDownloadDate(DateUtils.calculate_ISO8601(api.getLastDownloadDate().getTime())); |
|
| 119 |
ifc.setDownloadTotal(api.getLastDownloadTotal()); |
|
| 120 |
ifc.setDownloadObjId(api.getLastDownloadObjid()); |
|
| 121 |
} |
|
| 136 | 122 |
|
| 123 |
final StringTemplate st = new StringTemplate(IOUtils.toString(getClass() |
|
| 124 |
.getResourceAsStream("/eu/dnetlib/functionality/modular/ui/repositories/templates/getMetaWfs.xquery.st")));
|
|
| 125 |
st.setAttribute("dsId", dsId);
|
|
| 126 |
st.setAttribute("apiId", apiId);
|
|
| 127 |
|
|
| 128 |
final String s = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(st.toString()); |
|
| 129 |
|
|
| 130 |
final Document doc = new SAXReader().read(new StringReader(s)); |
|
| 131 |
|
|
| 137 | 132 |
for (final Object o : doc.selectNodes("//metaWF")) {
|
| 138 | 133 |
final Node n = (Node) o; |
| 139 | 134 |
|
| ... | ... | |
| 172 | 167 |
|
| 173 | 168 |
public List<VocabularyEntry> fetchVocabularyTerms(final String voc) throws ISLookUpException {
|
| 174 | 169 |
final String xquery = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code = '"
|
| 175 |
+ voc.trim() + "']//TERM return concat($x/@code, ' @@@ ', $x/@english_name)";
|
|
| 170 |
+ voc.trim() + "']//TERM return concat($x/@code, ' @@@ ', $x/@english_name)"; |
|
| 176 | 171 |
|
| 177 | 172 |
final List<VocabularyEntry> list = Lists.newArrayList(); |
| 178 | 173 |
for (final String s : serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery)) {
|
| ... | ... | |
| 192 | 187 |
} |
| 193 | 188 |
final Set<String> res = Sets.newHashSet(); |
| 194 | 189 |
if (parametersMap.containsKey(protocol.toLowerCase())) {
|
| 195 |
res.add("baseUrl");
|
|
| 196 | 190 |
for (final ProtocolParameter p : parametersMap.get(protocol.toLowerCase())) {
|
| 197 | 191 |
res.add(p.getName()); |
| 198 | 192 |
} |
Also available in: Unified diff
merge from eosc_services