Revision 50489
Added by Michele Artini about 6 years ago
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FindExistingRepositoriesJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes; |
|
| 2 |
|
|
| 3 |
import java.util.List; |
|
| 4 |
import java.util.Map; |
|
| 5 |
import javax.annotation.Resource; |
|
| 6 |
|
|
| 7 |
import com.google.common.collect.Maps; |
|
| 8 |
import com.googlecode.sarasvati.Arc; |
|
| 9 |
import com.googlecode.sarasvati.NodeToken; |
|
| 10 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
| 11 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
| 12 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
|
| 13 |
|
|
| 14 |
public class FindExistingRepositoriesJobNode extends SimpleJobNode {
|
|
| 15 |
|
|
| 16 |
@Resource |
|
| 17 |
private UniqueServiceLocator serviceLocator; |
|
| 18 |
|
|
| 19 |
private String existingReposParam = "existingRepos"; |
|
| 20 |
|
|
| 21 |
@Override |
|
| 22 |
protected String execute(final NodeToken token) throws Exception {
|
|
| 23 |
String xquery = |
|
| 24 |
"for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') " +
|
|
| 25 |
"return concat (" +
|
|
| 26 |
"$x//RESOURCE_IDENTIFIER/@value, " + |
|
| 27 |
"'@@@', " + |
|
| 28 |
"$x//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value)"; |
|
| 29 |
final List<String> list = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery); |
|
| 30 |
|
|
| 31 |
final Map<String, String> map = Maps.newHashMap(); |
|
| 32 |
for (String s : list) {
|
|
| 33 |
String[] arr = s.split("@@@");
|
|
| 34 |
if (arr.length == 2) {
|
|
| 35 |
map.put(arr[1], arr[0]); |
|
| 36 |
} |
|
| 37 |
} |
|
| 38 |
token.getEnv().setTransientAttribute(getExistingReposParam(), map); |
|
| 39 |
|
|
| 40 |
return Arc.DEFAULT_ARC; |
|
| 41 |
} |
|
| 42 |
|
|
| 43 |
public String getExistingReposParam() {
|
|
| 44 |
return existingReposParam; |
|
| 45 |
} |
|
| 46 |
|
|
| 47 |
public void setExistingReposParam(final String existingReposParam) {
|
|
| 48 |
this.existingReposParam = existingReposParam; |
|
| 49 |
} |
|
| 50 |
|
|
| 51 |
} |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FindComplianceRepositoriesJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes; |
|
| 2 |
|
|
| 3 |
import javax.xml.ws.wsaddressing.W3CEndpointReference; |
|
| 4 |
|
|
| 5 |
import com.googlecode.sarasvati.Arc; |
|
| 6 |
import com.googlecode.sarasvati.NodeToken; |
|
| 7 |
import eu.dnetlib.enabling.database.rmi.DatabaseService; |
|
| 8 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
| 9 |
import eu.dnetlib.enabling.resultset.XSLTMappedResultSetFactory; |
|
| 10 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
|
| 11 |
import org.antlr.stringtemplate.StringTemplate; |
|
| 12 |
import org.apache.commons.io.IOUtils; |
|
| 13 |
import org.apache.commons.logging.Log; |
|
| 14 |
import org.apache.commons.logging.LogFactory; |
|
| 15 |
import org.springframework.core.io.ClassPathResource; |
|
| 16 |
import org.springframework.core.io.Resource; |
|
| 17 |
|
|
| 18 |
public class FindComplianceRepositoriesJobNode extends SimpleJobNode {
|
|
| 19 |
|
|
| 20 |
private String dbName; |
|
| 21 |
|
|
| 22 |
// The following files are in "dnet-openaireplus-datasource-manager-service" module |
|
| 23 |
private final Resource sqlSelectDatasourcesTmpl = new ClassPathResource("/eu/dnetlib/enabling/datasources/getDatasources.sql.st");
|
|
| 24 |
private final Resource xsltFile = new ClassPathResource("/eu/dnetlib/enabling/datasources/repo_2_is.xslt");
|
|
| 25 |
|
|
| 26 |
private String eprParam = "repoEpr"; |
|
| 27 |
|
|
| 28 |
@javax.annotation.Resource |
|
| 29 |
private UniqueServiceLocator serviceLocator; |
|
| 30 |
|
|
| 31 |
private XSLTMappedResultSetFactory xsltMappedResultSetFactory; |
|
| 32 |
|
|
| 33 |
private static final Log log = LogFactory.getLog(FindComplianceRepositoriesJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM |
|
| 34 |
|
|
| 35 |
@Override |
|
| 36 |
protected String execute(final NodeToken token) throws Exception {
|
|
| 37 |
final StringTemplate st = new StringTemplate(IOUtils.toString(sqlSelectDatasourcesTmpl.getInputStream())); |
|
| 38 |
// st.setAttribute( |
|
| 39 |
// "condition", |
|
| 40 |
// "datasourceclass = 'entityregistry' OR (openairecompatibilityclass IS NOT null AND openairecompatibilityclass != 'UNKNOWN' AND openairecompatibilityclass != 'notCompatible')"); |
|
| 41 |
final String sql = st.toString(); |
|
| 42 |
final String xslt = IOUtils.toString(xsltFile.getInputStream()); |
|
| 43 |
|
|
| 44 |
log.debug("Executing query: " + sql);
|
|
| 45 |
|
|
| 46 |
final W3CEndpointReference epr = serviceLocator.getService(DatabaseService.class).searchSQL(dbName, sql); |
|
| 47 |
final W3CEndpointReference mapped = xsltMappedResultSetFactory.createMappedResultSet(epr, xslt); |
|
| 48 |
|
|
| 49 |
token.getEnv().setTransientAttribute(getEprParam(), mapped); |
|
| 50 |
|
|
| 51 |
return Arc.DEFAULT_ARC; |
|
| 52 |
} |
|
| 53 |
|
|
| 54 |
public String getDbName() {
|
|
| 55 |
return dbName; |
|
| 56 |
} |
|
| 57 |
|
|
| 58 |
public void setDbName(final String dbName) {
|
|
| 59 |
this.dbName = dbName; |
|
| 60 |
} |
|
| 61 |
|
|
| 62 |
public XSLTMappedResultSetFactory getXsltMappedResultSetFactory() {
|
|
| 63 |
return xsltMappedResultSetFactory; |
|
| 64 |
} |
|
| 65 |
|
|
| 66 |
public void setXsltMappedResultSetFactory(final XSLTMappedResultSetFactory xsltMappedResultSetFactory) {
|
|
| 67 |
this.xsltMappedResultSetFactory = xsltMappedResultSetFactory; |
|
| 68 |
} |
|
| 69 |
|
|
| 70 |
public String getEprParam() {
|
|
| 71 |
return eprParam; |
|
| 72 |
} |
|
| 73 |
|
|
| 74 |
public void setEprParam(final String eprParam) {
|
|
| 75 |
this.eprParam = eprParam; |
|
| 76 |
} |
|
| 77 |
|
|
| 78 |
} |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repohi/UpdateOpenaireMetaWfStatusJobNode.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes.repohi; |
| 2 | 2 |
|
| 3 |
import javax.annotation.Resource;
|
|
| 3 |
import org.springframework.beans.factory.annotation.Autowired;
|
|
| 4 | 4 |
|
| 5 |
import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerService;
|
|
| 5 |
import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;
|
|
| 6 | 6 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
| 7 | 7 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
| 8 | 8 |
import eu.dnetlib.msro.workflows.nodes.repohi.UpdateMetaWfStatusJobNode; |
| 9 | 9 |
|
| 10 | 10 |
public class UpdateOpenaireMetaWfStatusJobNode extends UpdateMetaWfStatusJobNode {
|
| 11 | 11 |
|
| 12 |
@Resource
|
|
| 12 |
@Autowired
|
|
| 13 | 13 |
private UniqueServiceLocator serviceLocator; |
| 14 |
@Autowired |
|
| 15 |
private LocalOpenaireDatasourceManager dsManager; |
|
| 14 | 16 |
|
| 15 | 17 |
@Override |
| 16 | 18 |
protected void updateDatasource(final String dsId, final String ifaceId) throws Exception {
|
| ... | ... | |
| 21 | 23 |
if (openaireDsId.equals("openaire____::bootstrap")) {
|
| 22 | 24 |
super.updateDatasource(dsId, ifaceId); |
| 23 | 25 |
} else {
|
| 24 |
serviceLocator.getService(DatasourceManagerService.class).updateActivationStatus(openaireDsId, ifaceId, true); |
|
| 26 |
dsManager.setManaged(openaireDsId, true); |
|
| 27 |
dsManager.setActive(openaireDsId, ifaceId, true); |
|
| 25 | 28 |
} |
| 26 | 29 |
} |
| 27 | 30 |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/repobye/DeleteOpenaireMetaWfJobNode.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes.repobye; |
| 2 | 2 |
|
| 3 |
import javax.annotation.Resource;
|
|
| 3 |
import org.springframework.beans.factory.annotation.Autowired;
|
|
| 4 | 4 |
|
| 5 |
import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerService;
|
|
| 5 |
import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;
|
|
| 6 | 6 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
| 7 | 7 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
| 8 | 8 |
import eu.dnetlib.msro.workflows.nodes.repobye.DeleteMetaWfJobNode; |
| 9 | 9 |
|
| 10 | 10 |
public class DeleteOpenaireMetaWfJobNode extends DeleteMetaWfJobNode {
|
| 11 | 11 |
|
| 12 |
@Resource
|
|
| 12 |
@Autowired
|
|
| 13 | 13 |
private UniqueServiceLocator serviceLocator; |
| 14 | 14 |
|
| 15 |
@Autowired |
|
| 16 |
private LocalOpenaireDatasourceManager dsManager; |
|
| 17 |
|
|
| 15 | 18 |
@Override |
| 16 |
protected void updateIfaceActivationStatus(final String dsId, final String ifaceId, final boolean active) throws Exception {
|
|
| 19 |
protected void setActivationStatus(final String dsId, final String ifaceId, final boolean active) throws Exception {
|
|
| 17 | 20 |
|
| 18 | 21 |
final String openaireDsId = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery( |
| 19 | 22 |
"/*[.//RESOURCE_IDENTIFIER/@value = '" + dsId + "']//FIELD/value[../key='OpenAireDataSourceId']/text()"); |
| 20 | 23 |
|
| 21 | 24 |
if (openaireDsId.equals("openaire____::bootstrap")) {
|
| 22 |
super.updateIfaceActivationStatus(dsId, ifaceId, active);
|
|
| 25 |
dsManager.setActive(dsId, ifaceId, active);
|
|
| 23 | 26 |
} else {
|
| 24 |
serviceLocator.getService(DatasourceManagerService.class).updateActivationStatus(openaireDsId, ifaceId, active);
|
|
| 27 |
dsManager.setActive(openaireDsId, ifaceId, active);
|
|
| 25 | 28 |
} |
| 26 | 29 |
} |
| 27 | 30 |
} |
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/PatchHostedByJobNode.java | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import java.io.StringReader; |
| 4 | 4 |
import java.util.Map; |
| 5 |
|
|
| 5 | 6 |
import javax.annotation.Resource; |
| 6 | 7 |
import javax.xml.ws.wsaddressing.W3CEndpointReference; |
| 7 | 8 |
|
| 9 |
import org.apache.commons.lang.StringUtils; |
|
| 10 |
import org.apache.commons.logging.Log; |
|
| 11 |
import org.apache.commons.logging.LogFactory; |
|
| 12 |
import org.dom4j.Document; |
|
| 13 |
import org.dom4j.DocumentException; |
|
| 14 |
import org.dom4j.io.SAXReader; |
|
| 15 |
import org.springframework.beans.factory.annotation.Required; |
|
| 16 |
import org.springframework.beans.factory.annotation.Value; |
|
| 17 |
|
|
| 8 | 18 |
import com.google.common.collect.Maps; |
| 9 | 19 |
import com.googlecode.sarasvati.Arc; |
| 10 | 20 |
import com.googlecode.sarasvati.NodeToken; |
| 21 |
|
|
| 11 | 22 |
import eu.dnetlib.enabling.database.rmi.DatabaseException; |
| 12 | 23 |
import eu.dnetlib.enabling.database.rmi.DatabaseService; |
| 13 | 24 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
| ... | ... | |
| 15 | 26 |
import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory; |
| 16 | 27 |
import eu.dnetlib.enabling.resultset.client.utils.EPRUtils; |
| 17 | 28 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
| 18 |
import org.apache.commons.lang.StringUtils; |
|
| 19 |
import org.apache.commons.logging.Log; |
|
| 20 |
import org.apache.commons.logging.LogFactory; |
|
| 21 |
import org.dom4j.Document; |
|
| 22 |
import org.dom4j.DocumentException; |
|
| 23 |
import org.dom4j.io.SAXReader; |
|
| 24 |
import org.springframework.beans.factory.annotation.Required; |
|
| 25 |
import org.springframework.beans.factory.annotation.Value; |
|
| 26 | 29 |
|
| 27 | 30 |
public class PatchHostedByJobNode extends SimpleJobNode {
|
| 28 | 31 |
|
| ... | ... | |
| 57 | 60 |
|
| 58 | 61 |
Map<String, HostedByEntry> hostedByEntryMap; |
| 59 | 62 |
|
| 60 |
if (!StringUtils.isBlank(keyTypeFilter)) |
|
| 63 |
if (!StringUtils.isBlank(keyTypeFilter)) {
|
|
| 61 | 64 |
hostedByEntryMap = loadHostedByMapFromType(keyTypeFilter); |
| 62 |
else
|
|
| 65 |
} else {
|
|
| 63 | 66 |
hostedByEntryMap = loadHostedByMap(datasourceId); |
| 67 |
} |
|
| 64 | 68 |
|
| 65 | 69 |
final W3CEndpointReference epr = mappedResultSetFactory.createMappedResultSet(inputEpr, new PatchHostedBy(hostedByEntryMap, |
| 66 | 70 |
getXpathEntry(), counters)); |
| ... | ... | |
| 72 | 76 |
} |
| 73 | 77 |
|
| 74 | 78 |
private Map<String, HostedByEntry> loadHostedByMap(final String datasourceId) throws DocumentException, DatabaseException {
|
| 75 |
final String sql = "SELECT d.id, d.officialname, p.entry from %s p JOIN datasources d ON (p.datasourceid = d.id) WHERE p.oa_source_id= '%s'"; |
|
| 79 |
final String sql = "SELECT d.id, d.officialname, p.entry from %s p JOIN dsm_datasources d ON (p.datasourceid = d.id) WHERE p.oa_source_id= '%s'";
|
|
| 76 | 80 |
return getHostedByEntryMapByQuery(datasourceId, sql); |
| 77 | 81 |
} |
| 78 | 82 |
|
| 79 | 83 |
private Map<String, HostedByEntry> loadHostedByMapFromType(final String keyType) throws DocumentException, DatabaseException {
|
| 80 |
final String sql = "SELECT d.id, d.officialname, p.entry from %s p JOIN datasources d ON (p.datasourceid = d.id) WHERE p.key_type= '%s'"; |
|
| 84 |
final String sql = "SELECT d.id, d.officialname, p.entry from %s p JOIN dsm_datasources d ON (p.datasourceid = d.id) WHERE p.key_type= '%s'";
|
|
| 81 | 85 |
return getHostedByEntryMapByQuery(keyType, sql); |
| 82 | 86 |
} |
| 83 | 87 |
|
| ... | ... | |
| 87 | 91 |
final W3CEndpointReference epr = serviceLocator.getService(DatabaseService.class).searchSQL(getDbName(), |
| 88 | 92 |
String.format(sql, getHostedbyMapTable(), datasourceId)); |
| 89 | 93 |
final SAXReader reader = new SAXReader(); |
| 90 |
for (String s : resultSetClientFactory.getClient(epr)) {
|
|
| 94 |
for (final String s : resultSetClientFactory.getClient(epr)) {
|
|
| 91 | 95 |
final Document doc = reader.read(new StringReader(s)); |
| 92 | 96 |
final String entry = doc.valueOf("//FIELD[@name='entry']");
|
| 93 | 97 |
final String dsId = doc.valueOf("//FIELD[@name='id']");
|
| ... | ... | |
| 110 | 114 |
/** |
| 111 | 115 |
* Setter for property 'inputEprParam'. |
| 112 | 116 |
* |
| 113 |
* @param inputEprParam Value to set for property 'inputEprParam'. |
|
| 117 |
* @param inputEprParam |
|
| 118 |
* Value to set for property 'inputEprParam'. |
|
| 114 | 119 |
*/ |
| 115 | 120 |
public void setInputEprParam(final String inputEprParam) {
|
| 116 | 121 |
this.inputEprParam = inputEprParam; |
| ... | ... | |
| 128 | 133 |
/** |
| 129 | 134 |
* Setter for property 'outputEprParam'. |
| 130 | 135 |
* |
| 131 |
* @param outputEprParam Value to set for property 'outputEprParam'. |
|
| 136 |
* @param outputEprParam |
|
| 137 |
* Value to set for property 'outputEprParam'. |
|
| 132 | 138 |
*/ |
| 133 | 139 |
public void setOutputEprParam(final String outputEprParam) {
|
| 134 | 140 |
this.outputEprParam = outputEprParam; |
| ... | ... | |
| 146 | 152 |
/** |
| 147 | 153 |
* Setter for property 'dbName'. |
| 148 | 154 |
* |
| 149 |
* @param dbName Value to set for property 'dbName'. |
|
| 155 |
* @param dbName |
|
| 156 |
* Value to set for property 'dbName'. |
|
| 150 | 157 |
*/ |
| 151 | 158 |
public void setDbName(final String dbName) {
|
| 152 | 159 |
this.dbName = dbName; |
| ... | ... | |
| 172 | 179 |
/** |
| 173 | 180 |
* Setter for property 'mappedResultSetFactory'. |
| 174 | 181 |
* |
| 175 |
* @param mappedResultSetFactory Value to set for property 'mappedResultSetFactory'. |
|
| 182 |
* @param mappedResultSetFactory |
|
| 183 |
* Value to set for property 'mappedResultSetFactory'. |
|
| 176 | 184 |
*/ |
| 177 | 185 |
@Required |
| 178 | 186 |
public void setMappedResultSetFactory(final MappedResultSetFactory mappedResultSetFactory) {
|
| ... | ... | |
| 191 | 199 |
/** |
| 192 | 200 |
* Setter for property 'resultSetClientFactory'. |
| 193 | 201 |
* |
| 194 |
* @param resultSetClientFactory Value to set for property 'resultSetClientFactory'. |
|
| 202 |
* @param resultSetClientFactory |
|
| 203 |
* Value to set for property 'resultSetClientFactory'. |
|
| 195 | 204 |
*/ |
| 196 | 205 |
@Required |
| 197 | 206 |
public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
|
| ... | ... | |
| 210 | 219 |
/** |
| 211 | 220 |
* Setter for property 'countersParam'. |
| 212 | 221 |
* |
| 213 |
* @param countersParam Value to set for property 'countersParam'. |
|
| 222 |
* @param countersParam |
|
| 223 |
* Value to set for property 'countersParam'. |
|
| 214 | 224 |
*/ |
| 215 | 225 |
public void setCountersParam(final String countersParam) {
|
| 216 | 226 |
this.countersParam = countersParam; |
| ... | ... | |
| 224 | 234 |
} |
| 225 | 235 |
|
| 226 | 236 |
/** |
| 227 |
* @param hostedbyMapTable the hostedbyMapTable to set |
|
| 237 |
* @param hostedbyMapTable |
|
| 238 |
* the hostedbyMapTable to set |
|
| 228 | 239 |
*/ |
| 229 | 240 |
public void setHostedbyMapTable(final String hostedbyMapTable) {
|
| 230 | 241 |
this.hostedbyMapTable = hostedbyMapTable; |
| ... | ... | |
| 246 | 257 |
} |
| 247 | 258 |
|
| 248 | 259 |
/** |
| 249 |
* @param overrideDataSourceId the overrideDataSourceId to set |
|
| 260 |
* @param overrideDataSourceId |
|
| 261 |
* the overrideDataSourceId to set |
|
| 250 | 262 |
*/ |
| 251 | 263 |
public void setOverrideDataSourceId(final String overrideDataSourceId) {
|
| 252 | 264 |
this.overrideDataSourceId = overrideDataSourceId; |
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/hostedby/UpsertHostedByApisJobNode.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes.hostedby; |
| 2 | 2 |
|
| 3 | 3 |
import java.util.Map.Entry; |
| 4 |
import javax.annotation.Resource; |
|
| 5 | 4 |
|
| 5 |
import org.apache.commons.logging.Log; |
|
| 6 |
import org.apache.commons.logging.LogFactory; |
|
| 7 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 8 |
|
|
| 6 | 9 |
import com.googlecode.sarasvati.Arc; |
| 7 | 10 |
import com.googlecode.sarasvati.NodeToken; |
| 8 |
import eu.dnetlib.enabling.datasources.rmi.DatasourceDesc; |
|
| 9 |
import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerService;
|
|
| 10 |
import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerServiceException;
|
|
| 11 |
import eu.dnetlib.enabling.datasources.rmi.IfaceDesc;
|
|
| 12 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
|
|
| 11 |
|
|
| 12 |
import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager;
|
|
| 13 |
import eu.dnetlib.enabling.datasources.common.Api;
|
|
| 14 |
import eu.dnetlib.enabling.datasources.common.ApiParam;
|
|
| 15 |
import eu.dnetlib.enabling.datasources.common.DsmException;
|
|
| 13 | 16 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
| 14 | 17 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
| 15 |
import org.apache.commons.logging.Log; |
|
| 16 |
import org.apache.commons.logging.LogFactory; |
|
| 17 | 18 |
|
| 18 | 19 |
public class UpsertHostedByApisJobNode extends SimpleJobNode {
|
| 19 | 20 |
|
| ... | ... | |
| 21 | 22 |
|
| 22 | 23 |
private static final String HOSTED_BY_COMPLIANCE = "hostedBy"; |
| 23 | 24 |
|
| 24 |
@Resource
|
|
| 25 |
private UniqueServiceLocator serviceLocator;
|
|
| 25 |
@Autowired
|
|
| 26 |
private LocalOpenaireDatasourceManager dsManager;
|
|
| 26 | 27 |
|
| 27 | 28 |
private String countersParam; |
| 28 | 29 |
private String mdId; |
| ... | ... | |
| 33 | 34 |
@Override |
| 34 | 35 |
protected String execute(final NodeToken token) throws Exception {
|
| 35 | 36 |
final HostedByCounters counters = (HostedByCounters) token.getEnv().getTransientAttribute(countersParam); |
| 36 |
final DatasourceManagerService dsManager = serviceLocator.getService(DatasourceManagerService.class); |
|
| 37 | 37 |
final String date = DateUtils.now_ISO8601(); |
| 38 | 38 |
final String namespacePrefix = token.getEnv().getAttribute("namespacePrefix");
|
| 39 | 39 |
|
| 40 | 40 |
log.info(counters); |
| 41 | 41 |
|
| 42 |
for (Entry<String, Integer> e : counters.getCounters().entrySet()) {
|
|
| 43 |
updateHostedByApi(dsManager, e.getKey(), namespacePrefix, date, e.getValue());
|
|
| 42 |
for (final Entry<String, Integer> e : counters.getCounters().entrySet()) {
|
|
| 43 |
updateHostedByApi(e.getKey(), namespacePrefix, date, e.getValue()); |
|
| 44 | 44 |
} |
| 45 | 45 |
|
| 46 | 46 |
return Arc.DEFAULT_ARC; |
| 47 | 47 |
} |
| 48 | 48 |
|
| 49 |
private void updateHostedByApi(final DatasourceManagerService dsManager, |
|
| 50 |
final String dsId, |
|
| 49 |
private void updateHostedByApi(final String dsId, |
|
| 51 | 50 |
final String namepsacePrefix, |
| 52 | 51 |
final String date, |
| 53 | 52 |
final int size) {
|
| 54 | 53 |
log.info("Verifying hostedBy api in ds: " + dsId);
|
| 55 | 54 |
try {
|
| 56 |
final DatasourceDesc ds = dsManager.getDatasource(dsId); |
|
| 57 | 55 |
|
| 58 |
for (IfaceDesc iface : ds.getInterfaces()) {
|
|
| 59 |
if (HOSTED_BY_COMPLIANCE.equals(iface.getCompliance())) {
|
|
| 60 |
return; |
|
| 61 |
} |
|
| 56 |
for (final Api<ApiParam> iface : dsManager.getApis(dsId)) {
|
|
| 57 |
if (HOSTED_BY_COMPLIANCE.equals(iface.getCompatibility())) { return; }
|
|
| 62 | 58 |
} |
| 63 | 59 |
|
| 64 |
final IfaceDesc iface = new IfaceDesc(); |
|
| 65 |
iface.setId("api_________::" + dsId + "::hostedBy");
|
|
| 66 |
iface.setTypology(ds.getDatasourceClass()); |
|
| 67 |
iface.setCompliance(HOSTED_BY_COMPLIANCE); |
|
| 68 |
iface.setAccessProtocol("UNKNOWN");
|
|
| 69 |
iface.setContentDescription("metadata");
|
|
| 70 |
iface.setBaseUrl("");
|
|
| 71 |
iface.setActive(false); |
|
| 72 |
iface.setRemovable(true); |
|
| 73 |
dsManager.addInterface(dsId, iface); |
|
| 74 |
} catch (DatasourceManagerServiceException e) {
|
|
| 60 |
final Api<ApiParam> api = new Api<>(); |
|
| 61 |
api.setDatasource(dsId); |
|
| 62 |
api.setId("api_________::" + dsId + "::hostedBy");
|
|
| 63 |
api.setTypology(dsManager.getDs(dsId).getTypology()); |
|
| 64 |
api.setCompatibility(HOSTED_BY_COMPLIANCE); |
|
| 65 |
api.setProtocol("UNKNOWN");
|
|
| 66 |
api.setContentdescription("metadata");
|
|
| 67 |
api.setBaseurl("");
|
|
| 68 |
api.setActive(false); |
|
| 69 |
api.setRemovable(true); |
|
| 70 |
dsManager.addApi(api); |
|
| 71 |
} catch (final DsmException e) {
|
|
| 75 | 72 |
log.warn("Error setting hostedBy api of ds: " + dsId, e);
|
| 76 | 73 |
} |
| 77 | 74 |
} |
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/FilterManagedDatasourcesJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes; |
|
| 2 |
|
|
| 3 |
import java.io.StringReader; |
|
| 4 |
import java.util.Set; |
|
| 5 |
import javax.xml.ws.wsaddressing.W3CEndpointReference; |
|
| 6 |
|
|
| 7 |
import com.googlecode.sarasvati.Arc; |
|
| 8 |
import com.googlecode.sarasvati.NodeToken; |
|
| 9 |
import eu.dnetlib.enabling.datasources.LocalOpenaireDatasourceManager; |
|
| 10 |
import eu.dnetlib.enabling.resultset.MappedResultSetFactory; |
|
| 11 |
import eu.dnetlib.enabling.resultset.client.utils.EPRUtils; |
|
| 12 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
|
| 13 |
import org.apache.commons.logging.Log; |
|
| 14 |
import org.apache.commons.logging.LogFactory; |
|
| 15 |
import org.dom4j.Document; |
|
| 16 |
import org.dom4j.DocumentException; |
|
| 17 |
import org.dom4j.Element; |
|
| 18 |
import org.dom4j.io.SAXReader; |
|
| 19 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 20 |
|
|
| 21 |
public class FilterManagedDatasourcesJobNode extends SimpleJobNode {
|
|
| 22 |
|
|
| 23 |
private static final Log log = LogFactory.getLog(FilterManagedDatasourcesJobNode.class); |
|
| 24 |
|
|
| 25 |
private String inputEprParam; |
|
| 26 |
private String outputEprParam; |
|
| 27 |
|
|
| 28 |
@Autowired |
|
| 29 |
private LocalOpenaireDatasourceManager dsManager; |
|
| 30 |
|
|
| 31 |
@Autowired |
|
| 32 |
private MappedResultSetFactory mappedResultSetFactory; |
|
| 33 |
|
|
| 34 |
@Override |
|
| 35 |
protected String execute(final NodeToken token) throws Exception {
|
|
| 36 |
final W3CEndpointReference inputEpr = new EPRUtils().getEpr(token.getEnv().getAttribute(getInputEprParam())); |
|
| 37 |
|
|
| 38 |
final Set<String> managedDatasources = dsManager.listManagedDatasourceIds(); |
|
| 39 |
|
|
| 40 |
log.info(String.format("found %s managed datasources", managedDatasources.size()));
|
|
| 41 |
|
|
| 42 |
final W3CEndpointReference outputEpr = mappedResultSetFactory.createMappedResultSet(inputEpr, s -> filterManaged(s, managedDatasources)); |
|
| 43 |
|
|
| 44 |
token.getEnv().setAttribute(getOutputEprParam(), outputEpr.toString()); |
|
| 45 |
|
|
| 46 |
return Arc.DEFAULT_ARC; |
|
| 47 |
} |
|
| 48 |
|
|
| 49 |
/** |
|
| 50 |
* Extracts the datasource id from the input record and checks its existence in the given set. |
|
| 51 |
* @param data |
|
| 52 |
* @param filter |
|
| 53 |
* @return The |
|
| 54 |
* @throws IllegalStateException |
|
| 55 |
*/ |
|
| 56 |
private String filterManaged(final String data, Set<String> filter) throws IllegalStateException {
|
|
| 57 |
try {
|
|
| 58 |
final Document doc = new SAXReader().read(new StringReader(data)); |
|
| 59 |
|
|
| 60 |
final String dsId = doc.valueOf("/record/metadata/ROWS/ROW[@table = 'dsm_datasources']/FIELD[@name = 'id']/text()");
|
|
| 61 |
if (filter.contains(dsId)) {
|
|
| 62 |
doc.selectSingleNode("/record/metadata/ROWS").detach();
|
|
| 63 |
((Element) doc.selectSingleNode("/record/metadata")).addElement("ROWS");
|
|
| 64 |
return doc.asXML(); |
|
| 65 |
} else return data; |
|
| 66 |
} catch (DocumentException e) {
|
|
| 67 |
throw new IllegalStateException(e); |
|
| 68 |
} |
|
| 69 |
} |
|
| 70 |
|
|
| 71 |
public String getInputEprParam() {
|
|
| 72 |
return inputEprParam; |
|
| 73 |
} |
|
| 74 |
|
|
| 75 |
public void setInputEprParam(final String inputEprParam) {
|
|
| 76 |
this.inputEprParam = inputEprParam; |
|
| 77 |
} |
|
| 78 |
|
|
| 79 |
public String getOutputEprParam() {
|
|
| 80 |
return outputEprParam; |
|
| 81 |
} |
|
| 82 |
|
|
| 83 |
public void setOutputEprParam(final String outputEprParam) {
|
|
| 84 |
this.outputEprParam = outputEprParam; |
|
| 85 |
} |
|
| 86 |
|
|
| 87 |
} |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/UpdateRepositoryProfilesJobNode.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes; |
| 2 | 2 |
|
| 3 |
import java.io.StringReader; |
|
| 4 |
import java.util.Map; |
|
| 5 |
import javax.annotation.Resource; |
|
| 6 |
import javax.xml.ws.wsaddressing.W3CEndpointReference; |
|
| 3 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 7 | 4 |
|
| 8 | 5 |
import com.googlecode.sarasvati.Arc; |
| 9 | 6 |
import com.googlecode.sarasvati.NodeToken; |
| 10 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; |
|
| 11 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
|
|
| 12 |
import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
|
|
| 13 |
import eu.dnetlib.msro.workflows.nodes.ProgressJobNode;
|
|
| 7 |
|
|
| 8 |
import eu.dnetlib.enabling.datasources.common.Api;
|
|
| 9 |
import eu.dnetlib.enabling.datasources.common.Datasource;
|
|
| 10 |
import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager;
|
|
| 14 | 11 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
| 15 |
import eu.dnetlib.msro.workflows.resultset.ProcessCountingResultSetFactory; |
|
| 16 |
import eu.dnetlib.msro.workflows.util.ProgressProvider; |
|
| 17 |
import eu.dnetlib.msro.workflows.util.ResultsetProgressProvider; |
|
| 18 |
import eu.dnetlib.msro.workflows.util.WorkflowsConstants; |
|
| 19 |
import org.apache.commons.logging.Log; |
|
| 20 |
import org.apache.commons.logging.LogFactory; |
|
| 21 |
import org.dom4j.Document; |
|
| 22 |
import org.dom4j.io.SAXReader; |
|
| 23 | 12 |
|
| 24 |
public class UpdateRepositoryProfilesJobNode extends SimpleJobNode implements ProgressJobNode {
|
|
| 13 |
public class UpdateRepositoryProfilesJobNode extends SimpleJobNode {
|
|
| 25 | 14 |
|
| 26 |
private static final String REPOSITORY_SERVICE_RESOURCE_TYPE = "RepositoryServiceResourceType"; |
|
| 15 |
@Autowired |
|
| 16 |
private LocalDatasourceManager<Datasource<?, ?>, Api<?>> dsManager; |
|
| 27 | 17 |
|
| 28 |
@Resource |
|
| 29 |
private ResultSetClientFactory resultSetClientFactory; |
|
| 30 |
|
|
| 31 |
@Resource |
|
| 32 |
private UniqueServiceLocator serviceLocator; |
|
| 33 |
|
|
| 34 |
@Resource |
|
| 35 |
private ProcessCountingResultSetFactory processCountingResultSetFactory; |
|
| 36 |
|
|
| 37 |
private String eprParam = "repoEpr"; |
|
| 38 |
private String existingReposParam = "existingRepos"; |
|
| 39 |
private ResultsetProgressProvider progressProvider; |
|
| 40 |
|
|
| 41 |
private static final Log log = LogFactory.getLog(UpdateRepositoryProfilesJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM |
|
| 42 |
|
|
| 43 | 18 |
@Override |
| 44 | 19 |
protected String execute(final NodeToken token) throws Exception {
|
| 45 |
// datasourceID -> profileID |
|
| 46 |
@SuppressWarnings("unchecked") final Map<String, String> existingRepos =
|
|
| 47 |
(Map<String, String>) token.getFullEnv().getTransientAttribute(getExistingReposParam()); |
|
| 48 |
final W3CEndpointReference epr = (W3CEndpointReference) token.getFullEnv().getTransientAttribute(getEprParam()); |
|
| 20 |
dsManager.regenerateProfiles(); |
|
| 49 | 21 |
|
| 50 |
this.progressProvider = processCountingResultSetFactory.createProgressProvider(token.getProcess(), epr); |
|
| 51 |
|
|
| 52 |
final SAXReader reader = new SAXReader(); |
|
| 53 |
|
|
| 54 |
int countUpdated = 0; |
|
| 55 |
int countInserted = 0; |
|
| 56 |
int countProfileErrors = 0; |
|
| 57 |
|
|
| 58 |
log.info("Adding/updating repository profiles...");
|
|
| 59 |
|
|
| 60 |
final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class); |
|
| 61 |
|
|
| 62 |
for (String profile : resultSetClientFactory.getClient(this.progressProvider.getEpr())) {
|
|
| 63 |
try {
|
|
| 64 |
final Document doc = reader.read(new StringReader(profile)); |
|
| 65 |
final String dsId = doc.valueOf("//EXTRA_FIELDS/FIELD[./key='OpenAireDataSourceId']/value");
|
|
| 66 |
|
|
| 67 |
log.debug("Registering/updating profile:\n " + profile + "\n");
|
|
| 68 |
|
|
| 69 |
if (existingRepos.containsKey(dsId)) {
|
|
| 70 |
final String profId = existingRepos.get(dsId); |
|
| 71 |
doc.selectSingleNode("//RESOURCE_IDENTIFIER/@value").setText(profId);
|
|
| 72 |
registry.updateProfile(profId, doc.asXML(), REPOSITORY_SERVICE_RESOURCE_TYPE); |
|
| 73 |
log.debug("Profile " + profId + " UPDATED for ds " + dsId);
|
|
| 74 |
countUpdated++; |
|
| 75 |
} else {
|
|
| 76 |
final String profId = registry.registerProfile(doc.asXML()); |
|
| 77 |
log.debug("Valid Profile " + profId + " REGISTERED for ds " + dsId);
|
|
| 78 |
countInserted++; |
|
| 79 |
} |
|
| 80 |
} catch (Exception e) {
|
|
| 81 |
log.error("INVALID PROFILE: " + profile, e);
|
|
| 82 |
countProfileErrors++; |
|
| 83 |
} |
|
| 84 |
} |
|
| 85 |
|
|
| 86 |
log.info(" - updated_profiles: " + countUpdated);
|
|
| 87 |
log.info(" - inserted_profiles: " + countInserted);
|
|
| 88 |
log.info(" - profiles_with_errors: " + countProfileErrors);
|
|
| 89 |
|
|
| 90 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "updated_profiles", countUpdated); |
|
| 91 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "inserted_profiles", countInserted); |
|
| 92 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "profiles_with_errors", countProfileErrors); |
|
| 93 |
|
|
| 94 | 22 |
return Arc.DEFAULT_ARC; |
| 95 | 23 |
} |
| 96 | 24 |
|
| 97 |
public String getEprParam() {
|
|
| 98 |
return eprParam; |
|
| 99 |
} |
|
| 100 |
|
|
| 101 |
public void setEprParam(final String eprParam) {
|
|
| 102 |
this.eprParam = eprParam; |
|
| 103 |
} |
|
| 104 |
|
|
| 105 |
public String getExistingReposParam() {
|
|
| 106 |
return existingReposParam; |
|
| 107 |
} |
|
| 108 |
|
|
| 109 |
public void setExistingReposParam(final String existingReposParam) {
|
|
| 110 |
this.existingReposParam = existingReposParam; |
|
| 111 |
} |
|
| 112 |
|
|
| 113 |
@Override |
|
| 114 |
public ProgressProvider getProgressProvider() {
|
|
| 115 |
return progressProvider; |
|
| 116 |
} |
|
| 117 |
|
|
| 118 | 25 |
} |
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_EntityRegistry_Native.xml | ||
|---|---|---|
| 9 | 9 |
<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/> |
| 10 | 10 |
</HEADER> |
| 11 | 11 |
<BODY> |
| 12 |
<WORKFLOW_NAME>Aggregate Metadata (datasource, organization and person entities) [Ingestion]</WORKFLOW_NAME>
|
|
| 12 |
<WORKFLOW_NAME>Aggregate Metadata (datasource and organization entities) [Ingestion]</WORKFLOW_NAME>
|
|
| 13 | 13 |
<WORKFLOW_INFO> |
| 14 | 14 |
<FIELD name="Action">Aggregate Metadata</FIELD> |
| 15 | 15 |
<FIELD name="Consequence IS">Ingestion</FIELD> |
| 16 | 16 |
<FIELD name="Datasource class">EntityRegistry, Aggregator</FIELD> |
| 17 |
<FIELD name="Content">datasource, organization and person entities</FIELD>
|
|
| 17 |
<FIELD name="Content">datasource and organization entities</FIELD>
|
|
| 18 | 18 |
</WORKFLOW_INFO> |
| 19 | 19 |
<WORKFLOW_TYPE>REPO_HI</WORKFLOW_TYPE> |
| 20 | 20 |
<WORKFLOW_PRIORITY>20</WORKFLOW_PRIORITY> |
| ... | ... | |
| 42 | 42 |
<NODE name="createMetaWf" type="RegisterMetaWf"> |
| 43 | 43 |
<DESCRIPTION>Create MetaWorkflow</DESCRIPTION> |
| 44 | 44 |
<PARAMETERS> |
| 45 |
<PARAM name="wfName" managedBy="system" required="true" type="string">Aggregate Metadata (datasource, organization and person entities) [Ingestion]</PARAM>
|
|
| 45 |
<PARAM name="wfName" managedBy="system" required="true" type="string">Aggregate Metadata (datasource and organization entities) [Ingestion]</PARAM>
|
|
| 46 | 46 |
</PARAMETERS> |
| 47 | 47 |
<ARCS> |
| 48 | 48 |
<ARC to="createGMF"/> |
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/is/update-repositories.xml | ||
|---|---|---|
| 12 | 12 |
<WORKFLOW_TYPE>UpdateIS</WORKFLOW_TYPE> |
| 13 | 13 |
<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY> |
| 14 | 14 |
<CONFIGURATION start="auto"> |
| 15 |
<NODE name="findComplianceRepos" type="FindComplianceRepositories" isStart="true">
|
|
| 15 |
<NODE name="updateRepoProfiles" type="UpdateRepositoryProfiles" isStart="true">>
|
|
| 16 | 16 |
<DESCRIPTION/> |
| 17 |
<PARAMETERS> |
|
| 18 |
<PARAM name="dbName" managedBy="user" type="string" required="true">dnet_openaireplus</PARAM> |
|
| 19 |
<PARAM name="eprParam" managedBy="system" type="string" required="true">repoEpr</PARAM> |
|
| 20 |
</PARAMETERS> |
|
| 21 |
<ARCS> |
|
| 22 |
<ARC to="waitRepos"/> |
|
| 23 |
</ARCS> |
|
| 24 |
</NODE> |
|
| 25 |
|
|
| 26 |
<NODE name="findExistingRepos" type="FindExistingRepositories" isStart="true"> |
|
| 27 |
<DESCRIPTION/> |
|
| 28 |
<PARAMETERS> |
|
| 29 |
<PARAM name="existingReposParam" managedBy="system" type="string" required="true">existingRepos</PARAM> |
|
| 30 |
</PARAMETERS> |
|
| 31 |
<ARCS> |
|
| 32 |
<ARC to="waitRepos"/> |
|
| 33 |
</ARCS> |
|
| 34 |
</NODE> |
|
| 35 |
|
|
| 36 |
<NODE name="waitRepos" isJoin="true"> |
|
| 37 |
<DESCRIPTION/> |
|
| 38 | 17 |
<PARAMETERS/> |
| 39 | 18 |
<ARCS> |
| 40 |
<ARC to="updateRepoProfiles"/> |
|
| 41 |
</ARCS> |
|
| 42 |
</NODE> |
|
| 43 |
|
|
| 44 |
<NODE name="updateRepoProfiles" type="UpdateRepositoryProfiles"> |
|
| 45 |
<DESCRIPTION/> |
|
| 46 |
<PARAMETERS> |
|
| 47 |
<PARAM name="eprParam" managedBy="system" type="string" required="true">repoEpr</PARAM> |
|
| 48 |
<PARAM name="existingReposParam" managedBy="system" type="string" required="true">existingRepos</PARAM> |
|
| 49 |
</PARAMETERS> |
|
| 50 |
<ARCS> |
|
| 51 | 19 |
<ARC to="success"/> |
| 52 | 20 |
</ARCS> |
| 53 | 21 |
</NODE> |
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasources.sql | ||
|---|---|---|
| 1 | 1 |
SELECT |
| 2 | 2 |
d.id AS datasourceid, |
| 3 |
--regexp_replace(d.id, '^.*::', '') AS originalid, |
|
| 4 |
-- array_agg(DISTINCT (d.id, i.pid)) as identities, |
|
| 5 |
ARRAY [d.id, i.pid] AS identities, |
|
| 6 |
--SELECT ARRAY[dd.id, ii.pid] as identities from datasources dd left outer join datasourcepids dps2 on (dps2.datasource = dd.id) |
|
| 7 |
-- left outer join identities ii on (ii.pid = dps2.pid) |
|
| 8 |
|
|
| 9 |
-- d.id AS originalid, |
|
| 3 |
d.id || array_agg(di.pid) AS identities, |
|
| 10 | 4 |
d.officialname AS officialname, |
| 11 | 5 |
d.englishname AS englishname, |
| 12 |
|
|
| 13 | 6 |
CASE |
| 14 |
WHEN (array_agg(DISTINCT tcc.compliance :: TEXT) @> ARRAY ['driver', 'openaire2.0'])
|
|
| 15 |
THEN |
|
| 16 |
'driver-openaire2.0@@@OpenAIRE 2.0+ (DRIVER OA, EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
|
| 17 |
WHEN (array_agg(DISTINCT tcc.compliance :: TEXT) @> ARRAY ['driver'])
|
|
| 18 |
THEN |
|
| 19 |
'driver@@@OpenAIRE Basic (DRIVER OA)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
|
| 20 |
WHEN (array_agg(DISTINCT tcc.compliance :: TEXT) @> ARRAY ['openaire2.0'])
|
|
| 21 |
THEN |
|
| 22 |
'openaire2.0@@@OpenAIRE 2.0 (EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
|
| 23 |
WHEN (array_agg(DISTINCT tcc.compliance :: TEXT) @> ARRAY ['openaire3.0'])
|
|
| 24 |
THEN |
|
| 25 |
'openaire3.0@@@OpenAIRE 3.0 (OA, funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
|
| 26 |
WHEN (array_agg(DISTINCT tcc.compliance :: TEXT) @> ARRAY ['openaire2.0_data'])
|
|
| 27 |
THEN |
|
| 28 |
'openaire2.0_data@@@OpenAIRE Data (funded, referenced datasets)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
|
| 29 |
WHEN (array_agg(DISTINCT tcc.compliance :: TEXT) @> ARRAY ['native'])
|
|
| 30 |
THEN |
|
| 31 |
'native@@@proprietary@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
|
| 32 |
WHEN (array_agg(DISTINCT tcc.compliance :: TEXT) @> ARRAY ['hostedBy'])
|
|
| 33 |
THEN |
|
| 34 |
'hostedBy@@@collected from a compatible aggregator@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
|
| 35 |
WHEN (array_agg(DISTINCT tcc.compliance :: TEXT) @> ARRAY ['notCompatible'])
|
|
| 36 |
THEN |
|
| 7 |
WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility):: TEXT) @> ARRAY ['driver', 'openaire2.0'])
|
|
| 8 |
THEN
|
|
| 9 |
'driver-openaire2.0@@@OpenAIRE 2.0+ (DRIVER OA, EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
|
|
| 10 |
WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['driver'])
|
|
| 11 |
THEN
|
|
| 12 |
'driver@@@OpenAIRE Basic (DRIVER OA)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
|
|
| 13 |
WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire2.0'])
|
|
| 14 |
THEN
|
|
| 15 |
'openaire2.0@@@OpenAIRE 2.0 (EC funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
|
|
| 16 |
WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire3.0'])
|
|
| 17 |
THEN
|
|
| 18 |
'openaire3.0@@@OpenAIRE 3.0 (OA, funding)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
|
|
| 19 |
WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['openaire2.0_data'])
|
|
| 20 |
THEN
|
|
| 21 |
'openaire2.0_data@@@OpenAIRE Data (funded, referenced datasets)@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
|
|
| 22 |
WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['native'])
|
|
| 23 |
THEN
|
|
| 24 |
'native@@@proprietary@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
|
|
| 25 |
WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['hostedBy'])
|
|
| 26 |
THEN
|
|
| 27 |
'hostedBy@@@collected from a compatible aggregator@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel'
|
|
| 28 |
WHEN (array_agg(DISTINCT COALESCE (a.compatibility_override, a.compatibility) :: TEXT) @> ARRAY ['notCompatible'])
|
|
| 29 |
THEN
|
|
| 37 | 30 |
'notCompatible@@@under validation@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
| 38 | 31 |
ELSE |
| 39 | 32 |
'UNKNOWN@@@not available@@@dnet:datasourceCompatibilityLevel@@@dnet:datasourceCompatibilityLevel' |
| 40 | 33 |
END AS openairecompatibility, |
| 41 |
|
|
| 42 | 34 |
d.websiteurl AS websiteurl, |
| 43 | 35 |
d.logourl AS logourl, |
| 44 |
d.contactemail AS contactemail, |
|
| 45 |
array_agg(DISTINCT CASE |
|
| 46 |
WHEN apc.param = 'baseUrl' AND api.protocolclass = 'oai' |
|
| 47 |
THEN |
|
| 48 |
CASE WHEN COALESCE(apc.edited, '') = '' |
|
| 49 |
THEN apc.original |
|
| 50 |
ELSE apc.edited END |
|
| 51 |
ELSE |
|
| 52 |
NULL |
|
| 53 |
END) AS accessinfopackage, |
|
| 54 |
-- we need baseUrl param only -when available-, but we can not use WHERE clause on apc.param, otherwise we exclude datasources without a baseUrl param and the returned rows are less than we expect. |
|
| 36 |
array_agg(DISTINCT CASE WHEN a.protocol = 'oai' THEN a.baseurl ELSE NULL END) AS accessinfopackage, |
|
| 55 | 37 |
d.latitude AS latitude, |
| 56 | 38 |
d.longitude AS longitude, |
| 57 | 39 |
d.namespaceprefix AS namespaceprefix, |
| 58 |
d.od_numberofitems AS odnumberofitems,
|
|
| 59 |
d.od_numberofitemsdate AS odnumberofitemsdate,
|
|
| 60 |
array_agg(DISTINCT s.name || '###' || sc.code || '@@@' || sc.name || '@@@' || ss.code || '@@@' || ss.name) AS subjects,
|
|
| 40 |
NULL AS odnumberofitems,
|
|
| 41 |
NULL AS odnumberofitemsdate,
|
|
| 42 |
(SELECT array_agg(s|| '###unknown@@@unknown@@@dnet:subjects@@@dnet:subjects') FROM UNNEST(d.subjects::text[]) AS s) AS subjects,
|
|
| 61 | 43 |
d.description AS description, |
| 62 |
d.od_policies AS odpolicies, |
|
| 63 |
ARRAY(SELECT trim(s) |
|
| 64 |
FROM unnest(string_to_array(d.od_languages, ',')) AS s) AS odlanguages, |
|
| 65 |
ARRAY(SELECT trim(s) |
|
| 66 |
FROM unnest(string_to_array(d.od_contenttypes, '-')) AS s) AS odcontenttypes, |
|
| 67 |
d.inferred AS inferred, |
|
| 68 |
d.deletedbyinference AS deletedbyinference, |
|
| 69 |
d.trust AS trust, |
|
| 70 |
d.inferenceprovenance AS inferenceprovenance, |
|
| 44 |
NULL AS odpolicies, |
|
| 45 |
d.languages AS odlanguages, |
|
| 46 |
d.od_contenttypes AS odcontenttypes, |
|
| 47 |
false AS inferred, |
|
| 48 |
false AS deletedbyinference, |
|
| 49 |
0.9 AS trust, |
|
| 50 |
NULL AS inferenceprovenance, |
|
| 71 | 51 |
d.dateofcollection AS dateofcollection, |
| 72 | 52 |
d.dateofvalidation AS dateofvalidation, |
| 73 |
|
|
| 74 |
-- re3data fields |
|
| 53 |
-- re3data fields |
|
| 75 | 54 |
d.releasestartdate AS releasestartdate, |
| 76 | 55 |
d.releaseenddate AS releaseenddate, |
| 77 | 56 |
d.missionstatementurl AS missionstatementurl, |
| ... | ... | |
| 86 | 65 |
d.qualitymanagementkind AS qualitymanagementkind, |
| 87 | 66 |
d.pidsystems AS pidsystems, |
| 88 | 67 |
d.certificates AS certificates, |
| 89 |
array_agg(DISTINCT p.name || '&&&' || p.url) AS policies, |
|
| 90 |
-- end of re3data fields |
|
| 91 |
|
|
| 68 |
ARRAY[]::text[] AS policies, |
|
| 92 | 69 |
dc.id AS collectedfromid, |
| 93 | 70 |
dc.officialname AS collectedfromname, |
| 71 |
d.typology || '@@@' || d.typology || '@@@dnet:datasource_typologies@@@dnet:datasource_typologies' AS datasourcetype, |
|
| 72 |
'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction |
|
| 94 | 73 |
|
| 95 |
tc.code || '@@@' || tc.name || '@@@' || ts.code || '@@@' || ts.name AS datasourcetype, |
|
| 96 |
pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name AS provenanceaction |
|
| 74 |
FROM dsm_datasources d |
|
| 97 | 75 |
|
| 98 |
-- merged into identities above |
|
| 99 |
-- array_agg(DISTINCT i.pid || '###' || i.issuertypeclass) as pid |
|
| 76 |
LEFT OUTER JOIN dsm_datasources dc on (d.collectedfrom = dc.id) |
|
| 77 |
LEFT OUTER JOIN dsm_api a ON (d.id = a.datasource) |
|
| 78 |
LEFT OUTER JOIN dsm_datasourcepids di ON (d.id = di.datasource) |
|
| 100 | 79 |
|
| 101 |
|
|
| 102 |
FROM datasources d |
|
| 103 |
LEFT OUTER JOIN class tc ON (tc.code = d.datasourceclass) |
|
| 104 |
LEFT OUTER JOIN scheme ts ON (ts.code = d.datasourcescheme) |
|
| 105 |
|
|
| 106 |
LEFT OUTER JOIN class pac ON (pac.code = d.provenanceactionclass) |
|
| 107 |
LEFT OUTER JOIN scheme pas ON (pas.code = d.provenanceactionscheme) |
|
| 108 |
|
|
| 109 |
LEFT OUTER JOIN datasourcepids dps ON (dps.datasource = d.id) |
|
| 110 |
LEFT OUTER JOIN identities i ON (i.pid = dps.pid) |
|
| 111 |
|
|
| 112 |
LEFT OUTER JOIN datasources dc ON (dc.id = d.collectedfrom) |
|
| 113 |
LEFT OUTER JOIN api ON (api.datasource = d.id) |
|
| 114 |
LEFT OUTER JOIN apicollections apc ON (apc.api = api.id) |
|
| 115 |
|
|
| 116 |
LEFT OUTER JOIN temp_compliances tcc ON (tcc.api = api.id) |
|
| 117 |
|
|
| 118 |
LEFT OUTER JOIN datasource_subject ds ON (ds.datasource = d.id) |
|
| 119 |
LEFT OUTER JOIN subjects s ON (s.id = ds.subject) |
|
| 120 |
|
|
| 121 |
LEFT OUTER JOIN class sc ON (sc.code = s.semanticclass) |
|
| 122 |
LEFT OUTER JOIN scheme ss ON (ss.code = s.semanticscheme) |
|
| 123 |
|
|
| 124 |
LEFT OUTER JOIN datasource_policy dp ON (dp.datasource = d.id) |
|
| 125 |
LEFT OUTER JOIN policies p ON (p.id = dp.policy) |
|
| 126 |
|
|
| 127 |
-- These clauses have been commented to allow the import of all the datasource on the index |
|
| 128 |
-- WHERE d.datasourceclass != 'entityregistry' |
|
| 129 |
-- AND d.openairecompatibilityclass IS NOT null |
|
| 130 |
-- AND d.openairecompatibilityclass != 'UNKNOWN' |
|
| 131 |
-- AND d.openairecompatibilityclass != 'notCompatible' |
|
| 132 |
|
|
| 133 | 80 |
GROUP BY |
| 134 | 81 |
d.id, |
| 135 | 82 |
d.officialname, |
| ... | ... | |
| 139 | 86 |
d.contactemail, |
| 140 | 87 |
d.namespaceprefix, |
| 141 | 88 |
d.description, |
| 142 |
d.od_numberofitems, |
|
| 143 |
d.od_numberofitemsdate, |
|
| 144 |
d.od_policies, |
|
| 145 |
d.od_languages, |
|
| 146 |
d.od_contenttypes, |
|
| 147 | 89 |
d.latitude, |
| 148 | 90 |
d.longitude, |
| 149 |
d.inferred, |
|
| 150 |
d.deletedbyinference, |
|
| 151 |
d.trust, |
|
| 152 |
d.inferenceprovenance, |
|
| 153 | 91 |
d.dateofcollection, |
| 154 | 92 |
d.dateofvalidation, |
| 155 |
dc.id, |
|
| 156 | 93 |
d.releasestartdate, |
| 157 | 94 |
d.releaseenddate, |
| 158 | 95 |
d.missionstatementurl, |
| ... | ... | |
| 167 | 104 |
d.qualitymanagementkind, |
| 168 | 105 |
d.pidsystems, |
| 169 | 106 |
d.certificates, |
| 170 |
dc.officialname, |
|
| 171 |
tc.code, tc.name, ts.code, ts.name, |
|
| 172 |
pac.code, pac.name, pas.code, pas.name, |
|
| 173 |
sc.code, sc.name, ss.code, ss.name, |
|
| 174 |
i.pid |
|
| 107 |
dc.id, |
|
| 108 |
dc.officialname |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourceOrganization.sql | ||
|---|---|---|
| 1 | 1 |
SELECT |
| 2 | 2 |
dor.datasource AS datasource, |
| 3 | 3 |
dor.organization AS organization, |
| 4 |
dor.startdate AS startdate,
|
|
| 5 |
dor.enddate AS enddate,
|
|
| 6 |
dor.inferred AS inferred,
|
|
| 7 |
dor.deletedbyinference AS deletedbyinference,
|
|
| 8 |
dor.trust AS trust,
|
|
| 9 |
dor.inferenceprovenance AS inferenceprovenance,
|
|
| 4 |
NULL AS startdate,
|
|
| 5 |
NULL AS enddate,
|
|
| 6 |
false AS inferred,
|
|
| 7 |
false AS deletedbyinference,
|
|
| 8 |
0.9 AS trust,
|
|
| 9 |
NULL AS inferenceprovenance,
|
|
| 10 | 10 |
|
| 11 |
doc.code || '@@@' || doc.name || '@@@' || dos.code || '@@@' || dos.name AS semantics,
|
|
| 12 |
pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name AS provenanceaction
|
|
| 11 |
'providedBy@@@provided by@@@dnet:datasources_organizations_typologies@@@dnet:datasources_organizations_typologies' AS semantics,
|
|
| 12 |
d.provenanceaction || '@@@' || d.provenanceaction || '@@@dnet:provenanceActions@@@dnet:provenanceActions' AS provenanceaction
|
|
| 13 | 13 |
|
| 14 |
FROM datasource_organization dor |
|
| 14 |
FROM dsm_datasource_organization dor |
|
| 15 |
LEFT OUTER JOIN dsm_datasources d ON (dor.datasource = d.id) |
|
| 15 | 16 |
|
| 16 |
LEFT OUTER JOIN datasources d ON (dor.datasource = d.id) |
|
| 17 |
|
|
| 18 |
LEFT OUTER JOIN class pac ON (pac.code = d.provenanceactionclass) |
|
| 19 |
LEFT OUTER JOIN scheme pas ON (pas.code = d.provenanceactionscheme) |
|
| 20 |
|
|
| 21 |
LEFT OUTER JOIN class doc ON (doc.code = dor.semanticclass) |
|
| 22 |
LEFT OUTER JOIN scheme dos ON (dos.code = dor.semanticscheme) |
|
| 23 |
|
|
| 24 |
-- These clauses have been commented to allow the import of all the datasource on the index |
|
| 25 |
-- WHERE d.datasourceclass != 'entityregistry' |
|
| 26 |
-- AND d.openairecompatibilityclass IS NOT null |
|
| 27 |
-- AND d.openairecompatibilityclass != 'UNKNOWN' |
|
| 28 |
-- AND d.openairecompatibilityclass != 'notCompatible' |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryDatasourcesForSize.sql | ||
|---|---|---|
| 1 | 1 |
SELECT count(*) |
| 2 |
FROM datasources; |
|
| 2 |
FROM dsm_datasources; |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/hbase/queryOrganizations.sql | ||
|---|---|---|
| 1 |
CREATE OR REPLACE VIEW orgs AS ( |
|
| 2 |
SELECT |
|
| 3 |
o.id AS organizationid, |
|
| 4 |
o.legalshortname AS legalshortname, |
|
| 5 |
o.legalname AS legalname, |
|
| 6 |
o.websiteurl AS websiteurl, |
|
| 7 |
o.logourl AS logourl, |
|
| 8 |
o.ec_legalbody AS eclegalbody, |
|
| 9 |
o.ec_legalperson AS eclegalperson, |
|
| 10 |
o.ec_nonprofit AS ecnonprofit, |
|
| 11 |
o.ec_researchorganization AS ecresearchorganization, |
|
| 12 |
o.ec_highereducation AS echighereducation, |
|
| 13 |
o.ec_internationalorganizationeurinterests AS ecinternationalorganizationeurinterests, |
|
| 14 |
o.ec_internationalorganization AS ecinternationalorganization, |
|
| 15 |
o.ec_enterprise AS ecenterprise, |
|
| 16 |
o.ec_smevalidated AS ecsmevalidated, |
|
| 17 |
o.ec_nutscode AS ecnutscode, |
|
| 18 |
o.dateofcollection AS dateofcollection, |
|
| 19 |
o.inferred AS inferred, |
|
| 20 |
o.deletedbyinference AS deletedbyinference, |
|
| 21 |
o.trust AS trust, |
|
| 22 |
o.inferenceprovenance AS inferenceprovenance, |
|
| 23 |
dc.id AS collectedfromid, |
|
| 24 |
dc.officialname AS collectedfromname, |
|
| 25 |
|
|
| 26 |
cc.code || '@@@' || cc.name || '@@@' || cs.code || '@@@' || cs.name AS country, |
|
| 27 |
pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name AS provenanceaction, |
|
| 28 |
|
|
| 29 |
array_agg(DISTINCT i.pid || '###' || i.issuertypeclass) AS pid |
|
| 30 |
|
|
| 31 |
FROM organizations o |
|
| 32 |
LEFT OUTER JOIN class cc ON (cc.code = o.countryclass) |
|
| 33 |
LEFT OUTER JOIN scheme cs ON (cs.code = o.countryscheme) |
|
| 34 |
|
|
| 35 |
LEFT OUTER JOIN class pac ON (pac.code = o.provenanceactionclass) |
|
| 36 |
LEFT OUTER JOIN scheme pas ON (pas.code = o.provenanceactionscheme) |
|
| 37 |
|
|
| 38 |
LEFT OUTER JOIN organizationpids op ON (op.organization = o.id) |
|
| 39 |
LEFT OUTER JOIN identities i ON (i.pid = op.pid) |
|
| 40 |
|
|
| 41 |
LEFT OUTER JOIN datasources dc ON (dc.id = o.collectedfrom) |
|
| 42 |
|
|
| 43 |
GROUP BY |
|
| 44 |
o.id, |
|
| 45 |
o.legalshortname, |
|
| 46 |
o.legalname, |
|
| 47 |
o.websiteurl, |
|
| 48 |
o.logourl, |
|
| 49 |
o.ec_legalbody, |
|
| 50 |
o.ec_legalperson, |
|
| 51 |
o.ec_nonprofit, |
|
| 52 |
o.ec_researchorganization, |
|
| 53 |
o.ec_highereducation, |
|
| 54 |
o.ec_internationalorganizationeurinterests, |
|
| 55 |
o.ec_internationalorganization, |
|
| 56 |
o.ec_enterprise, |
|
| 57 |
o.ec_smevalidated, |
|
| 58 |
o.ec_nutscode, |
|
| 59 |
o.dateofcollection, |
|
| 60 |
o.inferred, |
|
| 61 |
o.deletedbyinference, |
|
| 62 |
o.trust, |
|
| 63 |
o.inferenceprovenance, |
|
| 64 |
dc.id, |
|
| 65 |
dc.officialname, |
|
| 66 |
cc.code, cc.name, cs.code, cs.name, |
|
| 67 |
pac.code, pac.name, pas.code, pas.name |
|
| 68 |
); |
|
| 69 |
|
|
| 70 |
SELECT * FROM orgs |
|
| 71 |
UNION |
|
| 1 | 72 |
SELECT |
| 2 |
o.id AS organizationid,
|
|
| 3 |
o.legalshortname AS legalshortname,
|
|
| 4 |
o.legalname AS legalname,
|
|
| 5 |
o.websiteurl AS websiteurl,
|
|
| 6 |
o.logourl AS logourl,
|
|
| 7 |
o.ec_legalbody AS eclegalbody,
|
|
| 8 |
o.ec_legalperson AS eclegalperson,
|
|
| 9 |
o.ec_nonprofit AS ecnonprofit,
|
|
| 10 |
o.ec_researchorganization AS ecresearchorganization,
|
|
| 11 |
o.ec_highereducation AS echighereducation,
|
|
| 12 |
o.ec_internationalorganizationeurinterests AS ecinternationalorganizationeurinterests,
|
|
| 13 |
o.ec_internationalorganization AS ecinternationalorganization,
|
|
| 14 |
o.ec_enterprise AS ecenterprise,
|
|
| 15 |
o.ec_smevalidated AS ecsmevalidated,
|
|
| 16 |
o.ec_nutscode AS ecnutscode,
|
|
| 17 |
o.dateofcollection AS dateofcollection,
|
|
| 18 |
o.inferred AS inferred,
|
|
| 19 |
o.deletedbyinference AS deletedbyinference,
|
|
| 20 |
o.trust AS trust,
|
|
| 21 |
o.inferenceprovenance AS inferenceprovenance,
|
|
| 22 |
dc.id AS collectedfromid,
|
|
| 23 |
dc.officialname AS collectedfromname,
|
|
| 73 |
o.id AS organizationid,
|
|
| 74 |
o.legalshortname AS legalshortname,
|
|
| 75 |
o.legalname AS legalname,
|
|
| 76 |
o.websiteurl AS websiteurl,
|
|
| 77 |
o.logourl AS logourl,
|
|
| 78 |
null AS eclegalbody,
|
|
| 79 |
null AS eclegalperson,
|
|
| 80 |
null AS ecnonprofit,
|
|
| 81 |
null AS ecresearchorganization,
|
|
| 82 |
null AS echighereducation,
|
|
| 83 |
null AS ecinternationalorganizationeurinterests,
|
|
| 84 |
null AS ecinternationalorganization,
|
|
| 85 |
null AS ecenterprise,
|
|
| 86 |
null AS ecsmevalidated,
|
|
| 87 |
null AS ecnutscode,
|
|
| 88 |
o.dateofcollection AS dateofcollection,
|
|
| 89 |
false AS inferred,
|
|
| 90 |
false AS deletedbyinference,
|
|
| 91 |
0.9 AS trust,
|
|
| 92 |
'' AS inferenceprovenance,
|
|
| 93 |
d.id AS collectedfromid,
|
|
| 94 |
d.officialname AS collectedfromname,
|
|
| 24 | 95 |
|
| 25 |
cc.code || '@@@' || cc.name || '@@@' || cs.code || '@@@' || cs.name AS country,
|
|
| 26 |
pac.code || '@@@' || pac.name || '@@@' || pas.code || '@@@' || pas.name AS provenanceaction,
|
|
| 96 |
o.country || '@@@' || o.country || '@@@dnet:countries@@@dnet:countries' AS country,
|
|
| 97 |
'sysimport:crosswalk:entityregistry@@@sysimport:crosswalk:entityregistry@@@dnet:provenance_actions@@@dnet:provenance_actions' AS provenanceaction,
|
|
| 27 | 98 |
|
| 28 |
array_agg(DISTINCT i.pid || '###' || i.issuertypeclass) AS pid |
|
| 99 |
ARRAY[]::text[] AS pid |
|
| 100 |
FROM dsm_organizations o |
|
| 101 |
LEFT OUTER JOIN dsm_datasources d ON (d.id = o.collectedfrom); |
|
| 29 | 102 |
|
| 30 |
FROM organizations o |
|
| 31 |
LEFT OUTER JOIN class cc ON (cc.code = o.countryclass) |
|
| 32 |
LEFT OUTER JOIN scheme cs ON (cs.code = o.countryscheme) |
|
| 33 | 103 |
|
| 34 |
LEFT OUTER JOIN class pac ON (pac.code = o.provenanceactionclass) |
|
| 35 |
LEFT OUTER JOIN scheme pas ON (pas.code = o.provenanceactionscheme) |
|
| 36 | 104 |
|
| 37 |
LEFT OUTER JOIN organizationpids op ON (op.organization = o.id) |
|
| 38 |
LEFT OUTER JOIN identities i ON (i.pid = op.pid) |
|
| 39 | 105 |
|
| 40 |
LEFT OUTER JOIN datasources dc ON (dc.id = o.collectedfrom) |
|
| 41 | 106 |
|
| 42 |
GROUP BY |
|
| 43 |
o.id, |
|
| 44 |
o.legalshortname, |
|
| 45 |
o.legalname, |
|
| 46 |
o.websiteurl, |
|
| 47 |
o.logourl, |
|
| 48 |
o.ec_legalbody, |
|
| 49 |
o.ec_legalperson, |
|
| 50 |
o.ec_nonprofit, |
|
| 51 |
o.ec_researchorganization, |
|
| 52 |
o.ec_highereducation, |
|
| 53 |
o.ec_internationalorganizationeurinterests, |
|
| 54 |
o.ec_internationalorganization, |
|
| 55 |
o.ec_enterprise, |
|
| 56 |
o.ec_smevalidated, |
|
| 57 |
o.ec_nutscode, |
|
| 58 |
o.dateofcollection, |
|
| 59 |
o.inferred, |
|
| 60 |
o.deletedbyinference, |
|
| 61 |
o.trust, |
|
| 62 |
o.inferenceprovenance, |
|
| 63 |
dc.id, |
|
| 64 |
dc.officialname, |
|
| 65 |
cc.code, cc.name, cs.code, cs.name, |
|
| 66 |
pac.code, pac.name, pas.code, pas.name |
|
| 67 |
|
|
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/xslt/opendoar_2_db.xsl | ||
|---|---|---|
| 6 | 6 |
<xsl:param name="namespacePrefix"/> |
| 7 | 7 |
|
| 8 | 8 |
<xsl:template match="/"> |
| 9 |
<record |
|
| 10 |
> |
|
| 9 |
<record> |
|
| 11 | 10 |
|
| 12 | 11 |
<xsl:copy-of select=".//*[local-name()='header']"/> |
| 13 | 12 |
<metadata> |
| 14 | 13 |
<xsl:variable name="rid" select="normalize-space(//repository/@rID)"/> |
| 15 | 14 |
<xsl:variable name="datasourceId" select="concat($namespacePrefix, '::', $rid)"/> |
| 15 |
<xsl:variable name="apiId" select="concat('api_________::', $datasourceId, '::0')"/>
|
|
| 16 | 16 |
<xsl:variable name="oUrl" select="normalize-space(//oUrl)"/> |
| 17 | 17 |
<xsl:variable name="oName" select="normalize-space(//oName)"/> |
| 18 | 18 |
<xsl:variable name="organizationId" select="translate(concat($namespacePrefix, '::', $oName), ' ', '_')"/> |
| ... | ... | |
| 45 | 45 |
<xsl:value-of select="."/> |
| 46 | 46 |
</xsl:for-each> |
| 47 | 47 |
</xsl:variable> |
| 48 |
|
|
| 49 |
<xsl:variable name="subjects"> |
|
| 50 |
<xsl:for-each select=".//clTitle"> |
|
| 51 |
<xsl:if test="position() > 1">,</xsl:if> |
|
| 52 |
<xsl:value-of select="concat('"', ., '"')"/>
|
|
| 53 |
</xsl:for-each> |
|
| 54 |
</xsl:variable> |
|
| 55 |
|
|
| 48 | 56 |
<ROWS> |
| 49 |
<ROW table="datasources"> |
|
| 50 |
<FIELD name="id">
|
|
| 57 |
<ROW table="dsm_datasources">
|
|
| 58 |
<FIELD name="_dnet_resource_identifier_">
|
|
| 51 | 59 |
<xsl:value-of select="$datasourceId"/> |
| 52 | 60 |
</FIELD> |
| 53 |
<FIELD name="_dnet_resource_identifier_">
|
|
| 61 |
<FIELD name="id">
|
|
| 54 | 62 |
<xsl:value-of select="$datasourceId"/> |
| 55 | 63 |
</FIELD> |
| 56 | 64 |
<FIELD name="officialname"> |
| ... | ... | |
| 59 | 67 |
<FIELD name="englishname"> |
| 60 | 68 |
<xsl:value-of select="normalize-space(//rAcronym)"/> |
| 61 | 69 |
</FIELD> |
| 62 |
<FIELD name="od_contenttypes">
|
|
| 63 |
<xsl:value-of select="normalize-space($contenttypes)"/>
|
|
| 70 |
<FIELD name="websiteurl">
|
|
| 71 |
<xsl:value-of select="normalize-space(//rUrl)"/>
|
|
| 64 | 72 |
</FIELD> |
| 65 |
<FIELD name="od_languages"> |
|
| 66 |
<xsl:value-of select="normalize-space($languages)"/> |
|
| 67 |
</FIELD> |
|
| 68 |
<FIELD name="od_numberofitems"> |
|
| 69 |
<xsl:value-of select="normalize-space(//rNumOfItems)"/> |
|
| 70 |
</FIELD> |
|
| 71 |
<FIELD name="od_numberofitemsdate"> |
|
| 72 |
<xsl:value-of select="normalize-space(//rDateHarvested)"/> |
|
| 73 |
</FIELD> |
|
| 74 |
<FIELD name="description"> |
|
| 75 |
<xsl:value-of select="normalize-space(//rDescription)"/> |
|
| 76 |
</FIELD> |
|
| 77 |
<FIELD name="contactEmail"> |
|
| 73 |
<FIELD name="contactemail"> |
|
| 78 | 74 |
<xsl:value-of select="normalize-space(//pEmail)"/> |
| 79 | 75 |
</FIELD> |
| 80 | 76 |
<FIELD name="latitude" type="float"> |
| ... | ... | |
| 83 | 79 |
<FIELD name="longitude" type="float"> |
| 84 | 80 |
<xsl:value-of select="normalize-space(//paLongitude)"/> |
| 85 | 81 |
</FIELD> |
| 86 |
<FIELD name="websiteurl"> |
|
| 87 |
<xsl:value-of select="normalize-space(//rUrl)"/> |
|
| 88 |
</FIELD> |
|
| 89 |
<FIELD name="logourl"></FIELD> |
|
| 90 | 82 |
<FIELD name="namespaceprefix"> |
| 91 | 83 |
<xsl:value-of select="dnet:generateNsPrefix('od', $rid)"/>
|
| 92 | 84 |
</FIELD> |
| 93 |
<FIELD name="datasourceclass">
|
|
| 94 |
<xsl:value-of select="$repositoryType"/>
|
|
| 85 |
<FIELD name="languages">
|
|
| 86 |
<xsl:value-of select="normalize-space($languages)"/>
|
|
| 95 | 87 |
</FIELD> |
| 96 |
<FIELD name="datasourcescheme">dnet:datasource_typologies</FIELD> |
|
| 97 |
<FIELD name="provenanceActionClass">sysimport:crosswalk:entityregistry</FIELD> |
|
| 98 |
<FIELD name="provenanceActionScheme">dnet:provenanceActions</FIELD> |
|
| 99 |
<FIELD name="typology"> |
|
| 100 |
<xsl:value-of select="normalize-space(//rSoftWareName)"/> |
|
| 88 |
<FIELD name="od_contenttypes"> |
|
| 89 |
<xsl:value-of select="normalize-space($contenttypes)"/> |
|
| 101 | 90 |
</FIELD> |
| 102 |
<FIELD name="optional1"></FIELD> |
|
| 103 |
<FIELD name="optional2"></FIELD> |
|
| 104 | 91 |
<FIELD name="collectedfrom"> |
| 105 | 92 |
<xsl:value-of select="$parentDatasourceId"/> |
| 106 | 93 |
</FIELD> |
| 94 |
<FIELD name="typology"> |
|
| 95 |
<xsl:value-of select="$repositoryType"/> |
|
| 96 |
</FIELD> |
|
| 97 |
<FIELD name="provenanceaction">sysimport:crosswalk:entityregistry</FIELD> |
|
| 98 |
<FIELD name="platform"> |
|
| 99 |
<xsl:value-of select="normalize-space(//rSoftWareName)"/> |
|
| 100 |
</FIELD> |
|
| 101 |
<FIELD name="description"> |
|
| 102 |
<xsl:value-of select="normalize-space(//rDescription)"/> |
|
| 103 |
</FIELD> |
|
| 104 |
|
|
| 105 |
<FIELD name="subjects"> |
|
| 106 |
<xsl:value-of select="concat('{', normalize-space($subjects), '}')"/>
|
|
| 107 |
</FIELD> |
|
| 107 | 108 |
</ROW> |
| 108 | 109 |
|
| 109 |
<xsl:for-each select=".//class"> |
|
| 110 |
<xsl:variable name="subjectId" select="normalize-space(concat($parentDatasourceId, '::', ./clCode))"/> |
|
| 111 |
<xsl:variable name="subjectValue" select="normalize-space(./clTitle)"/> |
|
| 112 |
<ROW table="subjects"> |
|
| 113 |
<FIELD name="id"> |
|
| 114 |
<xsl:value-of select="$subjectId"/> |
|
| 115 |
</FIELD> |
|
| 116 |
<FIELD name="name"> |
|
| 117 |
<xsl:value-of select="$subjectValue"/> |
|
| 118 |
</FIELD> |
|
| 119 |
<FIELD name="semanticclass">dnet:od_subjects</FIELD> |
|
| 120 |
<FIELD name="semanticscheme">dnet:subject_classification_typologies</FIELD> |
|
| 121 |
<FIELD name="_dnet_resource_identifier_"> |
|
| 122 |
<xsl:value-of select="$subjectId"/> |
|
| 123 |
</FIELD> |
|
| 124 |
</ROW> |
|
| 125 |
<ROW table="datasource_subject"> |
|
| 126 |
<FIELD name="datasource"> |
|
| 127 |
<xsl:value-of select="$datasourceId"/> |
|
| 128 |
</FIELD> |
|
| 129 |
<FIELD name="subject"> |
|
| 130 |
<xsl:value-of select="$subjectId"/> |
|
| 131 |
</FIELD> |
|
| 132 |
<FIELD name="_dnet_resource_identifier_"> |
|
| 133 |
<xsl:value-of select="concat($datasourceId, '@@', $subjectId)"/> |
|
| 134 |
</FIELD> |
|
| 135 |
</ROW> |
|
| 136 |
</xsl:for-each> |
|
| 137 |
|
|
| 138 |
<xsl:variable name="apiId" select="concat('api_________::', $datasourceId, '::0')"/>
|
|
| 139 |
<ROW table="api"> |
|
| 140 |
<FIELD name="id"> |
|
| 110 |
<ROW table="dsm_api"> |
|
| 111 |
<FIELD name="_dnet_resource_identifier_"> |
|
| 141 | 112 |
<xsl:value-of select="$apiId"/> |
| 142 | 113 |
</FIELD> |
| 143 |
<FIELD name="_dnet_resource_identifier_">
|
|
| 114 |
<FIELD name="id">
|
|
| 144 | 115 |
<xsl:value-of select="$apiId"/> |
| 145 | 116 |
</FIELD> |
| 146 |
<FIELD name="protocolclass">oai</FIELD>
|
|
| 117 |
<FIELD name="protocol">oai</FIELD> |
|
| 147 | 118 |
<FIELD name="datasource"> |
| 148 | 119 |
<xsl:value-of select="$datasourceId"/> |
| 149 | 120 |
</FIELD> |
| 150 |
<FIELD name="contentdescriptionclass">metadata</FIELD>
|
|
| 151 |
<FIELD name="typologyclass">
|
|
| 121 |
<FIELD name="contentdescription">metadata</FIELD> |
|
| 122 |
<FIELD name="typology"> |
|
| 152 | 123 |
<xsl:value-of select="$repositoryType"/> |
| 153 | 124 |
</FIELD> |
| 125 |
<FIELD name="baseurl"> |
|
| 126 |
<xsl:value-of select="normalize-space(//rOaiBaseUrl)"/> |
|
| 127 |
</FIELD> |
|
| 128 |
<FIELD name="metadata_identifier_path">//*[local-name()='header']/*[local-name()='identifier']</FIELD> |
|
| 154 | 129 |
</ROW> |
| 155 | 130 |
|
| 156 |
<ROW table="apicollections"> |
|
| 157 |
<FIELD name="api"> |
|
| 158 |
<xsl:value-of select="$apiId"/> |
|
| 159 |
</FIELD> |
|
| 160 |
<FIELD name="param">baseUrl</FIELD> |
|
| 131 |
<ROW table="dsm_apiparams"> |
|
| 161 | 132 |
<FIELD name="_dnet_resource_identifier_"> |
| 162 |
<xsl:value-of select="concat($apiId, '@@baseUrl')"/>
|
|
| 133 |
<xsl:value-of select="concat($apiId, '@@format')"/>
|
|
| 163 | 134 |
</FIELD> |
| 164 |
<xsl:choose> |
|
| 165 |
<xsl:when test="string-length(normalize-space(//rOaiBaseUrl)) > 0"> |
|
| 166 |
<FIELD name="original"> |
|
| 167 |
<xsl:value-of select="normalize-space(//rOaiBaseUrl)"/> |
|
| 168 |
</FIELD> |
|
| 169 |
</xsl:when> |
|
| 170 |
<xsl:otherwise> |
|
| 171 |
<FIELD name="original">unknown</FIELD> |
|
| 172 |
</xsl:otherwise> |
|
| 173 |
</xsl:choose> |
|
| 174 |
</ROW> |
|
| 175 |
|
|
| 176 |
<ROW table="apicollections"> |
|
| 177 | 135 |
<FIELD name="api"> |
| 178 | 136 |
<xsl:value-of select="$apiId"/> |
| 179 | 137 |
</FIELD> |
| 180 | 138 |
<FIELD name="param">format</FIELD> |
| 181 |
<FIELD name="_dnet_resource_identifier_"> |
|
| 182 |
<xsl:value-of select="concat($apiId, '@@format')"/> |
|
| 183 |
</FIELD> |
|
| 184 |
<FIELD name="original">oai_dc</FIELD> |
|
| 139 |
<FIELD name="value">oai_dc</FIELD> |
|
| 185 | 140 |
</ROW> |
| 186 | 141 |
|
| 187 |
<ROW table="apicollections"> |
|
| 188 |
<FIELD name="api"> |
|
| 189 |
<xsl:value-of select="$apiId"/> |
|
| 190 |
</FIELD> |
|
| 191 |
<FIELD name="param">metadata_identifier_path</FIELD> |
|
| 142 |
<ROW table="dsm_organizations"> |
|
| 192 | 143 |
<FIELD name="_dnet_resource_identifier_"> |
| 193 |
<xsl:value-of select="concat($apiId, '@@metadata_identifier_path')"/>
|
|
| 144 |
<xsl:value-of select="$organizationId"/>
|
|
| 194 | 145 |
</FIELD> |
| 195 |
<FIELD name="original">//*[local-name()='header']/*[local-name()='identifier']</FIELD> |
|
| 196 |
<FIELD name="accessparam" type="boolean">false</FIELD> |
|
| 197 |
</ROW> |
|
| 198 |
|
|
| 199 |
<ROW table="organizations"> |
|
| 200 | 146 |
<FIELD name="id"> |
| 201 | 147 |
<xsl:value-of select="$organizationId"/> |
| 202 | 148 |
</FIELD> |
| 203 |
<FIELD name="_dnet_resource_identifier_"> |
|
| 204 |
<xsl:value-of select="$organizationId"/> |
|
| 205 |
</FIELD> |
|
| 206 | 149 |
<FIELD name="legalname"> |
| 207 | 150 |
<xsl:value-of select="$oName"/> |
| 208 | 151 |
</FIELD> |
| ... | ... | |
| 211 | 154 |
</FIELD> |
| 212 | 155 |
<FIELD name="websiteurl"> |
| 213 | 156 |
<xsl:choose> |
| 214 |
<xsl:when test="starts-with(normalize-space(//oUrl), 'http')">
|
|
| 215 |
<xsl:value-of select="normalize-space(//oUrl)"/>
|
|
| 157 |
<xsl:when test="starts-with(normalize-space($oUrl), 'http')">
|
|
| 158 |
<xsl:value-of select="normalize-space($oUrl)"/>
|
|
| 216 | 159 |
</xsl:when> |
| 217 |
<xsl:when test="string-length(normalize-space(//oUrl)) > 0">
|
|
| 218 |
<xsl:value-of select="concat('http://', normalize-space(//oUrl))"/>
|
|
| 160 |
<xsl:when test="string-length(normalize-space($oUrl)) > 0">
|
|
| 161 |
<xsl:value-of select="concat('http://', normalize-space($oUrl))"/>
|
|
| 219 | 162 |
</xsl:when> |
| 220 | 163 |
</xsl:choose> |
| 221 | 164 |
</FIELD> |
| 222 |
<FIELD name="countryClass"> |
|
| 165 |
|
|
| 166 |
<FIELD name="country"> |
|
| 223 | 167 |
<xsl:choose> |
| 224 | 168 |
<xsl:when test="normalize-space(//cIsoCode) = 'GB'">UK</xsl:when> |
| 225 | 169 |
<xsl:otherwise> |
| ... | ... | |
| 227 | 171 |
</xsl:otherwise> |
| 228 | 172 |
</xsl:choose> |
| 229 | 173 |
</FIELD> |
| 230 |
<FIELD name="countryScheme">dnet:countries</FIELD> |
|
| 174 |
|
|
| 231 | 175 |
<FIELD name="collectedfrom"> |
| 232 | 176 |
<xsl:value-of select="$parentDatasourceId"/> |
| 233 | 177 |
</FIELD> |
| 234 |
<FIELD name="provenanceActionClass">sysimport:crosswalk:entityregistry</FIELD> |
|
| 235 |
<FIELD name="provenanceActionScheme">dnet:provenanceActions</FIELD> |
|
| 236 |
<FIELD name="trust" type="float">0.9</FIELD> |
|
| 178 |
<FIELD name="provenanceaction">sysimport:crosswalk:entityregistry</FIELD> |
|
| 237 | 179 |
</ROW> |
| 238 | 180 |
|
| 239 |
<ROW table="datasource_organization"> |
|
| 181 |
<ROW table="dsm_datasource_organization"> |
|
| 182 |
<FIELD name="_dnet_resource_identifier_"> |
|
| 183 |
<xsl:value-of select="concat($datasourceId, '@@', $organizationId)"/> |
|
| 184 |
</FIELD> |
|
| 240 | 185 |
<FIELD name="datasource"> |
| 241 | 186 |
<xsl:value-of select="$datasourceId"/> |
| 242 | 187 |
</FIELD> |
| 243 | 188 |
<FIELD name="organization"> |
| 244 | 189 |
<xsl:value-of select="$organizationId"/> |
| 245 | 190 |
</FIELD> |
| 246 |
<FIELD name="_dnet_resource_identifier_"> |
|
| 247 |
<xsl:value-of select="concat($datasourceId,'@@',$organizationId)"/> |
|
| 248 |
</FIELD> |
|
| 249 | 191 |
</ROW> |
| 250 | 192 |
|
Also available in: Unified diff
merged branch dsm into trunk