Revision 52478
Added by Alessia Bardi over 5 years ago
| modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/parthenos/workflows/nodes/ReadVirtuosoByTypesJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.parthenos.workflows.nodes; |
|
| 2 |
|
|
| 3 |
import java.util.Iterator; |
|
| 4 |
|
|
| 5 |
import com.google.common.collect.Iterators; |
|
| 6 |
import eu.dnetlib.msro.workflows.graph.Arc; |
|
| 7 |
import eu.dnetlib.msro.workflows.procs.Env; |
|
| 8 |
import eu.dnetlib.rmi.common.ResultSet; |
|
| 9 |
import org.apache.commons.logging.Log; |
|
| 10 |
import org.apache.commons.logging.LogFactory; |
|
| 11 |
import org.springframework.web.client.RestTemplate; |
|
| 12 |
|
|
| 13 |
/** |
|
| 14 |
* Created by Alessia Bardi on 26/01/2018. |
|
| 15 |
* |
|
| 16 |
* @author Alessia Bardi |
|
| 17 |
*/ |
|
| 18 |
public class ReadVirtuosoByTypesJobNode extends ReadVirtuosoJobNode {
|
|
| 19 |
|
|
| 20 |
private static final Log log = LogFactory.getLog(ReadVirtuosoByTypesJobNode.class); |
|
| 21 |
|
|
| 22 |
private static String CRMNamespace = "http://www.cidoc-crm.org/cidoc-crm/"; |
|
| 23 |
private static String CRMdigNamespace = "http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/"; |
|
| 24 |
private static String CRMpeNamespace = "http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"; |
|
| 25 |
|
|
| 26 |
@Override |
|
| 27 |
protected String execute(final Env env){
|
|
| 28 |
log.info("Using virtuoso reader at : " + getVirtuosoReaderAPIUrl());
|
|
| 29 |
RestTemplate restTemplate = new RestTemplate(getClientHttpRequestFactory()); |
|
| 30 |
|
|
| 31 |
VirtuosoParthenosByTypeIterator iteratorD14 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMdigNamespace).typeName("D14_Software").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
|
|
| 32 |
VirtuosoParthenosByTypeIterator iteratorE39 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMNamespace).typeName("E39_Actor").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
|
|
| 33 |
VirtuosoParthenosByTypeIterator iteratorE78 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMNamespace).typeName("E78_Collection").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
|
|
| 34 |
VirtuosoParthenosByTypeIterator iteratorPE1 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMpeNamespace).typeName("PE1_Service").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
|
|
| 35 |
VirtuosoParthenosByTypeIterator iteratorPE18 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMpeNamespace).typeName("PE18_Dataset").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
|
|
| 36 |
VirtuosoParthenosByTypeIterator iteratorPE28 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMpeNamespace).typeName("PE28_Curation_Plan").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
|
|
| 37 |
VirtuosoParthenosByTypeIterator iteratorPE35 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMpeNamespace).typeName("PE35_Project").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
|
|
| 38 |
|
|
| 39 |
Iterator<String> it = Iterators.concat(iteratorD14, iteratorE39, iteratorE78, iteratorPE1, iteratorPE18, iteratorPE28, iteratorPE35); |
|
| 40 |
ResultSet<String> rs = getRsFactory().createResultSet(() -> it); |
|
| 41 |
|
|
| 42 |
env.setAttribute("virtuoso_rs", rs);
|
|
| 43 |
|
|
| 44 |
return Arc.DEFAULT_ARC; |
|
| 45 |
} |
|
| 46 |
} |
|
| modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/parthenos/workflows/nodes/ReadVirtuosoByTypeJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.parthenos.workflows.nodes; |
|
| 2 |
|
|
| 3 |
import eu.dnetlib.msro.workflows.graph.Arc; |
|
| 4 |
import eu.dnetlib.msro.workflows.procs.Env; |
|
| 5 |
import eu.dnetlib.rmi.common.ResultSet; |
|
| 6 |
import org.apache.commons.logging.Log; |
|
| 7 |
import org.apache.commons.logging.LogFactory; |
|
| 8 |
import org.springframework.web.client.RestTemplate; |
|
| 9 |
|
|
| 10 |
/** |
|
| 11 |
* Created by Alessia Bardi on 26/01/2018. |
|
| 12 |
* |
|
| 13 |
* @author Alessia Bardi |
|
| 14 |
*/ |
|
| 15 |
public class ReadVirtuosoByTypeJobNode extends ReadVirtuosoJobNode {
|
|
| 16 |
|
|
| 17 |
private static final Log log = LogFactory.getLog(ReadVirtuosoByTypeJobNode.class); |
|
| 18 |
|
|
| 19 |
private String typeNamespace; |
|
| 20 |
private String typeName; |
|
| 21 |
|
|
| 22 |
@Override |
|
| 23 |
protected String execute(final Env env){
|
|
| 24 |
log.info("Using virtuoso reader at : " + getVirtuosoReaderAPIUrl());
|
|
| 25 |
RestTemplate restTemplate = new RestTemplate(getClientHttpRequestFactory()); |
|
| 26 |
|
|
| 27 |
VirtuosoParthenosByTypeIterator iterator = new VirtuosoParthenosByTypeIterator().typeNamespace(typeNamespace).typeName(typeName).datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate); |
|
| 28 |
log.debug(String.format("Created iterator for %s:%s : ", typeNamespace, typeName ));
|
|
| 29 |
ResultSet<String> rs = getRsFactory().createResultSet(() -> iterator); |
|
| 30 |
|
|
| 31 |
env.setAttribute("virtuoso_rs", rs);
|
|
| 32 |
|
|
| 33 |
return Arc.DEFAULT_ARC; |
|
| 34 |
} |
|
| 35 |
|
|
| 36 |
public String getTypeNamespace() {
|
|
| 37 |
return typeNamespace; |
|
| 38 |
} |
|
| 39 |
|
|
| 40 |
public void setTypeNamespace(final String typeNamespace) {
|
|
| 41 |
this.typeNamespace = typeNamespace; |
|
| 42 |
} |
|
| 43 |
|
|
| 44 |
public String getTypeName() {
|
|
| 45 |
return typeName; |
|
| 46 |
} |
|
| 47 |
|
|
| 48 |
public void setTypeName(final String typeName) {
|
|
| 49 |
this.typeName = typeName; |
|
| 50 |
} |
|
| 51 |
} |
|
| modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/parthenos/workflows/nodes/VirtuosoParthenosIterator.java | ||
|---|---|---|
| 186 | 186 |
try {
|
| 187 | 187 |
verifyStarted(); |
| 188 | 188 |
} catch (Exception e) {
|
| 189 |
executor.shutdownNow();
|
|
| 189 |
getExecutor().shutdownNow();
|
|
| 190 | 190 |
throw new CollectorServiceRuntimeException(e); |
| 191 | 191 |
} |
| 192 | 192 |
switch(currentElement){
|
| ... | ... | |
| 203 | 203 |
|
| 204 | 204 |
@Override |
| 205 | 205 |
public String doNext() {
|
| 206 |
try {
|
|
| 207 |
verifyStarted(); |
|
| 208 |
} catch (Exception e) {
|
|
| 209 |
executor.shutdownNow(); |
|
| 210 |
throw new CollectorServiceRuntimeException(e); |
|
| 206 |
if(!hasNext()) {
|
|
| 207 |
log.error("Next called but hasNext is false", new NoSuchElementException());
|
|
| 208 |
throw new NoSuchElementException(); |
|
| 211 | 209 |
} |
| 212 | 210 |
switch(currentElement){
|
| 213 | 211 |
case TERMINATOR: |
| modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/parthenos/workflows/nodes/applicationContext-parthenos-msro-nodes.xml | ||
|---|---|---|
| 25 | 25 |
<bean id="wfNodeRegisterClarinWorkflowFromTemplate" class="eu.dnetlib.parthenos.workflows.nodes.RegisterClarinWorkflowFromTemplateJobNode" scope="prototype"/> |
| 26 | 26 |
|
| 27 | 27 |
<bean id="wfNodeReadVirtuoso" class="eu.dnetlib.parthenos.workflows.nodes.ReadVirtuosoJobNode" scope="prototype"/> |
| 28 |
<bean id="wfNodeReadVirtuosoByTypes" class="eu.dnetlib.parthenos.workflows.nodes.ReadVirtuosoByTypesJobNode" scope="prototype"/>
|
|
| 28 |
<bean id="wfNodeReadVirtuosoByType" class="eu.dnetlib.parthenos.workflows.nodes.ReadVirtuosoByTypeJobNode" scope="prototype"/>
|
|
| 29 | 29 |
|
| 30 | 30 |
</beans> |
| modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/publish/solr_publish_template.xml | ||
|---|---|---|
| 18 | 18 |
<PARAM name="interpretation" description="Interpretation of the MDFormat" required="true" type="string"/> |
| 19 | 19 |
</PARAMETERS> |
| 20 | 20 |
<WORKFLOW> |
| 21 |
<NODE name="readFromVirtuoso" type="ReadVirtuosoByTypes" isStart="true">
|
|
| 21 |
<NODE name="readSoftwareFromVirtuoso" type="ReadVirtuosoByType" isStart="true">
|
|
| 22 | 22 |
<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION> |
| 23 | 23 |
<PARAMETERS> |
| 24 | 24 |
<PARAM name="datasourceName" ref="dsName"/> |
| 25 | 25 |
<PARAM name="datasourceInterface" ref="interface"/> |
| 26 | 26 |
<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/> |
| 27 |
<PARAM name="typeNamespace" value="http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/"/> |
|
| 28 |
<PARAM name="typeName" value="D14_Software"/> |
|
| 27 | 29 |
</PARAMETERS> |
| 28 | 30 |
<ARCS> |
| 29 |
<ARC to="index"/> |
|
| 31 |
<ARC to="indexSoftware"/>
|
|
| 30 | 32 |
</ARCS> |
| 31 | 33 |
</NODE> |
| 32 |
<NODE name="index" type="UpdateIndex"> |
|
| 34 |
<NODE name="indexSoftware" type="UpdateIndex">
|
|
| 33 | 35 |
<DESCRIPTION>Feed index records</DESCRIPTION> |
| 34 | 36 |
<PARAMETERS> |
| 35 | 37 |
<PARAM name="eprParam" value="virtuoso_rs"/> |
| ... | ... | |
| 40 | 42 |
<PARAM name="feedingType" ref="feedingType"/> |
| 41 | 43 |
</PARAMETERS> |
| 42 | 44 |
<ARCS> |
| 45 |
<ARC to="readActorsFromVirtuoso"/> |
|
| 46 |
</ARCS> |
|
| 47 |
</NODE> |
|
| 48 |
<NODE name="readActorsFromVirtuoso" type="ReadVirtuosoByType"> |
|
| 49 |
<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION> |
|
| 50 |
<PARAMETERS> |
|
| 51 |
<PARAM name="datasourceName" ref="dsName"/> |
|
| 52 |
<PARAM name="datasourceInterface" ref="interface"/> |
|
| 53 |
<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/> |
|
| 54 |
<PARAM name="typeNamespace" value="http://www.cidoc-crm.org/cidoc-crm/"/> |
|
| 55 |
<PARAM name="typeName" value="E39_Actor"/> |
|
| 56 |
</PARAMETERS> |
|
| 57 |
<ARCS> |
|
| 58 |
<ARC to="indexActors"/> |
|
| 59 |
</ARCS> |
|
| 60 |
</NODE> |
|
| 61 |
<NODE name="indexActors" type="UpdateIndex"> |
|
| 62 |
<DESCRIPTION>Feed index records</DESCRIPTION> |
|
| 63 |
<PARAMETERS> |
|
| 64 |
<PARAM name="eprParam" value="virtuoso_rs"/> |
|
| 65 |
<PARAM name="indexId" ref="indexId"/> |
|
| 66 |
<PARAM name="format" value="CIDOC"/> |
|
| 67 |
<PARAM name="interpretation" ref="interpretation"/> |
|
| 68 |
<PARAM name="layout" value="index"/> |
|
| 69 |
<PARAM name="feedingType" value="INCREMENTAL"/> |
|
| 70 |
</PARAMETERS> |
|
| 71 |
<ARCS> |
|
| 72 |
<ARC to="readCollectionsFromVirtuoso"/> |
|
| 73 |
</ARCS> |
|
| 74 |
</NODE> |
|
| 75 |
<NODE name="readCollectionsFromVirtuoso" type="ReadVirtuosoByType"> |
|
| 76 |
<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION> |
|
| 77 |
<PARAMETERS> |
|
| 78 |
<PARAM name="datasourceName" ref="dsName"/> |
|
| 79 |
<PARAM name="datasourceInterface" ref="interface"/> |
|
| 80 |
<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/> |
|
| 81 |
<PARAM name="typeNamespace" value="http://www.cidoc-crm.org/cidoc-crm/"/> |
|
| 82 |
<PARAM name="typeName" value="E78_Collection"/> |
|
| 83 |
</PARAMETERS> |
|
| 84 |
<ARCS> |
|
| 85 |
<ARC to="indexCollections"/> |
|
| 86 |
</ARCS> |
|
| 87 |
</NODE> |
|
| 88 |
<NODE name="indexCollections" type="UpdateIndex"> |
|
| 89 |
<DESCRIPTION>Feed index records</DESCRIPTION> |
|
| 90 |
<PARAMETERS> |
|
| 91 |
<PARAM name="eprParam" value="virtuoso_rs"/> |
|
| 92 |
<PARAM name="indexId" ref="indexId"/> |
|
| 93 |
<PARAM name="format" value="CIDOC"/> |
|
| 94 |
<PARAM name="interpretation" ref="interpretation"/> |
|
| 95 |
<PARAM name="layout" value="index"/> |
|
| 96 |
<PARAM name="feedingType" value="INCREMENTAL"/> |
|
| 97 |
</PARAMETERS> |
|
| 98 |
<ARCS> |
|
| 99 |
<ARC to="readServicesFromVirtuoso"/> |
|
| 100 |
</ARCS> |
|
| 101 |
</NODE> |
|
| 102 |
<NODE name="readServicesFromVirtuoso" type="ReadVirtuosoByType"> |
|
| 103 |
<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION> |
|
| 104 |
<PARAMETERS> |
|
| 105 |
<PARAM name="datasourceName" ref="dsName"/> |
|
| 106 |
<PARAM name="datasourceInterface" ref="interface"/> |
|
| 107 |
<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/> |
|
| 108 |
<PARAM name="typeNamespace" value="http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"/> |
|
| 109 |
<PARAM name="typeName" value="PE1_Service"/> |
|
| 110 |
</PARAMETERS> |
|
| 111 |
<ARCS> |
|
| 112 |
<ARC to="indexServices"/> |
|
| 113 |
</ARCS> |
|
| 114 |
</NODE> |
|
| 115 |
<NODE name="indexServices" type="UpdateIndex"> |
|
| 116 |
<DESCRIPTION>Feed index records</DESCRIPTION> |
|
| 117 |
<PARAMETERS> |
|
| 118 |
<PARAM name="eprParam" value="virtuoso_rs"/> |
|
| 119 |
<PARAM name="indexId" ref="indexId"/> |
|
| 120 |
<PARAM name="format" value="CIDOC"/> |
|
| 121 |
<PARAM name="interpretation" ref="interpretation"/> |
|
| 122 |
<PARAM name="layout" value="index"/> |
|
| 123 |
<PARAM name="feedingType" value="INCREMENTAL"/> |
|
| 124 |
</PARAMETERS> |
|
| 125 |
<ARCS> |
|
| 126 |
<ARC to="readDatasetsFromVirtuoso"/> |
|
| 127 |
</ARCS> |
|
| 128 |
</NODE> |
|
| 129 |
<NODE name="readDatasetsFromVirtuoso" type="ReadVirtuosoByType"> |
|
| 130 |
<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION> |
|
| 131 |
<PARAMETERS> |
|
| 132 |
<PARAM name="datasourceName" ref="dsName"/> |
|
| 133 |
<PARAM name="datasourceInterface" ref="interface"/> |
|
| 134 |
<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/> |
|
| 135 |
<PARAM name="typeNamespace" value="http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"/> |
|
| 136 |
<PARAM name="typeName" value="PE18_Dataset"/> |
|
| 137 |
</PARAMETERS> |
|
| 138 |
<ARCS> |
|
| 139 |
<ARC to="indexDatasets"/> |
|
| 140 |
</ARCS> |
|
| 141 |
</NODE> |
|
| 142 |
<NODE name="indexDatasets" type="UpdateIndex"> |
|
| 143 |
<DESCRIPTION>Feed index records</DESCRIPTION> |
|
| 144 |
<PARAMETERS> |
|
| 145 |
<PARAM name="eprParam" value="virtuoso_rs"/> |
|
| 146 |
<PARAM name="indexId" ref="indexId"/> |
|
| 147 |
<PARAM name="format" value="CIDOC"/> |
|
| 148 |
<PARAM name="interpretation" ref="interpretation"/> |
|
| 149 |
<PARAM name="layout" value="index"/> |
|
| 150 |
<PARAM name="feedingType" value="INCREMENTAL"/> |
|
| 151 |
</PARAMETERS> |
|
| 152 |
<ARCS> |
|
| 153 |
<ARC to="readCurationPlans"/> |
|
| 154 |
</ARCS> |
|
| 155 |
</NODE> |
|
| 156 |
<NODE name="readCurationPlans" type="ReadVirtuosoByType"> |
|
| 157 |
<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION> |
|
| 158 |
<PARAMETERS> |
|
| 159 |
<PARAM name="datasourceName" ref="dsName"/> |
|
| 160 |
<PARAM name="datasourceInterface" ref="interface"/> |
|
| 161 |
<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/> |
|
| 162 |
<PARAM name="typeNamespace" value="http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"/> |
|
| 163 |
<PARAM name="typeName" value="PE28_Curation_Plan"/> |
|
| 164 |
</PARAMETERS> |
|
| 165 |
<ARCS> |
|
| 166 |
<ARC to="indexCurationPlans"/> |
|
| 167 |
</ARCS> |
|
| 168 |
</NODE> |
|
| 169 |
<NODE name="indexCurationPlans" type="UpdateIndex"> |
|
| 170 |
<DESCRIPTION>Feed index records</DESCRIPTION> |
|
| 171 |
<PARAMETERS> |
|
| 172 |
<PARAM name="eprParam" value="virtuoso_rs"/> |
|
| 173 |
<PARAM name="indexId" ref="indexId"/> |
|
| 174 |
<PARAM name="format" value="CIDOC"/> |
|
| 175 |
<PARAM name="interpretation" ref="interpretation"/> |
|
| 176 |
<PARAM name="layout" value="index"/> |
|
| 177 |
<PARAM name="feedingType" value="INCREMENTAL"/> |
|
| 178 |
</PARAMETERS> |
|
| 179 |
<ARCS> |
|
| 180 |
<ARC to="readProjects"/> |
|
| 181 |
</ARCS> |
|
| 182 |
</NODE> |
|
| 183 |
<NODE name="readProjects" type="ReadVirtuosoByType"> |
|
| 184 |
<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION> |
|
| 185 |
<PARAMETERS> |
|
| 186 |
<PARAM name="datasourceName" ref="dsName"/> |
|
| 187 |
<PARAM name="datasourceInterface" ref="interface"/> |
|
| 188 |
<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/> |
|
| 189 |
<PARAM name="typeNamespace" value="http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"/> |
|
| 190 |
<PARAM name="typeName" value="PE35_Project"/> |
|
| 191 |
</PARAMETERS> |
|
| 192 |
<ARCS> |
|
| 193 |
<ARC to="indexProjects"/> |
|
| 194 |
</ARCS> |
|
| 195 |
</NODE> |
|
| 196 |
<NODE name="indexProjects" type="UpdateIndex"> |
|
| 197 |
<DESCRIPTION>Feed index records</DESCRIPTION> |
|
| 198 |
<PARAMETERS> |
|
| 199 |
<PARAM name="eprParam" value="virtuoso_rs"/> |
|
| 200 |
<PARAM name="indexId" ref="indexId"/> |
|
| 201 |
<PARAM name="format" value="CIDOC"/> |
|
| 202 |
<PARAM name="interpretation" ref="interpretation"/> |
|
| 203 |
<PARAM name="layout" value="index"/> |
|
| 204 |
<PARAM name="feedingType" value="INCREMENTAL"/> |
|
| 205 |
</PARAMETERS> |
|
| 206 |
<ARCS> |
|
| 43 | 207 |
<ARC to="success"/> |
| 44 | 208 |
</ARCS> |
| 45 | 209 |
</NODE> |
Also available in: Unified diff
Revised publishign workflows for the index, as it seems we are having troubles when the resultset is created from Iterables.concat