Project

General

Profile

« Previous | Next » 

Revision 52478

Revised publishign workflows for the index, as it seems we are having troubles when the resultset is created from Iterables.concat

View differences:

modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/parthenos/workflows/nodes/ReadVirtuosoByTypesJobNode.java
1
package eu.dnetlib.parthenos.workflows.nodes;
2

  
3
import java.util.Iterator;
4

  
5
import com.google.common.collect.Iterators;
6
import eu.dnetlib.msro.workflows.graph.Arc;
7
import eu.dnetlib.msro.workflows.procs.Env;
8
import eu.dnetlib.rmi.common.ResultSet;
9
import org.apache.commons.logging.Log;
10
import org.apache.commons.logging.LogFactory;
11
import org.springframework.web.client.RestTemplate;
12

  
13
/**
14
 * Created by Alessia Bardi on 26/01/2018.
15
 *
16
 * @author Alessia Bardi
17
 */
18
public class ReadVirtuosoByTypesJobNode extends ReadVirtuosoJobNode {
19

  
20
	private static final Log log = LogFactory.getLog(ReadVirtuosoByTypesJobNode.class);
21

  
22
	private static String CRMNamespace = "http://www.cidoc-crm.org/cidoc-crm/";
23
	private static String CRMdigNamespace = "http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/";
24
	private static String CRMpeNamespace = "http://parthenos.d4science.org/CRMext/CRMpe.rdfs/";
25

  
26
	@Override
27
	protected String execute(final Env env){
28
		log.info("Using virtuoso reader at : " + getVirtuosoReaderAPIUrl());
29
		RestTemplate restTemplate = new RestTemplate(getClientHttpRequestFactory());
30

  
31
		VirtuosoParthenosByTypeIterator iteratorD14 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMdigNamespace).typeName("D14_Software").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
32
		VirtuosoParthenosByTypeIterator iteratorE39 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMNamespace).typeName("E39_Actor").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
33
		VirtuosoParthenosByTypeIterator iteratorE78 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMNamespace).typeName("E78_Collection").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
34
		VirtuosoParthenosByTypeIterator iteratorPE1 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMpeNamespace).typeName("PE1_Service").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
35
		VirtuosoParthenosByTypeIterator iteratorPE18 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMpeNamespace).typeName("PE18_Dataset").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
36
		VirtuosoParthenosByTypeIterator iteratorPE28 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMpeNamespace).typeName("PE28_Curation_Plan").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
37
		VirtuosoParthenosByTypeIterator iteratorPE35 = new VirtuosoParthenosByTypeIterator().typeNamespace(CRMpeNamespace).typeName("PE35_Project").datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
38

  
39
		Iterator<String> it = Iterators.concat(iteratorD14, iteratorE39, iteratorE78, iteratorPE1, iteratorPE18, iteratorPE28, iteratorPE35);
40
		ResultSet<String> rs = getRsFactory().createResultSet(() -> it);
41

  
42
		env.setAttribute("virtuoso_rs", rs);
43

  
44
		return Arc.DEFAULT_ARC;
45
	}
46
}
modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/parthenos/workflows/nodes/ReadVirtuosoByTypeJobNode.java
1
package eu.dnetlib.parthenos.workflows.nodes;
2

  
3
import eu.dnetlib.msro.workflows.graph.Arc;
4
import eu.dnetlib.msro.workflows.procs.Env;
5
import eu.dnetlib.rmi.common.ResultSet;
6
import org.apache.commons.logging.Log;
7
import org.apache.commons.logging.LogFactory;
8
import org.springframework.web.client.RestTemplate;
9

  
10
/**
11
 * Created by Alessia Bardi on 26/01/2018.
12
 *
13
 * @author Alessia Bardi
14
 */
15
public class ReadVirtuosoByTypeJobNode extends ReadVirtuosoJobNode {
16

  
17
	private static final Log log = LogFactory.getLog(ReadVirtuosoByTypeJobNode.class);
18

  
19
	private String typeNamespace;
20
	private String typeName;
21

  
22
	@Override
23
	protected String execute(final Env env){
24
		log.info("Using virtuoso reader at : " + getVirtuosoReaderAPIUrl());
25
		RestTemplate restTemplate = new RestTemplate(getClientHttpRequestFactory());
26

  
27
		VirtuosoParthenosByTypeIterator iterator = new VirtuosoParthenosByTypeIterator().typeNamespace(typeNamespace).typeName(typeName).datasourceInterface(getDatasourceInterface()).virtuosoReaderAPIUrl(getVirtuosoReaderAPIUrl()).datasourceName(getDatasourceName()).restTemplate(restTemplate);
28
		log.debug(String.format("Created iterator for %s:%s : ", typeNamespace, typeName ));
29
		ResultSet<String> rs = getRsFactory().createResultSet(() -> iterator);
30

  
31
		env.setAttribute("virtuoso_rs", rs);
32

  
33
		return Arc.DEFAULT_ARC;
34
	}
35

  
36
	public String getTypeNamespace() {
37
		return typeNamespace;
38
	}
39

  
40
	public void setTypeNamespace(final String typeNamespace) {
41
		this.typeNamespace = typeNamespace;
42
	}
43

  
44
	public String getTypeName() {
45
		return typeName;
46
	}
47

  
48
	public void setTypeName(final String typeName) {
49
		this.typeName = typeName;
50
	}
51
}
modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/parthenos/workflows/nodes/VirtuosoParthenosIterator.java
186 186
		try {
187 187
			verifyStarted();
188 188
		} catch (Exception e) {
189
			executor.shutdownNow();
189
			getExecutor().shutdownNow();
190 190
			throw new CollectorServiceRuntimeException(e);
191 191
		}
192 192
		switch(currentElement){
......
203 203

  
204 204
	@Override
205 205
	public String doNext() {
206
		try {
207
			verifyStarted();
208
		} catch (Exception e) {
209
			executor.shutdownNow();
210
			throw new CollectorServiceRuntimeException(e);
206
		if(!hasNext()) {
207
			log.error("Next called but hasNext is false", new NoSuchElementException());
208
			throw new NoSuchElementException();
211 209
		}
212 210
		switch(currentElement){
213 211
		case TERMINATOR:
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/parthenos/workflows/nodes/applicationContext-parthenos-msro-nodes.xml
25 25
	<bean id="wfNodeRegisterClarinWorkflowFromTemplate" class="eu.dnetlib.parthenos.workflows.nodes.RegisterClarinWorkflowFromTemplateJobNode" scope="prototype"/>
26 26

  
27 27
	<bean id="wfNodeReadVirtuoso" class="eu.dnetlib.parthenos.workflows.nodes.ReadVirtuosoJobNode" scope="prototype"/>
28
	<bean id="wfNodeReadVirtuosoByTypes" class="eu.dnetlib.parthenos.workflows.nodes.ReadVirtuosoByTypesJobNode" scope="prototype"/>
28
	<bean id="wfNodeReadVirtuosoByType" class="eu.dnetlib.parthenos.workflows.nodes.ReadVirtuosoByTypeJobNode" scope="prototype"/>
29 29

  
30 30
</beans>
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/publish/solr_publish_template.xml
18 18
				<PARAM name="interpretation" description="Interpretation of the MDFormat" required="true" type="string"/>
19 19
			</PARAMETERS>
20 20
			<WORKFLOW>
21
				<NODE name="readFromVirtuoso" type="ReadVirtuosoByTypes" isStart="true">
21
				<NODE name="readSoftwareFromVirtuoso" type="ReadVirtuosoByType" isStart="true">
22 22
					<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION>
23 23
					<PARAMETERS>
24 24
						<PARAM name="datasourceName" ref="dsName"/>
25 25
						<PARAM name="datasourceInterface" ref="interface"/>
26 26
						<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/>
27
						<PARAM name="typeNamespace" value="http://www.ics.forth.gr/isl/CRMext/CRMdig.rdfs/"/>
28
						<PARAM name="typeName" value="D14_Software"/>
27 29
					</PARAMETERS>
28 30
					<ARCS>
29
						<ARC to="index"/>
31
						<ARC to="indexSoftware"/>
30 32
					</ARCS>
31 33
				</NODE>
32
				<NODE name="index" type="UpdateIndex">
34
				<NODE name="indexSoftware" type="UpdateIndex">
33 35
					<DESCRIPTION>Feed index records</DESCRIPTION>
34 36
					<PARAMETERS>
35 37
						<PARAM name="eprParam" value="virtuoso_rs"/>
......
40 42
						<PARAM name="feedingType" ref="feedingType"/>
41 43
					</PARAMETERS>
42 44
					<ARCS>
45
						<ARC to="readActorsFromVirtuoso"/>
46
					</ARCS>
47
				</NODE>
48
				<NODE name="readActorsFromVirtuoso" type="ReadVirtuosoByType">
49
					<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION>
50
					<PARAMETERS>
51
						<PARAM name="datasourceName" ref="dsName"/>
52
						<PARAM name="datasourceInterface" ref="interface"/>
53
						<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/>
54
						<PARAM name="typeNamespace" value="http://www.cidoc-crm.org/cidoc-crm/"/>
55
						<PARAM name="typeName" value="E39_Actor"/>
56
					</PARAMETERS>
57
					<ARCS>
58
						<ARC to="indexActors"/>
59
					</ARCS>
60
				</NODE>
61
				<NODE name="indexActors" type="UpdateIndex">
62
					<DESCRIPTION>Feed index records</DESCRIPTION>
63
					<PARAMETERS>
64
						<PARAM name="eprParam" value="virtuoso_rs"/>
65
						<PARAM name="indexId" ref="indexId"/>
66
						<PARAM name="format" value="CIDOC"/>
67
						<PARAM name="interpretation" ref="interpretation"/>
68
						<PARAM name="layout" value="index"/>
69
						<PARAM name="feedingType" value="INCREMENTAL"/>
70
					</PARAMETERS>
71
					<ARCS>
72
						<ARC to="readCollectionsFromVirtuoso"/>
73
					</ARCS>
74
				</NODE>
75
				<NODE name="readCollectionsFromVirtuoso" type="ReadVirtuosoByType">
76
					<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION>
77
					<PARAMETERS>
78
						<PARAM name="datasourceName" ref="dsName"/>
79
						<PARAM name="datasourceInterface" ref="interface"/>
80
						<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/>
81
						<PARAM name="typeNamespace" value="http://www.cidoc-crm.org/cidoc-crm/"/>
82
						<PARAM name="typeName" value="E78_Collection"/>
83
					</PARAMETERS>
84
					<ARCS>
85
						<ARC to="indexCollections"/>
86
					</ARCS>
87
				</NODE>
88
				<NODE name="indexCollections" type="UpdateIndex">
89
					<DESCRIPTION>Feed index records</DESCRIPTION>
90
					<PARAMETERS>
91
						<PARAM name="eprParam" value="virtuoso_rs"/>
92
						<PARAM name="indexId" ref="indexId"/>
93
						<PARAM name="format" value="CIDOC"/>
94
						<PARAM name="interpretation" ref="interpretation"/>
95
						<PARAM name="layout" value="index"/>
96
						<PARAM name="feedingType" value="INCREMENTAL"/>
97
					</PARAMETERS>
98
					<ARCS>
99
						<ARC to="readServicesFromVirtuoso"/>
100
					</ARCS>
101
				</NODE>
102
				<NODE name="readServicesFromVirtuoso" type="ReadVirtuosoByType">
103
				<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION>
104
				<PARAMETERS>
105
					<PARAM name="datasourceName" ref="dsName"/>
106
					<PARAM name="datasourceInterface" ref="interface"/>
107
					<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/>
108
					<PARAM name="typeNamespace" value="http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"/>
109
					<PARAM name="typeName" value="PE1_Service"/>
110
				</PARAMETERS>
111
				<ARCS>
112
					<ARC to="indexServices"/>
113
				</ARCS>
114
			</NODE>
115
				<NODE name="indexServices" type="UpdateIndex">
116
					<DESCRIPTION>Feed index records</DESCRIPTION>
117
					<PARAMETERS>
118
						<PARAM name="eprParam" value="virtuoso_rs"/>
119
						<PARAM name="indexId" ref="indexId"/>
120
						<PARAM name="format" value="CIDOC"/>
121
						<PARAM name="interpretation" ref="interpretation"/>
122
						<PARAM name="layout" value="index"/>
123
						<PARAM name="feedingType" value="INCREMENTAL"/>
124
					</PARAMETERS>
125
					<ARCS>
126
						<ARC to="readDatasetsFromVirtuoso"/>
127
					</ARCS>
128
				</NODE>
129
				<NODE name="readDatasetsFromVirtuoso" type="ReadVirtuosoByType">
130
					<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION>
131
					<PARAMETERS>
132
						<PARAM name="datasourceName" ref="dsName"/>
133
						<PARAM name="datasourceInterface" ref="interface"/>
134
						<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/>
135
						<PARAM name="typeNamespace" value="http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"/>
136
						<PARAM name="typeName" value="PE18_Dataset"/>
137
					</PARAMETERS>
138
					<ARCS>
139
						<ARC to="indexDatasets"/>
140
					</ARCS>
141
				</NODE>
142
				<NODE name="indexDatasets" type="UpdateIndex">
143
					<DESCRIPTION>Feed index records</DESCRIPTION>
144
					<PARAMETERS>
145
						<PARAM name="eprParam" value="virtuoso_rs"/>
146
						<PARAM name="indexId" ref="indexId"/>
147
						<PARAM name="format" value="CIDOC"/>
148
						<PARAM name="interpretation" ref="interpretation"/>
149
						<PARAM name="layout" value="index"/>
150
						<PARAM name="feedingType" value="INCREMENTAL"/>
151
					</PARAMETERS>
152
					<ARCS>
153
						<ARC to="readCurationPlans"/>
154
					</ARCS>
155
				</NODE>
156
				<NODE name="readCurationPlans" type="ReadVirtuosoByType">
157
					<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION>
158
					<PARAMETERS>
159
						<PARAM name="datasourceName" ref="dsName"/>
160
						<PARAM name="datasourceInterface" ref="interface"/>
161
						<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/>
162
						<PARAM name="typeNamespace" value="http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"/>
163
						<PARAM name="typeName" value="PE28_Curation_Plan"/>
164
					</PARAMETERS>
165
					<ARCS>
166
						<ARC to="indexCurationPlans"/>
167
					</ARCS>
168
				</NODE>
169
				<NODE name="indexCurationPlans" type="UpdateIndex">
170
					<DESCRIPTION>Feed index records</DESCRIPTION>
171
					<PARAMETERS>
172
						<PARAM name="eprParam" value="virtuoso_rs"/>
173
						<PARAM name="indexId" ref="indexId"/>
174
						<PARAM name="format" value="CIDOC"/>
175
						<PARAM name="interpretation" ref="interpretation"/>
176
						<PARAM name="layout" value="index"/>
177
						<PARAM name="feedingType" value="INCREMENTAL"/>
178
					</PARAMETERS>
179
					<ARCS>
180
						<ARC to="readProjects"/>
181
					</ARCS>
182
				</NODE>
183
				<NODE name="readProjects" type="ReadVirtuosoByType">
184
					<DESCRIPTION>Fetch records from Virtuoso</DESCRIPTION>
185
					<PARAMETERS>
186
						<PARAM name="datasourceName" ref="dsName"/>
187
						<PARAM name="datasourceInterface" ref="interface"/>
188
						<PARAM name="virtuosoReaderAPIUrl" ref="virtuosoReaderAPIUrl"/>
189
						<PARAM name="typeNamespace" value="http://parthenos.d4science.org/CRMext/CRMpe.rdfs/"/>
190
						<PARAM name="typeName" value="PE35_Project"/>
191
					</PARAMETERS>
192
					<ARCS>
193
						<ARC to="indexProjects"/>
194
					</ARCS>
195
				</NODE>
196
				<NODE name="indexProjects" type="UpdateIndex">
197
					<DESCRIPTION>Feed index records</DESCRIPTION>
198
					<PARAMETERS>
199
						<PARAM name="eprParam" value="virtuoso_rs"/>
200
						<PARAM name="indexId" ref="indexId"/>
201
						<PARAM name="format" value="CIDOC"/>
202
						<PARAM name="interpretation" ref="interpretation"/>
203
						<PARAM name="layout" value="index"/>
204
						<PARAM name="feedingType" value="INCREMENTAL"/>
205
					</PARAMETERS>
206
					<ARCS>
43 207
						<ARC to="success"/>
44 208
					</ARCS>
45 209
				</NODE>

Also available in: Unified diff