Project

General

Profile

« Previous | Next » 

Revision 49000

Publish/unpublish on virtuoso using the API provided by the springboot app dnet-parthenos-publisher

View differences:

modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/publish/virtuoso_template.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<RESOURCE_PROFILE>
3
	<HEADER>
4
		<RESOURCE_IDENTIFIER value="b7a6c826-6d5e-46c7-a553-4c0fc1d9ab72_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
5
		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
6
		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
7
		<RESOURCE_URI value=""/>
8
		<DATE_OF_CREATION value="2017-07-12T17:00:00.0Z"/>
9
	</HEADER>
10
	<BODY>
11
		<CONFIGURATION>
12
			<PARAMETERS>
13
				<PARAM name="dsId" description="Datasource Id" required="true" type="string"/>
14
				<PARAM name="interface" description="Datasource Interface" required="true" type="string"/>
15
				<PARAM name="cleanMdstoreId" description="Store for transformed records" required="true" type="string"/>
16
			</PARAMETERS>
17
			<WORKFLOW>
18
				<NODE name="fetchMdStore" type="FetchMDStoreRecords" isStart="true">
19
					<DESCRIPTION>Fetch records from MDStore</DESCRIPTION>
20
					<PARAMETERS>
21
						<PARAM name="mdId" ref="cleanMdstoreId"/>
22
						<PARAM name="eprParam" value="clean_epr"/>
23
					</PARAMETERS>
24
					<ARCS>
25
						<ARC to="virtuoso"/>
26
					</ARCS>
27
				</NODE>
28
				<NODE name="virtuoso" type="StoreVirtuoso">
29
					<DESCRIPTION>Feed records into Virtuoso</DESCRIPTION>
30
					<PARAMETERS>
31
						<PARAM name="inputEprParam" value="clean_epr"/>
32
						<PARAM name="interface" value="interface"/>
33
					</PARAMETERS>
34
					<ARCS>
35
						<ARC to="success"/>
36
					</ARCS>
37
				</NODE>
38
			</WORKFLOW>
39
		</CONFIGURATION>
40
	</BODY>
41
</RESOURCE_PROFILE>
modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/parthenos/workflows/nodes/PublishVirtuosoJobNode.java
1
package eu.dnetlib.parthenos.workflows.nodes;
2

  
3
import java.util.List;
4
import java.util.Map;
5

  
6
import com.google.common.collect.Lists;
7
import com.google.common.collect.Maps;
8
import com.google.gson.Gson;
9
import eu.dnetlib.enabling.resultset.client.ResultSetClient;
10
import eu.dnetlib.msro.workflows.graph.Arc;
11
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
12
import eu.dnetlib.msro.workflows.procs.Env;
13
import eu.dnetlib.rmi.common.ResultSet;
14
import eu.dnetlib.rmi.manager.MSROException;
15
import org.apache.commons.logging.Log;
16
import org.apache.commons.logging.LogFactory;
17
import org.apache.http.HttpResponse;
18
import org.apache.http.NameValuePair;
19
import org.apache.http.client.HttpClient;
20
import org.apache.http.client.entity.UrlEncodedFormEntity;
21
import org.apache.http.client.methods.HttpPost;
22
import org.apache.http.impl.client.HttpClients;
23
import org.apache.http.message.BasicNameValuePair;
24
import org.springframework.beans.factory.annotation.Autowired;
25

  
26
public class PublishVirtuosoJobNode extends SimpleJobNode {
27

  
28
	private static final Log log = LogFactory.getLog(PublishVirtuosoJobNode.class);
29

  
30
	private String inputEprParam;
31

  
32
	@Autowired
33
	private ResultSetClient resultSetClient;
34

  
35
	private String publisherEndpoint;
36

  
37

  
38
	@Override
39
	protected String execute(final Env env) throws Exception {
40

  
41
		final ResultSet<?> rsIn = env.getAttribute(getInputEprParam(), ResultSet.class);
42
		if ((rsIn == null)) { throw new MSROException("InputEprParam (" + getInputEprParam()+ ") not found in ENV"); }
43

  
44
		int countAll = 0;
45
		int countOk = 0;
46
		Map<Integer,Integer> errors = Maps.newHashMap();
47
		log.info("Pulisher endpoint: "+getPublisherEndpoint());
48

  
49
		//let's start sequencially
50
		for(String record : getResultSetClient().iter(rsIn, String.class)){
51

  
52
			HttpPost post = new HttpPost(getPublisherEndpoint());
53
			List<NameValuePair> params = Lists.newArrayList();
54
			params.add(new BasicNameValuePair("record", record));
55
			params.add(new BasicNameValuePair("parthenosTarget", "VIRTUOSO"));
56
			UrlEncodedFormEntity ent = new UrlEncodedFormEntity(params, "UTF-8");
57
			post.setEntity(ent);
58
			HttpClient client = HttpClients.createDefault();
59
			HttpResponse responsePOST = client.execute(post);
60
			countAll++;
61
			int statusCode = responsePOST.getStatusLine().getStatusCode();
62
			switch(statusCode){
63
			case 200: countOk++;
64
			default:
65
				errors.merge(statusCode, 1, Integer::sum);
66
			}
67

  
68
		}
69
		env.setAttribute("log:countOk", countOk);
70
		env.setAttribute("log:countAll", countAll);
71
		env.setAttribute("log:errorsMap", new Gson().toJson(errors));
72

  
73
		return Arc.DEFAULT_ARC;
74

  
75
	}
76

  
77
	public String getInputEprParam() {
78
		return this.inputEprParam;
79
	}
80

  
81
	public void setInputEprParam(final String inputEprParam) {
82
		this.inputEprParam = inputEprParam;
83
	}
84

  
85
	public String getPublisherEndpoint() {
86
		return publisherEndpoint;
87
	}
88

  
89
	public void setPublisherEndpoint(final String publisherEndpoint) {
90
		this.publisherEndpoint = publisherEndpoint;
91
	}
92

  
93
	public ResultSetClient getResultSetClient() {
94
		return resultSetClient;
95
	}
96

  
97
	public void setResultSetClient(final ResultSetClient resultSetClient) {
98
		this.resultSetClient = resultSetClient;
99
	}
100

  
101
}
modules/dnet-parthenos/trunk/src/main/java/eu/dnetlib/parthenos/workflows/nodes/UnpublishVirtuosoJobNode.java
1
package eu.dnetlib.parthenos.workflows.nodes;
2

  
3
import java.net.URI;
4

  
5
import eu.dnetlib.msro.workflows.graph.Arc;
6
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;
7
import eu.dnetlib.msro.workflows.procs.Env;
8
import org.apache.commons.logging.Log;
9
import org.apache.commons.logging.LogFactory;
10
import org.apache.http.HttpResponse;
11
import org.apache.http.client.HttpClient;
12
import org.apache.http.client.methods.HttpGet;
13
import org.apache.http.client.utils.URIBuilder;
14
import org.apache.http.impl.client.HttpClients;
15
import org.apache.http.util.EntityUtils;
16

  
17
public class UnpublishVirtuosoJobNode extends SimpleJobNode {
18

  
19
	private static final Log log = LogFactory.getLog(UnpublishVirtuosoJobNode.class);
20

  
21
	private String datasourceInterface;
22
	private String publisherEndpoint;
23

  
24
	@Override
25
	protected String execute(final Env env) throws Exception {
26
		URI getURI = new URIBuilder(getPublisherEndpoint()).addParameter("datasourceApi",  getDatasourceInterface()).addParameter("parthenosTarget", "VIRTUOSO").build();
27
		HttpClient c = HttpClients.createDefault();
28
		HttpResponse res = c.execute(new HttpGet(getURI));
29
		String nTriples = EntityUtils.toString(res.getEntity());
30

  
31
		log.info("Deleted " + nTriples + " triples in Virtuoso server");
32

  
33
		env.setAttribute("triples", nTriples);
34
		return Arc.DEFAULT_ARC;
35

  
36
	}
37

  
38
	public String getDatasourceInterface() {
39
		return datasourceInterface;
40
	}
41

  
42
	public void setDatasourceInterface(final String datasourceInterface) {
43
		this.datasourceInterface = datasourceInterface;
44
	}
45

  
46
	public String getPublisherEndpoint() {
47
		return publisherEndpoint;
48
	}
49

  
50
	public void setPublisherEndpoint(final String publisherEndpoint) {
51
		this.publisherEndpoint = publisherEndpoint;
52
	}
53
}
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/parthenos/workflows/repo-hi/publish_wf.xml.st
26 26
                <PARAM name="OAIInterpretation"     description="OAI Interpretation"                required="true" managedBy="system"  type="string">transformed</PARAM>
27 27
                <PARAM name="OAIDBName"             description="OAI Database Name"                 required="true" managedBy="system"  type="string">oaistore_parthenos</PARAM>
28 28
                <PARAM name="OAIConfiguredOnly"     description="Count only configured OAI sets"    required="true" managedBy="user"    type="boolean">false</PARAM>
29

  
29
                <PARAM name="publisherEndpoint"     description="Parthenos Publisher Endpoint"      required="true" managedBy="user"    type="string"/>
30 30
            </PARAMETERS>
31 31
            <WORKFLOW>
32 32
                <NODE name="index" type="LaunchWorkflowTemplate" isStart="true" >
......
54 54
                        <PARAM name="wfTemplateId" value="b7a6c826-6d5e-46c7-a553-4c0fc1d9ab72_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ==" />
55 55
                        <PARAM name="wfTemplateParams">
56 56
                            <MAP>
57
                                <ENTRY key="dsId"               value="$dsId$" />
57
                                <ENTRY key="publisherEndpoint"  ref="publisherEndpoint" />
58 58
                                <ENTRY key="interface"          value="$interface$" />
59 59
                                <ENTRY key="cleanMdstoreId"     ref="cleanMdstoreId" />
60 60
                            </MAP>
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/parthenos/workflows/nodes/applicationContext-parthenos-msro-nodes.xml
9 9
		class="eu.dnetlib.parthenos.workflows.nodes.ReportReaderJobNode"
10 10
		scope="prototype"/>
11 11

  
12
	<bean id="wfNodePublishVirtuoso"
13
	      class="eu.dnetlib.parthenos.workflows.nodes.PublishVirtuosoJobNode"
14
	      scope="prototype"/>
12 15

  
16
	<bean id="wfNodeUnpublishVirtuoso"
17
	      class="eu.dnetlib.parthenos.workflows.nodes.UnpublishVirtuosoJobNode"
18
	      scope="prototype"/>
19

  
13 20
</beans>
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/TransformationRuleDSResources/TransformationRuleDSResourceType/culturaitalia_musei-actors_416_mapping.xml
1
<RESOURCE_PROFILE>
2
	<HEADER>
3
		<RESOURCE_IDENTIFIER value="97b32f6d-6d9e-4c4c-a1d9-b388c7465422_VHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZXMvVHJhbnNmb3JtYXRpb25SdWxlRFNSZXNvdXJjZVR5cGU="/>
4
		<RESOURCE_TYPE value="TransformationRuleDSResourceType"/>
5
		<RESOURCE_KIND value="TransformationRuleDSResources"/>
6
		<RESOURCE_URI value=""/>
7
		<DATE_OF_CREATION value="2017-08-10T18:14:22+01:00"/>
8
	</HEADER>
9
	<BODY>
10
		<CONFIGURATION>
11
			<IMPORTED/>
12
			<SCRIPT>
13
				<TITLE>Cultura Italia Musei Actor 416 Mapping</TITLE>
14
				<CODE><![CDATA[
15

  
16
                ]]></CODE>
17
			</SCRIPT>
18
		</CONFIGURATION>
19
		<STATUS/>
20
		<SECURITY_PARAMETERS/>
21
	</BODY>
22
</RESOURCE_PROFILE>
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/publish/virtuoso_publish_template.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<RESOURCE_PROFILE>
3
	<HEADER>
4
		<RESOURCE_IDENTIFIER value="b7a6c826-6d5e-46c7-a553-4c0fc1d9ab72_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
5
		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
6
		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
7
		<RESOURCE_URI value=""/>
8
		<DATE_OF_CREATION value="2017-07-12T17:00:00.0Z"/>
9
	</HEADER>
10
	<BODY>
11
		<CONFIGURATION>
12
			<PARAMETERS>
13
				<PARAM name="publisherEndpoint" description="Parthenos Publisher Endpoint" required="true" type="string"/>
14
				<PARAM name="cleanMdstoreId" description="Store for transformed records" required="true" type="string"/>
15
			</PARAMETERS>
16
			<WORKFLOW>
17
				<NODE name="fetchMdStore" type="FetchMDStoreRecords" isStart="true">
18
					<DESCRIPTION>Fetch records from MDStore</DESCRIPTION>
19
					<PARAMETERS>
20
						<PARAM name="mdId" ref="cleanMdstoreId"/>
21
						<PARAM name="eprParam" value="clean_epr"/>
22
					</PARAMETERS>
23
					<ARCS>
24
						<ARC to="publish"/>
25
					</ARCS>
26
				</NODE>
27
				<NODE name="publish" type="PublishVirtuoso">
28
					<DESCRIPTION>Feed records into Virtuoso</DESCRIPTION>
29
					<PARAMETERS>
30
						<PARAM name="inputEprParam" value="clean_epr"/>
31
						<PARAM name="publisherEndpoint" ref="publisherEndpoint"/>
32
					</PARAMETERS>
33
					<ARCS>
34
						<ARC to="success"/>
35
					</ARCS>
36
				</NODE>
37
			</WORKFLOW>
38
		</CONFIGURATION>
39
	</BODY>
40
</RESOURCE_PROFILE>
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/unpublish/virtuoso_unpublish_template.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<RESOURCE_PROFILE>
3
	<HEADER>
4
		<RESOURCE_IDENTIFIER value="e36ccdf8-6ae4-4b67-a326-360243423468_V29ya2Zsb3dUZW1wbGF0ZURTUmVzb3VyY2VzL1dvcmtmbG93VGVtcGxhdGVEU1Jlc291cmNlVHlwZQ=="/>
5
		<RESOURCE_TYPE value="WorkflowTemplateDSResourceType"/>
6
		<RESOURCE_KIND value="WorkflowTemplateDSResources"/>
7
		<RESOURCE_URI value=""/>
8
		<DATE_OF_CREATION value="2017-08-29T17:00:00.0Z"/>
9
	</HEADER>
10
	<BODY>
11
		<CONFIGURATION>
12
			<PARAMETERS>
13
				<PARAM name="publisherEndpoint" description="Parthenos Publisher Endpoint" required="true" type="string"/>
14
				<PARAM name="interface" description="Datasource Interface" required="true" type="string"/>
15
			</PARAMETERS>
16
			<WORKFLOW>
17
				<NODE name="unpublish" type="UnpublishVirtuoso">
18
					<DESCRIPTION>Feed records into Virtuoso</DESCRIPTION>
19
					<PARAMETERS>
20
						<PARAM name="publisherEndpoint" value="publisherEndpoint"/>
21
						<PARAM name="datasourceInterface" value="interface"/>
22
					</PARAMETERS>
23
					<ARCS>
24
						<ARC to="success"/>
25
					</ARCS>
26
				</NODE>
27
			</WORKFLOW>
28
		</CONFIGURATION>
29
	</BODY>
30
</RESOURCE_PROFILE>
modules/dnet-parthenos/trunk/pom.xml
25 25
			<groupId>org.springframework</groupId>
26 26
			<artifactId>spring-core</artifactId>
27 27
			<version>${spring.version}</version>
28
			<scope>test</scope>
28
			<scope>provided</scope>
29 29
		</dependency>
30 30
		<dependency>
31 31
			<groupId>eu.dnetlib</groupId>
modules/dnet-parthenos/trunk/dnet-parthenos.iml
19 19
    <orderEntry type="sourceFolder" forTests="false" />
20 20
    <orderEntry type="library" name="Maven: junit:junit:4.12" level="project" />
21 21
    <orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
22
    <orderEntry type="library" scope="TEST" name="Maven: org.springframework:spring-core:4.2.5.RELEASE" level="project" />
22
    <orderEntry type="library" scope="PROVIDED" name="Maven: org.springframework:spring-core:4.2.5.RELEASE" level="project" />
23 23
    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" />
24 24
    <orderEntry type="module" module-name="dnet-data-services" />
25 25
    <orderEntry type="module" module-name="dnet-core-components" />
......
70 70
    <orderEntry type="library" name="Maven: net.sf.saxon:Saxon-HE:9.5.1-5" level="project" />
71 71
    <orderEntry type="library" name="Maven: jaxen:jaxen:1.1.6" level="project" />
72 72
    <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
73
    <orderEntry type="library" name="Maven: com.ximpleware:vtd-xml:2.13.2" level="project" />
73
    <orderEntry type="library" name="Maven: com.ximpleware:vtd-xml:2.13.4" level="project" />
74 74
    <orderEntry type="library" name="Maven: com.mycila:xmltool:3.3" level="project" />
75 75
    <orderEntry type="library" name="Maven: joda-time:joda-time:2.9.9" level="project" />
76 76
    <orderEntry type="library" name="Maven: org.springframework:spring-test:4.2.5.RELEASE" level="project" />

Also available in: Unified diff