Project

General

Profile

« Previous | Next » 

Revision 36526

Transformation wf nodes

View differences:

modules/dnet-node-services/trunk/src/main/java/eu/dnetlib/msro/worker/nodes/transform/GroovyUnaryFunction.java
1
package eu.dnetlib.msro.worker.nodes.transform;
2

  
3
import java.util.Map;
4

  
5
import com.google.common.base.Function;
6

  
7
public abstract class GroovyUnaryFunction implements Function<String, String> {
8

  
9
	private Map<String, String> params;
10

  
11
	@Override
12
	abstract public String apply(String input);
13

  
14
	public Map<String, String> getParams() {
15
		return params;
16
	}
17

  
18
	public void setParams(final Map<String, String> params) {
19
		this.params = params;
20
	}
21

  
22
}
modules/dnet-node-services/trunk/src/main/java/eu/dnetlib/msro/worker/nodes/transform/ApplyXsltJobNode.java
29 29
	protected String execute(final NodeToken token) throws Exception {
30 30
		@SuppressWarnings("unchecked")
31 31
		final ResultSet<String> inputRs = (ResultSet<String>) token.getEnv().getTransientAttribute(inputResultSetParam);
32
		if (inputRs == null) { throw new ManagerServiceException("InputEprParam (" + inputResultSetParam + ") not found in ENV"); }
32
		if (inputRs == null) { throw new ManagerServiceException("InputResultSetParam (" + inputResultSetParam + ") not found in ENV"); }
33 33

  
34 34
		final Map<String, String> xsltParams = new HashMap<String, String>();
35 35

  
modules/dnet-node-services/trunk/src/main/java/eu/dnetlib/msro/worker/nodes/transform/CleanJobNode.java
32 32
	protected String execute(final NodeToken token) throws Exception {
33 33
		@SuppressWarnings("unchecked")
34 34
		final ResultSet<String> inputRs = (ResultSet<String>) token.getEnv().getTransientAttribute(inputResultSetParam);
35
		if (inputRs == null) { throw new ManagerServiceException("InputEprParam (" + inputResultSetParam + ") not found in ENV"); }
35
		if (inputRs == null) { throw new ManagerServiceException("InputResultSetParam (" + inputResultSetParam + ") not found in ENV"); }
36 36

  
37 37
		final CleaningRule rule = isClient.getResourceByCode(ruleId, CleaningRule.class);
38 38

  
modules/dnet-node-services/trunk/src/main/java/eu/dnetlib/msro/worker/nodes/transform/GroovyJobNode.java
1 1
package eu.dnetlib.msro.worker.nodes.transform;
2 2

  
3
import java.io.IOException;
4
import java.util.Map;
5

  
6 3
import javax.annotation.Resource;
7
import javax.xml.ws.wsaddressing.W3CEndpointReference;
8 4

  
9
import org.apache.commons.logging.Log;
10
import org.apache.commons.logging.LogFactory;
5
import org.springframework.beans.factory.annotation.Autowired;
11 6
import org.springframework.beans.factory.annotation.Required;
12 7

  
13
import com.google.common.collect.Maps;
14 8
import com.googlecode.sarasvati.Arc;
15 9
import com.googlecode.sarasvati.NodeToken;
16 10

  
17
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
18

  
19
import eu.dnetlib.enabling.resultset.MappedResultSetFactory;
20
import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
21
import eu.dnetlib.msro.rmi.ManagerServiceException;
11
import eu.dnetlib.common.services.locators.DnetServiceLocator;
12
import eu.dnetlib.data.resultSet.ResultSetFactory;
13
import eu.dnetlib.data.resultSet.groovy.GroovyFactory;
14
import eu.dnetlib.enabling.datastructures.GroovyRule;
15
import eu.dnetlib.enabling.is.client.InformationServiceClient;
22 16
import eu.dnetlib.msro.worker.nodes.SimpleJobNode;
23
import groovy.lang.GroovyShell;
24
import groovy.util.GroovyScriptEngine;
17
import eu.dnetlib.rmi.objects.resultSet.ResultSet;
18
import eu.dnetlib.rmi.soap.exceptions.ManagerServiceException;
25 19

  
26 20
public class GroovyJobNode extends SimpleJobNode {
27 21

  
28
	private static final Log log = LogFactory.getLog(GroovyJobNode.class);
22
	private String inputResultSetParam;
23
	private String outputResultSetParam;
24
	private String ruleId;
29 25

  
30
	/**
31
	 * used to transform the records using Groovy.
32
	 */
26
	@Autowired
27
	private InformationServiceClient isClient;
33 28

  
34
	private MappedResultSetFactory mappedResultSetFactory;
29
	@Autowired
30
	private ResultSetFactory resultSetFactory;
35 31

  
36
	private String inputEprParam;
37
	private String outputEprParam;
38
	private String transformationRuleId;
39
	// private String groovyParams;
32
	private GroovyFactory groovyFactory;
40 33

  
41 34
	@Resource
42 35
	private DnetServiceLocator serviceLocator;
43 36

  
44
	private Map<String, String> retrieveGroovyParameter() {
45
		Map<String, String> out = Maps.newHashMap();
46

  
47
		String query = "for $x in collection('/db/DRIVER/GroovyProcessingDSResource/GroovyProcessingDSResourceType')"
48
				+ "where $x[.//RESOURCE_IDENTIFIER/@value='" + transformationRuleId + "']"
49
				+ "return concat($x//GROOVY_CLASSPATH/text(),':::',$x//GROOVY_DNETCLASS/text())";
50
		try {
51
			String result = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(query).get(0);
52
			if (result == null) { return null; }
53
			String[] data = result.trim().split(":::");
54
			if (data.length == 2) {
55
				out.put("classpath", data[0]);
56
				out.put("mainClass", data[1]);
57
			}
58

  
59
			return out;
60
		} catch (Exception e) {
61
			log.error(e);
62
			return null;
63
		}
64
	}
65

  
66 37
	@Override
67 38
	protected String execute(final NodeToken token) throws Exception {
68
		final String inputEprString = token.getEnv().getAttribute(inputEprParam);
69
		if (inputEprString == null || inputEprString.isEmpty()) { throw new ManagerServiceException("InputEprParam (" + inputEprParam + ") not found in ENV"); }
70
		final W3CEndpointReference inputEpr = new EPRUtils().getEpr(inputEprString);
71
		String groovyClasspath, groovyDnetClass;
72
		Map<String, String> prop = retrieveGroovyParameter();
73
		groovyClasspath = prop.get("classpath");
74
		groovyDnetClass = prop.get("mainClass");
75
		W3CEndpointReference epr = transformGroovy(inputEpr, groovyClasspath, groovyDnetClass, parseJsonParameters(token));
76
		token.getEnv().setAttribute(outputEprParam, epr.toString());
77
		return Arc.DEFAULT_ARC;
78
	}
39
		@SuppressWarnings("unchecked")
40
		final ResultSet<String> inputRs = (ResultSet<String>) token.getEnv().getTransientAttribute(inputResultSetParam);
41
		if (inputRs == null) { throw new ManagerServiceException("InputResultSetParam (" + inputResultSetParam + ") not found in ENV"); }
79 42

  
80
	private W3CEndpointReference transformGroovy(final W3CEndpointReference source,
81
			final String groovyClasspath,
82
			final String groovyDnetClass,
83
			final Map<String, String> params) throws ClassNotFoundException, IOException {
43
		final GroovyRule rule = isClient.getResourceByCode(ruleId, GroovyRule.class);
84 44

  
85
		GroovyScriptEngine gse = new GroovyScriptEngine(groovyClasspath);
86
		gse.getGroovyClassLoader().loadClass(groovyDnetClass);
87
		log.info("***********************************************");
88
		log.info("Loaded Groovy classes:");
89
		for (Class<?> c : gse.getGroovyClassLoader().getLoadedClasses()) {
90
			log.info(c.getCanonicalName());
91
		}
92
		log.info("***********************************************");
93
		GroovyShell groovyShell = new GroovyShell(gse.getGroovyClassLoader());
45
		final ResultSet<String> outputRs = resultSetFactory.transform(inputRs, groovyFactory.createGroovyFunction(rule, null));
94 46

  
95
		Object go = groovyShell.evaluate("new " + groovyDnetClass + "()");
96
		if (go instanceof GroovyUnaryFunction) {
97
			GroovyUnaryFunction groovyUnaryFunction = (GroovyUnaryFunction) go;
98
			if (params != null) {
99
				groovyUnaryFunction.setParams(params);
100
			}
101
			return mappedResultSetFactory.createMappedResultSet(source, groovyUnaryFunction);
102
		} else {
103
			throw new RuntimeException("Groovy object " + go + " is not supported");
104
		}
47
		token.getEnv().setAttribute(outputResultSetParam, outputRs.toString());
48

  
49
		return Arc.DEFAULT_ARC;
105 50
	}
106 51

  
107
	public MappedResultSetFactory getMappedResultSetFactory() {
108
		return mappedResultSetFactory;
52
	public String getInputResultSetParam() {
53
		return inputResultSetParam;
109 54
	}
110 55

  
111
	@Required
112
	public void setMappedResultSetFactory(final MappedResultSetFactory mappedResultSetFactory) {
113
		this.mappedResultSetFactory = mappedResultSetFactory;
56
	public void setInputResultSetParam(final String inputResultSetParam) {
57
		this.inputResultSetParam = inputResultSetParam;
114 58
	}
115 59

  
116
	public String getInputEprParam() {
117
		return inputEprParam;
60
	public String getOutputResultSetParam() {
61
		return outputResultSetParam;
118 62
	}
119 63

  
120
	public void setInputEprParam(final String inputEprParam) {
121
		this.inputEprParam = inputEprParam;
64
	public void setOutputResultSetParam(final String outputResultSetParam) {
65
		this.outputResultSetParam = outputResultSetParam;
122 66
	}
123 67

  
124
	public String getOutputEprParam() {
125
		return outputEprParam;
68
	public String getRuleId() {
69
		return ruleId;
126 70
	}
127 71

  
128
	public void setOutputEprParam(final String outputEprParam) {
129
		this.outputEprParam = outputEprParam;
72
	public void setRuleId(final String ruleId) {
73
		this.ruleId = ruleId;
130 74
	}
131 75

  
132
	public String getTransformationRuleId() {
133
		return transformationRuleId;
76
	public GroovyFactory getGroovyFactory() {
77
		return groovyFactory;
134 78
	}
135 79

  
136
	public void setTransformationRuleId(final String transformationRuleId) {
137
		this.transformationRuleId = transformationRuleId;
80
	@Required
81
	public void setGroovyFactory(final GroovyFactory groovyFactory) {
82
		this.groovyFactory = groovyFactory;
138 83
	}
139 84

  
140 85
}
modules/dnet-node-services/trunk/src/main/java/eu/dnetlib/msro/worker/nodes/transform/MdBuilderJobNode.java
1 1
package eu.dnetlib.msro.worker.nodes.transform;
2 2

  
3
import java.io.IOException;
4 3
import java.net.URLEncoder;
5
import java.util.List;
6 4

  
7
import javax.xml.ws.wsaddressing.W3CEndpointReference;
8

  
9 5
import org.antlr.stringtemplate.StringTemplate;
10 6
import org.apache.commons.io.IOUtils;
11 7
import org.apache.commons.logging.Log;
12 8
import org.apache.commons.logging.LogFactory;
9
import org.springframework.beans.factory.annotation.Autowired;
13 10
import org.springframework.beans.factory.annotation.Required;
14 11
import org.springframework.core.io.Resource;
15 12

  
16 13
import com.googlecode.sarasvati.Arc;
17 14
import com.googlecode.sarasvati.NodeToken;
18 15

  
19
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
20
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
21

  
22
import eu.dnetlib.enabling.resultset.XSLTMappedResultSetFactory;
23
import eu.dnetlib.enabling.resultset.client.utils.EPRUtils;
24
import eu.dnetlib.msro.rmi.ManagerServiceException;
16
import eu.dnetlib.common.services.locators.DnetServiceLocator;
17
import eu.dnetlib.data.resultSet.ResultSetFactory;
18
import eu.dnetlib.enabling.datastructures.Datasource;
19
import eu.dnetlib.miscutils.ApplyXslt;
25 20
import eu.dnetlib.msro.worker.nodes.SimpleJobNode;
21
import eu.dnetlib.rmi.objects.dsmanager.IfaceDesc;
22
import eu.dnetlib.rmi.objects.resultSet.ResultSet;
23
import eu.dnetlib.rmi.soap.DatasourceManagerService;
24
import eu.dnetlib.rmi.soap.exceptions.ManagerServiceException;
26 25

  
27 26
public class MdBuilderJobNode extends SimpleJobNode {
28 27

  
......
30 29

  
31 30
	private Resource mdBuilderTemplateXslt;
32 31

  
33
	private String inputEprParam;
34
	private String outputEprParam;
32
	private String inputResultSetParam;
33
	private String outputResultSetParam;
35 34
	private String datasourceId;
36 35
	private String datasourceInterface;
37 36

  
38
	private XSLTMappedResultSetFactory xsltMappedResultSetFactory;
37
	@Autowired
38
	private ResultSetFactory resultSetFactory;
39 39

  
40 40
	@javax.annotation.Resource
41 41
	private DnetServiceLocator serviceLocator;
42 42

  
43 43
	@Override
44 44
	protected String execute(final NodeToken token) throws ManagerServiceException {
45
		final String inputEpr = token.getEnv().getAttribute(inputEprParam);
45
		try {
46
			@SuppressWarnings("unchecked")
47
			final ResultSet<String> inputRs = (ResultSet<String>) token.getEnv().getTransientAttribute(inputResultSetParam);
46 48

  
47
		if ((inputEpr == null) || inputEpr.isEmpty()) throw new ManagerServiceException("InputEprParam (" + inputEprParam + ") not found in ENV");
48
		StringTemplate st = null;
49
		try {
49
			final Datasource ds = serviceLocator.getService(DatasourceManagerService.class).getDatasource(datasourceId);
50
			final IfaceDesc ifc = findIface(ds, datasourceInterface);
51

  
52
			if (inputRs == null) { throw new ManagerServiceException("inputResultSetParam (" + inputResultSetParam + ") not found in ENV"); }
53

  
54
			StringTemplate st = null;
50 55
			st = new StringTemplate(IOUtils.toString(getMdBuilderTemplateXslt().getInputStream()));
51 56
			st.setAttribute("datasourceId", datasourceId);
52
			st.setAttribute("xpath", getMetadataIdentifierPath());
53
			st.setAttribute("baseurl", URLEncoder.encode(getBaseUrl(), "UTF-8"));
54
			st.setAttribute("metadatanamespace", getMetadataNamespace());
57
			st.setAttribute("xpath", ifc.getExtraFields().get("metadata_identifier_path"));
58
			st.setAttribute("baseurl", URLEncoder.encode(ifc.getBaseUrl(), "UTF-8"));
59
			st.setAttribute("metadatanamespace", getMetadataNamespace(ifc.getAccessParams().get("format")));
55 60

  
56 61
			/*
57 62
			 * If namespacePrefix has been already pushed to env by some custom JobNode e.g. ObtainOpenaireDataSourceParamsJobNode then push
......
61 66
			if (token.getEnv().hasAttribute("namespacePrefix")) {
62 67
				st.setAttribute("namespacePrefix", token.getEnv().getAttribute("namespacePrefix"));
63 68
			} else {
64
				List<String> namespacePrefix;
65
				String xQuery = "/*[.//RESOURCE_IDENTIFIER/@value='" + datasourceId + "']//EXTRA_FIELDS/FIELD/value[../key='NamespacePrefix']/string()";
66
				namespacePrefix = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xQuery);
67
				if (namespacePrefix.size() != 0) {
68
					st.setAttribute("namespacePrefix", namespacePrefix.get(0));
69
				} else {
70
					xQuery = "/*[.//RESOURCE_IDENTIFIER/@value='" + datasourceId + "']//DATASOURCE_ORIGINAL_ID/string()";
71
					namespacePrefix = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xQuery);
72
					if (namespacePrefix.size() != 0) {
73
						st.setAttribute("namespacePrefix", namespacePrefix.get(0));
74
					} else {
75
						st.setAttribute("namespacePrefix", datasourceId);
76
					}
77
				}
69
				st.setAttribute("namespacePrefix", ds.getNamespacePrefix());
78 70
			}
79 71

  
80
			final W3CEndpointReference epr = xsltMappedResultSetFactory.createMappedResultSet(new EPRUtils().getEpr(inputEpr), st.toString());
72
			final ResultSet<String> rs = resultSetFactory.transform(inputRs, new ApplyXslt(st.toString()));
81 73

  
82
			token.getEnv().setAttribute(outputEprParam, epr.toString());
74
			token.getEnv().setTransientAttribute(outputResultSetParam, rs);
83 75

  
84 76
			return Arc.DEFAULT_ARC;
85
		} catch (ISLookUpException e) {
86
			throw new ManagerServiceException("Error while initializing mdBuilder template (" + getMdBuilderTemplateXslt().getFilename() + ") for datasource "
87
					+ datasourceId, e);
88
		} catch (IOException e) {
89
			throw new ManagerServiceException("Error parsing template: " + getMdBuilderTemplateXslt().getFilename(), e);
90 77
		} catch (Exception e) {
91
			log.error(st.toString());
92
			throw new ManagerServiceException(e);
78
			final String msg = "Error creating mdBuilder (" + getMdBuilderTemplateXslt().getFilename() + ") for datasource " + datasourceId;
79
			log.error(msg, e);
80
			throw new ManagerServiceException(msg, e);
93 81
		}
94 82
	}
95 83

  
96
	private String getBaseUrl() throws ISLookUpException {
97
		String xQuery = "/*[.//RESOURCE_IDENTIFIER/@value='{datasourceId}']//INTERFACE[@id='{interfaceId}']//BASE_URL/string()";
98
		xQuery = xQuery.replace("{interfaceId}", datasourceInterface).replace("{datasourceId}", datasourceId);
99
		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xQuery);
84
	private IfaceDesc findIface(final Datasource ds, final String ifaceId) throws ManagerServiceException {
85
		for (IfaceDesc ifc : ds.getInterfaces()) {
86
			if (ifaceId.equals(ifc.getId())) { return ifc; }
87
		}
88
		throw new ManagerServiceException("Missing ds interface: " + ifaceId);
100 89
	}
101 90

  
102
	private String getMetadataIdentifierPath() throws ISLookUpException {
103
		String xQuery = "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') "
104
				+ "return $x//INTERFACE[@id='{interfaceId}']/INTERFACE_EXTRA_FIELD[@name='metadata_identifier_path']/string()";
105
		xQuery = xQuery.replace("{interfaceId}", datasourceInterface);
106
		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xQuery);
107
	}
91
	private String getMetadataNamespace(final String format) {
92
		// TODO
108 93

  
109
	private String getMetadataNamespace() {
110
		try {
111
			String xQuery = "let $x := /*[.//RESOURCE_IDENTIFIER/@value='{datasourceId}']//INTERFACE[@id='{interfaceId}']/ACCESS_PROTOCOL/@format/string() "
112
					+ "return /*[.//RESOURCE_TYPE/@value='MetadataFormatDSResourceType']//METADATAFORMAT[@Prefix=$x]/@NameSpace/string()";
113
			xQuery = xQuery.replace("{interfaceId}", datasourceInterface).replace("{datasourceId}", datasourceId);
114
			return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xQuery);
115
		} catch (ISLookUpException e) {
116
			log.error("The interface is not OAI or the format is not found in the MetadataFormatDSResourceType, thus metadata format in the <about> section "
117
					+ "cannot managed here and it will be leaved empty (for the time being)");
118
			return "";
119
		}
94
		return "http://aaa";
120 95
	}
121 96

  
122
	public String getInputEprParam() {
123
		return inputEprParam;
97
	public String getInputResultSetParam() {
98
		return inputResultSetParam;
124 99
	}
125 100

  
126
	public void setInputEprParam(final String inputEprParam) {
127
		this.inputEprParam = inputEprParam;
101
	public void setInputResultSetParam(final String inputResultSetParam) {
102
		this.inputResultSetParam = inputResultSetParam;
128 103
	}
129 104

  
130
	public String getOutputEprParam() {
131
		return outputEprParam;
105
	public String getOutputResultSetParam() {
106
		return outputResultSetParam;
132 107
	}
133 108

  
134
	public void setOutputEprParam(final String outputEprParam) {
135
		this.outputEprParam = outputEprParam;
109
	public void setOutputResultSetParam(final String outputResultSetParam) {
110
		this.outputResultSetParam = outputResultSetParam;
136 111
	}
137 112

  
138
	public XSLTMappedResultSetFactory getXsltMappedResultSetFactory() {
139
		return xsltMappedResultSetFactory;
140
	}
141

  
142 113
	public String getDatasourceId() {
143 114
		return datasourceId;
144 115
	}
......
155 126
		this.datasourceInterface = datasourceInterface;
156 127
	}
157 128

  
158
	@Required
159
	public void setXsltMappedResultSetFactory(final XSLTMappedResultSetFactory xsltMappedResultSetFactory) {
160
		this.xsltMappedResultSetFactory = xsltMappedResultSetFactory;
161
	}
162

  
163 129
	public Resource getMdBuilderTemplateXslt() {
164 130
		return mdBuilderTemplateXslt;
165 131
	}
modules/dnet-node-services/trunk/src/main/java/eu/dnetlib/msro/worker/nodes/transform/TransformJobNode.java
32 32
	protected String execute(final NodeToken token) throws Exception {
33 33
		@SuppressWarnings("unchecked")
34 34
		final ResultSet<String> inputRs = (ResultSet<String>) token.getEnv().getTransientAttribute(inputResultSetParam);
35
		if (inputRs == null) { throw new ManagerServiceException("InputEprParam (" + inputResultSetParam + ") not found in ENV"); }
35
		if (inputRs == null) { throw new ManagerServiceException("InputResultSetParam (" + inputResultSetParam + ") not found in ENV"); }
36 36

  
37 37
		final TransformationRule rule = isClient.getResourceByCode(ruleId, TransformationRule.class);
38 38

  

Also available in: Unified diff