Project

General

Profile

« Previous | Next » 

Revision 47197

X3M Transformation node must be able to handle multiple 3M mappings

View differences:

modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/parthenos/workflows/repo-hi/x3m_aggregation_wf.xml.st
26 26

  
27 27
                <PARAM name="transformationMode"    description="Incremental or refresh mode"       managedBy="user"    type="string"   function="validValues(['REFRESH','INCREMENTAL'])">INCREMENTAL</PARAM>
28 28
                <PARAM name="verboseTransformationLogging"  description="Enable verbose logging of X3M"     required="false"    managedBy="user"    type="boolean"/>
29
				<PARAM name="mappingProfileId"              description="X3M mapping rules"                 required="true"     managedBy="user"    category="TRANSFORMATION_RULE_ID"   type="string"   function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
30
                <PARAM name="mappingPolicyProfileId"        description="Mapping policy to apply by X3M"    required="false"    managedBy="user"    category="TRANSFORMATION_RULE_ID"   type="string"   function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
29
				<PARAM name="mappingProfiles"              description="X3M mapping rules"                 required="true"     managedBy="user"    category="TRANSFORMATION_RULE_ID"   type="list"   function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
30
                <PARAM name="mappingPolicyProfile"        description="Mapping policy to apply by X3M"    required="false"    managedBy="user"    category="TRANSFORMATION_RULE_ID"   type="string"   function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
31 31

  
32 32
				<PARAM name="cleaningRuleId"                description="Cleaning rule"                     required="true"     managedBy="user"    category="CLEANER_RULE_ID"          type="string"   function="listProfiles('CleanerDSResourceType', '//CLEANER_NAME')"/>
33 33
				<PARAM name="cleanMdstoreId"                description="Store for cleaned records"         required="true"     managedBy="system"  category="MDSTORE_ID"     />
......
75 75
                                <ENTRY key="enableSchemaValidation"         ref="enableSchemaValidation"/>
76 76
                                <ENTRY key="useDeclaredSchema"              ref="useDeclaredSchema"/>
77 77
                                <ENTRY key="schemaURL"                      ref="schemaURL"/>
78
                                <ENTRY key="mappingPolicyProfileId"         ref="mappingPolicyProfileId"/>
79
                                <ENTRY key="mappingProfileId"               ref="mappingProfileId"/>
78
                                <ENTRY key="mappingPolicyProfile"           ref="mappingPolicyProfile"/>
79
                                <ENTRY key="mappingProfiles"                ref="mappingProfiles"/>
80 80
                                <ENTRY key="verboseTransformationLogging"   ref="verboseTransformationLogging"/>
81 81
                                <ENTRY key="cleaningRuleId"                 ref="cleaningRuleId"/>
82 82
                                <ENTRY key="transformationMode"             ref="transformationMode" />
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/transform_x3m_template.xml
17 17
				<PARAM name="enableSchemaValidation" description="Perform record validation against XML schema" required="true" type="boolean"/>
18 18
				<PARAM name="useDeclaredSchema" description="Validate according to the schema decalred in each record" required="true" type="boolean" />
19 19
				<PARAM name="schemaURL"  description="XML schema URL for validation"  required="false" type="string" />
20
				<PARAM name="mappingPolicyProfileId" description="Mapping policy to apply by X3M" required="false" type="string" />
21
				<PARAM name="mappingProfileId" description="X3M mapping rules" required="true" type="string" />
20
				<PARAM name="mappingPolicyProfile" description="Mapping policy to apply by X3M" required="false" type="string" />
21
				<PARAM name="mappingProfiles" description="X3M mapping rules" required="true" type="list" />
22 22
				<PARAM name="verboseTransformationLogging" description="Enable verbose logging of X3M" required="false" type="boolean"/>
23 23
				<PARAM name="cleaningRuleId" description="Cleaning rule" required="true" type="string" />
24 24
			</PARAMETERS>
......
73 73
					<PARAMETERS>
74 74
						<PARAM name="inputEprParam" value="valid_epr"/>
75 75
						<PARAM name="outputEprParam" value="trans_epr"/>
76
						<PARAM name="mappingPolicyProfileId" ref="mappingPolicyProfileId"/>
77
						<PARAM name="mappingProfileId" ref="mappingProfileId"/>
76
						<PARAM name="mappingPolicyProfileId" ref="mappingPolicyProfile"/>
77
						<PARAM name="mappingProfileIds" ref="mappingProfiles"/>
78 78
						<PARAM name="verboseLogging" ref="verboseTransformationLogging"/>
79 79
					</PARAMETERS>
80 80
					<ARCS>
modules/dnet-msro-service/branches/saxonHE/src/test/java/eu/dnetlib/x3m/ApplyX3MappingTest.java
28 28
		String m = getString(forthMappingPath);
29 29
		String r = getString(forthInputPath);
30 30

  
31
		ApplyX3Mapping x3m = new ApplyX3Mapping(m, null, true);
31
		ApplyX3Mapping x3m = new ApplyX3Mapping(new String[]{m}, null, true);
32 32
		String res = x3m.apply(r);
33 33
		System.out.println(res);
34 34
	}
......
38 38
		String m = getString(mappingPath);
39 39
		String g = getString(policyPath);
40 40
		String r = getString(recordPath);
41
		ApplyX3Mapping x3m = new ApplyX3Mapping(m, g, true);
41
		ApplyX3Mapping x3m = new ApplyX3Mapping(new String[]{m}, g, true);
42 42
		String res = x3m.apply(r);
43 43
		System.out.println(res);
44 44

  
......
49 49
		String m = getString(mappingPath);
50 50
		String g = getString(policyPath);
51 51

  
52
		ApplyX3Mapping x3m = new ApplyX3Mapping(m, g, true);
52
		ApplyX3Mapping x3m = new ApplyX3Mapping(new String[]{m}, g, true);
53 53
		for(String path : allInputFiles){
54 54
			String res = x3m.apply(getString(path));
55 55
			System.out.println(res);
modules/dnet-msro-service/branches/saxonHE/src/main/java/eu/dnetlib/msro/workflows/nodes/transform/X3MTransformJobNode.java
42 42
	private String outputEprParam;
43 43

  
44 44
	private String mappingPolicyProfileId;
45
	private String mappingProfileId;
45
	private String mappingProfileIds;
46 46

  
47 47
	private boolean verboseLogging;
48 48

  
......
61 61

  
62 62
	@Override
63 63
	protected String execute(final Env env) throws Exception {
64
		log.info("Mapping profile id: " + mappingProfileId);
64
		log.info("Mapping profile ids: " + mappingProfileIds);
65 65
		log.info("Mapping Policy profile id: " + mappingPolicyProfileId);
66
		final String mapping = getProfileCode(mappingProfileId);
66
		final String[] mappings = getMappingsCode(mappingProfileIds.split(","));
67 67
		final String policy = getProfileCode(mappingPolicyProfileId);
68 68

  
69 69
		LocalDateTime now = LocalDateTime.now();
......
84 84
			String provenanceFooter = extractFromRecord(record, xpathSelectorFooter);
85 85
			Instant endExtraction = Instant.now();
86 86

  
87
			ApplyX3Mapping mappingFunction = new ApplyX3Mapping(mapping, policy, verboseLogging);
87
			ApplyX3Mapping mappingFunction = new ApplyX3Mapping(mappings, policy, verboseLogging);
88 88

  
89 89
			Instant startTransform = Instant.now();
90 90
			String transformed = mappingFunction.apply(metadata);
......
103 103
		return Arc.DEFAULT_ARC;
104 104
	}
105 105

  
106
	private String[] getMappingsCode(String[] mappingIds) throws ISLookUpException {
107
		String[] mappings = new String[mappingIds.length];
108
		for(int i =0; i < mappingIds.length; i++){
109
			mappings[i] = getProfileCode(mappingIds[i]);
110
		}
111
		return mappings;
112
	}
113

  
106 114
	protected String buildXML(String header, String transformationDate, String metadata, String provenance) {
107 115
		Instant start = Instant.now();
108 116
		try {
......
166 174
		String xquery = "string(collection('/db/DRIVER/TransformationRuleDSResources')//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value ='" +
167 175
				profId + "']//CODE)";
168 176
		List<String> res = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery);
169
		if (res.isEmpty()) {
177
		if (res.isEmpty() || StringUtils.isBlank(res.get(0))) {
170 178
			throw new RuntimeException("Can't find transformation rule CODE for " + profId);
171 179
		}
172 180
		return res.get(0);
......
196 204
		this.mappingPolicyProfileId = mappingPolicyProfileId;
197 205
	}
198 206

  
199
	public String getMappingProfileId() {
200
		return mappingProfileId;
207
	public String getMappingProfileIds() {
208
		return mappingProfileIds;
201 209
	}
202 210

  
203
	public void setMappingProfileId(final String mappingProfileId) {
204
		this.mappingProfileId = mappingProfileId;
211
	public void setMappingProfileIds(final String mappingProfileIds) {
212
		this.mappingProfileIds = mappingProfileIds;
205 213
	}
206 214

  
207 215
	public boolean isVerboseLogging() {
modules/dnet-msro-service/branches/saxonHE/src/main/java/eu/dnetlib/msro/workflows/nodes/transform/ApplyX3Mapping.java
18 18
public class ApplyX3Mapping implements Function<String, String> {
19 19

  
20 20
	private static final Log log = LogFactory.getLog(ApplyX3Mapping.class);
21
	private String mapping;
21
	private String[] mappings;
22 22
	private String generatorPolicy;
23 23
	private boolean verboseLogging;
24 24

  
25
	public ApplyX3Mapping(final String mapping, final String generatorPolicy, final boolean verboseLogging) {
26
		this.mapping = mapping;
25
	public ApplyX3Mapping(final String[] mappings, final String generatorPolicy, final boolean verboseLogging) {
26
		this.mappings = mappings;
27 27
		this.generatorPolicy = generatorPolicy;
28 28
		this.verboseLogging = verboseLogging;
29 29
	}
30 30

  
31 31
	@Override
32 32
	public String apply(final String metadata) {
33
		try (InputStream mappingStream = getStream(mapping);
34
				InputStream policyStream = getStream(generatorPolicy);
33
		InputStream[] mappingStreams = new InputStream[mappings.length];
34
		try {
35
			for (int i = 0; i < mappings.length; i++) {
36
				mappingStreams[i] = getStream(mappings[i]);
37
				log.debug(mappings[i]);
38
			}
39
		}catch(IOException e){
40
			log.error("Can't create mappingStreams for mappings");
41
			closeStreams(mappingStreams);
42
			throw new RuntimeException(e);
43
		}
44
		try (   InputStream policyStream = getStream(generatorPolicy);
35 45
				InputStream metadataStream = getStream(metadata);
36 46
				final ByteArrayOutputStream os = new ByteArrayOutputStream()){
37
			X3MLEngineFactory x3mEngineFactory = getConfiguredX3MEngineFactory(mappingStream, policyStream);
47
			X3MLEngineFactory x3mEngineFactory = getConfiguredX3MEngineFactory(mappingStreams, policyStream);
38 48
			x3mEngineFactory.withInput(metadataStream).withOutput(os, OutputFormat.RDF_XML);
39 49
			x3mEngineFactory.execute();
40 50
			return new String(os.toByteArray());
41 51
		} catch (Exception e) {
42 52
			throw new RuntimeException(e);
43 53
		}
54
		finally {
55
			closeStreams(mappingStreams);
56
		}
44 57
	}
45 58

  
46
	private X3MLEngineFactory getConfiguredX3MEngineFactory(InputStream mapping, InputStream policy) {
47
		X3MLEngineFactory x3mEngineFactory = X3MLEngineFactory.create().withMappings(mapping);
59
	private void closeStreams(InputStream[] streams){
60
		for(InputStream s: streams){
61
			try {
62
				if(s != null) s.close();
63
			} catch (IOException e) {
64
				log.error("Can't close stream"+e.getMessage());
65
			}
66
		}
67
	}
68

  
69
	private X3MLEngineFactory getConfiguredX3MEngineFactory(InputStream[] mappings, InputStream policy) {
70
		X3MLEngineFactory x3mEngineFactory = X3MLEngineFactory.create().withMappings(mappings);
48 71
		if (policy != null) {
49 72
			x3mEngineFactory.withGeneratorPolicy(policy);
50 73
		}

Also available in: Unified diff