Revision 47197
Added by Alessia Bardi almost 7 years ago
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/parthenos/workflows/repo-hi/x3m_aggregation_wf.xml.st | ||
---|---|---|
26 | 26 |
|
27 | 27 |
<PARAM name="transformationMode" description="Incremental or refresh mode" managedBy="user" type="string" function="validValues(['REFRESH','INCREMENTAL'])">INCREMENTAL</PARAM> |
28 | 28 |
<PARAM name="verboseTransformationLogging" description="Enable verbose logging of X3M" required="false" managedBy="user" type="boolean"/> |
29 |
<PARAM name="mappingProfileId" description="X3M mapping rules" required="true" managedBy="user" category="TRANSFORMATION_RULE_ID" type="string" function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
|
|
30 |
<PARAM name="mappingPolicyProfileId" description="Mapping policy to apply by X3M" required="false" managedBy="user" category="TRANSFORMATION_RULE_ID" type="string" function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
|
|
29 |
<PARAM name="mappingProfiles" description="X3M mapping rules" required="true" managedBy="user" category="TRANSFORMATION_RULE_ID" type="list" function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/>
|
|
30 |
<PARAM name="mappingPolicyProfile" description="Mapping policy to apply by X3M" required="false" managedBy="user" category="TRANSFORMATION_RULE_ID" type="string" function="listProfiles('TransformationRuleDSResourceType', '//TITLE')"/> |
|
31 | 31 |
|
32 | 32 |
<PARAM name="cleaningRuleId" description="Cleaning rule" required="true" managedBy="user" category="CLEANER_RULE_ID" type="string" function="listProfiles('CleanerDSResourceType', '//CLEANER_NAME')"/> |
33 | 33 |
<PARAM name="cleanMdstoreId" description="Store for cleaned records" required="true" managedBy="system" category="MDSTORE_ID" /> |
... | ... | |
75 | 75 |
<ENTRY key="enableSchemaValidation" ref="enableSchemaValidation"/> |
76 | 76 |
<ENTRY key="useDeclaredSchema" ref="useDeclaredSchema"/> |
77 | 77 |
<ENTRY key="schemaURL" ref="schemaURL"/> |
78 |
<ENTRY key="mappingPolicyProfileId" ref="mappingPolicyProfileId"/>
|
|
79 |
<ENTRY key="mappingProfileId" ref="mappingProfileId"/>
|
|
78 |
<ENTRY key="mappingPolicyProfile" ref="mappingPolicyProfile"/>
|
|
79 |
<ENTRY key="mappingProfiles" ref="mappingProfiles"/>
|
|
80 | 80 |
<ENTRY key="verboseTransformationLogging" ref="verboseTransformationLogging"/> |
81 | 81 |
<ENTRY key="cleaningRuleId" ref="cleaningRuleId"/> |
82 | 82 |
<ENTRY key="transformationMode" ref="transformationMode" /> |
modules/dnet-parthenos/trunk/src/main/resources/eu/dnetlib/bootstrap/profiles/workflows/transform_x3m_template.xml | ||
---|---|---|
17 | 17 |
<PARAM name="enableSchemaValidation" description="Perform record validation against XML schema" required="true" type="boolean"/> |
18 | 18 |
<PARAM name="useDeclaredSchema" description="Validate according to the schema decalred in each record" required="true" type="boolean" /> |
19 | 19 |
<PARAM name="schemaURL" description="XML schema URL for validation" required="false" type="string" /> |
20 |
<PARAM name="mappingPolicyProfileId" description="Mapping policy to apply by X3M" required="false" type="string" />
|
|
21 |
<PARAM name="mappingProfileId" description="X3M mapping rules" required="true" type="string" />
|
|
20 |
<PARAM name="mappingPolicyProfile" description="Mapping policy to apply by X3M" required="false" type="string" /> |
|
21 |
<PARAM name="mappingProfiles" description="X3M mapping rules" required="true" type="list" />
|
|
22 | 22 |
<PARAM name="verboseTransformationLogging" description="Enable verbose logging of X3M" required="false" type="boolean"/> |
23 | 23 |
<PARAM name="cleaningRuleId" description="Cleaning rule" required="true" type="string" /> |
24 | 24 |
</PARAMETERS> |
... | ... | |
73 | 73 |
<PARAMETERS> |
74 | 74 |
<PARAM name="inputEprParam" value="valid_epr"/> |
75 | 75 |
<PARAM name="outputEprParam" value="trans_epr"/> |
76 |
<PARAM name="mappingPolicyProfileId" ref="mappingPolicyProfileId"/>
|
|
77 |
<PARAM name="mappingProfileId" ref="mappingProfileId"/>
|
|
76 |
<PARAM name="mappingPolicyProfileId" ref="mappingPolicyProfile"/> |
|
77 |
<PARAM name="mappingProfileIds" ref="mappingProfiles"/>
|
|
78 | 78 |
<PARAM name="verboseLogging" ref="verboseTransformationLogging"/> |
79 | 79 |
</PARAMETERS> |
80 | 80 |
<ARCS> |
modules/dnet-msro-service/branches/saxonHE/src/test/java/eu/dnetlib/x3m/ApplyX3MappingTest.java | ||
---|---|---|
28 | 28 |
String m = getString(forthMappingPath); |
29 | 29 |
String r = getString(forthInputPath); |
30 | 30 |
|
31 |
ApplyX3Mapping x3m = new ApplyX3Mapping(m, null, true);
|
|
31 |
ApplyX3Mapping x3m = new ApplyX3Mapping(new String[]{m}, null, true);
|
|
32 | 32 |
String res = x3m.apply(r); |
33 | 33 |
System.out.println(res); |
34 | 34 |
} |
... | ... | |
38 | 38 |
String m = getString(mappingPath); |
39 | 39 |
String g = getString(policyPath); |
40 | 40 |
String r = getString(recordPath); |
41 |
ApplyX3Mapping x3m = new ApplyX3Mapping(m, g, true);
|
|
41 |
ApplyX3Mapping x3m = new ApplyX3Mapping(new String[]{m}, g, true);
|
|
42 | 42 |
String res = x3m.apply(r); |
43 | 43 |
System.out.println(res); |
44 | 44 |
|
... | ... | |
49 | 49 |
String m = getString(mappingPath); |
50 | 50 |
String g = getString(policyPath); |
51 | 51 |
|
52 |
ApplyX3Mapping x3m = new ApplyX3Mapping(m, g, true);
|
|
52 |
ApplyX3Mapping x3m = new ApplyX3Mapping(new String[]{m}, g, true);
|
|
53 | 53 |
for(String path : allInputFiles){ |
54 | 54 |
String res = x3m.apply(getString(path)); |
55 | 55 |
System.out.println(res); |
modules/dnet-msro-service/branches/saxonHE/src/main/java/eu/dnetlib/msro/workflows/nodes/transform/X3MTransformJobNode.java | ||
---|---|---|
42 | 42 |
private String outputEprParam; |
43 | 43 |
|
44 | 44 |
private String mappingPolicyProfileId; |
45 |
private String mappingProfileId; |
|
45 |
private String mappingProfileIds;
|
|
46 | 46 |
|
47 | 47 |
private boolean verboseLogging; |
48 | 48 |
|
... | ... | |
61 | 61 |
|
62 | 62 |
@Override |
63 | 63 |
protected String execute(final Env env) throws Exception { |
64 |
log.info("Mapping profile id: " + mappingProfileId);
|
|
64 |
log.info("Mapping profile ids: " + mappingProfileIds);
|
|
65 | 65 |
log.info("Mapping Policy profile id: " + mappingPolicyProfileId); |
66 |
final String mapping = getProfileCode(mappingProfileId);
|
|
66 |
final String[] mappings = getMappingsCode(mappingProfileIds.split(","));
|
|
67 | 67 |
final String policy = getProfileCode(mappingPolicyProfileId); |
68 | 68 |
|
69 | 69 |
LocalDateTime now = LocalDateTime.now(); |
... | ... | |
84 | 84 |
String provenanceFooter = extractFromRecord(record, xpathSelectorFooter); |
85 | 85 |
Instant endExtraction = Instant.now(); |
86 | 86 |
|
87 |
ApplyX3Mapping mappingFunction = new ApplyX3Mapping(mapping, policy, verboseLogging); |
|
87 |
ApplyX3Mapping mappingFunction = new ApplyX3Mapping(mappings, policy, verboseLogging);
|
|
88 | 88 |
|
89 | 89 |
Instant startTransform = Instant.now(); |
90 | 90 |
String transformed = mappingFunction.apply(metadata); |
... | ... | |
103 | 103 |
return Arc.DEFAULT_ARC; |
104 | 104 |
} |
105 | 105 |
|
106 |
private String[] getMappingsCode(String[] mappingIds) throws ISLookUpException { |
|
107 |
String[] mappings = new String[mappingIds.length]; |
|
108 |
for(int i =0; i < mappingIds.length; i++){ |
|
109 |
mappings[i] = getProfileCode(mappingIds[i]); |
|
110 |
} |
|
111 |
return mappings; |
|
112 |
} |
|
113 |
|
|
106 | 114 |
protected String buildXML(String header, String transformationDate, String metadata, String provenance) { |
107 | 115 |
Instant start = Instant.now(); |
108 | 116 |
try { |
... | ... | |
166 | 174 |
String xquery = "string(collection('/db/DRIVER/TransformationRuleDSResources')//RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value ='" + |
167 | 175 |
profId + "']//CODE)"; |
168 | 176 |
List<String> res = serviceLocator.getService(ISLookUpService.class).quickSearchProfile(xquery); |
169 |
if (res.isEmpty()) { |
|
177 |
if (res.isEmpty() || StringUtils.isBlank(res.get(0))) {
|
|
170 | 178 |
throw new RuntimeException("Can't find transformation rule CODE for " + profId); |
171 | 179 |
} |
172 | 180 |
return res.get(0); |
... | ... | |
196 | 204 |
this.mappingPolicyProfileId = mappingPolicyProfileId; |
197 | 205 |
} |
198 | 206 |
|
199 |
public String getMappingProfileId() { |
|
200 |
return mappingProfileId; |
|
207 |
public String getMappingProfileIds() {
|
|
208 |
return mappingProfileIds;
|
|
201 | 209 |
} |
202 | 210 |
|
203 |
public void setMappingProfileId(final String mappingProfileId) {
|
|
204 |
this.mappingProfileId = mappingProfileId;
|
|
211 |
public void setMappingProfileIds(final String mappingProfileIds) {
|
|
212 |
this.mappingProfileIds = mappingProfileIds;
|
|
205 | 213 |
} |
206 | 214 |
|
207 | 215 |
public boolean isVerboseLogging() { |
modules/dnet-msro-service/branches/saxonHE/src/main/java/eu/dnetlib/msro/workflows/nodes/transform/ApplyX3Mapping.java | ||
---|---|---|
18 | 18 |
public class ApplyX3Mapping implements Function<String, String> { |
19 | 19 |
|
20 | 20 |
private static final Log log = LogFactory.getLog(ApplyX3Mapping.class); |
21 |
private String mapping;
|
|
21 |
private String[] mappings;
|
|
22 | 22 |
private String generatorPolicy; |
23 | 23 |
private boolean verboseLogging; |
24 | 24 |
|
25 |
public ApplyX3Mapping(final String mapping, final String generatorPolicy, final boolean verboseLogging) {
|
|
26 |
this.mapping = mapping;
|
|
25 |
public ApplyX3Mapping(final String[] mappings, final String generatorPolicy, final boolean verboseLogging) {
|
|
26 |
this.mappings = mappings;
|
|
27 | 27 |
this.generatorPolicy = generatorPolicy; |
28 | 28 |
this.verboseLogging = verboseLogging; |
29 | 29 |
} |
30 | 30 |
|
31 | 31 |
@Override |
32 | 32 |
public String apply(final String metadata) { |
33 |
try (InputStream mappingStream = getStream(mapping); |
|
34 |
InputStream policyStream = getStream(generatorPolicy); |
|
33 |
InputStream[] mappingStreams = new InputStream[mappings.length]; |
|
34 |
try { |
|
35 |
for (int i = 0; i < mappings.length; i++) { |
|
36 |
mappingStreams[i] = getStream(mappings[i]); |
|
37 |
log.debug(mappings[i]); |
|
38 |
} |
|
39 |
}catch(IOException e){ |
|
40 |
log.error("Can't create mappingStreams for mappings"); |
|
41 |
closeStreams(mappingStreams); |
|
42 |
throw new RuntimeException(e); |
|
43 |
} |
|
44 |
try ( InputStream policyStream = getStream(generatorPolicy); |
|
35 | 45 |
InputStream metadataStream = getStream(metadata); |
36 | 46 |
final ByteArrayOutputStream os = new ByteArrayOutputStream()){ |
37 |
X3MLEngineFactory x3mEngineFactory = getConfiguredX3MEngineFactory(mappingStream, policyStream); |
|
47 |
X3MLEngineFactory x3mEngineFactory = getConfiguredX3MEngineFactory(mappingStreams, policyStream);
|
|
38 | 48 |
x3mEngineFactory.withInput(metadataStream).withOutput(os, OutputFormat.RDF_XML); |
39 | 49 |
x3mEngineFactory.execute(); |
40 | 50 |
return new String(os.toByteArray()); |
41 | 51 |
} catch (Exception e) { |
42 | 52 |
throw new RuntimeException(e); |
43 | 53 |
} |
54 |
finally { |
|
55 |
closeStreams(mappingStreams); |
|
56 |
} |
|
44 | 57 |
} |
45 | 58 |
|
46 |
private X3MLEngineFactory getConfiguredX3MEngineFactory(InputStream mapping, InputStream policy) { |
|
47 |
X3MLEngineFactory x3mEngineFactory = X3MLEngineFactory.create().withMappings(mapping); |
|
59 |
private void closeStreams(InputStream[] streams){ |
|
60 |
for(InputStream s: streams){ |
|
61 |
try { |
|
62 |
if(s != null) s.close(); |
|
63 |
} catch (IOException e) { |
|
64 |
log.error("Can't close stream"+e.getMessage()); |
|
65 |
} |
|
66 |
} |
|
67 |
} |
|
68 |
|
|
69 |
private X3MLEngineFactory getConfiguredX3MEngineFactory(InputStream[] mappings, InputStream policy) { |
|
70 |
X3MLEngineFactory x3mEngineFactory = X3MLEngineFactory.create().withMappings(mappings); |
|
48 | 71 |
if (policy != null) { |
49 | 72 |
x3mEngineFactory.withGeneratorPolicy(policy); |
50 | 73 |
} |
Also available in: Unified diff
X3M Transformation node must be able to handle multiple 3M mappings