Revision 62365
Added by Claudio Atzori over 1 year ago
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/core/xsl/XsltConstants.java | ||
---|---|---|
1 |
package eu.dnetlib.data.collective.transformation.core.xsl; |
|
2 |
|
|
3 |
/** |
|
4 |
* |
|
5 |
* @author js |
|
6 |
* |
|
7 |
*/ |
|
8 |
public class XsltConstants { |
|
9 |
|
|
10 |
public static final String applyTemplates = "xsl:apply-templates"; |
|
11 |
public static final String attribute = "xsl:attribute"; |
|
12 |
public static final String callTemplate = "xsl:call-template"; |
|
13 |
public static final String choose = "xsl:choose"; |
|
14 |
public static final String copy = "xsl:copy"; |
|
15 |
public static final String copyOf = "xsl:copy-of"; |
|
16 |
public static final String element = "xsl:element"; |
|
17 |
public static final String forEach = "xsl:for-each"; |
|
18 |
public static final String ifCondition = "xsl:if"; |
|
19 |
public static final String message = "xsl:message"; |
|
20 |
public static final String otherwise = "xsl:otherwise"; |
|
21 |
public static final String param = "xsl:param"; |
|
22 |
public static final String template = "xsl:template"; |
|
23 |
public static final String text = "xsl:text"; |
|
24 |
public static final String valueOf = "xsl:value-of"; |
|
25 |
public static final String variable = "xsl:variable"; |
|
26 |
public static final String when = "xsl:when"; |
|
27 |
public static final String withParam = "xsl:with-param"; |
|
28 |
|
|
29 |
public static final String nsXsl = "xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\""; |
|
30 |
public static final String extFuncNS = "TransformationFunction"; |
|
31 |
|
|
32 |
} |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/core/schema/SchemaElement.java | ||
---|---|---|
1 |
package eu.dnetlib.data.collective.transformation.core.schema; |
|
2 |
|
|
3 |
import java.util.LinkedList; |
|
4 |
import java.util.List; |
|
5 |
|
|
6 |
/** |
|
7 |
* @author jochen |
|
8 |
* |
|
9 |
*/ |
|
10 |
public class SchemaElement { |
|
11 |
|
|
12 |
private String targetNamespace; |
|
13 |
private String name; |
|
14 |
private boolean isRepeatable; |
|
15 |
private boolean isRoot; |
|
16 |
private boolean containsSimpleType; |
|
17 |
private int minOccurs; |
|
18 |
private int maxOccurs; |
|
19 |
private List<SchemaElement> childList = new LinkedList<SchemaElement>(); |
|
20 |
private List<SchemaAttribute> attributeList = new LinkedList<SchemaAttribute>(); |
|
21 |
private Namespace namespace; |
|
22 |
|
|
23 |
/** |
|
24 |
* @return the targetNamespace |
|
25 |
*/ |
|
26 |
public String getTargetNamespace() { |
|
27 |
return targetNamespace; |
|
28 |
} |
|
29 |
/** |
|
30 |
* @param targetNamespace the targetNamespace to set |
|
31 |
*/ |
|
32 |
public void setTargetNamespace(String targetNamespace) { |
|
33 |
this.targetNamespace = targetNamespace; |
|
34 |
} |
|
35 |
/** |
|
36 |
* @return the isRepeatable |
|
37 |
*/ |
|
38 |
public boolean isRepeatable() { |
|
39 |
return isRepeatable; |
|
40 |
} |
|
41 |
/** |
|
42 |
* @param isRepeatable the isRepeatable to set |
|
43 |
*/ |
|
44 |
public void setRepeatable(boolean isRepeatable) { |
|
45 |
this.isRepeatable = isRepeatable; |
|
46 |
} |
|
47 |
/** |
|
48 |
* @return the isMandatory |
|
49 |
*/ |
|
50 |
public boolean isMandatory() { |
|
51 |
if (minOccurs > 0) return true; |
|
52 |
return false; |
|
53 |
} |
|
54 |
|
|
55 |
/** |
|
56 |
* @return the minOccurs |
|
57 |
*/ |
|
58 |
public int getMinOccurs() { |
|
59 |
return minOccurs; |
|
60 |
} |
|
61 |
/** |
|
62 |
* @param minOccurs the minOccurs to set |
|
63 |
*/ |
|
64 |
public void setMinOccurs(int minOccurs) { |
|
65 |
this.minOccurs = minOccurs; |
|
66 |
} |
|
67 |
/** |
|
68 |
* @return the maxOccurs |
|
69 |
*/ |
|
70 |
public int getMaxOccurs() { |
|
71 |
return maxOccurs; |
|
72 |
} |
|
73 |
/** |
|
74 |
* @param maxOccurs the maxOccurs to set |
|
75 |
*/ |
|
76 |
public void setMaxOccurs(int maxOccurs) { |
|
77 |
this.maxOccurs = maxOccurs; |
|
78 |
} |
|
79 |
/** |
|
80 |
* @return the childList |
|
81 |
*/ |
|
82 |
public List<SchemaElement> getChildList() { |
|
83 |
return childList; |
|
84 |
} |
|
85 |
/** |
|
86 |
* @param childList the childList to set |
|
87 |
*/ |
|
88 |
public void setChildList(List<SchemaElement> childList) { |
|
89 |
this.childList = childList; |
|
90 |
} |
|
91 |
|
|
92 |
/** |
|
93 |
* @param name the name of the element to set |
|
94 |
*/ |
|
95 |
public void setName(String name) { |
|
96 |
this.name = name; |
|
97 |
} |
|
98 |
|
|
99 |
/** |
|
100 |
* @return the name of this element |
|
101 |
*/ |
|
102 |
public String getName() { |
|
103 |
return name; |
|
104 |
} |
|
105 |
|
|
106 |
/** |
|
107 |
* sets true if this element contains a simpleType, false else |
|
108 |
* @param containsSimpleType |
|
109 |
*/ |
|
110 |
public void setContainsSimpleType(boolean containsSimpleType) { |
|
111 |
this.containsSimpleType = containsSimpleType; |
|
112 |
} |
|
113 |
|
|
114 |
/** |
|
115 |
* @return the containsSimpleType |
|
116 |
*/ |
|
117 |
public boolean containsSimpleType() { |
|
118 |
return containsSimpleType; |
|
119 |
} |
|
120 |
|
|
121 |
/** |
|
122 |
* @param isRoot the isRoot to set |
|
123 |
*/ |
|
124 |
public void setRoot(boolean isRoot) { |
|
125 |
this.isRoot = isRoot; |
|
126 |
} |
|
127 |
|
|
128 |
/** |
|
129 |
* @return the isRoot |
|
130 |
*/ |
|
131 |
public boolean isRoot() { |
|
132 |
return isRoot; |
|
133 |
} |
|
134 |
/** |
|
135 |
* @param namespace the namespace to set |
|
136 |
*/ |
|
137 |
public void setNamespace(Namespace namespace) { |
|
138 |
this.namespace = namespace; |
|
139 |
} |
|
140 |
/** |
|
141 |
* @return the namespace |
|
142 |
*/ |
|
143 |
public Namespace getNamespace() { |
|
144 |
return namespace; |
|
145 |
} |
|
146 |
/** |
|
147 |
* @return the attributeList |
|
148 |
*/ |
|
149 |
public List<SchemaAttribute> getAttributeList() { |
|
150 |
return attributeList; |
|
151 |
} |
|
152 |
/** |
|
153 |
* @param attributeList the attributeList to set |
|
154 |
*/ |
|
155 |
public void addAttribute(SchemaAttribute aAttribute) { |
|
156 |
this.attributeList.add(aAttribute); |
|
157 |
} |
|
158 |
|
|
159 |
|
|
160 |
} |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/pom.xml | ||
---|---|---|
1 |
<?xml version="1.0" ?> |
|
2 |
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> |
|
3 |
<parent> |
|
4 |
<groupId>eu.dnetlib</groupId> |
|
5 |
<artifactId>dnet45-parent</artifactId> |
|
6 |
<version>1.0.0</version> |
|
7 |
</parent> |
|
8 |
<modelVersion>4.0.0</modelVersion> |
|
9 |
<groupId>eu.dnetlib</groupId> |
|
10 |
<artifactId>unibi-data-collective-transformation-common</artifactId> |
|
11 |
<packaging>jar</packaging> |
|
12 |
<version>3.0.3</version> |
|
13 |
<scm> |
|
14 |
<developerConnection>scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3</developerConnection> |
|
15 |
</scm> |
|
16 |
<dependencies> |
|
17 |
<dependency> |
|
18 |
<groupId>commons-beanutils</groupId> |
|
19 |
<artifactId>commons-beanutils</artifactId> |
|
20 |
<version>1.8.3</version> |
|
21 |
</dependency> |
|
22 |
<dependency> |
|
23 |
<groupId>org.apache.commons</groupId> |
|
24 |
<artifactId>commons-text</artifactId> |
|
25 |
<version>1.3</version> |
|
26 |
</dependency> |
|
27 |
<dependency> |
|
28 |
<groupId>junit</groupId> |
|
29 |
<artifactId>junit</artifactId> |
|
30 |
<version>${junit.version}</version> |
|
31 |
<scope>test</scope> |
|
32 |
</dependency> |
|
33 |
<dependency> |
|
34 |
<groupId>org.mockito</groupId> |
|
35 |
<artifactId>mockito-core</artifactId> |
|
36 |
<version>${mockito.version}</version> |
|
37 |
<scope>test</scope> |
|
38 |
</dependency> |
|
39 |
<dependency> |
|
40 |
<groupId>org.svenson</groupId> |
|
41 |
<artifactId>svenson-json</artifactId> |
|
42 |
<version>[1.4.0,1.5.0)</version> |
|
43 |
</dependency> |
|
44 |
<dependency> |
|
45 |
<groupId>org.slf4j</groupId> |
|
46 |
<artifactId>slf4j-api</artifactId> |
|
47 |
<version>1.6.4</version> |
|
48 |
<scope>compile</scope> |
|
49 |
</dependency> |
|
50 |
<dependency> |
|
51 |
<groupId>org.slf4j</groupId> |
|
52 |
<artifactId>slf4j-log4j12</artifactId> |
|
53 |
<version>1.6.4</version> |
|
54 |
<scope>runtime</scope> |
|
55 |
</dependency> |
|
56 |
<dependency> |
|
57 |
<groupId>com.google.guava</groupId> |
|
58 |
<artifactId>guava</artifactId> |
|
59 |
<version>${google.guava.version}</version> |
|
60 |
</dependency> |
|
61 |
<dependency> |
|
62 |
<groupId>eu.dnetlib</groupId> |
|
63 |
<artifactId>cnr-service-common</artifactId> |
|
64 |
<version>[2.1.0,3.0.0)</version> |
|
65 |
</dependency> |
|
66 |
<dependency> |
|
67 |
<groupId>eu.dnetlib</groupId> |
|
68 |
<artifactId>dnet-openaire-datasource-manager</artifactId> |
|
69 |
<version>[2.0.0,3.0.0)</version> |
|
70 |
</dependency> |
|
71 |
<dependency> |
|
72 |
<groupId>com.sun.xsom</groupId> |
|
73 |
<artifactId>xsom</artifactId> |
|
74 |
<version>20110809</version> |
|
75 |
</dependency> |
|
76 |
</dependencies> |
|
77 |
</project> |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/utils/BlacklistConsumer.java | ||
---|---|---|
1 |
package eu.dnetlib.data.collective.transformation.utils; |
|
2 |
|
|
3 |
import java.io.BufferedReader; |
|
4 |
import java.io.IOException; |
|
5 |
import java.io.InputStream; |
|
6 |
import java.io.InputStreamReader; |
|
7 |
import java.net.URL; |
|
8 |
import java.util.LinkedList; |
|
9 |
import java.util.List; |
|
10 |
|
|
11 |
public class BlacklistConsumer { |
|
12 |
|
|
13 |
public List<String> getBlackList(String apiURL){ |
|
14 |
List<String> blacklist = new LinkedList<String>(); |
|
15 |
try{ |
|
16 |
URL blacklistApi = new URL(apiURL); |
|
17 |
InputStream in = blacklistApi.openStream(); |
|
18 |
BufferedReader reader = new BufferedReader(new InputStreamReader(in)); |
|
19 |
String line; |
|
20 |
while((line = reader.readLine()) != null) { |
|
21 |
blacklist.add(line); |
|
22 |
} |
|
23 |
System.out.println(blacklist.size()); |
|
24 |
System.out.println(blacklist.get(0)); |
|
25 |
}catch(IOException e){ |
|
26 |
throw new IllegalStateException("error in blacklist api: " + e.getMessage()); |
|
27 |
} |
|
28 |
return blacklist; |
|
29 |
} |
|
30 |
} |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/rulelanguage/util/Converter.java | ||
---|---|---|
1 |
package eu.dnetlib.data.collective.transformation.rulelanguage.util; |
|
2 |
|
|
3 |
import java.util.LinkedList; |
|
4 |
import java.util.List; |
|
5 |
import java.util.Map; |
|
6 |
import java.util.StringTokenizer; |
|
7 |
|
|
8 |
/** |
|
9 |
* @author jochen |
|
10 |
* |
|
11 |
*/ |
|
12 |
public class Converter { |
|
13 |
|
|
14 |
private static final String xpathExpr = "xpath:\""; |
|
15 |
//private static final String labelExpr = "label:"; |
|
16 |
|
|
17 |
public static String getXpathFromLabelExpr(String aElement){ |
|
18 |
// TODO validate the argument -> consisting of 3 tokens, delimited by dot |
|
19 |
StringTokenizer tokenizer = new StringTokenizer(aElement, "."); |
|
20 |
List<String> tokenList = new LinkedList<String>(); |
|
21 |
while (tokenizer.hasMoreTokens()){ |
|
22 |
tokenList.add(tokenizer.nextToken()); |
|
23 |
} |
|
24 |
StringBuilder builder = new StringBuilder(); |
|
25 |
builder.append("//"); // the xpath-expr |
|
26 |
builder.append(tokenList.get(0) + ":"); // the namespace |
|
27 |
builder.append(tokenList.get(2)); // the elementname |
|
28 |
return builder.toString(); |
|
29 |
} |
|
30 |
|
|
31 |
/** |
|
32 |
* extracts a xpath-expression made in a production rule |
|
33 |
* @param aElement |
|
34 |
* @return xpath expression |
|
35 |
*/ |
|
36 |
public static String getXpathFromXpathExpr(String aElement){ |
|
37 |
String xpath = ""; |
|
38 |
if (aElement.startsWith(xpathExpr)){ |
|
39 |
xpath = aElement.substring(xpathExpr.length(), aElement.length() - 1); |
|
40 |
} |
|
41 |
return xpath; |
|
42 |
} |
|
43 |
|
|
44 |
public static boolean isXpathReturningString(String aXpathExpr){ |
|
45 |
String[] functions = {"concat", "normalize-space", "translate", "substring"}; |
|
46 |
for (String fct: functions) |
|
47 |
if (aXpathExpr.startsWith(fct)) return true; |
|
48 |
return false; |
|
49 |
} |
|
50 |
|
|
51 |
public static String getUnquotedString(String aValue){ |
|
52 |
return aValue.substring(1, aValue.length() - 1); |
|
53 |
} |
|
54 |
|
|
55 |
/** |
|
56 |
* returns a list of name-space declarations used in xsl |
|
57 |
* @param nsPrefixMap - a map of name-space prefixes and their uris |
|
58 |
* @return list of name-space declarations |
|
59 |
*/ |
|
60 |
public static List<String> getBoundPrefixes(Map<String, String> nsPrefixMap){ |
|
61 |
List<String> nsList = new LinkedList<String>(); |
|
62 |
for (String key: nsPrefixMap.keySet()){ |
|
63 |
nsList.add("xmlns:" + key + "=" + "\"" + nsPrefixMap.get(key) + "\" "); |
|
64 |
} |
|
65 |
return nsList; |
|
66 |
} |
|
67 |
} |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/rulelanguage/parser/FtScriptTreeConstants.java | ||
---|---|---|
1 |
/* Generated By:JavaCC: Do not edit this line. FtScriptTreeConstants.java Version 5.0 */ |
|
2 |
package eu.dnetlib.data.collective.transformation.rulelanguage.parser; |
|
3 |
|
|
4 |
public interface FtScriptTreeConstants |
|
5 |
{ |
|
6 |
public int JJTSTART = 0; |
|
7 |
public int JJTMYASSIGN = 1; |
|
8 |
public int JJTVOID = 2; |
|
9 |
public int JJTMYATTRIBUTE = 3; |
|
10 |
public int JJTMYCONDITION = 4; |
|
11 |
public int JJTMYEMPTY = 5; |
|
12 |
public int JJTMYIMPORT = 6; |
|
13 |
public int JJTMYNS = 7; |
|
14 |
public int JJTMYPREPROCESS = 8; |
|
15 |
public int JJTMYSET = 9; |
|
16 |
public int JJTMYSKIP = 10; |
|
17 |
public int JJTMYCOPY = 11; |
|
18 |
public int JJTMYOP = 12; |
|
19 |
public int JJTMYSCRIPT = 13; |
|
20 |
|
|
21 |
|
|
22 |
public String[] jjtNodeName = { |
|
23 |
"Start", |
|
24 |
"MyAssign", |
|
25 |
"void", |
|
26 |
"MyAttribute", |
|
27 |
"MyCondition", |
|
28 |
"MyEmpty", |
|
29 |
"MyImport", |
|
30 |
"MyNs", |
|
31 |
"MyPreprocess", |
|
32 |
"MySet", |
|
33 |
"MySkip", |
|
34 |
"MyCopy", |
|
35 |
"MyOp", |
|
36 |
"MyScript", |
|
37 |
}; |
|
38 |
} |
|
39 |
/* JavaCC - OriginalChecksum=c8056b53459b9b66f3a28fd32fead01c (do not edit this line) */ |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/core/xsl/AbstractXslElement.java | ||
---|---|---|
1 |
package eu.dnetlib.data.collective.transformation.core.xsl; |
|
2 |
|
|
3 |
import java.util.LinkedList; |
|
4 |
import java.util.List; |
|
5 |
|
|
6 |
/** |
|
7 |
* @author jochen |
|
8 |
* |
|
9 |
*/ |
|
10 |
public abstract class AbstractXslElement { |
|
11 |
|
|
12 |
private String functionName; |
|
13 |
protected List<String> attrList = new LinkedList<String>(); |
|
14 |
protected StringBuilder enclosedElements = new StringBuilder(); |
|
15 |
protected List<String> nsList = new LinkedList<String>(); |
|
16 |
|
|
17 |
|
|
18 |
public AbstractXslElement(String aFunctioName) { |
|
19 |
this.functionName = aFunctioName; |
|
20 |
} |
|
21 |
|
|
22 |
public String asXml(boolean isEmpty){ |
|
23 |
StringBuilder builder = new StringBuilder(); |
|
24 |
builder.append("<"); |
|
25 |
builder.append(functionName + " "); |
|
26 |
for (String ns: nsList){ |
|
27 |
builder.append(ns + " "); |
|
28 |
} |
|
29 |
|
|
30 |
for (String attr: attrList){ |
|
31 |
builder.append(attr); |
|
32 |
} |
|
33 |
if (isEmpty){ |
|
34 |
builder.append("/>"); |
|
35 |
}else{ |
|
36 |
builder.append(">"); |
|
37 |
builder.append(enclosedElements.toString()); |
|
38 |
builder.append("</"); |
|
39 |
builder.append(functionName + ">"); |
|
40 |
} |
|
41 |
return builder.toString(); |
|
42 |
} |
|
43 |
|
|
44 |
public String asXml() { |
|
45 |
return asXml(false); |
|
46 |
} |
|
47 |
|
|
48 |
} |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/rulelanguage/Rules.java | ||
---|---|---|
1 |
package eu.dnetlib.data.collective.transformation.rulelanguage; |
|
2 |
|
|
3 |
import java.util.Properties; |
|
4 |
|
|
5 |
import eu.dnetlib.data.collective.transformation.core.schema.SchemaElement; |
|
6 |
import eu.dnetlib.data.collective.transformation.rulelanguage.util.FunctionCall; |
|
7 |
|
|
8 |
/** |
|
9 |
* @author jochen |
|
10 |
* |
|
11 |
*/ |
|
12 |
public class Rules implements Comparable<Rules>, IRule{ |
|
13 |
|
|
14 |
public static final String staticRule = "static"; |
|
15 |
private String attribute = ""; |
|
16 |
private String targetField = ""; |
|
17 |
private String ruleDeclaration = ""; |
|
18 |
private String xpath = ""; |
|
19 |
private String constant = ""; |
|
20 |
private String namespace = ""; |
|
21 |
private String variable = ""; |
|
22 |
private String template = ""; |
|
23 |
private String templateMatch = ""; |
|
24 |
private FunctionCall funcCall; |
|
25 |
private Condition condition; |
|
26 |
private boolean isEmpty = false; |
|
27 |
private boolean isSkip = false; |
|
28 |
private SchemaElement targetSchemaElement; |
|
29 |
private String assignmentVariable = ""; |
|
30 |
private RulesSet rulesSet; |
|
31 |
private Properties properties = new Properties(); |
|
32 |
|
|
33 |
public Rules() { |
|
34 |
} |
|
35 |
|
|
36 |
/** |
|
37 |
* indicates if the rule is declared as static |
|
38 |
* @return true if static, false otherwise |
|
39 |
*/ |
|
40 |
public boolean isStatic(){ |
|
41 |
if (ruleDeclaration.equals(staticRule)){ |
|
42 |
return true; |
|
43 |
} |
|
44 |
return false; |
|
45 |
} |
|
46 |
|
|
47 |
/** |
|
48 |
* indicates if the rule defines a variable |
|
49 |
* @return true if variable is defined, false otherwise |
|
50 |
* @see eu.dnetlib.data.collective.transformation.rulelanguage.IRule#definesVariable() |
|
51 |
*/ |
|
52 |
public boolean definesVariable(){ |
|
53 |
if (variable.length() > 0) return true; |
|
54 |
return false; |
|
55 |
} |
|
56 |
|
|
57 |
public boolean definesTargetField(){ |
|
58 |
if (targetField.length() > 0) return true; |
|
59 |
return false; |
|
60 |
} |
|
61 |
|
|
62 |
/** |
|
63 |
* checks if this rule defines an attribute |
|
64 |
* @return true if defines attribute else false |
|
65 |
*/ |
|
66 |
public boolean definesAttribute(){ |
|
67 |
if (attribute.length() > 0) return true; |
|
68 |
return false; |
|
69 |
} |
|
70 |
|
|
71 |
public boolean definesTemplate(){ |
|
72 |
if (template.length() > 0) return true; |
|
73 |
return false; |
|
74 |
} |
|
75 |
|
|
76 |
@Override |
|
77 |
public boolean definesTemplateMatch() { |
|
78 |
if (templateMatch.length() > 0) return true; |
|
79 |
return false; |
|
80 |
} |
|
81 |
|
|
82 |
public void setXpath(String xpath) { |
|
83 |
this.xpath = xpath; |
|
84 |
} |
|
85 |
public String getXpath() { |
|
86 |
return xpath; |
|
87 |
} |
|
88 |
|
|
89 |
/** |
|
90 |
* sets the argument aVariable as the value of the rule |
|
91 |
* @param aVariable the variable as a reference to the value |
|
92 |
*/ |
|
93 |
public void setAssignmentVariable(String aVariable){ |
|
94 |
this.assignmentVariable = aVariable; |
|
95 |
} |
|
96 |
|
|
97 |
public String getAssignmentVariable(){ |
|
98 |
return this.assignmentVariable; |
|
99 |
} |
|
100 |
|
|
101 |
public void setNamespace(String namespace) { |
|
102 |
this.namespace = namespace; |
|
103 |
} |
|
104 |
|
|
105 |
public String getNamespace() { |
|
106 |
return namespace; |
|
107 |
} |
|
108 |
|
|
109 |
public void setConstant(String constant) { |
|
110 |
this.constant = constant; |
|
111 |
} |
|
112 |
|
|
113 |
public String getConstant() { |
|
114 |
return constant; |
|
115 |
} |
|
116 |
|
|
117 |
@Deprecated |
|
118 |
public void setTargetField(String targetField) { |
|
119 |
if (this.variable.length() > 0){ |
|
120 |
throw new IllegalStateException("Invalid rule definition: a rule is either defined as an output element or as a variable"); |
|
121 |
} |
|
122 |
this.targetField = targetField; |
|
123 |
} |
|
124 |
|
|
125 |
/* |
|
126 |
* @deprecated replaced by {@Link #getUniqueName()} |
|
127 |
*/ |
|
128 |
@Deprecated |
|
129 |
public String getTargetField() { |
|
130 |
return targetField; |
|
131 |
} |
|
132 |
|
|
133 |
public void setRuleDeclaration(String ruleDeclaration) { |
|
134 |
this.ruleDeclaration = ruleDeclaration; |
|
135 |
} |
|
136 |
|
|
137 |
public String getRuleDeclaration() { |
|
138 |
return ruleDeclaration; |
|
139 |
} |
|
140 |
|
|
141 |
/* |
|
142 |
* compares two rules objects based on their xpath, function and namespace names |
|
143 |
* @see java.lang.Comparable#compareTo(java.lang.Object) |
|
144 |
*/ |
|
145 |
public int compareTo(Rules o) { |
|
146 |
if ( |
|
147 |
o.targetField.equals(this.targetField) && |
|
148 |
o.variable.equals(this.variable) && |
|
149 |
o.template.equals(this.template) && |
|
150 |
o.templateMatch.equals(this.templateMatch) && |
|
151 |
o.ruleDeclaration.equals(this.ruleDeclaration) && |
|
152 |
o.namespace.equals(this.namespace) && |
|
153 |
o.constant.equals(this.constant) && |
|
154 |
o.xpath.equals(this.xpath)){ |
|
155 |
return 0; |
|
156 |
}else{ |
|
157 |
return -1; |
|
158 |
} |
|
159 |
} |
|
160 |
|
|
161 |
public void setFunctionCall(FunctionCall funcCall) { |
|
162 |
this.funcCall = funcCall; |
|
163 |
} |
|
164 |
|
|
165 |
public FunctionCall getFunctionCall() { |
|
166 |
return funcCall; |
|
167 |
} |
|
168 |
|
|
169 |
@Override |
|
170 |
public String getUniqueName() { |
|
171 |
if (this.definesVariable()) return this.variable; |
|
172 |
else if (this.definesTemplate()) return this.template; |
|
173 |
return this.targetField; |
|
174 |
} |
|
175 |
|
|
176 |
@Override |
|
177 |
public boolean hasCondition() { |
|
178 |
if (condition != null) return true; |
|
179 |
return false; |
|
180 |
} |
|
181 |
|
|
182 |
/** |
|
183 |
* @return the condition |
|
184 |
*/ |
|
185 |
public Condition getCondition() { |
|
186 |
return condition; |
|
187 |
} |
|
188 |
|
|
189 |
/** |
|
190 |
* @param condition the condition to set |
|
191 |
*/ |
|
192 |
public void setCondition(Condition condition) { |
|
193 |
this.condition = condition; |
|
194 |
} |
|
195 |
|
|
196 |
/** |
|
197 |
* @param variable the variable to set |
|
198 |
*/ |
|
199 |
public void setVariable(String variable) { |
|
200 |
if (this.targetField.length() > 0){ |
|
201 |
throw new IllegalStateException("Invalid rule definition: a rule is either defined as an output element or as a variable"); |
|
202 |
} |
|
203 |
this.variable = variable; |
|
204 |
} |
|
205 |
|
|
206 |
/** |
|
207 |
* @return the variable |
|
208 |
*/ |
|
209 |
public String getVariable() { |
|
210 |
return variable; |
|
211 |
} |
|
212 |
|
|
213 |
/** |
|
214 |
* @param isEmpty the isEmpty to set |
|
215 |
*/ |
|
216 |
public void setEmpty(boolean isEmpty) { |
|
217 |
this.isEmpty = isEmpty; |
|
218 |
} |
|
219 |
|
|
220 |
/** |
|
221 |
* @return the isEmpty |
|
222 |
*/ |
|
223 |
public boolean isEmpty() { |
|
224 |
return isEmpty; |
|
225 |
} |
|
226 |
|
|
227 |
/** |
|
228 |
* @param targetSchemaElement the targetSchemaElement to set |
|
229 |
*/ |
|
230 |
public void setTargetSchemaElement(SchemaElement targetSchemaElement) { |
|
231 |
this.targetSchemaElement = targetSchemaElement; |
|
232 |
} |
|
233 |
|
|
234 |
/** |
|
235 |
* @return the targetSchemaElement |
|
236 |
*/ |
|
237 |
public SchemaElement getTargetSchemaElement() { |
|
238 |
return targetSchemaElement; |
|
239 |
} |
|
240 |
|
|
241 |
/** |
|
242 |
* @return the template |
|
243 |
*/ |
|
244 |
public String getTemplate() { |
|
245 |
return template; |
|
246 |
} |
|
247 |
|
|
248 |
/** |
|
249 |
* @param template the template to set |
|
250 |
*/ |
|
251 |
public void setTemplate(String template) { |
|
252 |
this.template = template; |
|
253 |
} |
|
254 |
|
|
255 |
/** |
|
256 |
* @return the attribute |
|
257 |
*/ |
|
258 |
public String getAttribute() { |
|
259 |
return attribute; |
|
260 |
} |
|
261 |
|
|
262 |
/** |
|
263 |
* @param attribute the attribute to set |
|
264 |
*/ |
|
265 |
public void setAttribute(String attribute) { |
|
266 |
this.attribute = attribute; |
|
267 |
} |
|
268 |
|
|
269 |
/** |
|
270 |
* @return the rulesSet |
|
271 |
*/ |
|
272 |
public RulesSet getRulesSet() { |
|
273 |
return rulesSet; |
|
274 |
} |
|
275 |
|
|
276 |
/** |
|
277 |
* @param rulesSet the rulesSet to set |
|
278 |
*/ |
|
279 |
public void setRulesSet(RulesSet rulesSet) { |
|
280 |
this.rulesSet = rulesSet; |
|
281 |
} |
|
282 |
|
|
283 |
/* (non-Javadoc) |
|
284 |
* @see eu.dnetlib.data.collective.transformation.rulelanguage.IRule#hasSet() |
|
285 |
*/ |
|
286 |
@Override |
|
287 |
public boolean hasSet() { |
|
288 |
if (rulesSet != null) return true; |
|
289 |
return false; |
|
290 |
} |
|
291 |
|
|
292 |
public String getTemplateMatch() { |
|
293 |
return templateMatch; |
|
294 |
} |
|
295 |
|
|
296 |
public void setTemplateMatch(String templateMatch) { |
|
297 |
this.templateMatch = templateMatch; |
|
298 |
} |
|
299 |
|
|
300 |
public Properties getProperties() { |
|
301 |
return properties; |
|
302 |
} |
|
303 |
|
|
304 |
public void setProperties(Properties properties) { |
|
305 |
this.properties = properties; |
|
306 |
} |
|
307 |
|
|
308 |
public boolean isSkip() { |
|
309 |
return isSkip; |
|
310 |
} |
|
311 |
|
|
312 |
public void setSkip(boolean isSkip) { |
|
313 |
this.isSkip = isSkip; |
|
314 |
} |
|
315 |
|
|
316 |
} |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/core/schema/visitor/Visitor.java | ||
---|---|---|
1 |
package eu.dnetlib.data.collective.transformation.core.schema.visitor; |
|
2 |
|
|
3 |
|
|
4 |
import java.util.LinkedList; |
|
5 |
import java.util.List; |
|
6 |
|
|
7 |
import org.apache.commons.lang3.NotImplementedException; |
|
8 |
//import org.apache.commons.logging.Log; |
|
9 |
//import org.apache.commons.logging.LogFactory; |
|
10 |
|
|
11 |
import com.sun.xml.xsom.XSAnnotation; |
|
12 |
import com.sun.xml.xsom.XSAttGroupDecl; |
|
13 |
import com.sun.xml.xsom.XSAttributeDecl; |
|
14 |
import com.sun.xml.xsom.XSAttributeUse; |
|
15 |
import com.sun.xml.xsom.XSComplexType; |
|
16 |
import com.sun.xml.xsom.XSContentType; |
|
17 |
import com.sun.xml.xsom.XSElementDecl; |
|
18 |
import com.sun.xml.xsom.XSFacet; |
|
19 |
import com.sun.xml.xsom.XSIdentityConstraint; |
|
20 |
import com.sun.xml.xsom.XSModelGroup; |
|
21 |
import com.sun.xml.xsom.XSModelGroupDecl; |
|
22 |
import com.sun.xml.xsom.XSNotation; |
|
23 |
import com.sun.xml.xsom.XSParticle; |
|
24 |
import com.sun.xml.xsom.XSSchema; |
|
25 |
import com.sun.xml.xsom.XSSimpleType; |
|
26 |
import com.sun.xml.xsom.XSType; |
|
27 |
import com.sun.xml.xsom.XSWildcard; |
|
28 |
import com.sun.xml.xsom.XSXPath; |
|
29 |
import com.sun.xml.xsom.visitor.XSVisitor; |
|
30 |
|
|
31 |
import eu.dnetlib.data.collective.transformation.core.schema.SchemaAttribute; |
|
32 |
import eu.dnetlib.data.collective.transformation.core.schema.SchemaElement; |
|
33 |
|
|
34 |
/** |
|
35 |
* @author jochen |
|
36 |
* |
|
37 |
*/ |
|
38 |
public class Visitor implements XSVisitor { |
|
39 |
|
|
40 |
//private static Log log = LogFactory.getLog(Visitor.class); |
|
41 |
List<SchemaElement> schemaElements = new LinkedList<SchemaElement>(); |
|
42 |
SchemaElement currentElement; |
|
43 |
SchemaAttribute currentAttribute; |
|
44 |
|
|
45 |
@Override |
|
46 |
public void annotation(XSAnnotation arg0) { |
|
47 |
throw new NotImplementedException("TODO: annotation"); |
|
48 |
} |
|
49 |
|
|
50 |
@Override |
|
51 |
public void attGroupDecl(XSAttGroupDecl arg0) { |
|
52 |
throw new NotImplementedException("TODO attGroupDecl"); |
|
53 |
} |
|
54 |
|
|
55 |
@Override |
|
56 |
public void attributeDecl(XSAttributeDecl aAttributeDecl) { |
|
57 |
currentAttribute.setName(aAttributeDecl.getName()); |
|
58 |
//log.debug("visit attribute name: " + aAttributeDecl.getName()); |
|
59 |
//log.debug("visit attribute type: " + aAttributeDecl.getType()); |
|
60 |
throw new NotImplementedException("TODO attributeDecl"); |
|
61 |
} |
|
62 |
|
|
63 |
@Override |
|
64 |
public void attributeUse(XSAttributeUse aAttributeUse) { |
|
65 |
throw new NotImplementedException("TODO attributeUse"); |
|
66 |
} |
|
67 |
|
|
68 |
@Override |
|
69 |
public void complexType(XSComplexType aType) { |
|
70 |
if (aType.getDerivationMethod()== XSType.RESTRICTION){ |
|
71 |
XSContentTypeVisitorImpl contentTypeVisitor = new XSContentTypeVisitorImpl(); |
|
72 |
contentTypeVisitor.setVisitor(this); |
|
73 |
aType.getContentType().visit(contentTypeVisitor); |
|
74 |
}else{ |
|
75 |
// aType.getExplicitContent().visit(this); |
|
76 |
throw new NotImplementedException("visiting types other then 'RESTRICTION are not implemented'"); |
|
77 |
} |
|
78 |
} |
|
79 |
|
|
80 |
@Override |
|
81 |
public void facet(XSFacet arg0) { |
|
82 |
throw new NotImplementedException("TODO facet"); |
|
83 |
} |
|
84 |
|
|
85 |
@Override |
|
86 |
public void identityConstraint(XSIdentityConstraint arg0) { |
|
87 |
throw new NotImplementedException("TODO identityConstraint"); |
|
88 |
} |
|
89 |
|
|
90 |
@Override |
|
91 |
public void notation(XSNotation arg0) { |
|
92 |
throw new NotImplementedException("TODO notation"); |
|
93 |
} |
|
94 |
|
|
95 |
@Override |
|
96 |
public void schema(XSSchema arg0) { |
|
97 |
throw new NotImplementedException("TODO schema"); |
|
98 |
} |
|
99 |
|
|
100 |
@Override |
|
101 |
public void xpath(XSXPath arg0) { |
|
102 |
throw new NotImplementedException("TODO xpath"); |
|
103 |
} |
|
104 |
|
|
105 |
@Override |
|
106 |
public void elementDecl(XSElementDecl aElementDecl) { |
|
107 |
XSType type = aElementDecl.getType(); |
|
108 |
if (type.isLocal()){ |
|
109 |
// complete infos about the current element |
|
110 |
// log.debug("visitor element name: " + aElementDecl.getName()); |
|
111 |
currentElement.setName(aElementDecl.getName()); |
|
112 |
currentElement.setTargetNamespace(aElementDecl.getTargetNamespace()); |
|
113 |
type.visit(this); |
|
114 |
} |
|
115 |
} |
|
116 |
|
|
117 |
@Override |
|
118 |
public void modelGroup(XSModelGroup aGroup) { |
|
119 |
// a group of elements as childs of the root element |
|
120 |
for (XSParticle p: aGroup.getChildren()){ |
|
121 |
particle(p); |
|
122 |
} |
|
123 |
} |
|
124 |
|
|
125 |
@Override |
|
126 |
public void modelGroupDecl(XSModelGroupDecl arg0) { |
|
127 |
throw new NotImplementedException("TODO modelGroupDecl"); |
|
128 |
} |
|
129 |
|
|
130 |
@Override |
|
131 |
public void wildcard(XSWildcard arg0) { |
|
132 |
throw new NotImplementedException("TODO wildcard"); |
|
133 |
} |
|
134 |
|
|
135 |
@Override |
|
136 |
public void empty(XSContentType arg0) { |
|
137 |
throw new NotImplementedException("TODO empty"); |
|
138 |
} |
|
139 |
|
|
140 |
@Override |
|
141 |
public void particle(XSParticle aParticle) { |
|
142 |
// create a new schema element, add to the list of schema elements, set this element as current element |
|
143 |
SchemaElement element = new SchemaElement(); |
|
144 |
element.setMinOccurs(aParticle.getMinOccurs().intValue()); |
|
145 |
element.setMaxOccurs(aParticle.getMaxOccurs().intValue()); |
|
146 |
element.setRepeatable(aParticle.isRepeated()); |
|
147 |
schemaElements.add(element); |
|
148 |
currentElement = element; |
|
149 |
XSTermVisitorImpl termVisitor = new XSTermVisitorImpl(); |
|
150 |
termVisitor.setVisitor(this); |
|
151 |
aParticle.getTerm().visit(termVisitor); |
|
152 |
} |
|
153 |
|
|
154 |
|
|
155 |
@Override |
|
156 |
public void simpleType(XSSimpleType arg0) { |
|
157 |
throw new NotImplementedException("TODO simpleType"); |
|
158 |
} |
|
159 |
|
|
160 |
public List<SchemaElement> getElements(){ |
|
161 |
return this.schemaElements; |
|
162 |
} |
|
163 |
|
|
164 |
protected SchemaElement getCurrentElement(){ |
|
165 |
return currentElement; |
|
166 |
} |
|
167 |
|
|
168 |
} |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/rulelanguage/parser/ASTMySkip.java | ||
---|---|---|
1 |
/* Generated By:JJTree: Do not edit this line. ASTMySkip.java Version 4.3 */ |
|
2 |
/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=false,VISITOR=true,TRACK_TOKENS=false,NODE_PREFIX=AST,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ |
|
3 |
package eu.dnetlib.data.collective.transformation.rulelanguage.parser; |
|
4 |
|
|
5 |
import eu.dnetlib.data.collective.transformation.rulelanguage.Rules; |
|
6 |
|
|
7 |
public |
|
8 |
class ASTMySkip extends AbstractNode { |
|
9 |
public ASTMySkip(int id) { |
|
10 |
super(id); |
|
11 |
} |
|
12 |
|
|
13 |
public ASTMySkip(FtScript p, int id) { |
|
14 |
super(p, id); |
|
15 |
} |
|
16 |
|
|
17 |
|
|
18 |
/** Accept the visitor. **/ |
|
19 |
public Object jjtAccept(FtScriptVisitor visitor, Object data) { |
|
20 |
return visitor.visit(this, data); |
|
21 |
} |
|
22 |
|
|
23 |
public void skipRecord() { |
|
24 |
// TODO Auto-generated method stub |
|
25 |
|
|
26 |
} |
|
27 |
|
|
28 |
} |
|
29 |
/* JavaCC - OriginalChecksum=149f1dfbf543fde562b05c417044a68a (do not edit this line) */ |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/rulelanguage/parser/ASTMyPreprocess.java | ||
---|---|---|
1 |
/* Generated By:JJTree: Do not edit this line. ASTMyPreprocess.java Version 4.3 */ |
|
2 |
/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=false,VISITOR=true,TRACK_TOKENS=false,NODE_PREFIX=AST,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ |
|
3 |
package eu.dnetlib.data.collective.transformation.rulelanguage.parser; |
|
4 |
|
|
5 |
public class ASTMyPreprocess extends SimpleNode { |
|
6 |
private String id; |
|
7 |
private String funcName; |
|
8 |
private String parameter; |
|
9 |
|
|
10 |
public ASTMyPreprocess(int id) { |
|
11 |
super(id); |
|
12 |
} |
|
13 |
|
|
14 |
public ASTMyPreprocess(FtScript p, int id) { |
|
15 |
super(p, id); |
|
16 |
} |
|
17 |
|
|
18 |
/** Accept the visitor. **/ |
|
19 |
public Object jjtAccept(FtScriptVisitor visitor, Object data) { |
|
20 |
return visitor.visit(this, data); |
|
21 |
} |
|
22 |
|
|
23 |
public String getFunctionName(){ |
|
24 |
return this.funcName; |
|
25 |
} |
|
26 |
|
|
27 |
public String getParameter(){ |
|
28 |
return this.parameter; |
|
29 |
} |
|
30 |
|
|
31 |
public String getId(){ |
|
32 |
return id; |
|
33 |
} |
|
34 |
|
|
35 |
public void preprocess(String aPreprocessId, String aFunction, String aParameter) { |
|
36 |
this.id = aPreprocessId; |
|
37 |
this.funcName = aFunction; |
|
38 |
// unquote |
|
39 |
if (aParameter.length() > 0){ |
|
40 |
this.parameter = aParameter.substring(1, aParameter.length() - 1); |
|
41 |
} |
|
42 |
} |
|
43 |
|
|
44 |
public void preprocess(String aFunction, String aParameter){ |
|
45 |
this.funcName = aFunction; |
|
46 |
// unquote |
|
47 |
if (aParameter.length() > 0){ |
|
48 |
this.parameter = aParameter.substring(1, aParameter.length() - 1); |
|
49 |
} |
|
50 |
} |
|
51 |
|
|
52 |
public void preprocess(String aFunction) { |
|
53 |
this.funcName = aFunction; |
|
54 |
} |
|
55 |
} |
|
56 |
/* JavaCC - OriginalChecksum=b9229360af18a53de1ce87664846e442 (do not edit this line) */ |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/engine/SimpleTransformationEngine.java | ||
---|---|---|
1 |
package eu.dnetlib.data.collective.transformation.engine; |
|
2 |
|
|
3 |
import java.io.StringReader; |
|
4 |
import java.util.Iterator; |
|
5 |
import java.util.LinkedHashMap; |
|
6 |
import java.util.LinkedList; |
|
7 |
import java.util.List; |
|
8 |
import java.util.Map; |
|
9 |
|
|
10 |
import javax.xml.xpath.XPath; |
|
11 |
import javax.xml.xpath.XPathConstants; |
|
12 |
import javax.xml.xpath.XPathExpressionException; |
|
13 |
import javax.xml.xpath.XPathFactory; |
|
14 |
|
|
15 |
import org.apache.commons.lang3.StringUtils; |
|
16 |
import org.apache.commons.logging.Log; |
|
17 |
import org.apache.commons.logging.LogFactory; |
|
18 |
import org.springframework.core.io.Resource; |
|
19 |
import org.w3c.dom.Node; |
|
20 |
import org.xml.sax.InputSource; |
|
21 |
|
|
22 |
import eu.dnetlib.common.profile.ResourceDao; |
|
23 |
import eu.dnetlib.data.collective.transformation.IDatabaseConnector; |
|
24 |
import eu.dnetlib.data.collective.transformation.TransformationException; |
|
25 |
import eu.dnetlib.data.collective.transformation.VocabularyRegistry; |
|
26 |
import eu.dnetlib.data.collective.transformation.core.xsl.ext.TransformationFunctionProxy; |
|
27 |
import eu.dnetlib.data.collective.transformation.engine.core.ITransformation; |
|
28 |
import eu.dnetlib.data.collective.transformation.engine.functions.Convert; |
|
29 |
// import eu.dnetlib.data.collective.transformation.engine.functions.Dblookup; |
|
30 |
import eu.dnetlib.data.collective.transformation.engine.functions.Extract; |
|
31 |
import eu.dnetlib.data.collective.transformation.engine.functions.IFeatureExtraction; |
|
32 |
import eu.dnetlib.data.collective.transformation.engine.functions.ProcessingException; |
|
33 |
import eu.dnetlib.data.collective.transformation.engine.functions.RegularExpression; |
|
34 |
import eu.dnetlib.data.collective.transformation.engine.functions.RetrieveValue; |
|
35 |
import eu.dnetlib.data.collective.transformation.rulelanguage.util.FunctionCall; |
|
36 |
import eu.dnetlib.data.collective.transformation.utils.BlacklistConsumer; |
|
37 |
import eu.dnetlib.enabling.datasources.common.Api; |
|
38 |
import eu.dnetlib.enabling.datasources.common.Datasource; |
|
39 |
import eu.dnetlib.enabling.datasources.common.DsmException; |
|
40 |
import eu.dnetlib.enabling.datasources.common.LocalDatasourceManager; |
|
41 |
import net.sf.saxon.instruct.TerminationException; |
|
42 |
|
|
43 |
/** |
|
44 |
* @author jochen |
|
45 |
* |
|
46 |
*/ |
|
47 |
public class SimpleTransformationEngine { |
|
48 |
|
|
49 |
private LocalDatasourceManager<Datasource<?, ?, ?>, Api<?>> dsManager; |
|
50 |
|
|
51 |
private static Log log = LogFactory.getLog(SimpleTransformationEngine.class); |
|
52 |
private ITransformation transformation; |
|
53 |
private VocabularyRegistry vocabularyRegistry; |
|
54 |
private IDatabaseConnector databaseConnector; |
|
55 |
private ResourceDao resourceDao; |
|
56 |
private IFeatureExtraction featureExtraction; |
|
57 |
private final List<String> mdRecords = new LinkedList<>(); |
|
58 |
private long totalTransformedRecords = 0; |
|
59 |
private long totalIgnoredRecords = 0; |
|
60 |
private String mappingFile; |
|
61 |
private boolean stylesheetParamsCalculated = false; |
|
62 |
private boolean preprocessingDone = false; |
|
63 |
private final Map<String, String> stylesheetParams = new LinkedHashMap<>(); |
|
64 |
private Resource blacklistApi; |
|
65 |
private List<String> blacklistedRecords = new LinkedList<>(); |
|
66 |
|
|
67 |
/** |
|
68 |
* execute any preprocessings declared in the transformation script prior starting the transformation of records |
|
69 |
*/ |
|
70 |
public void preprocess(final String dataSourceId) { |
|
71 |
for (final Map<String, String> preprocMap : this.transformation.getRuleLanguageParser().getPreprocessings()) { |
|
72 |
final Iterator<String> it = preprocMap.keySet().iterator(); |
|
73 |
while (it.hasNext()) { |
|
74 |
final String function = it.next(); |
|
75 |
// if (function.equals("dblookup")) { |
|
76 |
// Dblookup fun = new Dblookup(); |
|
77 |
// fun.setDbConnector(databaseConnector); |
|
78 |
// try { |
|
79 |
// log.debug("preprocessingMap value: " + preprocMap.get(function)); |
|
80 |
// TransformationFunctionProxy.getInstance().setLookupRecord(fun.getResults(preprocMap.get(function))); |
|
81 |
// } catch (Exception e) { |
|
82 |
// log.debug(e.getMessage()); |
|
83 |
// throw new IllegalStateException(e); |
|
84 |
// } |
|
85 |
// } |
|
86 |
if (function.equals("blacklist")) { |
|
87 |
final BlacklistConsumer bc = new BlacklistConsumer(); |
|
88 |
try { |
|
89 |
blacklistedRecords = bc.getBlackList(blacklistApi.getURL() + dataSourceId); |
|
90 |
} catch (final Exception e) { |
|
91 |
throw new IllegalStateException("error in preprocess: " + e.getMessage()); |
|
92 |
} |
|
93 |
} |
|
94 |
} |
|
95 |
} |
|
96 |
log.debug("preprocessing done."); |
|
97 |
} |
|
98 |
|
|
99 |
/** |
|
100 |
* check if blacklistedRecords exist and if so check if the current record is blacklisted by its objIdentifier |
|
101 |
* |
|
102 |
* @param aRecord |
|
103 |
* @return |
|
104 |
* @throws XPathExpressionException |
|
105 |
* @throws ProcessingException |
|
106 |
*/ |
|
107 |
private boolean isBlacklistRecord(final String aRecord) { |
|
108 |
if (blacklistedRecords.size() == 0) { return false; } |
|
109 |
final XPath xpath = XPathFactory.newInstance().newXPath(); |
|
110 |
try { |
|
111 |
final Node root = (Node) xpath.evaluate("/", new InputSource(new StringReader(aRecord)), XPathConstants.NODE); |
|
112 |
final String objId = xpath.evaluate("//*[local-name()='objIdentifier']", root); |
|
113 |
if (blacklistedRecords.contains(objId)) { return true; } |
|
114 |
} catch (final Exception e) { |
|
115 |
throw new IllegalStateException("error in isBlacklistRecord: " + e.getMessage()); |
|
116 |
} |
|
117 |
return false; |
|
118 |
} |
|
119 |
|
|
120 |
/** |
|
121 |
* transforms a source record |
|
122 |
* |
|
123 |
* @param sourceRecord |
|
124 |
* the record to transform |
|
125 |
* @return transformed record |
|
126 |
*/ |
|
127 |
public String transform(final String sourceRecord) { |
|
128 |
final List<String> objectRecords = new LinkedList<>(); |
|
129 |
objectRecords.add(sourceRecord); |
|
130 |
int index = 0; |
|
131 |
mdRecords.clear(); |
|
132 |
initTransformationFunction(); |
|
133 |
|
|
134 |
if (!stylesheetParamsCalculated) { |
|
135 |
try { |
|
136 |
calculateStylesheetParams(sourceRecord); |
|
137 |
} catch (final Exception e) { |
|
138 |
throw new IllegalStateException("error in calculateStylesheetParams" + e.getMessage(), e); |
|
139 |
} |
|
140 |
} |
|
141 |
|
|
142 |
if (!preprocessingDone) { |
|
143 |
// xpath sourceRecord dataSourceid |
|
144 |
preprocess(stylesheetParams.get("varBlacklistDataSourceId")); |
|
145 |
preprocessingDone = true; |
|
146 |
} |
|
147 |
|
|
148 |
if (isBlacklistRecord(sourceRecord)) { |
|
149 |
try { |
|
150 |
mdRecords.add(transformation.transformRecord(sourceRecord, ITransformation.XSLSyntaxcheckfailed)); |
|
151 |
} catch (final Exception e) { |
|
152 |
log.fatal(sourceRecord); |
|
153 |
throw new IllegalStateException(e); |
|
154 |
} |
|
155 |
} else if (!transformation.getRuleLanguageParser().isXslStylesheet()) { |
|
156 |
// iterate over all rules which are functionCalls |
|
157 |
log.debug("functionCalls size: " + transformation.getRuleLanguageParser().getFunctionCalls().size()); |
|
158 |
for (final FunctionCall functionCall : transformation.getRuleLanguageParser().getFunctionCalls()) { |
|
159 |
preprocess(objectRecords, functionCall); |
|
160 |
} |
|
161 |
for (final String record : objectRecords) { |
|
162 |
// log.debug(record); |
|
163 |
try { |
|
164 |
log.debug("now run transformation for record with index: " + index); |
|
165 |
try { |
|
166 |
final String transformedRecord = transformation.transformRecord(record, index); |
|
167 |
mdRecords.add(transformedRecord); |
|
168 |
} catch (final TerminationException e) { |
|
169 |
log.debug("record transformation terminated."); |
|
170 |
final String failedRecord = transformation.transformRecord(record, ITransformation.XSLSyntaxcheckfailed); |
|
171 |
log.debug(failedRecord); |
|
172 |
totalIgnoredRecords++; |
|
173 |
mdRecords.add(failedRecord); |
|
174 |
} |
|
175 |
} catch (final TransformationException e) { |
|
176 |
log.error(sourceRecord); |
|
177 |
throw new IllegalStateException(e); |
|
178 |
} |
|
179 |
index++; |
|
180 |
} |
|
181 |
} else { |
|
182 |
for (final String record : objectRecords) { |
|
183 |
// test for init params and assign values |
|
184 |
try { |
|
185 |
log.debug("now run transformation for record with index: " + index); |
|
186 |
try { |
|
187 |
final String transformedRecord = transformation.transformRecord(record, stylesheetParams); |
|
188 |
mdRecords.add(transformedRecord); |
|
189 |
} catch (final TerminationException e) { |
|
190 |
final String failedRecord = transformation.transformRecord(record, ITransformation.XSLSyntaxcheckfailed); |
|
191 |
totalIgnoredRecords++; |
|
192 |
log.debug(failedRecord); |
|
193 |
mdRecords.add(failedRecord); |
|
194 |
} |
|
195 |
} catch (final TransformationException e) { |
|
196 |
log.error(sourceRecord); |
|
197 |
throw new IllegalStateException(e); |
|
198 |
} |
|
199 |
index++; |
|
200 |
} |
|
201 |
} |
|
202 |
|
|
203 |
totalTransformedRecords = totalTransformedRecords + mdRecords.size(); |
|
204 |
log.debug("objRecordSize: " + objectRecords.size() + ", mdRecordSize: " + mdRecords.size() + ", ignoredRecordSize: " + totalIgnoredRecords); |
|
205 |
return mdRecords.get(0); |
|
206 |
} |
|
207 |
|
|
208 |
private void calculateStylesheetParams(final String aRecord) throws XPathExpressionException, ProcessingException, DsmException { |
|
209 |
stylesheetParamsCalculated = true; |
|
210 |
final XPath xpath = XPathFactory.newInstance().newXPath(); |
|
211 |
final Node root = (Node) xpath.evaluate("/", new InputSource(new StringReader(aRecord)), XPathConstants.NODE); |
|
212 |
|
|
213 |
final String dsId = xpath.evaluate("//*[local-name()='repositoryId']", root); |
|
214 |
final String nsPrefix = xpath.evaluate("//*[local-name()='datasourceprefix']", root); |
|
215 |
|
|
216 |
final Datasource<?, ?, ?> ds; |
|
217 |
if (StringUtils.isNotBlank(nsPrefix)) { |
|
218 |
ds = dsManager.getDsByNsPrefix(nsPrefix); |
|
219 |
} else if (StringUtils.isNotBlank(dsId)) { |
|
220 |
ds = dsManager.getDs(dsId); |
|
221 |
} else { |
|
222 |
ds = null; |
|
223 |
} |
|
224 |
|
|
225 |
if (ds != null) { |
|
226 |
stylesheetParams.put("varOfficialName", ds.getOfficialname()); |
|
227 |
stylesheetParams.put("varDataSourceId", ds.getId()); |
|
228 |
stylesheetParams.put("varDsType", ds.getEoscDatasourceType()); |
|
229 |
} |
|
230 |
// if blacklist |
|
231 |
for (final Map<String, String> preprocMap : this.transformation.getRuleLanguageParser().getPreprocessings()) { |
|
232 |
final Iterator<String> it = preprocMap.keySet().iterator(); |
|
233 |
while (it.hasNext()) { |
|
234 |
final String function = it.next(); |
|
235 |
if (function.equals("blacklist")) { |
|
236 |
// TODO |
|
237 |
// stylesheetParams.put("varBlacklistDataSourceId", varBlacklistDataSourceId); |
|
238 |
} |
|
239 |
} |
|
240 |
} |
|
241 |
} |
|
242 |
|
|
243 |
private void initTransformationFunction() { |
|
244 |
if (this.vocabularyRegistry == null) { throw new IllegalStateException("vocabularyReg is null"); } |
|
245 |
final Convert convertFunction = new Convert(); |
|
246 |
convertFunction.setVocabularyRegistry(this.vocabularyRegistry); |
|
247 |
TransformationFunctionProxy.getInstance().setConvertFunction(convertFunction); |
|
248 |
|
|
249 |
} |
|
250 |
|
|
251 |
/** |
|
252 |
* preprocesses function if function is configured resp. |
|
253 |
* |
|
254 |
* @param records |
|
255 |
* list of object records |
|
256 |
* @param aFunctionCall |
|
257 |
*/ |
|
258 |
private void preprocess(final List<String> records, final FunctionCall aFunctionCall) { |
|
259 |
try { |
|
260 |
log.debug("preprocess"); |
|
261 |
if (transformation.getRuleLanguageParser() == null) { throw new IllegalStateException("rulelanguageparser not initialised"); } |
|
262 |
if (transformation.getRuleLanguageParser().getNamespaceDeclarations() == null) { throw new IllegalStateException("nsDecl is null"); } |
|
263 |
final PreProcessor preProc = new PreProcessor(); |
|
264 |
preProc.setConvertFunction(TransformationFunctionProxy.getInstance().getConvertFunction()); |
|
265 |
final RetrieveValue retrieveValue = new RetrieveValue(); |
|
266 |
retrieveValue.setResourceDao(resourceDao); |
|
267 |
retrieveValue.setDsManager(dsManager); |
|
268 |
preProc.setRetrieveFunction(retrieveValue); |
|
269 |
final RegularExpression regExpr = new RegularExpression(); |
|
270 |
preProc.setRegExprFunction(regExpr); |
|
271 |
final TransformationFunctionProxy functionProxy = TransformationFunctionProxy.getInstance(); |
|
272 |
preProc.setFunctionProxy(functionProxy); |
|
273 |
final Extract extractFunction = new Extract(); |
|
274 |
extractFunction.setFeatureExtraction(featureExtraction); |
|
275 |
preProc.setExtractFunction(extractFunction); |
|
276 |
if (aFunctionCall.doPreprocess() || aFunctionCall.isStatic()) { |
|
277 |
// log.debug("now call preprocess with: " + aFunctionCall.getExternalFunctionName() + " " + aFunctionCall.getUuid()); |
|
278 |
preProc.preprocess(aFunctionCall, records, transformation.getRuleLanguageParser().getNamespaceDeclarations(), transformation |
|
279 |
.getStaticTransformationResults(), transformation.getJobProperties(), transformation.getRuleLanguageParser().getVariableMappingRules()); |
|
280 |
// log.debug("preprocess end"); |
|
281 |
} else { |
|
282 |
log.debug("skip preprocessing for function: " + aFunctionCall.getExternalFunctionName()); |
|
283 |
} |
|
284 |
|
|
285 |
} catch (final Exception e) { |
|
286 |
throw new IllegalStateException(e); |
|
287 |
} |
|
288 |
|
|
289 |
} |
|
290 |
|
|
291 |
/** |
|
292 |
* @param transformation |
|
293 |
* the transformation to set |
|
294 |
*/ |
|
295 |
public void setTransformation(final ITransformation transformation) { |
|
296 |
this.transformation = transformation; |
|
297 |
} |
|
298 |
|
|
299 |
/** |
|
300 |
* @return the transformation |
|
301 |
*/ |
|
302 |
public ITransformation getTransformation() { |
|
303 |
return transformation; |
|
304 |
} |
|
305 |
|
|
306 |
/** |
|
307 |
* @param vocabularyRegistry |
|
308 |
* the vocabularyRegistry to set |
|
309 |
*/ |
|
310 |
public void setVocabularyRegistry(final VocabularyRegistry vocabularyRegistry) { |
|
311 |
this.vocabularyRegistry = vocabularyRegistry; |
|
312 |
} |
|
313 |
|
|
314 |
/** |
|
315 |
* @return the vocabularyRegistry |
|
316 |
*/ |
|
317 |
public VocabularyRegistry getVocabularyRegistry() { |
|
318 |
return vocabularyRegistry; |
|
319 |
} |
|
320 |
|
|
321 |
/** |
|
322 |
* @return the resourceDao |
|
323 |
*/ |
|
324 |
public ResourceDao getResourceDao() { |
|
325 |
return resourceDao; |
|
326 |
} |
|
327 |
|
|
328 |
/** |
|
329 |
* @param resourceDao |
|
330 |
* the resourceDao to set |
|
331 |
*/ |
|
332 |
public void setResourceDao(final ResourceDao resourceDao) { |
|
333 |
this.resourceDao = resourceDao; |
|
334 |
} |
|
335 |
|
|
336 |
/** |
|
337 |
* @param featureExtraction |
|
338 |
* the featureExtraction to set |
|
339 |
*/ |
|
340 |
public void setFeatureExtraction(final IFeatureExtraction featureExtraction) { |
|
341 |
this.featureExtraction = featureExtraction; |
|
342 |
} |
|
343 |
|
|
344 |
/** |
|
345 |
* @return the featureExtraction |
|
346 |
*/ |
|
347 |
public IFeatureExtraction getFeatureExtraction() { |
|
348 |
return featureExtraction; |
|
349 |
} |
|
350 |
|
|
351 |
/** |
|
352 |
* @return the databaseConnector |
|
353 |
*/ |
|
354 |
public IDatabaseConnector getDatabaseConnector() { |
|
355 |
return databaseConnector; |
|
356 |
} |
|
357 |
|
|
358 |
/** |
|
359 |
* @param databaseConnector |
|
360 |
* the databaseConnector to set |
|
361 |
*/ |
|
362 |
public void setDatabaseConnector(final IDatabaseConnector databaseConnector) { |
|
363 |
this.databaseConnector = databaseConnector; |
|
364 |
} |
|
365 |
|
|
366 |
public long getTotalTransformedRecords() { |
|
367 |
return this.totalTransformedRecords; |
|
368 |
} |
|
369 |
|
|
370 |
public long getTotalIgnoredRecords() { |
|
371 |
return this.totalIgnoredRecords; |
|
372 |
} |
|
373 |
|
|
374 |
/** |
|
375 |
* @return the mappingFile |
|
376 |
*/ |
|
377 |
public String getMappingFile() { |
|
378 |
return mappingFile; |
|
379 |
} |
|
380 |
|
|
381 |
/** |
|
382 |
* @param mappingFile |
|
383 |
* the mappingFile to set |
|
384 |
*/ |
|
385 |
public void setMappingFile(final String mappingFile) { |
|
386 |
this.mappingFile = mappingFile; |
|
387 |
} |
|
388 |
|
|
389 |
public Resource getBlacklistApi() { |
|
390 |
return blacklistApi; |
|
391 |
} |
|
392 |
|
|
393 |
public void setBlacklistApi(final Resource blacklistApi) { |
|
394 |
this.blacklistApi = blacklistApi; |
|
395 |
} |
|
396 |
|
|
397 |
public LocalDatasourceManager<Datasource<?, ?, ?>, Api<?>> getDsManager() { |
|
398 |
return dsManager; |
|
399 |
} |
|
400 |
|
|
401 |
public void setDsManager(final LocalDatasourceManager<Datasource<?, ?, ?>, Api<?>> dsManager) { |
|
402 |
this.dsManager = dsManager; |
|
403 |
} |
|
404 |
} |
modules/unibi-data-collective-transformation-common/tags/unibi-data-collective-transformation-common-3.0.3/src/main/java/eu/dnetlib/data/collective/transformation/rulelanguage/parser/ft.jj | ||
---|---|---|
1 |
/*@bgen(jjtree) Generated By:JJTree: Do not edit this line. ft.jj */ |
|
2 |
/*@egen*//** |
|
3 |
* JJTree file |
|
4 |
* NODE_PACKAGE = "eu.dnetlib.data.collective.transformation.rulelanguage.node"; |
|
5 |
*/ |
|
6 |
|
|
7 |
options { |
|
8 |
STATIC=false; |
|
9 |
|
|
10 |
|
Also available in: Unified diff
[maven-release-plugin] copy for tag unibi-data-collective-transformation-common-3.0.3