Revision 56401
Added by Miriam Baglioni about 5 years ago
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/deploy.info | ||
---|---|---|
1 |
{"type_source": "SVN", "goal": "package -U source:jar", |
|
2 |
"url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-openaireplus-mapping-utils/trunk/", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-openaireplus-mapping-utils"} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/openaire/exporter/model/ProjectTest.java | ||
---|---|---|
1 |
package eu.dnetlib.openaire.exporter.model; |
|
2 |
|
|
3 |
import org.junit.Before; |
|
4 |
import org.junit.Test; |
|
5 |
|
|
6 |
import static org.junit.Assert.assertEquals; |
|
7 |
|
|
8 |
public class ProjectTest { |
|
9 |
|
|
10 |
Project pMZOS; |
|
11 |
Project pFP7; |
|
12 |
|
|
13 |
@Before |
|
14 |
public void setUp() throws Exception { |
|
15 |
pMZOS = new Project() |
|
16 |
.setFunder("MZOS") |
|
17 |
.setJurisdiction("HR") |
|
18 |
.setFundingpathid("irb_hr______::MZOS") |
|
19 |
.setAcronym("") |
|
20 |
.setTitle("Project Title") |
|
21 |
.setCode("115-1152437-2500") |
|
22 |
.setStartdate("2007-01-01") |
|
23 |
.setEnddate("2009-01-01"); |
|
24 |
|
|
25 |
pFP7 = new Project() |
|
26 |
.setFunder("EC") |
|
27 |
.setJurisdiction("EU") |
|
28 |
.setFundingpathid("ec__________::EC::FP7::SP1::NMP") |
|
29 |
.setAcronym("REFFIBRE") |
|
30 |
.setTitle("Project Title") |
|
31 |
.setCode("604187") |
|
32 |
.setStartdate("2013-11-01") |
|
33 |
.setEnddate("20015-01-01"); |
|
34 |
} |
|
35 |
|
|
36 |
@Test |
|
37 |
public void testIdNamespaceMZOS(){ |
|
38 |
String ns = pMZOS.getIdnamespace(); |
|
39 |
assertEquals("info:eu-repo/grantAgreement/MZOS//115-1152437-2500/HR", ns); |
|
40 |
} |
|
41 |
|
|
42 |
@Test |
|
43 |
public void testIdNamespaceFP7(){ |
|
44 |
String ns = pFP7.getIdnamespace(); |
|
45 |
assertEquals("info:eu-repo/grantAgreement/EC/FP7/604187/EU", ns); |
|
46 |
} |
|
47 |
|
|
48 |
|
|
49 |
|
|
50 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/openaire/exporter/model/ProjectDetailTest.java | ||
---|---|---|
1 |
package eu.dnetlib.openaire.exporter.model; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.io.StringReader; |
|
5 |
|
|
6 |
import com.google.common.collect.Lists; |
|
7 |
import org.apache.commons.io.IOUtils; |
|
8 |
import org.springframework.core.io.ClassPathResource; |
|
9 |
import org.springframework.core.io.Resource; |
|
10 |
|
|
11 |
import static org.junit.Assert.assertEquals; |
|
12 |
|
|
13 |
/** |
|
14 |
* Created by claudio on 22/09/16. |
|
15 |
*/ |
|
16 |
public class ProjectDetailTest { |
|
17 |
|
|
18 |
private Resource projectsCsv = new ClassPathResource("/eu/dnetlib/openaire/exporter/model/projectDetails.csv"); |
|
19 |
private Resource projectsJson = new ClassPathResource("/eu/dnetlib/openaire/exporter/model/projectDetails.json"); |
|
20 |
|
|
21 |
//@Test |
|
22 |
public void testSerialisationCSV() throws IOException { |
|
23 |
final String csv = IOUtils.toString(projectsCsv.getInputStream()); |
|
24 |
doTest(csv, "csv"); |
|
25 |
} |
|
26 |
|
|
27 |
//@Test |
|
28 |
public void testSerialisationCSV2() throws IOException { |
|
29 |
final ProjectDetail p = new ProjectDetail() |
|
30 |
.setAcronym("acro") |
|
31 |
.setCode("01") |
|
32 |
.setOptional1("op1") |
|
33 |
.setOptional2("op2") |
|
34 |
.setProjectId("project_01") |
|
35 |
.setJsonextrainfo("extraInfo") |
|
36 |
.setFundingPath(Lists.newArrayList("fundingpath1", "fundingpath2")); |
|
37 |
|
|
38 |
doTest(p.asCSV(), "csv"); |
|
39 |
} |
|
40 |
|
|
41 |
//@Test |
|
42 |
public void testSerialisationJSON() throws IOException { |
|
43 |
final String json = IOUtils.toString(projectsJson.getInputStream()); |
|
44 |
doTest(json, "json"); |
|
45 |
} |
|
46 |
|
|
47 |
private void doTest(final String data, final String format) throws IOException { |
|
48 |
final StringReader reader = new StringReader(data); |
|
49 |
for(String line : IOUtils.readLines(reader)) { |
|
50 |
System.out.println("line: " + line); |
|
51 |
ProjectDetail p; |
|
52 |
String s = ""; |
|
53 |
switch (format) { |
|
54 |
case "csv": |
|
55 |
p = ProjectDetail.fromCSV(line); |
|
56 |
s = p.asCSV(); |
|
57 |
break; |
|
58 |
case "json": |
|
59 |
p = ProjectDetail.fromJson(line); |
|
60 |
s = p.asJson(); |
|
61 |
break; |
|
62 |
default: throw new IllegalArgumentException("invalid format: " + format); |
|
63 |
} |
|
64 |
System.out.println("serialisation: " + s); |
|
65 |
assertEquals(s, line); |
|
66 |
} |
|
67 |
} |
|
68 |
|
|
69 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/mapreduce/util/OafDecoderTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.mapreduce.util; |
|
2 |
|
|
3 |
import static org.junit.Assert.assertFalse; |
|
4 |
import static org.junit.Assert.assertNotNull; |
|
5 |
|
|
6 |
import java.util.List; |
|
7 |
|
|
8 |
import org.junit.Test; |
|
9 |
|
|
10 |
import eu.dnetlib.data.proto.KindProtos.Kind; |
|
11 |
import eu.dnetlib.miscutils.functional.xml.IndentXmlString; |
|
12 |
|
|
13 |
public class OafDecoderTest { |
|
14 |
|
|
15 |
@Test |
|
16 |
public void testAsXml() { |
|
17 |
|
|
18 |
final OafDecoder decoder = OafTest.embed(OafTest.getResult("50|id_1"), Kind.entity); |
|
19 |
|
|
20 |
assertNotNull(decoder); |
|
21 |
|
|
22 |
assertNotNull(decoder.asXml()); |
|
23 |
|
|
24 |
System.out.println(IndentXmlString.apply(decoder.asXml())); |
|
25 |
|
|
26 |
} |
|
27 |
|
|
28 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/mapreduce/util/OafRowKeyDecoderTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.mapreduce.util; |
|
2 |
|
|
3 |
import org.junit.Test; |
|
4 |
|
|
5 |
public class OafRowKeyDecoderTest { |
|
6 |
|
|
7 |
@Test |
|
8 |
public void test() { |
|
9 |
|
|
10 |
String id1 = "50|acnbad______::0a454baf9c61e63d42fb83ab549f8062"; |
|
11 |
|
|
12 |
OafRowKeyDecoder d = OafRowKeyDecoder.decode(id1); |
|
13 |
|
|
14 |
System.out.println(d.getId()); |
|
15 |
} |
|
16 |
|
|
17 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/mapreduce/util/OafRelDecoderTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.mapreduce.util; |
|
2 |
|
|
3 |
import eu.dnetlib.data.proto.DedupProtos.Dedup.RelName; |
|
4 |
import eu.dnetlib.data.proto.OafProtos.OafRel; |
|
5 |
import eu.dnetlib.data.proto.RelTypeProtos.RelType; |
|
6 |
import eu.dnetlib.data.proto.RelTypeProtos.SubRelType; |
|
7 |
import org.junit.Before; |
|
8 |
import org.junit.Test; |
|
9 |
|
|
10 |
import static org.junit.Assert.assertEquals; |
|
11 |
import static org.junit.Assert.assertNotNull; |
|
12 |
|
|
13 |
public class OafRelDecoderTest { |
|
14 |
|
|
15 |
private OafRel oafRel; |
|
16 |
|
|
17 |
@Before |
|
18 |
public void setUp() { |
|
19 |
oafRel = OafTest.getDedupRel("ID_1", "ID_2", RelType.resultResult, "isMergedIn"); |
|
20 |
} |
|
21 |
|
|
22 |
@Test |
|
23 |
public void testSetClass() { |
|
24 |
|
|
25 |
OafRelDecoder d1 = OafRelDecoder.decode(oafRel); |
|
26 |
|
|
27 |
assertNotNull(d1); |
|
28 |
assertEquals("isMergedIn", d1.getRelClass()); |
|
29 |
|
|
30 |
OafRelDecoder d2 = OafRelDecoder.decode(d1.setClassId("isMergedIn").build()); |
|
31 |
|
|
32 |
assertEquals("isMergedIn", d2.getRelClass()); |
|
33 |
assertEquals("isMergedIn", d2.getRelMetadata().getSemantics().getClassid()); |
|
34 |
assertEquals("isMergedIn", d2.getRelMetadata().getSemantics().getClassname()); |
|
35 |
|
|
36 |
} |
|
37 |
|
|
38 |
@Test |
|
39 |
public void testGetCF() { |
|
40 |
assertEquals("resultResult_dedup_isMergedIn", OafRelDecoder.getCFQ(RelType.resultResult, SubRelType.dedup, RelName.isMergedIn)); |
|
41 |
assertEquals("resultResult_dedup_isMergedIn", OafRelDecoder.getCFQ(RelType.resultResult, SubRelType.dedup, "isMergedIn")); |
|
42 |
} |
|
43 |
|
|
44 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/bulktag/CommunityConfigurationFactoryTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.bulktag; |
|
2 |
|
|
3 |
import com.google.gson.Gson; |
|
4 |
|
|
5 |
import eu.dnetlib.data.bulktag.selectioncriteria.VerbResolver; |
|
6 |
import eu.dnetlib.data.bulktag.selectioncriteria.VerbResolverFactory; |
|
7 |
import junit.framework.Assert; |
|
8 |
import org.apache.commons.io.IOUtils; |
|
9 |
import org.apache.commons.lang3.StringUtils; |
|
10 |
import org.dom4j.DocumentException; |
|
11 |
import org.junit.Before; |
|
12 |
import org.junit.Ignore; |
|
13 |
import org.junit.Test; |
|
14 |
import java.io.IOException; |
|
15 |
import java.lang.reflect.InvocationTargetException; |
|
16 |
import java.util.*; |
|
17 |
|
|
18 |
import static org.junit.Assert.assertEquals; |
|
19 |
import static org.junit.Assert.assertTrue; |
|
20 |
|
|
21 |
/** |
|
22 |
* Created by miriam on 03/08/2018. |
|
23 |
*/ |
|
24 |
|
|
25 |
|
|
26 |
public class CommunityConfigurationFactoryTest { |
|
27 |
|
|
28 |
private String xml; |
|
29 |
private String xml1; |
|
30 |
|
|
31 |
private final VerbResolver resolver = new VerbResolver(); |
|
32 |
|
|
33 |
@Before |
|
34 |
public void setUp() throws IOException, DocumentException { |
|
35 |
xml = IOUtils.toString(getClass().getResourceAsStream("community_configuration.xml")); |
|
36 |
xml1 = IOUtils.toString(getClass().getResourceAsStream("community_configuration_selcrit.xml")); |
|
37 |
|
|
38 |
} |
|
39 |
|
|
40 |
@Test |
|
41 |
public void parseTest() throws DocumentException { |
|
42 |
|
|
43 |
final CommunityConfiguration cc = CommunityConfigurationFactory.newInstance(xml); |
|
44 |
assertEquals(5,cc.size()); |
|
45 |
cc.getCommunityList().forEach(c -> assertTrue(StringUtils.isNoneBlank(c.getId()))); |
|
46 |
|
|
47 |
|
|
48 |
} |
|
49 |
|
|
50 |
@Test |
|
51 |
public void applyVerb() throws InvocationTargetException, IllegalAccessException, NoSuchMethodException, InstantiationException { |
|
52 |
Constraint sc = new Constraint(); |
|
53 |
String verb = "not_contains"; |
|
54 |
sc.setVerb("not_contains"); |
|
55 |
sc.setField("contributor"); |
|
56 |
sc.setValue("DARIAH"); |
|
57 |
sc.setSelection(resolver.getSelectionCriteria(sc.getVerb(),sc.getValue())); |
|
58 |
String metadata = "This work has been partially supported by DARIAH-EU infrastructure"; |
|
59 |
|
|
60 |
Assert.assertFalse(sc.verifyCriteria(metadata)); |
|
61 |
} |
|
62 |
|
|
63 |
|
|
64 |
@Test |
|
65 |
public void loadSelCriteriaTest() throws DocumentException { |
|
66 |
final CommunityConfiguration cc = CommunityConfigurationFactory.newInstance(xml1); |
|
67 |
Map<String,List<String>> param = new HashMap<>(); |
|
68 |
param.put("author",new ArrayList<>(Collections.singletonList("Pippo Pippi"))); |
|
69 |
param.put("description",new ArrayList<>(Collections.singletonList("This work has been partially supported by DARIAH-EU infrastructure"))); |
|
70 |
param.put("contributor",new ArrayList<>(Collections.singletonList("Pallino ha aiutato a scrivere il paper. Pallino lavora per DARIAH"))); |
|
71 |
List<String> comm = cc.getCommunityForDatasource("openaire____::1cfdb2e14977f31a98e0118283401f32",param); |
|
72 |
Assert.assertEquals(1,comm.size()); |
|
73 |
Assert.assertEquals("dariah",comm.get(0)); |
|
74 |
|
|
75 |
} |
|
76 |
|
|
77 |
@Test |
|
78 |
public void test4() throws DocumentException, IOException { |
|
79 |
final CommunityConfiguration cc = CommunityConfigurationFactory.fromJson(IOUtils.toString(getClass().getResourceAsStream("community_configuration_selcrit.json"))); |
|
80 |
cc.toString(); |
|
81 |
|
|
82 |
|
|
83 |
} |
|
84 |
|
|
85 |
@Test |
|
86 |
public void test5() throws IOException, DocumentException { |
|
87 |
|
|
88 |
final CommunityConfiguration cc = CommunityConfigurationFactory.newInstance(IOUtils.toString(getClass().getResourceAsStream("test.xml"))); |
|
89 |
|
|
90 |
System.out.println(cc.toJson()); |
|
91 |
} |
|
92 |
|
|
93 |
|
|
94 |
@Test |
|
95 |
public void test6() { |
|
96 |
String json = "{\"sc\":{\"cl\":{\"criteria\":[{\"ce\":{\"constraint\":[{\"verb\":\"contains\",\"field\":\"contributor\",\"value\":\"DARIAH\"}]}}]}}}"; |
|
97 |
|
|
98 |
String step1 = "{\"verb\":\"contains\",\"field\":\"contributor\",\"value\":\"DARIAH\"}"; |
|
99 |
|
|
100 |
Constraint c = new Gson().fromJson(step1, Constraint.class); |
|
101 |
|
|
102 |
String step2 = "{\"constraint\":[{\"verb\":\"contains\",\"field\":\"contributor\",\"value\":\"DARIAH\"}]}"; |
|
103 |
|
|
104 |
ConstraintEncapsulator ce = new Gson().fromJson(step2,ConstraintEncapsulator.class); |
|
105 |
|
|
106 |
|
|
107 |
String step3 = "{\"ce\":{\"constraint\":[{\"verb\":\"contains\",\"field\":\"contributor\",\"value\":\"DARIAH\"}]}}"; |
|
108 |
|
|
109 |
Constraints cons = new Gson().fromJson(step3,Constraints.class); |
|
110 |
|
|
111 |
String step4 = "{\"criteria\":[{\"ce\":{\"constraint\":[{\"verb\":\"contains\",\"field\":\"contributor\",\"value\":\"DARIAH\"}]}}]}"; |
|
112 |
|
|
113 |
ConstraintsList cl = new Gson().fromJson(step4,ConstraintsList.class); |
|
114 |
|
|
115 |
String step5 = "{\"cl\":{\"criteria\":[{\"ce\":{\"constraint\":[{\"verb\":\"contains\",\"field\":\"contributor\",\"value\":\"DARIAH\"}]}}]}}"; |
|
116 |
SelectionConstraints sl = new Gson().fromJson(step5,SelectionConstraints.class); |
|
117 |
|
|
118 |
|
|
119 |
} |
|
120 |
|
|
121 |
|
|
122 |
|
|
123 |
|
|
124 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/ParserToProtoIT.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.io.InputStream; |
|
5 |
import java.io.StringReader; |
|
6 |
import java.util.List; |
|
7 |
import java.util.Objects; |
|
8 |
import java.util.Properties; |
|
9 |
import java.util.concurrent.atomic.AtomicInteger; |
|
10 |
import java.util.function.Function; |
|
11 |
import java.util.stream.Collectors; |
|
12 |
import java.util.stream.StreamSupport; |
|
13 |
|
|
14 |
import com.google.protobuf.InvalidProtocolBufferException; |
|
15 |
import com.mongodb.client.MongoCollection; |
|
16 |
import com.mongodb.client.MongoDatabase; |
|
17 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
|
18 |
import eu.dnetlib.data.proto.OafProtos.OafEntity; |
|
19 |
import eu.dnetlib.data.transform.xml2.DatasetToProto; |
|
20 |
import eu.dnetlib.miscutils.collections.Pair; |
|
21 |
import eu.dnetlib.miscutils.datetime.HumanTime; |
|
22 |
import org.apache.commons.io.IOUtils; |
|
23 |
import org.apache.commons.lang3.time.StopWatch; |
|
24 |
import org.apache.commons.logging.Log; |
|
25 |
import org.apache.commons.logging.LogFactory; |
|
26 |
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; |
|
27 |
import org.bson.Document; |
|
28 |
import org.dom4j.DocumentException; |
|
29 |
import org.dom4j.io.SAXReader; |
|
30 |
import org.junit.Before; |
|
31 |
import org.junit.Test; |
|
32 |
import org.junit.runner.RunWith; |
|
33 |
import org.springframework.beans.factory.annotation.Autowired; |
|
34 |
import org.springframework.test.context.ContextConfiguration; |
|
35 |
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
|
36 |
|
|
37 |
import static org.junit.Assert.assertFalse; |
|
38 |
import static org.junit.Assert.assertNotNull; |
|
39 |
import static org.junit.Assert.assertTrue; |
|
40 |
|
|
41 |
@RunWith(SpringJUnit4ClassRunner.class) |
|
42 |
@ContextConfiguration(classes = { ConfigurationTestConfig.class }) |
|
43 |
public class ParserToProtoIT { |
|
44 |
|
|
45 |
private static final Log log = LogFactory.getLog(ParserToProtoIT.class); |
|
46 |
|
|
47 |
private static final String DATACITE = "datacite"; |
|
48 |
private static final String NARCIS = "narcis"; |
|
49 |
|
|
50 |
private static final int BATCH_SIZE = 10000; |
|
51 |
private static final int LOG_FREQ = 5000; |
|
52 |
private static final int LIMIT = 10000; |
|
53 |
public static final String BODY = "body"; |
|
54 |
|
|
55 |
private static String basePathProfiles = "/eu/dnetlib/test/profiles/TransformationRuleDSResources/TransformationRuleDSResourceType/"; |
|
56 |
|
|
57 |
private int batchSize = BATCH_SIZE; |
|
58 |
private int logFreq = LOG_FREQ; |
|
59 |
private int limit = LIMIT; |
|
60 |
|
|
61 |
@Autowired |
|
62 |
private MongoDatabase db; |
|
63 |
|
|
64 |
@Autowired() |
|
65 |
private Properties testProperties; |
|
66 |
|
|
67 |
@Before |
|
68 |
public void setUp() { |
|
69 |
logFreq = Integer.valueOf(testProperties.getProperty("test.logFreq", String.valueOf(LOG_FREQ))); |
|
70 |
batchSize = Integer.valueOf(testProperties.getProperty("test.batchSize", String.valueOf(BATCH_SIZE))); |
|
71 |
limit = Integer.valueOf(testProperties.getProperty("test.limit", String.valueOf(limit))); |
|
72 |
} |
|
73 |
|
|
74 |
@Test |
|
75 |
public void testParseDataciteWithVTD() throws IOException { |
|
76 |
doTest(s -> new Pair<>(s, new DatasetToProto().apply(s)), DATACITE); |
|
77 |
} |
|
78 |
|
|
79 |
@Test |
|
80 |
public void testParseDataciteWithXSLT() throws IOException { |
|
81 |
final String xslt = IOUtils.toString(loadFromTransformationProfile("odf2hbase.xml")); |
|
82 |
final XsltRowTransformer transformer = XsltRowTransformerFactory.newInstance(xslt); |
|
83 |
|
|
84 |
doTest(rowToOaf(transformer), DATACITE); |
|
85 |
} |
|
86 |
|
|
87 |
@Test |
|
88 |
public void testParseNarcisWithXSLT() throws IOException { |
|
89 |
final String xslt = IOUtils.toString(loadFromTransformationProfile("oaf2hbase.xml")); |
|
90 |
final XsltRowTransformer transformer = XsltRowTransformerFactory.newInstance(xslt); |
|
91 |
|
|
92 |
doTest(rowToOaf(transformer), NARCIS); |
|
93 |
} |
|
94 |
|
|
95 |
//// HELPERS |
|
96 |
|
|
97 |
private void doTest(final Function<String, Pair<String, Oaf>> mapper, final String collectionName) { |
|
98 |
final MongoCollection<Document> collection = db.getCollection(collectionName); |
|
99 |
|
|
100 |
final long collectionSize = collection.count(); |
|
101 |
log.info(String.format("found %s records in collection '%s'", collectionSize, collectionName)); |
|
102 |
|
|
103 |
final AtomicInteger read = new AtomicInteger(0); |
|
104 |
final DescriptiveStatistics stats = new DescriptiveStatistics(); |
|
105 |
|
|
106 |
final StopWatch recordTimer = new StopWatch(); |
|
107 |
final StopWatch totalTimer = StopWatch.createStarted(); |
|
108 |
|
|
109 |
StreamSupport.stream(collection.find().batchSize(batchSize).spliterator(), false) |
|
110 |
.limit(limit) |
|
111 |
.peek(d -> { |
|
112 |
if (read.addAndGet(1) % logFreq == 0) { |
|
113 |
log.info(String.format("records read so far %s", read.get())); |
|
114 |
//log.info(String.format("stats so far %s", stats.toString())); |
|
115 |
} |
|
116 |
}) |
|
117 |
.map(d -> (String) d.get("body")) |
|
118 |
.filter(Objects::nonNull) |
|
119 |
.collect(Collectors.toList()) // load them in memory first |
|
120 |
.stream() |
|
121 |
.peek(s -> recordTimer.start()) |
|
122 |
.map(mapper) |
|
123 |
.forEach(pair -> { |
|
124 |
recordTimer.stop(); |
|
125 |
stats.addValue(recordTimer.getTime()); |
|
126 |
recordTimer.reset(); |
|
127 |
|
|
128 |
assertNotNull(pair); |
|
129 |
assertTrue(pair.getValue().hasEntity()); |
|
130 |
|
|
131 |
try { |
|
132 |
final org.dom4j.Document doc = new SAXReader().read(new StringReader(pair.getKey())); |
|
133 |
final OafEntity entity = pair.getValue().getEntity(); |
|
134 |
|
|
135 |
//TODO add more asserts |
|
136 |
assertTrue(entity.getId().contains(doc.valueOf("/*[local-name() = 'record']/*[local-name() = 'header']/*[local-name() = 'objIdentifier']/text()"))); |
|
137 |
|
|
138 |
} catch (DocumentException e) { |
|
139 |
throw new IllegalArgumentException("unable to parse record " + pair.getKey(), e); |
|
140 |
} |
|
141 |
}); |
|
142 |
|
|
143 |
totalTimer.stop(); |
|
144 |
log.info(String.format("processed %s/%s records in %s", read.get(), collectionSize, HumanTime.exactly(totalTimer.getTime()))); |
|
145 |
log.info(stats.toString()); |
|
146 |
} |
|
147 |
|
|
148 |
private Function<String, Pair<String, Oaf>> rowToOaf(final XsltRowTransformer transformer) { |
|
149 |
return xml -> { |
|
150 |
final List<Row> rows = transformer.apply(xml); |
|
151 |
if (rows.isEmpty()) { |
|
152 |
return null; |
|
153 |
} |
|
154 |
|
|
155 |
return rows.stream() |
|
156 |
.filter(row -> row.getColumn(BODY) != null) |
|
157 |
.map(row -> row.getColumn(BODY)) |
|
158 |
.map(c -> c.getValue()) |
|
159 |
.map(b -> { |
|
160 |
try { |
|
161 |
return Oaf.parseFrom(b); |
|
162 |
} catch (InvalidProtocolBufferException e) { |
|
163 |
throw new IllegalStateException(e); |
|
164 |
} |
|
165 |
}) |
|
166 |
.filter(Objects::nonNull) |
|
167 |
.map(oaf -> new Pair<>(xml, oaf)) |
|
168 |
.findFirst() |
|
169 |
.get(); |
|
170 |
}; |
|
171 |
} |
|
172 |
|
|
173 |
private InputStream loadFromTransformationProfile(final String profilePath) { |
|
174 |
log.info("Loading xslt from: " + basePathProfiles + profilePath); |
|
175 |
InputStream profile = getClass().getResourceAsStream(basePathProfiles + profilePath); |
|
176 |
SAXReader saxReader = new SAXReader(); |
|
177 |
org.dom4j.Document doc = null; |
|
178 |
try { |
|
179 |
doc = saxReader.read(profile); |
|
180 |
} catch (DocumentException e) { |
|
181 |
e.printStackTrace(); |
|
182 |
throw new RuntimeException(e); |
|
183 |
} |
|
184 |
String xslt = doc.selectSingleNode("//SCRIPT/CODE/*[local-name()='stylesheet']").asXML(); |
|
185 |
//log.info(xslt); |
|
186 |
return IOUtils.toInputStream(xslt); |
|
187 |
} |
|
188 |
|
|
189 |
|
|
190 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/xml/OpenTrialsXsltFunctionsTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform.xml; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
|
|
5 |
import eu.dnetlib.data.transform.xml.OpenTrialsXsltFunctions.JsonProv; |
|
6 |
import org.junit.After; |
|
7 |
import org.junit.Before; |
|
8 |
import org.junit.Test; |
|
9 |
|
|
10 |
import static org.junit.Assert.assertEquals; |
|
11 |
|
|
12 |
|
|
13 |
/** |
|
14 |
* OpenTrialsXsltFunctions Tester. |
|
15 |
* |
|
16 |
*/ |
|
17 |
public class OpenTrialsXsltFunctionsTest { |
|
18 |
|
|
19 |
private String jsonProv = "[{\"url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT00378508\", \"sourceID\" : \"77eb42c5-0ec7-4e31-963a-5736b66f2d26\", \"sourceName\" : \"ictrp\"},{\"url\" : \"https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true\", \"sourceID\" : \"b389497c-0833-432b-a09b-930526b7b4d4\", \"sourceName\" : \"nct\"}]"; |
|
20 |
private String jsonProvWithNull = "[{\"url\" : \"\", \"sourceID\" : \"77eb42c5-0ec7-4e31-963a-5736b66f2d26\", \"sourceName\" : \"ictrp\"},{\"url\" : \"https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true\", \"sourceID\" : \"b389497c-0833-432b-a09b-930526b7b4d4\", \"sourceName\" : \"nct\"}]"; |
|
21 |
private String jidentifiers = "{112683,NCT00920439}"; |
|
22 |
|
|
23 |
|
|
24 |
private String jsonRecord = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\", \"is_primary\" : true},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]"; |
|
25 |
private String jsonRecordNull = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\"},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]"; |
|
26 |
private String jsonRecordVoid = "[{\"source_id\" : \"\", \"source_url\" : \"\", \"is_primary\" : \"\"}]"; |
|
27 |
private String jsonRecondPrimary = "[{\"source_id\" : \"nct\", \"source_url\" : \"https://clinicaltrials.gov/ct2/show/NCT02321059\", \"is_primary\" : false},{\"source_id\" : \"ictrp\", \"source_url\" : \"http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059\", \"is_primary\" : false}]"; |
|
28 |
|
|
29 |
private String jsonPeopleVoid ="[{\"person_name\" : null, \"person_id\" : null, \"person_role\" : null}]"; |
|
30 |
private String jsonPeopleOne = "[{\"person_name\" : \"Henk Verheul, M.D., PhD\", \"person_id\" : \"116438e9-f8b1-46e5-a1f8-20f851cab73c\", \"person_role\" : \"principal_investigator\"}]"; |
|
31 |
private String jsonPeopleMore = "[{\"person_name\" : \"Henk Verheul, M.D., PhD\", \"person_id\" : \"116438e9-f8b1-46e5-a1f8-20f851cab73c\", \"person_role\" : \"principal_investigator\"},{\"person_name\" : \"Miriam Pippolippo Baglioni, PhD\", \"person_id\" : \"fake\", \"person_role\" : \"principal_investigator\"}]"; |
|
32 |
|
|
33 |
private String jsonOrganizationVoid = "[{\"organization_name\" : null, \"organization_id\" : null, \"organization_role\" : null}]"; |
|
34 |
private String jsonOrganizationOne = "[{\"organization_name\" : \"Södertälje sjukhus AB\", \"organization_id\" : \"15f0d004-b82b-408c-8605-38a57352468d\", \"organization_role\" : \"sponsor\"}]"; |
|
35 |
private String jsonOrganizationMore = "[{\"organization_name\" : \"Södertälje sjukhus AB\", \"organization_id\" : \"15f0d004-b82b-408c-8605-38a57352468d\", \"organization_role\" : \"sponsor\"},{\"organization_name\" : \"Miriam Baglioni AB\", \"organization_id\" : \"fake\", \"organization_role\" : \"primary_sponsor\"}]"; |
|
36 |
|
|
37 |
private String jsonLocationVoid = "[{\"location_name\" : null}]"; |
|
38 |
private String jsonLocationOne = "[{\"location_name\" : \"China\"}]"; |
|
39 |
private String jsonLocationMore = "[{\"location_name\" : \"China\"},{\"location_name\" : \"North Korea\"}]"; |
|
40 |
|
|
41 |
@Before |
|
42 |
public void before() throws Exception { |
|
43 |
} |
|
44 |
|
|
45 |
@After |
|
46 |
public void after() throws Exception { |
|
47 |
} |
|
48 |
|
|
49 |
/** |
|
50 |
* Method: getProvs(String jsonProvList) |
|
51 |
*/ |
|
52 |
@Test |
|
53 |
public void testGetProvs() throws Exception { |
|
54 |
List<JsonProv> list = OpenTrialsXsltFunctions.getProvs(jsonProv); |
|
55 |
assertEquals(2, list.size()); |
|
56 |
} |
|
57 |
|
|
58 |
/** |
|
59 |
* Method: getMainIdentifierURL(String jsonProvList) |
|
60 |
*/ |
|
61 |
@Test |
|
62 |
public void testGetMainIdentifierURL() throws Exception { |
|
63 |
String url = OpenTrialsXsltFunctions.getMainIdentifierURL(jsonProv); |
|
64 |
assertEquals( "http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT00378508", url ); |
|
65 |
url = OpenTrialsXsltFunctions.getMainIdentifierURL(jsonProvWithNull); |
|
66 |
assertEquals("https://www.clinicaltrials.gov/ct2/show/NCT00378508?lup_e=02%2F04%2F2016&lup_s=01%2F01%2F2001&rank=175075&resultsxml=true", url); |
|
67 |
} |
|
68 |
|
|
69 |
@Test |
|
70 |
public void testGetPrimaryRecordUrl(){ |
|
71 |
String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecord); |
|
72 |
assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url); |
|
73 |
} |
|
74 |
|
|
75 |
@Test |
|
76 |
public void testGetPrimaryRecordID(){ |
|
77 |
String id = OpenTrialsXsltFunctions.getPrimaryRecordIdentifier(jsonRecord); |
|
78 |
assertEquals("nct", id); |
|
79 |
} |
|
80 |
|
|
81 |
@Test |
|
82 |
public void testGetPrimaryRecordUrlNull(){ |
|
83 |
String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecordNull); |
|
84 |
assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url); |
|
85 |
} |
|
86 |
|
|
87 |
@Test |
|
88 |
public void testGetPrimaryRecordUrlVoid(){ |
|
89 |
String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecordVoid); |
|
90 |
assertEquals("", url); |
|
91 |
} |
|
92 |
|
|
93 |
@Test |
|
94 |
public void testGetPrimaryRecordUrlNoPrimary(){ |
|
95 |
String url = OpenTrialsXsltFunctions.getPrimaryRecordUrl(jsonRecondPrimary); |
|
96 |
assertEquals("https://clinicaltrials.gov/ct2/show/NCT02321059", url); |
|
97 |
} |
|
98 |
@Test |
|
99 |
public void testGetPrimaryRecordIDNoPrimary(){ |
|
100 |
String id = OpenTrialsXsltFunctions.getPrimaryRecordIdentifier(jsonRecondPrimary); |
|
101 |
assertEquals("nct", id); |
|
102 |
} |
|
103 |
@Test |
|
104 |
public void testGetPrincipalInvestigatorsVoid(){ |
|
105 |
String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleVoid); |
|
106 |
assertEquals("",url); |
|
107 |
} |
|
108 |
|
|
109 |
|
|
110 |
@Test |
|
111 |
public void testGetPrincipalInvestigatorsOne(){ |
|
112 |
String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleOne); |
|
113 |
assertEquals("Verheul, Henk", url); |
|
114 |
} |
|
115 |
|
|
116 |
@Test |
|
117 |
public void testGetPrincipalInvestigatorsMore(){ |
|
118 |
String url = OpenTrialsXsltFunctions.getPrincipalInvestigators(jsonPeopleMore); |
|
119 |
assertEquals("Verheul, Henk@@Baglioni, Miriam Pippolippo", url); |
|
120 |
} |
|
121 |
|
|
122 |
|
|
123 |
|
|
124 |
@Test |
|
125 |
public void testgGetTrialOrganizationsVoid(){ |
|
126 |
String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationVoid); |
|
127 |
assertEquals("",url); |
|
128 |
} |
|
129 |
|
|
130 |
|
|
131 |
@Test |
|
132 |
public void testgGetTrialOrganizationsOne(){ |
|
133 |
String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationOne); |
|
134 |
assertEquals("Södertälje sjukhus AB@sponsor", url); |
|
135 |
} |
|
136 |
|
|
137 |
@Test |
|
138 |
public void testgGetTrialOrganizationsMore(){ |
|
139 |
String url = OpenTrialsXsltFunctions.getTrialOrganizations(jsonOrganizationMore); |
|
140 |
assertEquals("Södertälje sjukhus AB@sponsor@@Miriam Baglioni AB@sponsor", url); |
|
141 |
} |
|
142 |
|
|
143 |
@Test |
|
144 |
public void testgGetTrialLocationsVoid(){ |
|
145 |
String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationVoid); |
|
146 |
assertEquals("",url); |
|
147 |
} |
|
148 |
|
|
149 |
|
|
150 |
@Test |
|
151 |
public void testgGetTrialLocationsOne(){ |
|
152 |
String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationOne); |
|
153 |
assertEquals("China", url); |
|
154 |
} |
|
155 |
|
|
156 |
@Test |
|
157 |
public void testgGetTrialLocationsMore(){ |
|
158 |
String url = OpenTrialsXsltFunctions.getTrialLocations(jsonLocationMore); |
|
159 |
assertEquals("China@@North Korea", url); |
|
160 |
} |
|
161 |
|
|
162 |
@Test |
|
163 |
public void testGetNotPrimaryRecordUrlPrimary(){ |
|
164 |
String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecondPrimary); |
|
165 |
assertEquals("http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059", url); |
|
166 |
} |
|
167 |
|
|
168 |
@Test |
|
169 |
public void testGetNotPrimaryRecordUrlVoid(){ |
|
170 |
String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecordVoid); |
|
171 |
assertEquals("", url); |
|
172 |
} |
|
173 |
|
|
174 |
@Test |
|
175 |
public void testGetNotPrimaryRecordUrl(){ |
|
176 |
String url = OpenTrialsXsltFunctions.getNotPrimaryRecordUrl(jsonRecord); |
|
177 |
assertEquals("http://apps.who.int/trialsearch/Trial3.aspx?trialid=NCT02321059", url); |
|
178 |
} |
|
179 |
|
|
180 |
|
|
181 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/xml/FWFXsltFunctionsTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform.xml; |
|
2 |
|
|
3 |
import org.junit.After; |
|
4 |
import org.junit.Before; |
|
5 |
import org.junit.Test; |
|
6 |
|
|
7 |
import static org.junit.Assert.assertEquals; |
|
8 |
import static org.junit.Assert.assertNotNull; |
|
9 |
|
|
10 |
/** |
|
11 |
* |
|
12 |
* Created by miriam on 04/05/2017. |
|
13 |
*/ |
|
14 |
public class FWFXsltFunctionsTest { |
|
15 |
private String namesurname ="Gerhard SOMMER"; |
|
16 |
private String noSurname = "Gerhard"; |
|
17 |
private String noName = "SOMMER"; |
|
18 |
private String twoNames = "Gerhard Pippo SOMMER"; |
|
19 |
private String twoSurname = "Gerhard PIPPO SOMMER"; |
|
20 |
private String nonamesurname = ""; |
|
21 |
private String organization ="Universität Linz - Institut für Computational Perception; Universität für Musik und darstellende Kunst Graz - Institut 1: Komposition, Musiktheorie, Musikgeschichte und Dirigieren; Universität Mozarteum Salzburg - Institut für Musikalische Rezeptions- und Interpretationsgeschichte; Anton Bruckner Privatuniversität - Institut für Theorie und Geschichte der Musik; Eliette und Herbert von Karajan Institut - Eliette und Herbert von Karajan Institut"; |
|
22 |
|
|
23 |
@Before |
|
24 |
public void before() throws Exception { |
|
25 |
} |
|
26 |
|
|
27 |
@After |
|
28 |
public void after() throws Exception { |
|
29 |
} |
|
30 |
|
|
31 |
@Test |
|
32 |
public void testGetNamesNameNoNameSurname() throws Exception { |
|
33 |
String ret = FWFXsltFunctions.getName(nonamesurname,true); |
|
34 |
assertEquals("",ret ); |
|
35 |
} |
|
36 |
|
|
37 |
@Test |
|
38 |
public void testGetNamesSurnameNoNameSurname() throws Exception { |
|
39 |
String ret = FWFXsltFunctions.getName(nonamesurname,false); |
|
40 |
assertEquals("",ret ); |
|
41 |
} |
|
42 |
|
|
43 |
@Test |
|
44 |
public void testGetNamesNameTwoSurname() throws Exception { |
|
45 |
String ret = FWFXsltFunctions.getName(twoSurname,true); |
|
46 |
assertEquals("Gerhard",ret ); |
|
47 |
} |
|
48 |
|
|
49 |
@Test |
|
50 |
public void testGetNamesSurnameTwoSurname() throws Exception { |
|
51 |
String ret = FWFXsltFunctions.getName(twoSurname,false); |
|
52 |
assertEquals("PIPPO SOMMER",ret ); |
|
53 |
} |
|
54 |
|
|
55 |
@Test |
|
56 |
public void testGetNamesNameTwoNames() throws Exception { |
|
57 |
String ret = FWFXsltFunctions.getName(twoNames,true); |
|
58 |
assertEquals("Gerhard Pippo",ret ); |
|
59 |
} |
|
60 |
|
|
61 |
@Test |
|
62 |
public void testGetNamesSurnameTwoNames() throws Exception { |
|
63 |
String ret = FWFXsltFunctions.getName(twoNames,false); |
|
64 |
assertEquals("SOMMER",ret ); |
|
65 |
} |
|
66 |
|
|
67 |
/** |
|
68 |
* Method: getProvs(String jsonProvList) |
|
69 |
*/ |
|
70 |
@Test |
|
71 |
public void testGetNamesName() throws Exception { |
|
72 |
String ret = FWFXsltFunctions.getName(namesurname,true); |
|
73 |
assertEquals("Gerhard",ret ); |
|
74 |
} |
|
75 |
|
|
76 |
@Test |
|
77 |
public void testGetNamesSurname() throws Exception { |
|
78 |
String ret = FWFXsltFunctions.getName(namesurname,false); |
|
79 |
assertEquals("SOMMER",ret ); |
|
80 |
} |
|
81 |
|
|
82 |
@Test |
|
83 |
public void testGetNamesNameNoSurname() throws Exception { |
|
84 |
String ret = FWFXsltFunctions.getName(noSurname,true); |
|
85 |
assertEquals("Gerhard",ret ); |
|
86 |
} |
|
87 |
|
|
88 |
@Test |
|
89 |
public void testGetNamesSurnameNoSurname() throws Exception { |
|
90 |
String ret = FWFXsltFunctions.getName(noSurname,false); |
|
91 |
assertEquals("",ret ); |
|
92 |
} |
|
93 |
|
|
94 |
@Test |
|
95 |
public void testGetNamesNameNoName() throws Exception { |
|
96 |
String ret = FWFXsltFunctions.getName(noName,true); |
|
97 |
assertEquals("",ret ); |
|
98 |
} |
|
99 |
|
|
100 |
@Test |
|
101 |
public void testGetNamesSurnameNoName() throws Exception { |
|
102 |
String ret = FWFXsltFunctions.getName(noName,false); |
|
103 |
assertEquals("SOMMER",ret ); |
|
104 |
} |
|
105 |
|
|
106 |
@Test |
|
107 |
public void TestGetMd5()throws Exception{ |
|
108 |
String md5 = FWFXsltFunctions.getMd5(organization); |
|
109 |
System.out.println(md5); |
|
110 |
assertNotNull(md5); |
|
111 |
} |
|
112 |
|
|
113 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/xml/vtd/VtdParserToProtoTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform.xml.vtd; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.util.function.Function; |
|
5 |
|
|
6 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
|
7 |
import eu.dnetlib.data.transform.xml2.DatasetToProto; |
|
8 |
import eu.dnetlib.data.transform.xml2.PublicationToProto; |
|
9 |
import org.apache.commons.io.IOUtils; |
|
10 |
import org.apache.commons.logging.Log; |
|
11 |
import org.apache.commons.logging.LogFactory; |
|
12 |
import org.junit.Test; |
|
13 |
|
|
14 |
import static org.junit.Assert.assertNotNull; |
|
15 |
|
|
16 |
public class VtdParserToProtoTest { |
|
17 |
|
|
18 |
private static final Log log = LogFactory.getLog(VtdParserToProtoTest.class); |
|
19 |
|
|
20 |
@Test |
|
21 |
public void testParsePublication() throws IOException { |
|
22 |
doTest("/eu/dnetlib/data/transform/publication.xml", new PublicationToProto()); |
|
23 |
} |
|
24 |
|
|
25 |
@Test |
|
26 |
public void testParseDataset() throws IOException { |
|
27 |
doTest("/eu/dnetlib/data/transform/dataset.xml", new DatasetToProto()); |
|
28 |
} |
|
29 |
|
|
30 |
@Test |
|
31 |
public void testParseDataset2() throws IOException { |
|
32 |
doTest("/eu/dnetlib/data/transform/dataset2.xml", new DatasetToProto()); |
|
33 |
} |
|
34 |
|
|
35 |
private void doTest(final String filePath, Function<String, Oaf> f) throws IOException { |
|
36 |
final String xml = IOUtils.toString(getClass().getResourceAsStream(filePath)); |
|
37 |
|
|
38 |
assertNotNull(xml); |
|
39 |
|
|
40 |
final Oaf oaf = f.apply(xml); |
|
41 |
|
|
42 |
assertNotNull(oaf); |
|
43 |
|
|
44 |
log.info(oaf.toString()); |
|
45 |
} |
|
46 |
|
|
47 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/xml/vtd/ConfigurationTestConfig.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform.xml.vtd; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.io.InputStream; |
|
5 |
import java.util.Properties; |
|
6 |
|
|
7 |
import com.mongodb.MongoClient; |
|
8 |
import com.mongodb.client.MongoDatabase; |
|
9 |
import org.springframework.context.annotation.Bean; |
|
10 |
import org.springframework.context.annotation.Configuration; |
|
11 |
import org.springframework.core.io.ClassPathResource; |
|
12 |
|
|
13 |
@Configuration |
|
14 |
public class ConfigurationTestConfig { |
|
15 |
|
|
16 |
@Bean |
|
17 |
public MongoDatabase db() throws IOException { |
|
18 |
|
|
19 |
final Properties p = new Properties(); |
|
20 |
final ClassPathResource cp = new ClassPathResource("test.properties"); |
|
21 |
try (final InputStream stream = cp.getInputStream()) { |
|
22 |
p.load(stream); |
|
23 |
} |
|
24 |
|
|
25 |
final MongoClient mongo = new MongoClient( |
|
26 |
p.getProperty("mongodb.host"), |
|
27 |
Integer.valueOf(p.getProperty("mongodb.port"))); |
|
28 |
return mongo.getDatabase(p.getProperty("mongodb.dbname")); |
|
29 |
} |
|
30 |
|
|
31 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/xml/vtd/VtdParserToProtoIT.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform.xml.vtd; |
|
2 |
|
|
3 |
import com.mongodb.client.MongoCollection; |
|
4 |
import com.mongodb.client.MongoDatabase; |
|
5 |
import eu.dnetlib.data.transform.xml2.DatasetToProto; |
|
6 |
import org.apache.commons.lang3.time.StopWatch; |
|
7 |
import org.apache.commons.logging.Log; |
|
8 |
import org.apache.commons.logging.LogFactory; |
|
9 |
import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; |
|
10 |
import org.bson.Document; |
|
11 |
import org.junit.Test; |
|
12 |
import org.junit.runner.RunWith; |
|
13 |
import org.springframework.beans.factory.annotation.Autowired; |
|
14 |
import org.springframework.test.context.ContextConfiguration; |
|
15 |
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; |
|
16 |
|
|
17 |
import java.io.IOException; |
|
18 |
import java.util.Objects; |
|
19 |
import java.util.concurrent.atomic.AtomicInteger; |
|
20 |
import java.util.stream.StreamSupport; |
|
21 |
|
|
22 |
import static org.junit.Assert.assertNotNull; |
|
23 |
import static org.junit.Assert.assertTrue; |
|
24 |
|
|
25 |
@RunWith(SpringJUnit4ClassRunner.class) |
|
26 |
@ContextConfiguration(classes = { ConfigurationTestConfig.class }) |
|
27 |
public class VtdParserToProtoIT { |
|
28 |
|
|
29 |
private static final Log log = LogFactory.getLog(VtdParserToProtoIT.class); |
|
30 |
public static final String COLLECTION_NAME = "datacite"; |
|
31 |
private static final int BATCH_SIZE = 10000; |
|
32 |
public static final int LOG_FREQ = 5000; |
|
33 |
|
|
34 |
@Autowired |
|
35 |
private MongoDatabase db; |
|
36 |
|
|
37 |
@Test |
|
38 |
public void testParseDatacite() throws IOException { |
|
39 |
|
|
40 |
final MongoCollection<Document> collection = db.getCollection(COLLECTION_NAME); |
|
41 |
|
|
42 |
final long collectionSize = collection.count(); |
|
43 |
log.info(String.format("found %s records in collection '%s'", collectionSize, COLLECTION_NAME)); |
|
44 |
|
|
45 |
final AtomicInteger read = new AtomicInteger(0); |
|
46 |
final DescriptiveStatistics stats = new DescriptiveStatistics(); |
|
47 |
|
|
48 |
final StopWatch timer = new StopWatch(); |
|
49 |
|
|
50 |
final DatasetToProto mapper = new DatasetToProto(); |
|
51 |
StreamSupport.stream(collection.find().batchSize(BATCH_SIZE).spliterator(), false) |
|
52 |
.peek(d -> { |
|
53 |
if (read.addAndGet(1) % LOG_FREQ == 0) { |
|
54 |
log.info(String.format("records read so far %s", read.get())); |
|
55 |
log.info(String.format("stats so far %s", stats.toString())); |
|
56 |
} |
|
57 |
}) |
|
58 |
.map(d -> (String) d.get("body")) |
|
59 |
.filter(Objects::nonNull) |
|
60 |
.peek(s -> timer.start()) |
|
61 |
.map(mapper) |
|
62 |
.forEach(oaf -> { |
|
63 |
assertNotNull(oaf); |
|
64 |
assertTrue(oaf.hasEntity()); |
|
65 |
|
|
66 |
timer.stop(); |
|
67 |
stats.addValue(timer.getTime()); |
|
68 |
timer.reset(); |
|
69 |
}); |
|
70 |
|
|
71 |
log.info(String.format("processed %s/%s records", read.get(), collectionSize)); |
|
72 |
} |
|
73 |
|
|
74 |
|
|
75 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/xml/vtd/VtdUtilityParserTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform.xml.vtd; |
|
2 |
|
|
3 |
import java.io.InputStream; |
|
4 |
import java.util.List; |
|
5 |
|
|
6 |
import com.ximpleware.AutoPilot; |
|
7 |
import com.ximpleware.VTDGen; |
|
8 |
import com.ximpleware.VTDNav; |
|
9 |
import eu.dnetlib.data.transform.xml2.Node; |
|
10 |
import eu.dnetlib.data.transform.xml2.VtdUtilityParser; |
|
11 |
import org.apache.commons.io.IOUtils; |
|
12 |
import org.junit.Assert; |
|
13 |
import org.junit.Test; |
|
14 |
|
|
15 |
|
|
16 |
public class VtdUtilityParserTest { |
|
17 |
|
|
18 |
@Test |
|
19 |
public void testUtils1() { |
|
20 |
String xpath = VtdUtilityParser.xpath("a", "b", "c"); |
|
21 |
Assert.assertTrue("/*[local-name()='a']/*[local-name()='b']/*[local-name()='c']".equals(xpath)); |
|
22 |
} |
|
23 |
|
|
24 |
@Test |
|
25 |
public void testPartser() throws Exception { |
|
26 |
final InputStream resource = this.getClass().getResourceAsStream("/eu/dnetlib/data/transform/publication.xml"); |
|
27 |
final String record =IOUtils.toString(resource); |
|
28 |
final VTDGen vg = VtdUtilityParser.parseXml(record); |
|
29 |
final VTDNav vn = vg.getNav(); |
|
30 |
final AutoPilot ap = new AutoPilot(vn); |
|
31 |
|
|
32 |
List<Node> nodes = VtdUtilityParser.getNodes(ap, vn, "//*[local-name()='referenceaa']"); |
|
33 |
|
|
34 |
nodes.forEach(n -> Assert.assertTrue(n.getAttributes().keySet().size()>0)); |
|
35 |
|
|
36 |
System.out.println(VtdUtilityParser.countNodes(ap, vn, "count(//*[local-name()='CobjIdentifier'])")); |
|
37 |
|
|
38 |
|
|
39 |
|
|
40 |
|
|
41 |
} |
|
42 |
|
|
43 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/xml2/VtdParserToProtoTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform.xml2; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.util.function.Function; |
|
5 |
|
|
6 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
|
7 |
import org.apache.commons.io.IOUtils; |
|
8 |
import org.apache.commons.logging.Log; |
|
9 |
import org.apache.commons.logging.LogFactory; |
|
10 |
import org.junit.Test; |
|
11 |
|
|
12 |
import static org.junit.Assert.assertNotNull; |
|
13 |
|
|
14 |
public class VtdParserToProtoTest { |
|
15 |
|
|
16 |
private static final Log log = LogFactory.getLog(VtdParserToProtoTest.class); |
|
17 |
|
|
18 |
@Test |
|
19 |
public void testParsePublication() throws IOException { |
|
20 |
doTest("/eu/dnetlib/data/transform/publication.xml", new PublicationToProto()); |
|
21 |
} |
|
22 |
|
|
23 |
@Test |
|
24 |
public void testParseDataset() throws IOException { |
|
25 |
doTest("/eu/dnetlib/data/transform/dataset.xml", new DatasetToProto()); |
|
26 |
} |
|
27 |
|
|
28 |
@Test |
|
29 |
public void testParseDataset2() throws IOException { |
|
30 |
doTest("/eu/dnetlib/data/transform/dataset2.xml", new DatasetToProto()); |
|
31 |
} |
|
32 |
|
|
33 |
private void doTest(final String filePath, Function<String, Oaf> f) throws IOException { |
|
34 |
final String xml = IOUtils.toString(getClass().getResourceAsStream(filePath)); |
|
35 |
|
|
36 |
assertNotNull(xml); |
|
37 |
|
|
38 |
final Oaf oaf = f.apply(xml); |
|
39 |
|
|
40 |
assertNotNull(oaf); |
|
41 |
|
|
42 |
log.info(oaf.toString()); |
|
43 |
} |
|
44 |
|
|
45 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/xml2/VtdUtilityParserTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform.xml2; |
|
2 |
|
|
3 |
import java.io.InputStream; |
|
4 |
import java.util.List; |
|
5 |
|
|
6 |
import com.ximpleware.AutoPilot; |
|
7 |
import com.ximpleware.VTDGen; |
|
8 |
import com.ximpleware.VTDNav; |
|
9 |
import eu.dnetlib.data.transform.xml2.Node; |
|
10 |
import eu.dnetlib.data.transform.xml2.VtdUtilityParser; |
|
11 |
import org.apache.commons.io.IOUtils; |
|
12 |
import org.junit.Assert; |
|
13 |
import org.junit.Test; |
|
14 |
|
|
15 |
import static eu.dnetlib.data.transform.xml2.VtdUtilityParser.parseXml; |
|
16 |
|
|
17 |
public class VtdUtilityParserTest { |
|
18 |
|
|
19 |
@Test |
|
20 |
public void testUtils1() { |
|
21 |
String xpath = VtdUtilityParser.xpath("a", "b", "c"); |
|
22 |
Assert.assertTrue("/*[local-name()='a']/*[local-name()='b']/*[local-name()='c']".equals(xpath)); |
|
23 |
} |
|
24 |
|
|
25 |
@Test |
|
26 |
public void testPartser() throws Exception { |
|
27 |
final InputStream resource = this.getClass().getResourceAsStream("/eu/dnetlib/data/transform/publication.xml"); |
|
28 |
final String record =IOUtils.toString(resource); |
|
29 |
final VTDGen vg = parseXml(record); |
|
30 |
final VTDNav vn = vg.getNav(); |
|
31 |
final AutoPilot ap = new AutoPilot(vn); |
|
32 |
|
|
33 |
List<Node> nodes = VtdUtilityParser.getNodes(ap, vn, "//*[local-name()='referenceaa']"); |
|
34 |
|
|
35 |
nodes.forEach(n -> Assert.assertTrue(n.getAttributes().keySet().size()>0)); |
|
36 |
|
|
37 |
System.out.println(VtdUtilityParser.countNodes(ap, vn, "count(//*[local-name()='CobjIdentifier'])")); |
|
38 |
|
|
39 |
|
|
40 |
|
|
41 |
|
|
42 |
} |
|
43 |
|
|
44 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/SolrProtoMapperTest.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform; |
|
2 |
|
|
3 |
import static org.junit.Assert.assertFalse; |
|
4 |
import static org.junit.Assert.assertNotNull; |
|
5 |
|
|
6 |
import java.io.IOException; |
|
7 |
import java.io.StringWriter; |
|
8 |
|
|
9 |
import org.apache.commons.codec.binary.Base64; |
|
10 |
import org.apache.commons.io.IOUtils; |
|
11 |
import org.apache.commons.logging.Log; |
|
12 |
import org.apache.commons.logging.LogFactory; |
|
13 |
import org.apache.solr.common.SolrInputDocument; |
|
14 |
import org.apache.solr.common.SolrInputField; |
|
15 |
import org.dom4j.DocumentException; |
|
16 |
import org.junit.Before; |
|
17 |
import org.junit.Test; |
|
18 |
|
|
19 |
import com.google.protobuf.InvalidProtocolBufferException; |
|
20 |
import com.googlecode.protobuf.format.JsonFormat; |
|
21 |
|
|
22 |
import eu.dnetlib.data.mapreduce.util.OafTest; |
|
23 |
import eu.dnetlib.data.proto.KindProtos.Kind; |
|
24 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
|
25 |
import eu.dnetlib.data.proto.OafProtos.OafEntity; |
|
26 |
import eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory; |
|
27 |
|
|
28 |
public class SolrProtoMapperTest { |
|
29 |
|
|
30 |
private static final Log log = LogFactory.getLog(SolrProtoMapper.class); // NOPMD by marko on 11/24/08 5:02 PM |
|
31 |
private String fields; |
|
32 |
|
|
33 |
@Before |
|
34 |
public void setUp() throws IOException { |
|
35 |
final StringWriter sw = new StringWriter(); |
|
36 |
IOUtils.copy(getClass().getResourceAsStream("fields.xml"), sw); |
|
37 |
fields = sw.toString(); |
|
38 |
assertNotNull(fields); |
|
39 |
assertFalse(fields.isEmpty()); |
|
40 |
|
|
41 |
log.info(fields); |
|
42 |
} |
|
43 |
|
|
44 |
@Test |
|
45 |
public void testProto2SolrDocument() throws DocumentException, InvalidProtocolBufferException { |
|
46 |
final SolrProtoMapper mapper = new SolrProtoMapper(fields); |
|
47 |
|
|
48 |
assertNotNull(mapper); |
|
49 |
|
|
50 |
final OafEntity.Builder entity = OafTest.getResultBuilder("01"); |
|
51 |
entity.addChildren(OafTest.getResultBuilder("01_children")); |
|
52 |
|
|
53 |
final Oaf oaf = OafTest.embed(entity.build(), Kind.entity).getOaf(); |
|
54 |
|
|
55 |
assertNotNull(oaf.getEntity().getChildrenList()); |
|
56 |
assertFalse(oaf.getEntity().getChildrenList().isEmpty()); |
|
57 |
|
|
58 |
log.info("byte[] size: " + oaf.toByteArray().length); |
|
59 |
|
|
60 |
log.info("json size: " + JsonFormat.printToString(oaf).length()); |
|
61 |
|
|
62 |
log.info("base64 size: " + Base64.encodeBase64String(oaf.toByteArray()).length()); |
|
63 |
|
|
64 |
final byte[] decodeBase64 = Base64.decodeBase64(Base64.encodeBase64String(oaf.toByteArray())); |
|
65 |
|
|
66 |
log.info("decoded: " + JsonFormat.printToString(Oaf.parseFrom(decodeBase64))); |
|
67 |
|
|
68 |
final SolrInputDocument doc = mapper.map(oaf, InputDocumentFactory.getParsedDateField("2015-02-15"), "asd", "action-set"); |
|
69 |
|
|
70 |
assertNotNull(doc); |
|
71 |
|
|
72 |
for (final SolrInputField f : doc.values()) { |
|
73 |
log.info(f); |
|
74 |
} |
|
75 |
} |
|
76 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/java/eu/dnetlib/data/transform/ConfigurationTestConfig.java | ||
---|---|---|
1 |
package eu.dnetlib.data.transform; |
|
2 |
|
|
3 |
import java.io.IOException; |
|
4 |
import java.io.InputStream; |
|
5 |
import java.util.Properties; |
|
6 |
|
|
7 |
import com.mongodb.MongoClient; |
|
8 |
import com.mongodb.client.MongoDatabase; |
|
9 |
import org.springframework.context.annotation.Bean; |
|
10 |
import org.springframework.context.annotation.Configuration; |
|
11 |
import org.springframework.core.io.ClassPathResource; |
|
12 |
|
|
13 |
@Configuration |
|
14 |
public class ConfigurationTestConfig { |
|
15 |
|
|
16 |
@Bean |
|
17 |
public MongoDatabase db() throws IOException { |
|
18 |
|
|
19 |
final Properties p = testProperties(); |
|
20 |
|
|
21 |
final MongoClient mongo = new MongoClient( |
|
22 |
p.getProperty("mongodb.host"), |
|
23 |
Integer.valueOf(p.getProperty("mongodb.port"))); |
|
24 |
return mongo.getDatabase(p.getProperty("mongodb.dbname")); |
|
25 |
} |
|
26 |
|
|
27 |
@Bean |
|
28 |
public Properties testProperties() throws IOException { |
|
29 |
final Properties p = new Properties(); |
|
30 |
final ClassPathResource cp = new ClassPathResource("test.properties"); |
|
31 |
try (final InputStream stream = cp.getInputStream()) { |
|
32 |
p.load(stream); |
|
33 |
} |
|
34 |
return p; |
|
35 |
} |
|
36 |
|
|
37 |
} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/resources/eu/dnetlib/openaire/exporter/model/projectDetails.json | ||
---|---|---|
1 |
{"projectId":"corda_______::200130","acronym":"SUPERCOMPETITORS","code":"200130","optional1":"ERC-SG","optional2":"http://erc.europa.eu","jsonextrainfo":"{}","fundingPath":["\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003eec__________::EC\u003c/id\u003e\u003cshortname\u003eEC\u003c/shortname\u003e\u003cname\u003eEuropean Commission\u003c/name\u003e\u003cjurisdiction\u003eEU\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_2\u003e\u003cid\u003eec__________::EC::FP7::SP2::ERC\u003c/id\u003e\u003cdescription\u003eERC\u003c/description\u003e\u003cname\u003eERC\u003c/name\u003e\u003cclass\u003eec:program\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_1\u003e\u003cid\u003eec__________::EC::FP7::SP2\u003c/id\u003e\u003cdescription\u003eSP2-Ideas\u003c/description\u003e\u003cname\u003eSP2\u003c/name\u003e\u003cclass\u003eec:specificprogram\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003eec__________::EC::FP7\u003c/id\u003e\u003cdescription\u003eSEVENTH FRAMEWORK PROGRAMME\u003c/description\u003e\u003cname\u003eFP7\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003eec:frameworkprogram\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/parent\u003e\u003c/funding_level_2\u003e\u003c/fundingtree\u003e"]} |
|
2 |
{"projectId":"corda_______::200141","acronym":"QUESPACE","code":"200141","optional1":"ERC-SG","optional2":"http://erc.europa.eu","jsonextrainfo":"{}","fundingPath":["\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003eec__________::EC\u003c/id\u003e\u003cshortname\u003eEC\u003c/shortname\u003e\u003cname\u003eEuropean Commission\u003c/name\u003e\u003cjurisdiction\u003eEU\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_2\u003e\u003cid\u003eec__________::EC::FP7::SP2::ERC\u003c/id\u003e\u003cdescription\u003eERC\u003c/description\u003e\u003cname\u003eERC\u003c/name\u003e\u003cclass\u003eec:program\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_1\u003e\u003cid\u003eec__________::EC::FP7::SP2\u003c/id\u003e\u003cdescription\u003eSP2-Ideas\u003c/description\u003e\u003cname\u003eSP2\u003c/name\u003e\u003cclass\u003eec:specificprogram\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003eec__________::EC::FP7\u003c/id\u003e\u003cdescription\u003eSEVENTH FRAMEWORK PROGRAMME\u003c/description\u003e\u003cname\u003eFP7\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003eec:frameworkprogram\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/parent\u003e\u003c/funding_level_2\u003e\u003c/fundingtree\u003e"]} |
|
3 |
{"projectId":"corda_______::200165","acronym":"PROTEOMICS OF CHERNO","code":"200165","optional1":"MC-IRG","optional2":"http://cordis.europa.eu/fp7/people.htm","jsonextrainfo":"{}","fundingPath":["\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003eec__________::EC\u003c/id\u003e\u003cshortname\u003eEC\u003c/shortname\u003e\u003cname\u003eEuropean Commission\u003c/name\u003e\u003cjurisdiction\u003eEU\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_2\u003e\u003cid\u003eec__________::EC::FP7::SP3::PEOPLE\u003c/id\u003e\u003cdescription\u003eMarie-Curie Actions\u003c/description\u003e\u003cname\u003ePEOPLE\u003c/name\u003e\u003cclass\u003eec:program\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_1\u003e\u003cid\u003eec__________::EC::FP7::SP3\u003c/id\u003e\u003cdescription\u003eSP3-People\u003c/description\u003e\u003cname\u003eSP3\u003c/name\u003e\u003cclass\u003eec:specificprogram\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003eec__________::EC::FP7\u003c/id\u003e\u003cdescription\u003eSEVENTH FRAMEWORK PROGRAMME\u003c/description\u003e\u003cname\u003eFP7\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003eec:frameworkprogram\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/parent\u003e\u003c/funding_level_2\u003e\u003c/fundingtree\u003e"]} |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/resources/eu/dnetlib/openaire/exporter/model/projectDetails.csv | ||
---|---|---|
1 |
nih_________::3R01GM073898-02S1,,3R01GM073898-02S1,23188 $,,"{""orgname"":""UNIVERSITY OF CALIFORNIA SAN DIEGO"", ""activity"":""R01"", ""administeringic"":""GM"", ""serialnumber"":""73898"", ""coreprojectnum"":""R01GM073898""}","[""\u003cfundingtree\u003e\n \u003cfunder\u003e\n \u003cid\u003enih_________::NIH\u003c/id\u003e\n \u003cshortname\u003eNIH\u003c/shortname\u003e\n \u003cname\u003eNational Institutes of Health\u003c/name\u003e\n \u003cjurisdiction\u003eUS\u003c/jurisdiction\u003e\n \u003c/funder\u003e\n \u003cfunding_level_0\u003e\n \u003cid\u003enih_________::NIH::NATIONAL_INSTITUTE_OF_GENERAL_MEDICAL_SCIENCES\u003c/id\u003e\n \u003cname\u003eNATIONAL INSTITUTE OF GENERAL MEDICAL SCIENCES\u003c/name\u003e\n \u003cdescription\u003eNATIONAL INSTITUTE OF GENERAL MEDICAL SCIENCES\u003c/description\u003e\n \u003cparent/\u003e\n \u003cclass\u003enih:fundingStream\u003c/class\u003e\n \u003c/funding_level_0\u003e\n \u003c/fundingtree\u003e""]" |
|
2 |
corda_______::100202,RECOMP,100202,JTI-CP-ARTEMIS,http://cordis.europa.eu/fp7/home_en.html,{},"[""\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003eec__________::EC\u003c/id\u003e\u003cshortname\u003eEC\u003c/shortname\u003e\u003cname\u003eEuropean Commission\u003c/name\u003e\u003cjurisdiction\u003eEU\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_2\u003e\u003cid\u003eec__________::EC::FP7::SP1::SP1-JTI\u003c/id\u003e\u003cdescription\u003eJoint Technology Initiatives (Annex IV-SP1)\u003c/description\u003e\u003cname\u003eSP1-JTI\u003c/name\u003e\u003cclass\u003eec:program\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_1\u003e\u003cid\u003eec__________::EC::FP7::SP1\u003c/id\u003e\u003cdescription\u003eSP1-Cooperation\u003c/description\u003e\u003cname\u003eSP1\u003c/name\u003e\u003cclass\u003eec:specificprogram\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003eec__________::EC::FP7\u003c/id\u003e\u003cdescription\u003eSEVENTH FRAMEWORK PROGRAMME\u003c/description\u003e\u003cname\u003eFP7\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003eec:frameworkprogram\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/parent\u003e\u003c/funding_level_2\u003e\u003c/fundingtree\u003e""]" |
|
3 |
corda__h2020::633080,MACC-III,633080,SPACE,SPACE,{},"[""\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003eec__________::EC\u003c/id\u003e\u003cshortname\u003eEC\u003c/shortname\u003e\u003cname\u003eEuropean Commission\u003c/name\u003e\u003cjurisdiction\u003eEU\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_1\u003e\u003cid\u003eec__________::EC::H2020::CSA\u003c/id\u003e\u003cdescription\u003eCoordination and support action\u003c/description\u003e\u003cname\u003eCSA\u003c/name\u003e\u003cclass\u003eec:h2020toas\u003c/class\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003eec__________::EC::H2020\u003c/id\u003e\u003cname\u003eH2020\u003c/name\u003e\u003cdescription\u003eHorizon 2020 Framework Programme\u003c/description\u003e\u003cparent/\u003e\u003cclass\u003eec:h2020fundings\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/fundingtree\u003e""]" |
|
4 |
nsf_________::0000096,,0000096,,,{},"[""\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003ensf_________::NSF\u003c/id\u003e\u003cshortname\u003eNSF\u003c/shortname\u003e\u003cname\u003eNational Science Foundation\u003c/name\u003e\u003cjurisdiction\u003eUS\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_1\u003e\u003cid\u003ensf_________::NSF::OD::OD/OIA\u003c/id\u003e\u003cdescription\u003eOffice of Integrative Activities\u003c/description\u003e\u003cname\u003eOffice of Integrative Activities\u003c/name\u003e\u003cparent\u003e\u003cfunding_level_0\u003e\u003cid\u003ensf_________::NSF::OD\u003c/id\u003e\u003cdescription\u003eOffice of the Director\u003c/description\u003e\u003cname\u003eOffice of the Director\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003ensf:fundingStream\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/parent\u003e\u003c/funding_level_1\u003e\u003c/fundingtree\u003e""]" |
|
5 |
fct_________::100107,PTDC/SAU-ESA/100107/2008,100107,,,{},"[""\u003cfundingtree\u003e\u003cfunder\u003e\u003cid\u003efct_________::FCT\u003c/id\u003e\u003cshortname\u003eFCT\u003c/shortname\u003e\u003cname\u003eFundação para a Ciência e a Tecnologia, I.P.\u003c/name\u003e\u003cjurisdiction\u003ePT\u003c/jurisdiction\u003e\u003c/funder\u003e\u003cfunding_level_0\u003e\u003cid\u003efct_________::FCT::5876-PPCDTI\u003c/id\u003e\u003cdescription\u003e5876-PPCDTI\u003c/description\u003e\u003cname\u003e5876-PPCDTI\u003c/name\u003e\u003cparent/\u003e\u003cclass\u003efct:program\u003c/class\u003e\u003c/funding_level_0\u003e\u003c/fundingtree\u003e""]" |
modules/dnet-openaireplus-mapping-utils/tags/dnet-openaireplus-mapping-utils-6.3.29/src/test/resources/eu/dnetlib/data/bulktag/community_configuration_selcrit.xml | ||
---|---|---|
1 |
<communities> |
|
2 |
<community id="fet-fp7"> |
|
3 |
<oacommunity/> |
|
4 |
<subjects/> |
|
5 |
<datasources/> |
|
6 |
<zenodocommunities/> |
|
7 |
</community> |
|
8 |
<community id="fet-h2020"> |
|
9 |
<subjects/> |
|
10 |
<datasources/> |
|
11 |
<zenodocommunities/> |
|
12 |
</community> |
|
13 |
<community id="oa-pg"> |
|
14 |
<subjects/> |
|
15 |
<datasources/> |
|
16 |
<zenodocommunities/> |
|
17 |
</community> |
|
18 |
<community id="ee"> |
|
19 |
<subjects> |
|
20 |
<subject>SDG13 - Climate action</subject> |
|
21 |
<subject>SDG8 - Decent work and economic growth</subject> |
|
22 |
<subject>SDG15 - Life on land</subject> |
|
23 |
<subject>SDG2 - Zero hunger</subject> |
|
24 |
<subject>SDG17 - Partnerships for the goals</subject> |
|
25 |
<subject>SDG10 - Reduced inequalities</subject> |
|
26 |
<subject>SDG5 - Gender equality</subject> |
|
27 |
<subject>SDG12 - Responsible consumption and production</subject> |
|
28 |
<subject>SDG14 - Life below water</subject> |
|
29 |
<subject>SDG6 - Clean water and sanitation</subject> |
|
30 |
<subject>SDG11 - Sustainable cities and communities</subject> |
|
31 |
<subject>SDG1 - No poverty</subject> |
|
32 |
<subject>SDG3 - Good health and well being</subject> |
|
33 |
<subject>SDG7 - Affordable and clean energy</subject> |
|
34 |
<subject>SDG4 - Quality education</subject> |
|
35 |
<subject>SDG9 - Industry innovation and infrastructure</subject> |
|
36 |
<subject>SDG16 - Peace justice and strong institutions</subject> |
|
37 |
</subjects> |
|
38 |
<datasources/> |
|
39 |
<zenodocommunities> |
|
40 |
<zenodocommunity> |
|
41 |
<zenodoid>123</zenodoid> |
|
42 |
<selcriteria/> |
|
43 |
</zenodocommunity> |
|
44 |
</zenodocommunities> |
|
45 |
</community> |
|
46 |
<community id="dh-ch"> |
|
47 |
<subjects/> |
|
48 |
<datasources/> |
|
49 |
<zenodocommunities/> |
|
50 |
</community> |
|
51 |
<community id="fam"> |
|
52 |
<subjects/> |
|
53 |
<datasources/> |
|
54 |
<zenodocommunities/> |
|
55 |
</community> |
|
56 |
<community id="ni"> |
|
57 |
<subjects> |
|
58 |
<subject>brain mapping</subject> |
|
59 |
<subject>brain imaging</subject> |
|
60 |
<subject>electroencephalography</subject> |
|
61 |
<subject>arterial spin labelling</subject> |
|
62 |
<subject>brain fingerprinting</subject> |
|
63 |
<subject>brain</subject> |
|
64 |
<subject>neuroimaging</subject> |
|
65 |
<subject>Multimodal Brain Image Analysis</subject> |
|
66 |
<subject>fMRI</subject> |
|
67 |
<subject>neuroinformatics</subject> |
|
68 |
<subject>fetal brain</subject> |
|
69 |
<subject>brain ultrasonic imaging</subject> |
|
70 |
<subject>topographic brain mapping</subject> |
|
71 |
<subject>diffusion tensor imaging</subject> |
|
72 |
<subject>computerized knowledge assessment</subject> |
|
73 |
<subject>connectome mapping</subject> |
|
74 |
<subject>brain magnetic resonance imaging</subject> |
|
75 |
<subject>brain abnormalities</subject> |
|
76 |
</subjects> |
|
77 |
<datasources> |
|
78 |
<datasource> |
|
79 |
<openaireId>re3data_____::5b9bf9171d92df854cf3c520692e9122</openaireId> |
|
80 |
<selcriteria/> |
|
81 |
</datasource> |
|
82 |
<datasource> |
|
83 |
<openaireId>doajarticles::c7d3de67dc77af72f6747157441252ec</openaireId> |
|
84 |
<selcriteria/> |
|
85 |
</datasource> |
|
86 |
<datasource> |
|
87 |
<openaireId>re3data_____::8515794670370f49c1d176c399c714f5</openaireId> |
|
88 |
<selcriteria/> |
|
89 |
</datasource> |
|
90 |
<datasource> |
|
91 |
<openaireId>doajarticles::d640648c84b10d425f96f11c3de468f3</openaireId> |
|
92 |
<selcriteria/> |
|
93 |
</datasource> |
|
94 |
<datasource> |
|
95 |
<openaireId>doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a</openaireId> |
|
96 |
<selcriteria/> |
|
97 |
</datasource> |
|
98 |
</datasources> |
|
99 |
<zenodocommunities/> |
|
100 |
</community> |
|
101 |
<community id="mes"> |
|
102 |
<subjects> |
|
103 |
<subject>marine</subject> |
|
104 |
<subject>ocean</subject> |
|
105 |
<subject>fish</subject> |
|
106 |
<subject>aqua</subject> |
|
107 |
<subject>sea</subject> |
|
108 |
</subjects> |
|
109 |
<datasources> |
|
110 |
<datasource> |
|
111 |
<openaireId>re3data_____::9633d1e8c4309c833c2c442abeb0cfeb</openaireId> |
|
112 |
<selcriteria/> |
|
113 |
</datasource> |
|
114 |
</datasources> |
|
115 |
<zenodocommunities/> |
|
116 |
</community> |
|
117 |
<community id="aginfra"> |
|
118 |
<subjects> |
|
119 |
<subject>animal production and health</subject> |
|
120 |
<subject>fisheries and aquaculture</subject> |
|
121 |
<subject>food safety and human nutrition</subject> |
|
122 |
<subject>information management</subject> |
|
123 |
<subject>food technology</subject> |
|
124 |
<subject>agri-food education and extension</subject> |
|
125 |
<subject>natural resources and environment</subject> |
|
126 |
<subject>food system</subject> |
|
127 |
<subject>engineering technology and Research</subject> |
|
128 |
<subject>agriculture</subject> |
|
129 |
<subject>food safety risk assessment</subject> |
|
130 |
<subject>food security</subject> |
|
131 |
<subject>farming practices and systems</subject> |
|
132 |
<subject>plant production and protection</subject> |
|
133 |
<subject>agri-food economics and policy</subject> |
|
134 |
<subject>food distribution</subject> |
|
135 |
<subject>forestry</subject> |
|
136 |
</subjects> |
|
137 |
<datasources> |
|
138 |
<datasource> |
|
139 |
<openaireId>opendoar____::1a551829d50f1400b0dab21fdd969c04</openaireId> |
|
140 |
<selcriteria/> |
|
141 |
</datasource> |
|
142 |
<datasource> |
|
143 |
<openaireId>opendoar____::49af6c4e558a7569d80eee2e035e2bd7</openaireId> |
|
144 |
<selcriteria/> |
|
145 |
</datasource> |
|
146 |
<datasource> |
|
147 |
<openaireId>opendoar____::0266e33d3f546cb5436a10798e657d97</openaireId> |
|
148 |
<selcriteria/> |
|
149 |
</datasource> |
|
150 |
<datasource> |
|
151 |
<openaireId>opendoar____::fd4c2dc64ccb8496e6f1f94c85f30d06</openaireId> |
|
152 |
<selcriteria/> |
|
153 |
</datasource> |
|
154 |
<datasource> |
|
155 |
<openaireId>opendoar____::41bfd20a38bb1b0bec75acf0845530a7</openaireId> |
|
156 |
<selcriteria/> |
|
157 |
</datasource> |
|
158 |
<datasource> |
|
159 |
<openaireId>opendoar____::87ae6fb631f7c8a627e8e28785d9992d</openaireId> |
|
160 |
<selcriteria/> |
|
161 |
</datasource> |
|
162 |
</datasources> |
|
163 |
<zenodocommunities/> |
|
164 |
</community> |
|
165 |
<community id="clarin"> |
|
166 |
<oacommunity>oac_clarin</oacommunity> |
|
167 |
<subjects/> |
|
168 |
<datasources> |
|
169 |
<datasource> |
Also available in: Unified diff
[maven-release-plugin] copy for tag dnet-openaireplus-mapping-utils-6.3.29