Revision 53069
Added by Michele Artini over 5 years ago
modules/dnet-grid-application/trunk/src/test/java/eu/dnetlib/grid/process/utils/GridUtilsTest.java | ||
---|---|---|
8 | 8 |
import java.util.stream.Collectors; |
9 | 9 |
|
10 | 10 |
import org.junit.Before; |
11 |
import org.junit.Ignore; |
|
11 | 12 |
import org.junit.Test; |
12 | 13 |
|
13 | 14 |
import eu.dnetlib.grid.process.model.GridOrganization; |
14 | 15 |
|
16 |
@Ignore |
|
15 | 17 |
public class GridUtilsTest { |
16 | 18 |
|
17 | 19 |
@Before |
modules/dnet-grid-application/trunk/src/main/java/eu/dnetlib/grid/process/GridOrganizationsProcess.java | ||
---|---|---|
1 |
package eu.dnetlib.grid.process; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import java.util.Map; |
|
5 |
import java.util.Objects; |
|
6 |
import java.util.stream.Collectors; |
|
7 |
|
|
8 |
import eu.dnetlib.grid.process.model.GridOrganization; |
|
9 |
import eu.dnetlib.grid.process.utils.Datasource; |
|
10 |
import eu.dnetlib.grid.process.utils.GridCleaner; |
|
11 |
import eu.dnetlib.grid.process.utils.GridUtils; |
|
12 |
|
|
13 |
public class GridOrganizationsProcess { |
|
14 |
|
|
15 |
private static final Datasource gridDs = new Datasource("openaire____::grid", "GRID - Global Research Identifier Database", "grid________"); |
|
16 |
|
|
17 |
public static void main(final String[] args) { |
|
18 |
|
|
19 |
} |
|
20 |
|
|
21 |
protected static void processFile(final String jsonFile) throws Exception { |
|
22 |
final GridCleaner cleaner = new GridCleaner(); |
|
23 |
|
|
24 |
final Map<String, String> parents = GridUtils.streamOrganizations(jsonFile) |
|
25 |
.filter(Objects::nonNull) |
|
26 |
.filter(o -> o.getStatus().equals("active")) |
|
27 |
.collect(Collectors.toMap(GridOrganization::getId, GridUtils::findShortName)); |
|
28 |
|
|
29 |
GridUtils.streamOrganizations(jsonFile) |
|
30 |
.filter(Objects::nonNull) |
|
31 |
.filter(o -> o.getStatus().equals("active")) |
|
32 |
.map(cleaner) |
|
33 |
.map(org -> GridUtils.toProtos(org, parents, gridDs)) |
|
34 |
.flatMap(List::stream) |
|
35 |
.map(s -> s) // TODO OAF -> JSON |
|
36 |
.forEach(System.out::println); |
|
37 |
|
|
38 |
} |
|
39 |
|
|
40 |
} |
modules/dnet-grid-application/trunk/src/main/java/eu/dnetlib/grid/process/utils/GridUtils.java | ||
---|---|---|
23 | 23 |
import eu.dnetlib.data.proto.KindProtos.Kind; |
24 | 24 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
25 | 25 |
import eu.dnetlib.data.proto.OafProtos.OafEntity; |
26 |
import eu.dnetlib.data.proto.OafProtos.OafRel; |
|
26 | 27 |
import eu.dnetlib.data.proto.OrganizationProtos.Organization; |
27 | 28 |
import eu.dnetlib.data.proto.TypeProtos.Type; |
28 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
29 | 29 |
import eu.dnetlib.grid.process.model.GridOrganization; |
30 | 30 |
import eu.dnetlib.grid.process.model.GridResponse; |
31 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
32 |
import eu.dnetlib.miscutils.functional.hash.Hashing; |
|
31 | 33 |
|
32 | 34 |
public class GridUtils { |
33 | 35 |
|
... | ... | |
53 | 55 |
|
54 | 56 |
final String gridId = org.getId(); |
55 | 57 |
final String shortName = findShortName(org); |
58 |
final String parent = parents.get(gridId); |
|
59 |
final String mainName = calculateName(parent, org.getName()); |
|
60 |
final Set<String> otherNames = calculateNames(parent, org.getAcronyms()); |
|
56 | 61 |
|
57 |
final String mainName = calculateName(parents.get(gridId), org.getName()); |
|
58 |
final Set<String> names = calculateNames(parents.get(gridId), org.getAcronyms()); |
|
62 |
final Map<String, String> orgRels = otherNames.stream() |
|
63 |
.collect(Collectors.toMap(s -> calculateOpenaireId(ds.getPrefix(), gridId, s), s -> s)); |
|
64 |
orgRels.put(calculateOpenaireId(ds.getPrefix(), gridId, mainName), mainName); |
|
59 | 65 |
|
60 | 66 |
final Qualifier.Builder country = org.getAddresses().stream() |
61 | 67 |
.map(addr -> Qualifier.newBuilder() |
... | ... | |
68 | 74 |
.findFirst() |
69 | 75 |
.orElse(null); |
70 | 76 |
|
71 |
final String url = org.getLinks().stream().findFirst().orElse(null);
|
|
77 |
final String url = org.getLinks().stream().findFirst().orElse("");
|
|
72 | 78 |
|
73 | 79 |
final List<Oaf> res = new ArrayList<>(); |
74 | 80 |
|
75 |
res.add(toProto(gridId, mainName, shortName, url, country, org.getAcronyms(), ds)); |
|
76 |
res.addAll(names.stream() |
|
77 |
.map(name -> toProto(gridId, name, shortName, url, country, new ArrayList<>(), ds)) |
|
81 |
res.add(toProto(gridId, mainName, shortName, url, country, org.getAcronyms(), orgRels, ds)); |
|
82 |
|
|
83 |
res.addAll(otherNames.stream() |
|
84 |
.map(name -> toProto(gridId, name, shortName, url, country, new ArrayList<>(), orgRels, ds)) |
|
78 | 85 |
.collect(Collectors.toList())); |
79 | 86 |
|
80 | 87 |
return res; |
... | ... | |
86 | 93 |
final String url, |
87 | 94 |
final Qualifier.Builder country, |
88 | 95 |
final List<String> acronyms, |
96 |
final Map<String, String> orgRels, |
|
89 | 97 |
final Datasource ds) { |
90 | 98 |
|
99 |
// TODO ADD ALTERNATIVE NAMES AND RELS |
|
100 |
|
|
101 |
final String openaireId = calculateOpenaireId(ds.getPrefix(), gridId, name); |
|
102 |
|
|
103 |
final KeyValue collectedFrom = KeyValue.newBuilder() |
|
104 |
.setKey(ds.getId()) |
|
105 |
.setValue(ds.getName()).build(); |
|
106 |
|
|
107 |
final OafEntity.Builder entity = OafEntity.newBuilder() |
|
108 |
.setId(openaireId) |
|
109 |
.addPid(StructuredProperty.newBuilder() |
|
110 |
.setValue(gridId) |
|
111 |
.setQualifier(Qualifier.newBuilder() |
|
112 |
.setClassid("grid") |
|
113 |
.setClassname("grid") |
|
114 |
.setSchemeid("dnet:pid_types") |
|
115 |
.setSchemename("dnet:pid_types"))) |
|
116 |
.addCollectedfrom(collectedFrom) |
|
117 |
.setType(Type.organization) |
|
118 |
.setOrganization(Organization.newBuilder() |
|
119 |
.setMetadata(Organization.Metadata.newBuilder() |
|
120 |
.setLegalname(StringField.newBuilder().setValue(name)) |
|
121 |
.setLegalshortname(StringField.newBuilder().setValue(shortName)) |
|
122 |
.setWebsiteurl(StringField.newBuilder().setValue(url)) |
|
123 |
.addAllAlternativeNames(acronyms.stream().map(a -> StringField.newBuilder().setValue(a).build()).collect(Collectors.toList())) |
|
124 |
.setCountry(country))); |
|
125 |
|
|
126 |
orgRels.entrySet().stream() |
|
127 |
.filter(e -> !e.getKey().equals(openaireId)) |
|
128 |
.forEach(e -> entity.addCachedRel( |
|
129 |
OafRel.newBuilder() |
|
130 |
.setTarget(e.getKey()) |
|
131 |
// TODO ??? e.getValue() |
|
132 |
.addCollectedfrom(collectedFrom))); |
|
133 |
|
|
91 | 134 |
return Oaf.newBuilder() |
92 | 135 |
.setKind(Kind.entity) |
93 | 136 |
.setLastupdatetimestamp(DateUtils.now()) |
94 |
.setEntity(OafEntity.newBuilder() |
|
95 |
.addPid(StructuredProperty.newBuilder() |
|
96 |
.setValue(gridId) |
|
97 |
.setQualifier(Qualifier.newBuilder() |
|
98 |
.setClassid("grid") |
|
99 |
.setClassname("grid") |
|
100 |
.setSchemeid("dnet:pid_types") |
|
101 |
.setSchemename("dnet:pid_types"))) |
|
102 |
.addCollectedfrom(KeyValue.newBuilder() |
|
103 |
.setKey(ds.getId()) |
|
104 |
.setValue(ds.getName())) |
|
105 |
.setType(Type.organization) |
|
106 |
.setOrganization(Organization.newBuilder() |
|
107 |
.setMetadata(Organization.Metadata.newBuilder() |
|
108 |
.setLegalname(StringField.newBuilder().setValue(name)) |
|
109 |
.setLegalshortname(StringField.newBuilder().setValue(shortName)) |
|
110 |
|
|
111 |
.setWebsiteurl(StringField.newBuilder().setValue(url)) |
|
112 |
.setCountry(country)))) |
|
137 |
.setEntity(entity) |
|
113 | 138 |
.setDataInfo(DataInfo.newBuilder() |
114 |
.setTrust("0.99")
|
|
139 |
.setTrust("0.90")
|
|
115 | 140 |
.setInferred(false) |
116 | 141 |
.setProvenanceaction(Qualifier.newBuilder() |
117 | 142 |
.setClassid("UNKNOWN") |
... | ... | |
121 | 146 |
.build(); |
122 | 147 |
} |
123 | 148 |
|
149 |
private static String calculateOpenaireId(final String prefix, final String gridId, final String name) { |
|
150 |
return String.format("20|%s::%s", prefix, Hashing.md5(gridId + " " + name)); |
|
151 |
} |
|
152 |
|
|
124 | 153 |
private static String calculateName(final String parent, final String simpleName) { |
125 | 154 |
return StringUtils.isBlank(parent) ? simpleName : parent + " - " + simpleName; |
126 | 155 |
} |
modules/dnet-grid-application/trunk/src/main/java/eu/dnetlib/grid/process/GridOrganizationsApplication.java | ||
---|---|---|
1 |
package eu.dnetlib.grid.process; |
|
2 |
|
|
3 |
import java.util.List; |
|
4 |
import java.util.Map; |
|
5 |
import java.util.Objects; |
|
6 |
import java.util.stream.Collectors; |
|
7 |
|
|
8 |
import org.springframework.boot.CommandLineRunner; |
|
9 |
import org.springframework.boot.SpringApplication; |
|
10 |
import org.springframework.boot.autoconfigure.SpringBootApplication; |
|
11 |
|
|
12 |
import eu.dnetlib.grid.process.model.GridOrganization; |
|
13 |
import eu.dnetlib.grid.process.utils.Datasource; |
|
14 |
import eu.dnetlib.grid.process.utils.GridCleaner; |
|
15 |
import eu.dnetlib.grid.process.utils.GridUtils; |
|
16 |
|
|
17 |
@SpringBootApplication |
|
18 |
public class GridOrganizationsApplication implements CommandLineRunner { |
|
19 |
|
|
20 |
private static final Datasource gridDs = new Datasource("openaire____::grid", "GRID - Global Research Identifier Database", "grid________"); |
|
21 |
|
|
22 |
public static void main(final String[] args) { |
|
23 |
SpringApplication.run(GridOrganizationsApplication.class, args); |
|
24 |
} |
|
25 |
|
|
26 |
@Override |
|
27 |
public void run(final String... args) throws Exception { |
|
28 |
if (args.length == 1) { |
|
29 |
processFile(args[0]); |
|
30 |
} else { |
|
31 |
System.err.println(); |
|
32 |
System.err.println("Usage: java -jar <JARFILE> <GridJsonFile>"); |
|
33 |
System.err.println(); |
|
34 |
} |
|
35 |
} |
|
36 |
|
|
37 |
protected void processFile(final String jsonFile) throws Exception { |
|
38 |
final GridCleaner cleaner = new GridCleaner(); |
|
39 |
|
|
40 |
final Map<String, String> parents = GridUtils.streamOrganizations(jsonFile) |
|
41 |
.filter(Objects::nonNull) |
|
42 |
.filter(o -> o.getStatus().equals("active")) |
|
43 |
.collect(Collectors.toMap(GridOrganization::getId, GridUtils::findShortName)); |
|
44 |
|
|
45 |
GridUtils.streamOrganizations(jsonFile) |
|
46 |
.filter(Objects::nonNull) |
|
47 |
.filter(o -> o.getStatus().equals("active")) |
|
48 |
.map(cleaner) |
|
49 |
.map(org -> GridUtils.toProtos(org, parents, gridDs)) |
|
50 |
.flatMap(List::stream) |
|
51 |
.map(s -> s.getEntity().getId()) // TODO OAF -> JSON |
|
52 |
.forEach(System.out::println); |
|
53 |
|
|
54 |
} |
|
55 |
|
|
56 |
} |
modules/dnet-grid-application/trunk/pom.xml | ||
---|---|---|
26 | 26 |
<parent> |
27 | 27 |
<groupId>org.springframework.boot</groupId> |
28 | 28 |
<artifactId>spring-boot-starter-parent</artifactId> |
29 |
<version>2.0.0.RELEASE</version>
|
|
29 |
<version>2.0.3.RELEASE</version>
|
|
30 | 30 |
</parent> |
31 | 31 |
|
32 | 32 |
<repositories> |
... | ... | |
56 | 56 |
</repository> |
57 | 57 |
</repositories> |
58 | 58 |
|
59 |
<profiles> |
|
60 |
<profile> |
|
61 |
<id>java8-doclint-disabled</id> |
|
62 |
<activation> |
|
63 |
<jdk>[1.8,)</jdk> |
|
64 |
</activation> |
|
65 |
<properties> |
|
66 |
<javadoc.opts>-Xdoclint:none</javadoc.opts> |
|
67 |
</properties> |
|
68 |
</profile> |
|
69 |
</profiles> |
|
70 | 59 |
|
71 | 60 |
|
72 | 61 |
|
73 | 62 |
<!-- Add typical dependencies for a web application --> |
74 | 63 |
<dependencies> |
75 |
|
|
76 | 64 |
<dependency> |
65 |
<groupId>org.springframework.boot</groupId> |
|
66 |
<artifactId>spring-boot-starter</artifactId> |
|
67 |
</dependency> |
|
68 |
<dependency> |
|
77 | 69 |
<groupId>eu.dnetlib</groupId> |
78 | 70 |
<artifactId>dnet-openaire-data-protos</artifactId> |
79 |
<version>3.9.2</version>
|
|
71 |
<version>3.9.4-SNAPSHOT</version>
|
|
80 | 72 |
</dependency> |
81 | 73 |
<dependency> |
82 | 74 |
<groupId>eu.dnetlib</groupId> |
Also available in: Unified diff