Project

General

Profile

« Previous | Next » 

Revision 53069

View differences:

modules/dnet-grid-application/trunk/src/test/java/eu/dnetlib/grid/process/utils/GridUtilsTest.java
8 8
import java.util.stream.Collectors;
9 9

  
10 10
import org.junit.Before;
11
import org.junit.Ignore;
11 12
import org.junit.Test;
12 13

  
13 14
import eu.dnetlib.grid.process.model.GridOrganization;
14 15

  
16
@Ignore
15 17
public class GridUtilsTest {
16 18

  
17 19
	@Before
modules/dnet-grid-application/trunk/src/main/java/eu/dnetlib/grid/process/GridOrganizationsProcess.java
1
package eu.dnetlib.grid.process;
2

  
3
import java.util.List;
4
import java.util.Map;
5
import java.util.Objects;
6
import java.util.stream.Collectors;
7

  
8
import eu.dnetlib.grid.process.model.GridOrganization;
9
import eu.dnetlib.grid.process.utils.Datasource;
10
import eu.dnetlib.grid.process.utils.GridCleaner;
11
import eu.dnetlib.grid.process.utils.GridUtils;
12

  
13
public class GridOrganizationsProcess {
14

  
15
	private static final Datasource gridDs = new Datasource("openaire____::grid", "GRID - Global Research Identifier Database", "grid________");
16

  
17
	public static void main(final String[] args) {
18

  
19
	}
20

  
21
	protected static void processFile(final String jsonFile) throws Exception {
22
		final GridCleaner cleaner = new GridCleaner();
23

  
24
		final Map<String, String> parents = GridUtils.streamOrganizations(jsonFile)
25
				.filter(Objects::nonNull)
26
				.filter(o -> o.getStatus().equals("active"))
27
				.collect(Collectors.toMap(GridOrganization::getId, GridUtils::findShortName));
28

  
29
		GridUtils.streamOrganizations(jsonFile)
30
				.filter(Objects::nonNull)
31
				.filter(o -> o.getStatus().equals("active"))
32
				.map(cleaner)
33
				.map(org -> GridUtils.toProtos(org, parents, gridDs))
34
				.flatMap(List::stream)
35
				.map(s -> s) // TODO OAF -> JSON
36
				.forEach(System.out::println);
37

  
38
	}
39

  
40
}
modules/dnet-grid-application/trunk/src/main/java/eu/dnetlib/grid/process/utils/GridUtils.java
23 23
import eu.dnetlib.data.proto.KindProtos.Kind;
24 24
import eu.dnetlib.data.proto.OafProtos.Oaf;
25 25
import eu.dnetlib.data.proto.OafProtos.OafEntity;
26
import eu.dnetlib.data.proto.OafProtos.OafRel;
26 27
import eu.dnetlib.data.proto.OrganizationProtos.Organization;
27 28
import eu.dnetlib.data.proto.TypeProtos.Type;
28
import eu.dnetlib.miscutils.datetime.DateUtils;
29 29
import eu.dnetlib.grid.process.model.GridOrganization;
30 30
import eu.dnetlib.grid.process.model.GridResponse;
31
import eu.dnetlib.miscutils.datetime.DateUtils;
32
import eu.dnetlib.miscutils.functional.hash.Hashing;
31 33

  
32 34
public class GridUtils {
33 35

  
......
53 55

  
54 56
		final String gridId = org.getId();
55 57
		final String shortName = findShortName(org);
58
		final String parent = parents.get(gridId);
59
		final String mainName = calculateName(parent, org.getName());
60
		final Set<String> otherNames = calculateNames(parent, org.getAcronyms());
56 61

  
57
		final String mainName = calculateName(parents.get(gridId), org.getName());
58
		final Set<String> names = calculateNames(parents.get(gridId), org.getAcronyms());
62
		final Map<String, String> orgRels = otherNames.stream()
63
				.collect(Collectors.toMap(s -> calculateOpenaireId(ds.getPrefix(), gridId, s), s -> s));
64
		orgRels.put(calculateOpenaireId(ds.getPrefix(), gridId, mainName), mainName);
59 65

  
60 66
		final Qualifier.Builder country = org.getAddresses().stream()
61 67
				.map(addr -> Qualifier.newBuilder()
......
68 74
				.findFirst()
69 75
				.orElse(null);
70 76

  
71
		final String url = org.getLinks().stream().findFirst().orElse(null);
77
		final String url = org.getLinks().stream().findFirst().orElse("");
72 78

  
73 79
		final List<Oaf> res = new ArrayList<>();
74 80

  
75
		res.add(toProto(gridId, mainName, shortName, url, country, org.getAcronyms(), ds));
76
		res.addAll(names.stream()
77
				.map(name -> toProto(gridId, name, shortName, url, country, new ArrayList<>(), ds))
81
		res.add(toProto(gridId, mainName, shortName, url, country, org.getAcronyms(), orgRels, ds));
82

  
83
		res.addAll(otherNames.stream()
84
				.map(name -> toProto(gridId, name, shortName, url, country, new ArrayList<>(), orgRels, ds))
78 85
				.collect(Collectors.toList()));
79 86

  
80 87
		return res;
......
86 93
			final String url,
87 94
			final Qualifier.Builder country,
88 95
			final List<String> acronyms,
96
			final Map<String, String> orgRels,
89 97
			final Datasource ds) {
90 98

  
99
		// TODO ADD ALTERNATIVE NAMES AND RELS
100

  
101
		final String openaireId = calculateOpenaireId(ds.getPrefix(), gridId, name);
102

  
103
		final KeyValue collectedFrom = KeyValue.newBuilder()
104
				.setKey(ds.getId())
105
				.setValue(ds.getName()).build();
106

  
107
		final OafEntity.Builder entity = OafEntity.newBuilder()
108
				.setId(openaireId)
109
				.addPid(StructuredProperty.newBuilder()
110
						.setValue(gridId)
111
						.setQualifier(Qualifier.newBuilder()
112
								.setClassid("grid")
113
								.setClassname("grid")
114
								.setSchemeid("dnet:pid_types")
115
								.setSchemename("dnet:pid_types")))
116
				.addCollectedfrom(collectedFrom)
117
				.setType(Type.organization)
118
				.setOrganization(Organization.newBuilder()
119
						.setMetadata(Organization.Metadata.newBuilder()
120
								.setLegalname(StringField.newBuilder().setValue(name))
121
								.setLegalshortname(StringField.newBuilder().setValue(shortName))
122
								.setWebsiteurl(StringField.newBuilder().setValue(url))
123
								.addAllAlternativeNames(acronyms.stream().map(a -> StringField.newBuilder().setValue(a).build()).collect(Collectors.toList()))
124
								.setCountry(country)));
125

  
126
		orgRels.entrySet().stream()
127
				.filter(e -> !e.getKey().equals(openaireId))
128
				.forEach(e -> entity.addCachedRel(
129
						OafRel.newBuilder()
130
								.setTarget(e.getKey())
131
								// TODO ??? e.getValue()
132
								.addCollectedfrom(collectedFrom)));
133

  
91 134
		return Oaf.newBuilder()
92 135
				.setKind(Kind.entity)
93 136
				.setLastupdatetimestamp(DateUtils.now())
94
				.setEntity(OafEntity.newBuilder()
95
						.addPid(StructuredProperty.newBuilder()
96
								.setValue(gridId)
97
								.setQualifier(Qualifier.newBuilder()
98
										.setClassid("grid")
99
										.setClassname("grid")
100
										.setSchemeid("dnet:pid_types")
101
										.setSchemename("dnet:pid_types")))
102
						.addCollectedfrom(KeyValue.newBuilder()
103
								.setKey(ds.getId())
104
								.setValue(ds.getName()))
105
						.setType(Type.organization)
106
						.setOrganization(Organization.newBuilder()
107
								.setMetadata(Organization.Metadata.newBuilder()
108
										.setLegalname(StringField.newBuilder().setValue(name))
109
										.setLegalshortname(StringField.newBuilder().setValue(shortName))
110

  
111
										.setWebsiteurl(StringField.newBuilder().setValue(url))
112
										.setCountry(country))))
137
				.setEntity(entity)
113 138
				.setDataInfo(DataInfo.newBuilder()
114
						.setTrust("0.99")
139
						.setTrust("0.90")
115 140
						.setInferred(false)
116 141
						.setProvenanceaction(Qualifier.newBuilder()
117 142
								.setClassid("UNKNOWN")
......
121 146
				.build();
122 147
	}
123 148

  
149
	private static String calculateOpenaireId(final String prefix, final String gridId, final String name) {
150
		return String.format("20|%s::%s", prefix, Hashing.md5(gridId + " " + name));
151
	}
152

  
124 153
	private static String calculateName(final String parent, final String simpleName) {
125 154
		return StringUtils.isBlank(parent) ? simpleName : parent + " - " + simpleName;
126 155
	}
modules/dnet-grid-application/trunk/src/main/java/eu/dnetlib/grid/process/GridOrganizationsApplication.java
1
package eu.dnetlib.grid.process;
2

  
3
import java.util.List;
4
import java.util.Map;
5
import java.util.Objects;
6
import java.util.stream.Collectors;
7

  
8
import org.springframework.boot.CommandLineRunner;
9
import org.springframework.boot.SpringApplication;
10
import org.springframework.boot.autoconfigure.SpringBootApplication;
11

  
12
import eu.dnetlib.grid.process.model.GridOrganization;
13
import eu.dnetlib.grid.process.utils.Datasource;
14
import eu.dnetlib.grid.process.utils.GridCleaner;
15
import eu.dnetlib.grid.process.utils.GridUtils;
16

  
17
@SpringBootApplication
18
public class GridOrganizationsApplication implements CommandLineRunner {
19

  
20
	private static final Datasource gridDs = new Datasource("openaire____::grid", "GRID - Global Research Identifier Database", "grid________");
21

  
22
	public static void main(final String[] args) {
23
		SpringApplication.run(GridOrganizationsApplication.class, args);
24
	}
25

  
26
	@Override
27
	public void run(final String... args) throws Exception {
28
		if (args.length == 1) {
29
			processFile(args[0]);
30
		} else {
31
			System.err.println();
32
			System.err.println("Usage: java -jar <JARFILE> <GridJsonFile>");
33
			System.err.println();
34
		}
35
	}
36

  
37
	protected void processFile(final String jsonFile) throws Exception {
38
		final GridCleaner cleaner = new GridCleaner();
39

  
40
		final Map<String, String> parents = GridUtils.streamOrganizations(jsonFile)
41
				.filter(Objects::nonNull)
42
				.filter(o -> o.getStatus().equals("active"))
43
				.collect(Collectors.toMap(GridOrganization::getId, GridUtils::findShortName));
44

  
45
		GridUtils.streamOrganizations(jsonFile)
46
				.filter(Objects::nonNull)
47
				.filter(o -> o.getStatus().equals("active"))
48
				.map(cleaner)
49
				.map(org -> GridUtils.toProtos(org, parents, gridDs))
50
				.flatMap(List::stream)
51
				.map(s -> s.getEntity().getId()) // TODO OAF -> JSON
52
				.forEach(System.out::println);
53

  
54
	}
55

  
56
}
modules/dnet-grid-application/trunk/pom.xml
26 26
	<parent>
27 27
		<groupId>org.springframework.boot</groupId>
28 28
		<artifactId>spring-boot-starter-parent</artifactId>
29
		<version>2.0.0.RELEASE</version>
29
		<version>2.0.3.RELEASE</version>
30 30
	</parent>
31 31

  
32 32
	<repositories>
......
56 56
		</repository>
57 57
	</repositories>
58 58

  
59
	<profiles>
60
		<profile>
61
			<id>java8-doclint-disabled</id>
62
			<activation>
63
				<jdk>[1.8,)</jdk>
64
			</activation>
65
			<properties>
66
				<javadoc.opts>-Xdoclint:none</javadoc.opts>
67
			</properties>
68
		</profile>
69
	</profiles>
70 59

  
71 60

  
72 61

  
73 62
	<!-- Add typical dependencies for a web application -->
74 63
	<dependencies>
75

  
76 64
		<dependency>
65
			<groupId>org.springframework.boot</groupId>
66
			<artifactId>spring-boot-starter</artifactId>
67
		</dependency>
68
		<dependency>
77 69
			<groupId>eu.dnetlib</groupId>
78 70
			<artifactId>dnet-openaire-data-protos</artifactId>
79
			<version>3.9.2</version>
71
			<version>3.9.4-SNAPSHOT</version>
80 72
		</dependency>
81 73
		<dependency>
82 74
			<groupId>eu.dnetlib</groupId>

Also available in: Unified diff