Project

General

Profile

« Previous | Next » 

Revision 56340

View differences:

modules/dnet-orgs-database-application/trunk/src/test/java/eu/dnetlib/grid/process/utils/GridUtilsTest.java
1
package eu.dnetlib.grid.process.utils;
2

  
3
import static org.junit.Assert.assertEquals;
4

  
5
import java.io.FileInputStream;
6
import java.io.FileNotFoundException;
7
import java.io.InputStream;
8
import java.io.UnsupportedEncodingException;
9
import java.security.MessageDigest;
10
import java.security.NoSuchAlgorithmException;
11
import java.util.HashSet;
12
import java.util.Set;
13
import java.util.stream.Collectors;
14

  
15
import org.apache.commons.codec.binary.Hex;
16
import org.junit.Before;
17
import org.junit.Ignore;
18
import org.junit.Test;
19

  
20
import eu.dnetlib.grid.process.model.GridOrganization;
21
import eu.dnetlib.miscutils.functional.hash.Hashing;
22

  
23
@Ignore
24
public class GridUtilsTest {
25

  
26
	@Before
27
	public void setUp() throws Exception {}
28

  
29
	@Test
30
	@Ignore
31
	public void test() throws FileNotFoundException {
32
		// final InputStream is = getClass().getResourceAsStream("grid_example.json");
33
		final InputStream is = new FileInputStream("/Users/michele/Desktop/grid.json");
34
		int i = 0;
35
		final Set<String> s = new HashSet<>();
36
		for (final GridOrganization o : GridUtils.getOrganizations(is)) {
37
			System.out.println(++i + " " + o.getId());
38
			o.getAddresses().forEach(a -> s.add(a.getCountry_code()));
39
		}
40
		System.out.println(s.stream().collect(Collectors.joining(", ")));
41

  
42
	}
43

  
44
	@Test
45
	@Ignore
46
	public void testMd5() throws UnsupportedEncodingException, NoSuchAlgorithmException {
47
		final String md5_1 = Hashing.md5("grid");
48

  
49
		final MessageDigest md = MessageDigest.getInstance("MD5");
50
		md.update("grid".getBytes("UTF-8"));
51
		final String md5_2 = new String(Hex.encodeHex(md.digest()));
52

  
53
		assertEquals(md5_1, md5_2);
54
	}
55

  
56
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridNutsLevel.java
1
package eu.dnetlib.grid.process.model;
2

  
3
public class GridNutsLevel {
4

  
5
	private String name;
6
	private String code;
7

  
8
	public String getName() {
9
		return name;
10
	}
11

  
12
	public void setName(final String name) {
13
		this.name = name;
14
	}
15

  
16
	public String getCode() {
17
		return code;
18
	}
19

  
20
	public void setCode(final String code) {
21
		this.code = code;
22
	}
23
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridRel.java
1
package eu.dnetlib.grid.process.model;
2

  
3
public class GridRel {
4

  
5
	private String type;
6
	private String label;
7
	private String id;
8

  
9
	public String getType() {
10
		return type;
11
	}
12

  
13
	public void setType(final String type) {
14
		this.type = type;
15
	}
16

  
17
	public String getLabel() {
18
		return label;
19
	}
20

  
21
	public void setLabel(final String label) {
22
		this.label = label;
23
	}
24

  
25
	public String getId() {
26
		return id;
27
	}
28

  
29
	public void setId(final String id) {
30
		this.id = id;
31
	}
32

  
33
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridLicense.java
1
package eu.dnetlib.grid.process.model;
2

  
3
public class GridLicense {
4

  
5
	private String attribution;
6
	private String license;
7

  
8
	public String getAttribution() {
9
		return attribution;
10
	}
11

  
12
	public void setAttribution(final String attribution) {
13
		this.attribution = attribution;
14
	}
15

  
16
	public String getLicense() {
17
		return license;
18
	}
19

  
20
	public void setLicense(final String license) {
21
		this.license = license;
22
	}
23

  
24
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridOrganization.java
1
package eu.dnetlib.grid.process.model;
2

  
3
import java.util.ArrayList;
4
import java.util.LinkedHashMap;
5
import java.util.List;
6
import java.util.Map;
7

  
8
public class GridOrganization {
9

  
10
	private String id;
11
	private String status;
12
	private String redirect;
13
	private String name;
14
	private String wikipedia_url;
15
	private final String email_address = null;
16
	private final List<String> links = new ArrayList<>();
17
	private final List<String> aliases = new ArrayList<>();
18
	private final List<String> acronyms = new ArrayList<>();
19
	private final List<String> types = new ArrayList<>();
20
	private final List<String> ip_addresses = new ArrayList<>();
21
	private final List<GridAddress> addresses = new ArrayList<>();
22
	private final List<GridLabel> labels = new ArrayList<>();
23
	private long established;
24
	private final List<GridRel> relationships = new ArrayList<>();
25
	private final Map<String, GridExternalIdentifier> external_ids = new LinkedHashMap<>();
26
	private float weight;
27

  
28
	public String getId() {
29
		return id;
30
	}
31

  
32
	public void setId(final String id) {
33
		this.id = id;
34
	}
35

  
36
	public String getStatus() {
37
		return status;
38
	}
39

  
40
	public void setStatus(final String status) {
41
		this.status = status;
42
	}
43

  
44
	public String getRedirect() {
45
		return redirect;
46
	}
47

  
48
	public void setRedirect(final String redirect) {
49
		this.redirect = redirect;
50
	}
51

  
52
	public String getName() {
53
		return name;
54
	}
55

  
56
	public void setName(final String name) {
57
		this.name = name;
58
	}
59

  
60
	public String getWikipedia_url() {
61
		return wikipedia_url;
62
	}
63

  
64
	public void setWikipedia_url(final String wikipedia_url) {
65
		this.wikipedia_url = wikipedia_url;
66
	}
67

  
68
	public long getEstablished() {
69
		return established;
70
	}
71

  
72
	public void setEstablished(final long established) {
73
		this.established = established;
74
	}
75

  
76
	public float getWeight() {
77
		return weight;
78
	}
79

  
80
	public void setWeight(final float weight) {
81
		this.weight = weight;
82
	}
83

  
84
	public String getEmail_address() {
85
		return email_address;
86
	}
87

  
88
	public List<String> getLinks() {
89
		return links;
90
	}
91

  
92
	public List<String> getAliases() {
93
		return aliases;
94
	}
95

  
96
	public List<String> getAcronyms() {
97
		return acronyms;
98
	}
99

  
100
	public List<String> getTypes() {
101
		return types;
102
	}
103

  
104
	public List<String> getIp_addresses() {
105
		return ip_addresses;
106
	}
107

  
108
	public List<GridAddress> getAddresses() {
109
		return addresses;
110
	}
111

  
112
	public List<GridLabel> getLabels() {
113
		return labels;
114
	}
115

  
116
	public List<GridRel> getRelationships() {
117
		return relationships;
118
	}
119

  
120
	public Map<String, GridExternalIdentifier> getExternal_ids() {
121
		return external_ids;
122
	}
123

  
124
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridAddress.java
1
package eu.dnetlib.grid.process.model;
2

  
3
public class GridAddress {
4

  
5
	private String line_1;
6
	private String line_2;
7
	private String line_3 = null;
8
	private float lat;
9
	private float lng;
10
	private String postcode;
11
	private boolean primary;
12
	private String city;
13
	private String state;
14
	private String state_code;
15
	private String country;
16
	private String country_code;
17
	private GridCity geonames_city;
18

  
19
	public String getLine_1() {
20
		return line_1;
21
	}
22

  
23
	public void setLine_1(final String line_1) {
24
		this.line_1 = line_1;
25
	}
26

  
27
	public String getLine_2() {
28
		return line_2;
29
	}
30

  
31
	public void setLine_2(final String line_2) {
32
		this.line_2 = line_2;
33
	}
34

  
35
	public String getLine_3() {
36
		return line_3;
37
	}
38

  
39
	public void setLine_3(final String line_3) {
40
		this.line_3 = line_3;
41
	}
42

  
43
	public float getLat() {
44
		return lat;
45
	}
46

  
47
	public void setLat(final float lat) {
48
		this.lat = lat;
49
	}
50

  
51
	public float getLng() {
52
		return lng;
53
	}
54

  
55
	public void setLng(final float lng) {
56
		this.lng = lng;
57
	}
58

  
59
	public String getPostcode() {
60
		return postcode;
61
	}
62

  
63
	public void setPostcode(final String postcode) {
64
		this.postcode = postcode;
65
	}
66

  
67
	public boolean isPrimary() {
68
		return primary;
69
	}
70

  
71
	public void setPrimary(final boolean primary) {
72
		this.primary = primary;
73
	}
74

  
75
	public String getCity() {
76
		return city;
77
	}
78

  
79
	public void setCity(final String city) {
80
		this.city = city;
81
	}
82

  
83
	public String getState() {
84
		return state;
85
	}
86

  
87
	public void setState(final String state) {
88
		this.state = state;
89
	}
90

  
91
	public String getState_code() {
92
		return state_code;
93
	}
94

  
95
	public void setState_code(final String state_code) {
96
		this.state_code = state_code;
97
	}
98

  
99
	public String getCountry() {
100
		return country;
101
	}
102

  
103
	public void setCountry(final String country) {
104
		this.country = country;
105
	}
106

  
107
	public String getCountry_code() {
108
		return country_code;
109
	}
110

  
111
	public void setCountry_code(final String country_code) {
112
		this.country_code = country_code;
113
	}
114

  
115
	public GridCity getGeonames_city() {
116
		return geonames_city;
117
	}
118

  
119
	public void setGeonames_city(final GridCity geonames_city) {
120
		this.geonames_city = geonames_city;
121
	}
122

  
123
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridExternalIdentifier.java
1
package eu.dnetlib.grid.process.model;
2

  
3
import java.util.ArrayList;
4
import java.util.List;
5

  
6
public class GridExternalIdentifier {
7

  
8
	private String preferred = null;
9
	private List<String> all = new ArrayList<>();
10

  
11
	public String getPreferred() {
12
		return preferred;
13
	}
14

  
15
	public void setPreferred(final String preferred) {
16
		this.preferred = preferred;
17
	}
18

  
19
	public List<String> getAll() {
20
		return all;
21
	}
22

  
23
	public void setAll(final List<String> all) {
24
		this.all = all;
25
	}
26

  
27
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridGeoNamesAdmin.java
1
package eu.dnetlib.grid.process.model;
2

  
3
public class GridGeoNamesAdmin {
4

  
5
	private String name;
6
	private String ascii_name;
7
	private String code;
8

  
9
	public String getName() {
10
		return name;
11
	}
12

  
13
	public void setName(final String name) {
14
		this.name = name;
15
	}
16

  
17
	public String getAscii_name() {
18
		return ascii_name;
19
	}
20

  
21
	public void setAscii_name(final String ascii_name) {
22
		this.ascii_name = ascii_name;
23
	}
24

  
25
	public String getCode() {
26
		return code;
27
	}
28

  
29
	public void setCode(final String code) {
30
		this.code = code;
31
	}
32

  
33
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridCity.java
1
package eu.dnetlib.grid.process.model;
2

  
3
public class GridCity {
4

  
5
	private long id;
6
	private String city;
7
	private GridNutsLevel nuts_level1;
8
	private GridNutsLevel nuts_level2;
9
	private GridNutsLevel nuts_level3;
10
	private GridGeoNamesAdmin geonames_admin1;
11
	private GridGeoNamesAdmin geonames_admin2;
12
	private GridLicense license;
13

  
14
	public long getId() {
15
		return id;
16
	}
17

  
18
	public void setId(final long id) {
19
		this.id = id;
20
	}
21

  
22
	public String getCity() {
23
		return city;
24
	}
25

  
26
	public void setCity(final String city) {
27
		this.city = city;
28
	}
29

  
30
	public GridNutsLevel getNuts_level1() {
31
		return nuts_level1;
32
	}
33

  
34
	public void setNuts_level1(final GridNutsLevel nuts_level1) {
35
		this.nuts_level1 = nuts_level1;
36
	}
37

  
38
	public GridNutsLevel getNuts_level2() {
39
		return nuts_level2;
40
	}
41

  
42
	public void setNuts_level2(final GridNutsLevel nuts_level2) {
43
		this.nuts_level2 = nuts_level2;
44
	}
45

  
46
	public GridNutsLevel getNuts_level3() {
47
		return nuts_level3;
48
	}
49

  
50
	public void setNuts_level3(final GridNutsLevel nuts_level3) {
51
		this.nuts_level3 = nuts_level3;
52
	}
53

  
54
	public GridGeoNamesAdmin getGeonames_admin1() {
55
		return geonames_admin1;
56
	}
57

  
58
	public void setGeonames_admin1(final GridGeoNamesAdmin geonames_admin1) {
59
		this.geonames_admin1 = geonames_admin1;
60
	}
61

  
62
	public GridGeoNamesAdmin getGeonames_admin2() {
63
		return geonames_admin2;
64
	}
65

  
66
	public void setGeonames_admin2(final GridGeoNamesAdmin geonames_admin2) {
67
		this.geonames_admin2 = geonames_admin2;
68
	}
69

  
70
	public GridLicense getLicense() {
71
		return license;
72
	}
73

  
74
	public void setLicense(final GridLicense license) {
75
		this.license = license;
76
	}
77

  
78
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridResponse.java
1
package eu.dnetlib.grid.process.model;
2

  
3
public class GridResponse {
4

  
5
	private String version;
6
	private Iterable<GridOrganization> institutes;
7

  
8
	public String getVersion() {
9
		return version;
10
	}
11

  
12
	public void setVersion(final String version) {
13
		this.version = version;
14
	}
15

  
16
	public Iterable<GridOrganization> getInstitutes() {
17
		return institutes;
18
	}
19

  
20
	public void setInstitutes(final Iterable<GridOrganization> institutes) {
21
		this.institutes = institutes;
22
	}
23
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/model/GridLabel.java
1
package eu.dnetlib.grid.process.model;
2

  
3
public class GridLabel {
4

  
5
	private String label;
6
	private String iso639;
7

  
8
	public String getLabel() {
9
		return label;
10
	}
11

  
12
	public void setLabel(final String label) {
13
		this.label = label;
14
	}
15

  
16
	public String getIso639() {
17
		return iso639;
18
	}
19

  
20
	public void setIso639(final String iso639) {
21
		this.iso639 = iso639;
22
	}
23
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/utils/GridCleaner.java
1
package eu.dnetlib.grid.process.utils;
2

  
3
import java.util.function.UnaryOperator;
4

  
5
import eu.dnetlib.grid.process.model.GridOrganization;
6

  
7
public class GridCleaner implements UnaryOperator<GridOrganization> {
8

  
9
	public GridCleaner() {
10
		// TODO LOAD VOCABULARIES
11
	}
12

  
13
	@Override
14
	public GridOrganization apply(final GridOrganization t) {
15
		// TODO Auto-generated method stub
16
		return t;
17
	}
18

  
19
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/utils/GridUtils.java
1
package eu.dnetlib.grid.process.utils;
2

  
3
import java.io.FileInputStream;
4
import java.io.FileNotFoundException;
5
import java.io.InputStream;
6
import java.util.ArrayList;
7
import java.util.HashSet;
8
import java.util.List;
9
import java.util.Map;
10
import java.util.Set;
11
import java.util.stream.Collectors;
12
import java.util.stream.Stream;
13
import java.util.stream.StreamSupport;
14

  
15
import org.apache.commons.lang3.StringUtils;
16

  
17
import com.fasterxml.jackson.databind.ObjectMapper;
18

  
19
import eu.dnetlib.data.proto.FieldTypeProtos.DataInfo;
20
import eu.dnetlib.data.proto.FieldTypeProtos.KeyValue;
21
import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;
22
import eu.dnetlib.data.proto.FieldTypeProtos.StringField;
23
import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;
24
import eu.dnetlib.data.proto.KindProtos.Kind;
25
import eu.dnetlib.data.proto.OafProtos.Oaf;
26
import eu.dnetlib.data.proto.OafProtos.OafEntity;
27
import eu.dnetlib.data.proto.OafProtos.OafRel;
28
import eu.dnetlib.data.proto.OrganizationProtos.Organization;
29
import eu.dnetlib.data.proto.RelTypeProtos.RelType;
30
import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;
31
import eu.dnetlib.data.proto.TypeProtos.Type;
32
import eu.dnetlib.grid.process.model.GridLabel;
33
import eu.dnetlib.grid.process.model.GridOrganization;
34
import eu.dnetlib.grid.process.model.GridRel;
35
import eu.dnetlib.grid.process.model.GridResponse;
36
import eu.dnetlib.miscutils.datetime.DateUtils;
37
import eu.dnetlib.miscutils.functional.hash.Hashing;
38

  
39
public class GridUtils {
40

  
41
	private static final String TRUST_LEVEL = "0.91";
42

  
43
	public static Stream<GridOrganization> streamOrganizations(final String jsonFile) {
44
		try {
45
			return StreamSupport.stream(GridUtils.getOrganizations(new FileInputStream(jsonFile)).spliterator(), false);
46
		} catch (final FileNotFoundException e) {
47
			e.printStackTrace();
48
			return Stream.empty();
49
		}
50
	}
51

  
52
	protected static Iterable<GridOrganization> getOrganizations(final InputStream input) {
53
		try {
54
			return new ObjectMapper().readValue(input, GridResponse.class).getInstitutes();
55
		} catch (final Throwable e) {
56
			e.printStackTrace();
57
			return new ArrayList<>();
58
		}
59
	}
60

  
61
	public static List<Oaf> toProtos(final GridOrganization org, final Map<String, String> parents, final Datasource ds) {
62

  
63
		final String gridId = org.getId();
64
		final String parent = findParentName(org, parents);
65
		final String mainName = applyPrefix(parent, org.getName());
66
		final String shortName = findShortName(org);
67
		final String mainOpenaireId = calculateOpenaireId(ds.getPrefix(), gridId);
68
		final Set<String> alternativeNames = calculateAlternativeNames(org);
69

  
70
		final Map<String, String> orgNames = alternativeNames.stream()
71
				.map(s -> applyPrefix(parent, s))
72
				.collect(Collectors.toMap(
73
						s -> calculateOpenaireId(ds.getPrefix(), gridId, s),
74
						s -> s));
75
		orgNames.put(mainOpenaireId, mainName);
76

  
77
		final Qualifier country = org.getAddresses().stream()
78
				.map(addr -> Qualifier.newBuilder()
79
						.setClassid(addr.getCountry_code())
80
						.setClassname(addr.getCountry())
81
						.setSchemeid("dnet:countries")
82
						.setSchemename("dnet:countries"))
83
				.filter(q -> StringUtils.isNotBlank(q.getClassid()))
84
				.filter(q -> StringUtils.isNotBlank(q.getClassname()))
85
				.map(q -> q.build())
86
				.findFirst()
87
				.orElse(null);
88

  
89
		final KeyValue collectedFrom = KeyValue.newBuilder()
90
				.setKey(ds.getId())
91
				.setValue(ds.getName()).build();
92

  
93
		return orgNames.entrySet()
94
				.stream()
95
				.map(e -> toProtos(e.getKey(),
96
						gridId,
97
						e.getValue(),
98
						shortName,
99
						org.getLinks().stream().findFirst().orElse(""),
100
						country,
101
						e.getKey().equals(mainOpenaireId) ? alternativeNames : new HashSet<>(),
102
						orgNames.keySet(),
103
						collectedFrom))
104
				.flatMap(l -> l.stream())
105
				.collect(Collectors.toList());
106
	}
107

  
108
	private static List<Oaf> toProtos(final String openaireId,
109
			final String gridId,
110
			final String name,
111
			final String shortName,
112
			final String url,
113
			final Qualifier country,
114
			final Set<String> alternativeNames,
115
			final Set<String> orgRels,
116
			final KeyValue collectedFrom) {
117

  
118
		final OafEntity.Builder entity = OafEntity.newBuilder()
119
				.setId(openaireId)
120
				.addPid(StructuredProperty.newBuilder()
121
						.setValue(gridId)
122
						.setQualifier(Qualifier.newBuilder()
123
								.setClassid("grid")
124
								.setClassname("grid")
125
								.setSchemeid("dnet:pid_types")
126
								.setSchemename("dnet:pid_types")))
127
				.addCollectedfrom(collectedFrom)
128
				.setType(Type.organization)
129
				.setOrganization(Organization.newBuilder()
130
						.setMetadata(Organization.Metadata.newBuilder()
131
								.setLegalname(StringField.newBuilder().setValue(name))
132
								.setLegalshortname(StringField.newBuilder().setValue(shortName))
133
								.setWebsiteurl(StringField.newBuilder().setValue(url))
134
								.addAllAlternativeNames(
135
										alternativeNames.stream().map(a -> StringField.newBuilder().setValue(a).build()).collect(Collectors.toList()))
136
								.setCountry(country)));
137

  
138
		// Relations
139
		final List<Oaf> oafs = orgRels.stream()
140
				.filter(id -> !id.equals(openaireId))
141
				.map(id -> Oaf.newBuilder()
142
						.setKind(Kind.relation)
143
						.setLastupdatetimestamp(DateUtils.now())
144
						.setRel(OafRel.newBuilder()
145
								.setSource(openaireId)
146
								.setTarget(id)
147
								.setRelType(RelType.organizationOrganization)
148
								.setSubRelType(SubRelType.dedupSimilarity)
149
								.setRelClass("isSimilarTo")
150
								.setChild(false))
151
						.build())
152
				.collect(Collectors.toList());
153

  
154
		// Entity
155
		oafs.add(Oaf.newBuilder()
156
				.setKind(Kind.entity)
157
				.setLastupdatetimestamp(DateUtils.now())
158
				.setEntity(entity)
159
				.setDataInfo(DataInfo.newBuilder()
160
						.setTrust(TRUST_LEVEL)
161
						.setInferred(false)
162
						.setProvenanceaction(Qualifier.newBuilder()
163
								.setClassid("UNKNOWN")
164
								.setClassname("UNKNOWN")
165
								.setSchemeid("dnet:provenanceActions")
166
								.setSchemename("dnet:provenanceActions")))
167
				.build());
168

  
169
		return oafs;
170
	}
171

  
172
	private static String calculateOpenaireId(final String prefix, final String gridId) {
173
		return String.format("20|%s::%s", prefix, Hashing.md5(gridId));
174
	}
175

  
176
	private static String calculateOpenaireId(final String prefix, final String gridId, final String name) {
177
		return String.format("20|%s::%s", prefix, Hashing.md5(gridId + " " + name));
178
	}
179

  
180
	private static String applyPrefix(final String parent, final String simpleName) {
181
		return StringUtils.isBlank(parent) ? simpleName : parent + " - " + simpleName;
182
	}
183

  
184
	private static Set<String> calculateAlternativeNames(final GridOrganization org) {
185
		final Set<String> res = org.getLabels().stream()
186
				.map(GridLabel::getLabel)
187
				.collect(Collectors.toSet());
188
		res.addAll(org.getAcronyms());
189
		return res;
190
	}
191

  
192
	public static String findShortName(final GridOrganization org) {
193
		return org.getAcronyms()
194
				.stream()
195
				.filter(StringUtils::isNotBlank)
196
				.findFirst()
197
				.orElse(org.getName());
198
	}
199

  
200
	private static String findParentName(final GridOrganization org, final Map<String, String> parents) {
201
		return org.getRelationships()
202
				.stream()
203
				.filter(r -> r.getType().equalsIgnoreCase("Parent"))
204
				.map(GridRel::getId)
205
				.map(parents::get)
206
				.filter(StringUtils::isNotBlank)
207
				.findFirst()
208
				.orElse(null);
209
	}
210
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/utils/Datasource.java
1
package eu.dnetlib.grid.process.utils;
2

  
3
public class Datasource {
4

  
5
	private final String id;
6
	private final String name;
7
	private final String prefix;
8

  
9
	public Datasource(final String id, final String name, final String prefix) {
10
		this.id = id;
11
		this.name = name;
12
		this.prefix = prefix;
13
	}
14

  
15
	public String getId() {
16
		return id;
17
	}
18

  
19
	public String getName() {
20
		return name;
21
	}
22

  
23
	public String getPrefix() {
24
		return prefix;
25
	}
26

  
27
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/grid/process/GridOrganizationsApplication.java
1
package eu.dnetlib.grid.process;
2

  
3
import java.util.List;
4
import java.util.Map;
5
import java.util.Objects;
6
import java.util.stream.Collectors;
7

  
8
import org.springframework.boot.CommandLineRunner;
9
import org.springframework.boot.SpringApplication;
10
import org.springframework.boot.autoconfigure.SpringBootApplication;
11

  
12
import com.googlecode.protobuf.format.JsonFormat;
13

  
14
import eu.dnetlib.grid.process.model.GridOrganization;
15
import eu.dnetlib.grid.process.utils.Datasource;
16
import eu.dnetlib.grid.process.utils.GridCleaner;
17
import eu.dnetlib.grid.process.utils.GridUtils;
18
import eu.dnetlib.miscutils.functional.hash.Hashing;
19

  
20
@SpringBootApplication
21
public class GridOrganizationsApplication implements CommandLineRunner {
22

  
23
	private static final Datasource gridDs =
24
			new Datasource("10|openaire____::" + Hashing.md5("grid"), "GRID - Global Research Identifier Database", "grid________");
25

  
26
	public static void main(final String[] args) {
27
		SpringApplication.run(GridOrganizationsApplication.class, args);
28
	}
29

  
30
	@Override
31
	public void run(final String... args) throws Exception {
32

  
33
		if (args.length == 1) {
34
			processFile(args[0]);
35
		} else {
36
			System.err.println();
37
			System.err.println("Usage: java -jar <JARFILE> <GridJsonFile>");
38
			System.err.println();
39
		}
40
	}
41

  
42
	protected void processFile(final String jsonFile) throws Exception {
43
		final GridCleaner cleaner = new GridCleaner();
44

  
45
		final Map<String, String> parents = GridUtils.streamOrganizations(jsonFile)
46
				.filter(Objects::nonNull)
47
				.filter(o -> o.getStatus().equals("active"))
48
				.collect(Collectors.toMap(GridOrganization::getId, GridUtils::findShortName));
49

  
50
		final JsonFormat jsonFormat = new JsonFormat();
51

  
52
		GridUtils.streamOrganizations(jsonFile)
53
				.filter(Objects::nonNull)
54
				.filter(o -> o.getStatus().equals("active"))
55
				.map(cleaner)
56
				.map(org -> GridUtils.toProtos(org, parents, gridDs))
57
				.flatMap(List::stream)
58
				.map(jsonFormat::printToString)
59
				.forEach(System.out::println);
60

  
61
	}
62

  
63
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/repository/readonly/OrganizationViewRepository.java
1
package eu.dnetlib.organizations.repository.readonly;
2

  
3
import java.util.List;
4

  
5
import org.springframework.data.jpa.repository.Query;
6
import org.springframework.stereotype.Repository;
7

  
8
import eu.dnetlib.organizations.model.OrganizationView;
9

  
10
@Repository
11
public interface OrganizationViewRepository extends ReadOnlyRepository<OrganizationView, String> {
12

  
13
	@Query(value = "select distinct name from organization", nativeQuery = true)
14
	List<String> listNames();
15
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/repository/readonly/ReadOnlyRepository.java
1
package eu.dnetlib.organizations.repository.readonly;
2

  
3
import java.util.Optional;
4

  
5
import org.springframework.data.domain.Page;
6
import org.springframework.data.domain.Pageable;
7
import org.springframework.data.repository.NoRepositoryBean;
8
import org.springframework.data.repository.Repository;
9

  
10
@NoRepositoryBean
11
public interface ReadOnlyRepository<T, ID> extends Repository<T, ID> {
12

  
13
	Optional<T> findById(ID id);
14

  
15
	boolean existsById(ID id);
16

  
17
	Page<T> findAll(Pageable pageable);
18

  
19
	Iterable<T> findAll();
20

  
21
	long count();
22
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/model/OrganizationView.java
1
package eu.dnetlib.organizations.model;
2

  
3
import java.io.Serializable;
4
import java.util.Set;
5

  
6
import javax.persistence.Column;
7
import javax.persistence.Entity;
8
import javax.persistence.Id;
9
import javax.persistence.Table;
10

  
11
import org.hibernate.annotations.Type;
12
import org.hibernate.annotations.TypeDef;
13
import org.hibernate.annotations.TypeDefs;
14

  
15
import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
16
import com.vladmihalcea.hibernate.type.json.JsonStringType;
17

  
18
@Entity
19
@Table(name = "organizations_view")
20
@TypeDefs({
21
		@TypeDef(name = "json", typeClass = JsonStringType.class),
22
		@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
23
})
24
public class OrganizationView implements Serializable {
25

  
26
	/**
27
	 *
28
	 */
29
	private static final long serialVersionUID = -7864607073631041868L;
30

  
31
	@Id
32
	@Column(name = "id")
33
	private String id;
34

  
35
	@Column(name = "name")
36
	private String name;
37

  
38
	@Column(name = "lat")
39
	private Double lat;
40

  
41
	@Column(name = "lng")
42
	private Double lng;
43

  
44
	@Column(name = "city")
45
	private String city;
46

  
47
	@Column(name = "country")
48
	private String country;
49

  
50
	@Column(name = "source")
51
	private String source;
52

  
53
	@Type(type = "jsonb")
54
	@Column(name = "other_ids", columnDefinition = "jsonb")
55
	private Set<OtherIdentifier> otherIdentifiers;
56

  
57
	@Type(type = "jsonb")
58
	@Column(name = "other_names", columnDefinition = "jsonb")
59
	private Set<OtherName> otherNames;
60

  
61
	@Type(type = "jsonb")
62
	@Column(name = "relations", columnDefinition = "jsonb")
63
	private Set<Relation> relations;
64

  
65
	@Type(type = "jsonb")
66
	@Column(name = "acronyms", columnDefinition = "jsonb")
67
	private Set<String> acronyms;
68

  
69
	@Type(type = "jsonb")
70
	@Column(name = "types", columnDefinition = "jsonb")
71
	private Set<String> types;
72

  
73
	@Type(type = "jsonb")
74
	@Column(name = "urls", columnDefinition = "jsonb")
75
	private Set<String> urls;
76

  
77
	public String getId() {
78
		return id;
79
	}
80

  
81
	public void setId(final String id) {
82
		this.id = id;
83
	}
84

  
85
	public String getName() {
86
		return name;
87
	}
88

  
89
	public void setName(final String name) {
90
		this.name = name;
91
	}
92

  
93
	public Double getLat() {
94
		return lat;
95
	}
96

  
97
	public void setLat(final Double lat) {
98
		this.lat = lat;
99
	}
100

  
101
	public Double getLng() {
102
		return lng;
103
	}
104

  
105
	public void setLng(final Double lng) {
106
		this.lng = lng;
107
	}
108

  
109
	public String getCity() {
110
		return city;
111
	}
112

  
113
	public void setCity(final String city) {
114
		this.city = city;
115
	}
116

  
117
	public String getCountry() {
118
		return country;
119
	}
120

  
121
	public void setCountry(final String country) {
122
		this.country = country;
123
	}
124

  
125
	public String getSource() {
126
		return source;
127
	}
128

  
129
	public void setSource(final String source) {
130
		this.source = source;
131
	}
132

  
133
	public Set<OtherIdentifier> getOtherIdentifiers() {
134
		return otherIdentifiers;
135
	}
136

  
137
	public void setOtherIdentifiers(final Set<OtherIdentifier> otherIdentifiers) {
138
		this.otherIdentifiers = otherIdentifiers;
139
	}
140

  
141
	public Set<OtherName> getOtherNames() {
142
		return otherNames;
143
	}
144

  
145
	public void setOtherNames(final Set<OtherName> otherNames) {
146
		this.otherNames = otherNames;
147
	}
148

  
149
	public Set<Relation> getRelations() {
150
		return relations;
151
	}
152

  
153
	public void setRelations(final Set<Relation> relations) {
154
		this.relations = relations;
155
	}
156

  
157
	public Set<String> getAcronyms() {
158
		return acronyms;
159
	}
160

  
161
	public void setAcronyms(final Set<String> acronyms) {
162
		this.acronyms = acronyms;
163
	}
164

  
165
	public Set<String> getTypes() {
166
		return types;
167
	}
168

  
169
	public void setTypes(final Set<String> types) {
170
		this.types = types;
171
	}
172

  
173
	public Set<String> getUrls() {
174
		return urls;
175
	}
176

  
177
	public void setUrls(final Set<String> urls) {
178
		this.urls = urls;
179
	}
180

  
181
	public static long getSerialversionuid() {
182
		return serialVersionUID;
183
	}
184

  
185
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/model/OtherName.java
1
package eu.dnetlib.organizations.model;
2

  
3
import java.io.Serializable;
4
import java.util.Objects;
5

  
6
public class OtherName implements Serializable {
7

  
8
	/**
9
	 *
10
	 */
11
	private static final long serialVersionUID = -5212012138309835912L;
12

  
13
	private String name;
14

  
15
	private String lang;
16

  
17
	public String getName() {
18
		return name;
19
	}
20

  
21
	public void setName(final String name) {
22
		this.name = name;
23
	}
24

  
25
	public String getLang() {
26
		return lang;
27
	}
28

  
29
	public void setLang(final String lang) {
30
		this.lang = lang;
31
	}
32

  
33
	@Override
34
	public int hashCode() {
35
		return Objects.hash(lang, name);
36
	}
37

  
38
	@Override
39
	public boolean equals(final Object obj) {
40
		if (this == obj) { return true; }
41
		if (obj == null) { return false; }
42
		if (!(obj instanceof OtherName)) { return false; }
43
		final OtherName other = (OtherName) obj;
44
		return Objects.equals(lang, other.lang) && Objects.equals(name, other.name);
45
	}
46

  
47
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/model/OtherIdentifier.java
1
package eu.dnetlib.organizations.model;
2

  
3
import java.io.Serializable;
4
import java.util.Objects;
5

  
6
public class OtherIdentifier implements Serializable {
7

  
8
	/**
9
	 *
10
	 */
11
	private static final long serialVersionUID = -8290245353090885241L;
12

  
13
	private String id;
14
	private String type;
15

  
16
	public String getId() {
17
		return id;
18
	}
19

  
20
	public void setId(final String id) {
21
		this.id = id;
22
	}
23

  
24
	public String getType() {
25
		return type;
26
	}
27

  
28
	public void setType(final String type) {
29
		this.type = type;
30
	}
31

  
32
	@Override
33
	public int hashCode() {
34
		return Objects.hash(id, type);
35
	}
36

  
37
	@Override
38
	public boolean equals(final Object obj) {
39
		if (this == obj) { return true; }
40
		if (obj == null) { return false; }
41
		if (!(obj instanceof OtherIdentifier)) { return false; }
42
		final OtherIdentifier other = (OtherIdentifier) obj;
43
		return Objects.equals(id, other.id) && Objects.equals(type, other.type);
44
	}
45

  
46
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/model/Relation.java
1
package eu.dnetlib.organizations.model;
2

  
3
import java.io.Serializable;
4
import java.util.Objects;
5

  
6
public class Relation implements Serializable {
7

  
8
	/**
9
	 *
10
	 */
11
	private static final long serialVersionUID = -177324052187645001L;
12

  
13
	private String relatedOrgId;
14
	private String relatedOrgName;
15
	private String type;
16

  
17
	public String getRelatedOrgId() {
18
		return relatedOrgId;
19
	}
20

  
21
	public void setRelatedOrgId(final String relatedOrgId) {
22
		this.relatedOrgId = relatedOrgId;
23
	}
24

  
25
	public String getRelatedOrgName() {
26
		return relatedOrgName;
27
	}
28

  
29
	public void setRelatedOrgName(final String relatedOrgName) {
30
		this.relatedOrgName = relatedOrgName;
31
	}
32

  
33
	public String getType() {
34
		return type;
35
	}
36

  
37
	public void setType(final String type) {
38
		this.type = type;
39
	}
40

  
41
	@Override
42
	public int hashCode() {
43
		return Objects.hash(relatedOrgId, relatedOrgName, type);
44
	}
45

  
46
	@Override
47
	public boolean equals(final Object obj) {
48
		if (this == obj) { return true; }
49
		if (obj == null) { return false; }
50
		if (!(obj instanceof Relation)) { return false; }
51
		final Relation other = (Relation) obj;
52
		return Objects.equals(relatedOrgId, other.relatedOrgId) && Objects.equals(relatedOrgName, other.relatedOrgName) && Objects.equals(type, other.type);
53
	}
54
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/controller/SwaggerController.java
1
package eu.dnetlib.organizations.controller;
2

  
3
import org.springframework.stereotype.Controller;
4
import org.springframework.web.bind.annotation.RequestMapping;
5
import org.springframework.web.bind.annotation.RequestMethod;
6

  
7
@Controller
8
public class SwaggerController {
9

  
10
	@RequestMapping(value = { "/", "/apidoc", "/api-doc", "/doc", "/swagger" }, method = RequestMethod.GET)
11
	public String apiDoc() {
12
		return "redirect:swagger-ui.html";
13
	}
14
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java
1
package eu.dnetlib.organizations.controller;
2

  
3
import org.springframework.beans.factory.annotation.Autowired;
4
import org.springframework.web.bind.annotation.RequestMapping;
5
import org.springframework.web.bind.annotation.RequestMethod;
6
import org.springframework.web.bind.annotation.RequestParam;
7
import org.springframework.web.bind.annotation.RestController;
8

  
9
import eu.dnetlib.organizations.model.OrganizationView;
10
import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository;
11

  
12
@RestController
13
@RequestMapping("/api/organizations")
14
public class OrganizationController {
15

  
16
	@Autowired
17
	private OrganizationViewRepository organizationViewRepository;
18

  
19
	@RequestMapping(value = "/get}", method = RequestMethod.GET)
20
	public OrganizationView findById(@RequestParam final String id) {
21
		return organizationViewRepository.findById(id).get();
22
	}
23

  
24
}
modules/dnet-orgs-database-application/trunk/src/main/java/eu/dnetlib/organizations/MainApplication.java
1
package eu.dnetlib.organizations;
2

  
3
import org.slf4j.Logger;
4
import org.slf4j.LoggerFactory;
5
import org.springframework.boot.SpringApplication;
6
import org.springframework.boot.autoconfigure.SpringBootApplication;
7
import org.springframework.cache.annotation.EnableCaching;
8
import org.springframework.context.annotation.Bean;
9

  
10
import springfox.documentation.builders.ApiInfoBuilder;
11
import springfox.documentation.builders.RequestHandlerSelectors;
12
import springfox.documentation.service.ApiInfo;
13
import springfox.documentation.spi.DocumentationType;
14
import springfox.documentation.spring.web.plugins.Docket;
15
import springfox.documentation.swagger2.annotations.EnableSwagger2;
16

  
17
@SpringBootApplication
18
@EnableSwagger2
19
@EnableCaching
20
public class MainApplication {
21

  
22
	private static final Logger log = LoggerFactory.getLogger(MainApplication.class);
23

  
24
	public static void main(final String[] args) {
25
		SpringApplication.run(MainApplication.class, args);
26
	}
27

  
28
	@Bean
29
	public static Docket newSwaggerDocket() {
30
		log.info("Initializing SWAGGER...");
31

  
32
		return new Docket(DocumentationType.SWAGGER_2)
33
				.select()
34
				.apis(RequestHandlerSelectors.any())
35
				.paths(p -> p.startsWith("/api/"))
36
				.build().apiInfo(new ApiInfoBuilder()
37
						.title("D-Net Organizations Service APIs")
38
						.description("APIs documentation")
39
						.version("1.1")
40
						.contact(ApiInfo.DEFAULT_CONTACT)
41
						.license("Apache 2.0")
42
						.licenseUrl("http://www.apache.org/licenses/LICENSE-2.0")
43
						.build());
44

  
45
	}
46
}
modules/dnet-orgs-database-application/trunk/src/main/resources/sql/schema.sql
65 65
	org.city,
66 66
	org.country,
67 67
	org.source,
68
	COALESCE(jsonb_agg(DISTINCT jsonb_build_object('id', oid.otherid, 'type', oid.type))                                    FILTER (WHERE oid.otherid IS NOT NULL), '[]') AS other_ids,
69
	COALESCE(jsonb_agg(DISTINCT jsonb_build_object('name',          n.name, 'lang', n.lang))                                FILTER (WHERE n.name      IS NOT NULL), '[]') AS other_names,
70
	COALESCE(jsonb_agg(DISTINCT jsonb_build_object('related_to_id', r.id2,  'type', r.reltype, 'related_to_name', ro.name)) FILTER (WHERE r.id2       IS NOT NULL), '[]') AS relations,
71
	COALESCE(jsonb_agg(DISTINCT a.acronym)                                                                                  FILTER (WHERE a.acronym   IS NOT NULL), '[]') AS acronyms,	
72
	COALESCE(jsonb_agg(DISTINCT t.type)                                                                                     FILTER (WHERE t.type      IS NOT NULL), '[]') AS types,
73
	COALESCE(jsonb_agg(DISTINCT u.url)                                                                                      FILTER (WHERE u.url       IS NOT NULL), '[]') AS urls
68
	COALESCE(jsonb_agg(DISTINCT jsonb_build_object('id', oid.otherid, 'type', oid.type))                                   FILTER (WHERE oid.otherid IS NOT NULL), '[]') AS other_ids,
69
	COALESCE(jsonb_agg(DISTINCT jsonb_build_object('name',          n.name, 'lang', n.lang))                               FILTER (WHERE n.name      IS NOT NULL), '[]') AS other_names,
70
	COALESCE(jsonb_agg(DISTINCT jsonb_build_object('relatedOrgId',  r.id2,  'type', r.reltype, 'relatedOrgName', ro.name)) FILTER (WHERE r.id2       IS NOT NULL), '[]') AS relations,
71
	COALESCE(jsonb_agg(DISTINCT a.acronym)                                                                                 FILTER (WHERE a.acronym   IS NOT NULL), '[]') AS acronyms,	
72
	COALESCE(jsonb_agg(DISTINCT t.type)                                                                                    FILTER (WHERE t.type      IS NOT NULL), '[]') AS types,
73
	COALESCE(jsonb_agg(DISTINCT u.url)                                                                                     FILTER (WHERE u.url       IS NOT NULL), '[]') AS urls
74 74
FROM
75 75
    organizations org
76 76
    LEFT OUTER JOIN other_ids oid    ON (org.id = oid.id)
modules/dnet-orgs-database-application/trunk/src/main/resources/application.properties
1 1
spring.main.banner-mode = off
2
logging.level.root      = WARN
2
logging.level.root      = INFO
3

  
4
spring.datasource.url=jdbc:postgresql://localhost:5432/dnet_orgs
5
spring.datasource.username=
6
spring.datasource.password=
7

  
8
spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect
9

  
10
# Hibernate ddl auto (create, create-drop, validate, update)
11
spring.jpa.hibernate.ddl-auto = validate
12
spring.jpa.properties.hibernate.hbm2dll.extra_physical_table_types = MATERIALIZED VIEW
13
spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff