Project

General

Profile

« Previous | Next » 

Revision 63332

add new property and parsing methods for new json API responses

View differences:

modules/uoa-claims/trunk/pom.xml
93 93
			<artifactId>json-simple</artifactId>
94 94
			<version>1.1</version>
95 95
		</dependency>
96
		<dependency>
97
			<groupId>com.fasterxml.jackson.core</groupId>
98
			<artifactId>jackson-databind</artifactId>
99
			<version>2.8.2</version>
100
			<scope>compile</scope>
101
		</dependency>
96 102
	</dependencies>
97 103

  
98 104
	<!--3d answer: https://stackoverflow.com/questions/23260057/the-forked-vm-terminated-without-saying-properly-goodbye-vm-crash-or-system-exi-->
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/handler/ClaimHandler.java
235 235
                return result;
236 236
            }else if (collectedFrom.equals(ClaimUtils.OPENAIRE)){
237 237
                Result result = null;
238
                if(type.equals(ClaimUtils.PUBLICATION)){
239
                    result = indexResultHandler.fetchPublicationById(id);
238
                if(indexResultHandler.searchUtils.getNewJsonSearchAPIUrl() != null){
239
                    result = indexResultHandler.fetchResultById(id, type);
240
                }else {
241
                    if (type.equals(ClaimUtils.PUBLICATION)) {
242
                        result = indexResultHandler.fetchPublicationById(id);
240 243

  
241
                }else if(type.equals(ClaimUtils.DATASET)){
242
                    result = indexResultHandler.fetchDatasetById(id);
243
                }else if(type.equals(ClaimUtils.SOFTWARE)){
244
                    result = indexResultHandler.fetchSoftwareById(id);
245
                }else if(type.equals(ClaimUtils.OTHER)){
246
                    result = indexResultHandler.fetchOtherById(id);
244
                    } else if (type.equals(ClaimUtils.DATASET)) {
245
                        result = indexResultHandler.fetchDatasetById(id);
246
                    } else if (type.equals(ClaimUtils.SOFTWARE)) {
247
                        result = indexResultHandler.fetchSoftwareById(id);
248
                    } else if (type.equals(ClaimUtils.OTHER)) {
249
                        result = indexResultHandler.fetchOtherById(id);
250
                    }
247 251
                }
248 252
                if(result == null){
249 253
                    logger.error("Record with id:"+id + " and type " + type + " couldn't be fetched from openaire.");
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/handler/IndexResultHandler.java
2 2

  
3 3
import eu.dnetlib.data.claims.entity.Result;
4 4
import eu.dnetlib.data.claims.parser.OafParser;
5
import eu.dnetlib.data.claims.parser.OpenAIRERecordParser;
6
import eu.dnetlib.data.claims.utils.ClaimUtils;
5 7
import eu.dnetlib.data.claims.utils.SearchUtils;
6 8
import org.apache.logging.log4j.LogManager;
7 9
import org.apache.logging.log4j.Logger;
......
12 14
public class IndexResultHandler {
13 15

  
14 16
    SearchUtils searchUtils;
17
    OpenAIRERecordParser openAIRERecordParser = null;
18

  
15 19
    private Logger log = LogManager.getLogger(this.getClass());
16

  
20
    public Result fetchResultById(String id, String type) throws Exception {
21
        return openAIRERecordParser.parse2Result(searchUtils.fetchResultById(id, type));
22
    }
17 23
    private String fetchPublicationResult(String id) throws Exception {
18 24
        return searchUtils.fetchPublicationXmlFromIndex(id);
19 25
    }
......
99 105
    public void setSearchUtils(SearchUtils searchUtils) {
100 106
        this.searchUtils = searchUtils;
101 107
    }
108

  
109
    public OpenAIRERecordParser getOpenAIRERecordParser() {
110
        return openAIRERecordParser;
111
    }
112

  
113
    public void setOpenAIRERecordParser(OpenAIRERecordParser openAIRERecordParser) {
114
        this.openAIRERecordParser = openAIRERecordParser;
115
    }
102 116
}
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/handler/OrganizationHandler.java
2 2

  
3 3
import eu.dnetlib.data.claims.entity.Organization;
4 4
import eu.dnetlib.data.claims.parser.OafParser;
5
import eu.dnetlib.data.claims.parser.OpenAIRERecordParser;
5 6
import eu.dnetlib.data.claims.sql.SqlDAO;
6 7
import eu.dnetlib.data.claims.utils.QueryGenerator;
7 8
import eu.dnetlib.data.claims.utils.SearchUtils;
......
12 13
public class OrganizationHandler {
13 14
    SearchUtils searchUtils = null;
14 15
    OafParser oafParser = null;
16
    OpenAIRERecordParser openAIRERecordParser = null;
15 17
    SqlDAO sqlDAO = null;
16 18
    QueryGenerator queryGenerator = null;
17 19

  
......
22 24
    }
23 25

  
24 26
    public Organization fetchOrganizationByID(String id) throws Exception {
27
        if(searchUtils.getNewJsonSearchAPIUrl() != null){
28
            return openAIRERecordParser.parse2Organization(searchUtils.fetchOrganizationById(id));
29
        }
25 30
        return fetchOrganizationByOaf(fetchOrganizationById(id));
26 31
    }
27 32

  
......
65 70
        this.sqlDAO = sqlDAO;
66 71
    }
67 72

  
73
    public OpenAIRERecordParser getOpenAIRERecordParser() {
74
        return openAIRERecordParser;
75
    }
68 76

  
77
    public void setOpenAIRERecordParser(OpenAIRERecordParser openAIRERecordParser) {
78
        this.openAIRERecordParser = openAIRERecordParser;
79
    }
69 80
}
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/handler/ProjectHandler.java
2 2

  
3 3
import eu.dnetlib.data.claims.entity.Project;
4 4
import eu.dnetlib.data.claims.parser.OafParser;
5
import eu.dnetlib.data.claims.parser.OpenAIRERecordParser;
5 6
import eu.dnetlib.data.claims.sql.SQLStoreException;
6 7
import eu.dnetlib.data.claims.sql.SqlDAO;
7 8
import eu.dnetlib.data.claims.utils.QueryGenerator;
......
18 19
public class ProjectHandler {
19 20
    SearchUtils searchUtils = null;
20 21
    OafParser oafParser = null;
22
    OpenAIRERecordParser openAIRERecordParser = null;
21 23
    SqlDAO sqlDAO = null;
22 24
    QueryGenerator queryGenerator = null;
23 25

  
......
28 30
    }
29 31

  
30 32
    public Project fetchProjectByID(String id) throws Exception {
33
        if(searchUtils.getNewJsonSearchAPIUrl() != null) {
34
            return openAIRERecordParser.parse2Project(searchUtils.fetchProjectById(id));
35
        }
31 36
        return fetchProjectByOaf(fetchProjectById(id));
32 37
    }
33 38

  
......
85 90
        this.sqlDAO = sqlDAO;
86 91
    }
87 92

  
93
    public OpenAIRERecordParser getOpenAIRERecordParser() {
94
        return openAIRERecordParser;
95
    }
88 96

  
97
    public void setOpenAIRERecordParser(OpenAIRERecordParser openAIRERecordParser) {
98
        this.openAIRERecordParser = openAIRERecordParser;
99
    }
89 100
}
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/parser/OpenAIRERecordParser.java
1
package eu.dnetlib.data.claims.parser;
2

  
3
import com.fasterxml.jackson.databind.JsonNode;
4
import com.fasterxml.jackson.databind.ObjectMapper;
5
import eu.dnetlib.data.claims.entity.Organization;
6
import eu.dnetlib.data.claims.entity.Project;
7
import eu.dnetlib.data.claims.entity.Result;
8
import eu.dnetlib.data.claims.utils.ClaimUtils;
9
import org.apache.logging.log4j.LogManager;
10
import org.apache.logging.log4j.Logger;
11

  
12
import java.util.Optional;
13
import java.util.stream.StreamSupport;
14

  
15

  
16
public class OpenAIRERecordParser {
17

  
18
    /**
19
     * Gets the json response from crossref API
20
     * and returns a Result object
21
     * *
22
     *
23
     * @param json
24
     * @return Result or null
25
     */
26

  
27
    private static final Logger logger = LogManager.getLogger(OpenAIRERecordParser.class);
28

  
29
    public OpenAIRERecordParser() {
30
    }
31

  
32
    public Result parse2Result(String json) {
33
        if (json == null) {
34
            return null;
35
        }
36
        try {
37
            Result result = new Result();
38
            // Parse the JSON response into a JsonNode
39
            ObjectMapper objectMapper = new ObjectMapper();
40
            JsonNode root = objectMapper.readTree(json);
41
            // Manually map fields to your CustomPost class
42
            if (root.has("header")) {
43
                result.setOpenaireId(root.get("header").get("id").asText());
44
                result.setResultType(root.get("header").get("recordType").asText());
45
            }
46
            if (root.has("result")) {
47
                result.setAccessRights(root.get("result").has("bestaccessright") &&
48
                        root.get("result").get("bestaccessright").has("code")
49
                        ? root.get("result").get("bestaccessright").get("code").asText() : null);
50
                if (result.getAccessRights() != null && result.getAccessRights().equals("EMBARGO") && root.get("result").has("embargoenddate")) {
51
                    result.setEmbargoEndDate(root.get("result").get("embargoenddate").asText());
52
                }
53
                result.setTitle(root.get("result").has("maintitle") ? root.get("result").get("maintitle").asText() : null);
54
                if (root.get("result").has("author")) {
55
                    JsonNode authors = root.get("result").get("author");
56

  
57
                    // Iterate over the array elements
58
                    if (authors.isArray()) {
59
                        for (JsonNode authorNode : authors) {
60
                            if (authorNode.has("fullname") && authorNode.has("rank")) {
61
                                String ranking = authorNode.get("rank").asText();
62
                                String fullname = authorNode.get("fullname").asText();
63
                                result.getAuthors().put(ranking, fullname);
64
                            }
65
                        }
66
                    }
67
                }
68
            }
69
            if (root.has("pid")) {
70
                JsonNode pids = root.get("pid");
71
                if (pids.isArray()) {
72
                    for (JsonNode pidNode : pids) {
73
                        if (pidNode.has("typeCode") && pidNode.get("typeCode").asText().equals("doi") && result.getDoi() == null) {
74
                            result.setDoi(pidNode.get("value").asText());
75
                            result.setExternalUrl("http://dx.doi.org/" + result.getDoi());
76
                            break;
77
                        }
78
                    }
79
                }
80
            }
81
            if (result.getExternalUrl() == null) {
82
                if (root.has("instance")) {
83
                    JsonNode instanceNodes = root.get("instance");
84
                    Optional<JsonNode> openInstance = StreamSupport.stream(instanceNodes.spliterator(), false)
85
                            .filter(instanceNode -> instanceNode.has("accessright") && instanceNode.has("url")
86
                                    && (instanceNode.get("accessright").get("value").asText().equals("Open Access") || instanceNode.get("accessright").get("value").asText().equals("OPEN")))
87
                            .findFirst();
88
                    if (openInstance.isPresent()) {
89
                        result.setExternalUrl(openInstance.get().get("url").asText());
90
                    } else {
91
                        Optional<JsonNode> embargoInstance = StreamSupport.stream(instanceNodes.spliterator(), false)
92
                                .filter(instanceNode -> instanceNode.has("accessright") && instanceNode.has("url")
93
                                        && (instanceNode.get("accessright").get("value").asText().equals("Embargo") || instanceNode.get("accessright").get("value").asText().equals("EMBARGO")))
94
                                .findFirst();
95
                        if (embargoInstance.isPresent()) {
96
                            result.setExternalUrl(embargoInstance.get().get("url").asText());
97
                        } else {
98
                            Optional<JsonNode> closedInstance = StreamSupport.stream(instanceNodes.spliterator(), false)
99
                                    .filter(instanceNode -> instanceNode.has("accessright") && instanceNode.has("url")
100
                                            && (instanceNode.get("accessright").get("value").asText().equals("Closed Access") || instanceNode.get("accessright").get("value").asText().equals("CLOSED")))
101
                                    .findFirst();
102
                            if (embargoInstance.isPresent()) {
103
                                result.setExternalUrl(closedInstance.get().get("url").asText());
104
                            } else {
105
                                Optional<JsonNode> anyInstance = StreamSupport.stream(instanceNodes.spliterator(), false)
106
                                        .filter(instanceNode -> instanceNode.has("url"))
107
                                        .findFirst();
108
                                if (anyInstance.isPresent()) {
109
                                    result.setExternalUrl(closedInstance.get().get("url").asText());
110
                                }
111
                            }
112
                        }
113
                    }
114
                }
115
            }
116

  
117
            result.setCollectedFrom(ClaimUtils.COLLECTED_FROM_OPENAIRE);
118
            result.setRecordFormat(ClaimUtils.FORMAT_JSON);
119
            return result;
120

  
121
        } catch (Exception e) {
122
            e.printStackTrace();
123
            return null;
124
        }
125

  
126
    }
127

  
128
    public Project parse2Project(String json) {
129
        if (json == null) {
130
            return null;
131
        }
132
        try {
133
            Project project = new Project();
134
            ObjectMapper objectMapper = new ObjectMapper();
135
            JsonNode root = objectMapper.readTree(json);
136
            if (root.has("header")) {
137
                project.setOpenaireId(root.get("header").get("id").asText());
138
            }
139
            if (root.has("project")) {
140
                project.setCode(root.get("project").has("code") ? root.get("project").get("code").asText() : null);
141
                project.setName(root.get("project").has("title") ? root.get("project").get("title").asText() : null);
142
                project.setAcronym(root.get("project").has("acronym") ? root.get("project").get("acronym").asText() : null);
143
                if (root.get("project").has("funding") && root.get("project").get("funding").has("funder")) {
144
                    project.setFunderId(root.get("project").get("funding").get("funder").has("id")
145
                            ? root.get("project").get("funding").get("funder").get("id").asText() : null);
146
                    project.setFunderName(root.get("project").get("funding").get("funder").has("name")
147
                            ? root.get("project").get("funding").get("funder").get("name").asText() : null);
148
                    project.setFunderShortName(root.get("project").get("funding").get("funder").has("shortname")
149
                            ? root.get("project").get("funding").get("funder").get("shortname").asText() : project.getFunderId().split("::")[0]);
150
                }
151
            }
152

  
153
            return project;
154

  
155
        } catch (Exception e) {
156
            e.printStackTrace();
157
            return null;
158
        }
159

  
160
    }
161

  
162
    public Organization parse2Organization(String json) {
163
        if (json == null) {
164
            return null;
165
        }
166
        try {
167
            Organization organization = new Organization();
168
            // Parse the JSON response into a JsonNode
169
            ObjectMapper objectMapper = new ObjectMapper();
170
            JsonNode root = objectMapper.readTree(json);
171
            // Manually map fields to your CustomPost class
172
            if (root.has("header")) {
173
                organization.setOpenaireId(root.get("header").get("id").asText());
174
                if (root.get("header").has("originalId")) {
175
                    JsonNode originalIdNodes = root.get("header").get("originalId");
176
                    // Iterate over the array elements
177
                    if (originalIdNodes.isArray()) {
178
                        for (JsonNode originalId : originalIdNodes) {
179
                            organization.setOriginalId(originalId.asText());
180
                            break;
181
                        }
182
                    }
183
                }
184

  
185
            }
186
            if (root.has("pid")) {
187
                JsonNode pids = root.get("pid");
188

  
189
                // Iterate over the array elements
190
                if (pids.isArray()) {
191
                    for (JsonNode pidNode : pids) {
192
                        if (pidNode.has("typeCode") && pidNode.get("typeCode").asText().equals("ROR") && organization.getRORId() == null) {
193
                            organization.setRORId(pidNode.get("value").asText());
194
                        } else if (pidNode.has("typeCode") && pidNode.get("typeCode").asText().equals("GRID") && organization.getGRIDId() == null) {
195
                            organization.setGRIDId(pidNode.get("value").asText());
196
                        }
197
                    }
198
                }
199
            }
200

  
201
            if (root.has("organization")) {
202
                organization.setName(root.get("organization").has("legalname") ? root.get("organization").get("legalname").asText() : null);
203
                organization.setShortName(root.get("organization").has("legalshortname") ? root.get("organization").get("legalshortname").asText() : null);
204
                organization.setUrl(root.get("organization").has("websiteurl") ? root.get("organization").get("websiteurl").asText() : null);
205
                organization.setCountry(root.get("organization").has("country") &&
206
                        root.get("organization").get("country").has("code") ? root.get("organization").get("country").get("code").asText() : null);
207
            }
208

  
209
            return organization;
210

  
211
        } catch (Exception e) {
212
            e.printStackTrace();
213
            return null;
214
        }
215

  
216
    }
217

  
218
}
219

  
220

  
221

  
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/utils/SearchUtils.java
28 28
public class SearchUtils {
29 29
    private static final Logger logger = LogManager.getLogger(SearchUtils.class);
30 30
    public static String searchAPIUrl=null;//"http://beta.services.openaire.eu:8480/search/";
31
    public static String newJsonSearchAPIUrl=null;//"http://beta.services.openaire.eu:8480/search/";
31 32

  
32 33
    private static String crossrefUrl=null;
33 34
    private static String dataciteUrl=null;
......
39 40
    public SearchUtils(){
40 41

  
41 42
    }
43
    public String fetchResultById(String id, String type) throws Exception {
44
        String json =null;
45
        String requestUrl = newJsonSearchAPIUrl;
46
        if(type.equals(ClaimUtils.SOFTWARE) ) {
47
            requestUrl += "software/" + id;
42 48

  
49
        }else if(type.equals(ClaimUtils.DATASET)){
50
            requestUrl += "datasets/" + id;
51
        }else if (type.equals(ClaimUtils.OTHER)){
52
            requestUrl += "other/" + id;
53
        }else{
54
            requestUrl += "publications/" + id;
55
        }
56
        requestUrl +=  "?format=json";
57
        json = getRequest(requestUrl);
58
        if (json == null) {
59
            requestUrl =newJsonSearchAPIUrl +  "datasets/" + id + "?format=json";
60
            json = getRequest(requestUrl);
61
        }
62
        if(json == null){
63
            return null;
64
        }
65
        return json;
66
    }
67

  
68
    public String fetchProjectById(String id) throws Exception {
69
        String json = getRequest(newJsonSearchAPIUrl+"projects/" +id + "?format=json");
70

  
71
        if (json == null) {
72
            return null;
73
        }
74
        return json;
75
    }
76

  
77
    public String fetchOrganizationById(String id) throws Exception {
78

  
79
        String json = getRequest(newJsonSearchAPIUrl+"organizations/" +id + "?format=json");
80
        System.out.println(json);
81

  
82
        if (json == null) {
83
            return null;
84
        }
85
        return json;
86
    }
87

  
43 88
    public String fetchProjectXmlFromIndex(String id) throws Exception{
44 89
            return getRequest(getProjectApiUrl(id));
45 90
    }
......
288 333
    public static void setOrcidUrlSuffix(String orcidUrlSuffix) {
289 334
        SearchUtils.orcidUrlSuffix = orcidUrlSuffix;
290 335
    }
336

  
337
    public static String getNewJsonSearchAPIUrl() {
338
        return newJsonSearchAPIUrl;
339
    }
340

  
341
    public static void setNewJsonSearchAPIUrl(String newJsonSearchAPIUrl) {
342
        SearchUtils.newJsonSearchAPIUrl = newJsonSearchAPIUrl;
343
    }
291 344
}
292 345

  
293 346

  
modules/uoa-claims/trunk/src/main/resources/eu/dnetlib/data/claims/springContext-claims.properties
12 12
services.claims.communityAPI = https://dev-openaire.d4science.org/openaire/community/
13 13
services.claims.registryAPIForManagerEmails = http://mpagasas.di.uoa.gr:8080/dnet-role-management/admin/community/{community}?name=false
14 14
services.claims.searchAPIUrl = http://beta.services.openaire.eu:8480/search/
15
#services.claims.newJsonSearchAPIUrl = http://beta.services.openaire.eu:8480/test/rest/v2/api/
15 16
#email properties
16 17
## EMAIL SETTINGS
17 18
services.claims.mail.host = smtp.gmail.com
modules/uoa-claims/trunk/src/main/resources/eu/dnetlib/data/claims/springContext-claims.xml
39 39
    <bean id="projectHandler" class="eu.dnetlib.data.claims.handler.ProjectHandler">
40 40
        <property name="searchUtils" ref="searchUtils"/>
41 41
        <property name="oafParser" ref="oafParser"/>
42
        <property name="openAIRERecordParser" ref="openAIRERecordParser"/>
43

  
42 44
        <property name="queryGenerator" ref="queryGenerator"/>
43 45
        <property name="sqlDAO" ref="sqlDao"/>
44 46
    </bean>
45 47
    <bean id="organizationHandler" class="eu.dnetlib.data.claims.handler.OrganizationHandler">
46 48
        <property name="searchUtils" ref="searchUtils"/>
47 49
        <property name="oafParser" ref="oafParser"/>
50
        <property name="openAIRERecordParser" ref="openAIRERecordParser"/>
48 51
        <property name="queryGenerator" ref="queryGenerator"/>
49 52
        <property name="sqlDAO" ref="sqlDao"/>
50 53
    </bean>
......
68 71

  
69 72
    <bean id="indexResultHandler" class="eu.dnetlib.data.claims.handler.IndexResultHandler">
70 73
        <property name="searchUtils" ref="searchUtils"/>
74
        <property name="openAIRERecordParser" ref="openAIRERecordParser"/>
75

  
71 76
    </bean>
72 77
    <bean id="resultHandler" class="eu.dnetlib.data.claims.handler.ResultHandler">
73 78
        <property name="sqlDAO" ref="sqlDao"/>
......
83 88
    <!-- End of Handlers -->
84 89
    <!--Parsers -->
85 90
    <bean id="oafParser" class="eu.dnetlib.data.claims.parser.OafParser"></bean>
91
    <bean id="openAIRERecordParser" class="eu.dnetlib.data.claims.parser.OpenAIRERecordParser"></bean>
86 92
    <bean id="externalRecordParser" class="eu.dnetlib.data.claims.parser.ExternalRecordParser"></bean>
87 93
    <!-- end of Parsers -->
88 94
    <bean id="contextUtils" class="eu.dnetlib.data.claims.utils.ContextUtils">
89 95
        <property name="searchUtils" ref="searchUtils"/>
90 96
    </bean>
91 97
    <bean id="searchUtils" class="eu.dnetlib.data.claims.utils.SearchUtils">
98
        <property name="newJsonSearchAPIUrl" value="${services.claims.newJsonSearchAPIUrl:#{null}}" />
92 99
        <property name="searchAPIUrl" value="${services.claims.searchAPIUrl}"/>
93 100
        <property name="contextsAPIUrl" value="${services.claims.contextsAPIUrl}"/>
94 101
        <property name="crossrefUrl" value="${services.claims.crossrefUrl}"/>

Also available in: Unified diff