Project

General

Profile

« Previous | Next » 

Revision 49960

1. Change storeRepository method
2. Update converter of repository interface with extra fields.

View differences:

RepositoryApiImpl.java
1 1
package eu.dnetlib.repo.manager.service.controllers;
2 2

  
3
import com.fasterxml.jackson.databind.ObjectMapper;
3 4
import eu.dnetlib.domain.enabling.Vocabulary;
4 5
import eu.dnetlib.repo.manager.service.utils.Converter;
5 6
import eu.dnetlib.domain.data.Repository;
......
20 21
import org.springframework.web.util.UriComponents;
21 22
import org.springframework.web.util.UriComponentsBuilder;
22 23
import javax.annotation.PostConstruct;
24
import java.sql.Timestamp;
23 25
import java.util.*;
24 26
import java.util.concurrent.ConcurrentHashMap;
25 27
import eu.dnetlib.repo.manager.shared.*;
......
41 43

  
42 44
    private Map<String, Vocabulary> vocabularyMap = new ConcurrentHashMap<String, Vocabulary>();
43 45

  
44
    private Map<String,String> countriesMap = new HashMap<>();
45
    private Map<String,String> inverseCountriesMap = new HashMap<>();
46
    private Map<String, String> countriesMap = new HashMap<>();
47
    private Map<String, String> inverseCountriesMap = new HashMap<>();
46 48

  
47 49

  
50
    //String updateQuery = "UPDATE datasources SET englishname = '" + repo.getEnglishName() + "', logourl = '" + repo.getLogoUrl() + "', timezone = '" + repo.getTimezone() + "', registeredby = '" + repo.getRegisteredBy() + "', activationid = '" + repo.getActivationId() + "', contactemail = '" + repo.getContactEmail() + "', datasourceclass = '" + repo.getDatasourceClass() + "' WHERE id = '" + repo.getId() + "'";
51

  
52

  
48 53
    @PostConstruct
49
    private void init(){
54
    private void init() {
50 55
        LOGGER.debug("Initialization method of repository api!");
51 56

  
52 57
        restTemplate = new RestTemplate();
......
57 62
        }
58 63

  
59 64
        Country[] countries = getCountries();
60
        for(Country c: countries){
61
            countriesMap.put(c.getName(),c.getCode());
62
            inverseCountriesMap.put(c.getCode(),c.getName());
65
        for (Country c : countries) {
66
            countriesMap.put(c.getName(), c.getCode());
67
            inverseCountriesMap.put(c.getCode(), c.getName());
63 68
        }
64 69

  
65 70

  
......
74 79
        UriComponents uriComponents = UriComponentsBuilder
75 80
                .fromHttpUrl(baseAddress + "/ds/list/")
76 81
                .path("/{page}/{size}/")
77
                .build().expand(page,size).encode();
82
                .build().expand(page, size).encode();
78 83

  
79 84
        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
80 85
        List<String> ids = new ArrayList<>();
81
        while(!rs.equals("[]")){
82
            
86
        while (!rs.equals("[]")) {
87

  
83 88
            ids.addAll(getIdsWithNonEmptyAggregations(rs));
84 89

  
85
            LOGGER.debug("Checked " + (page+1)*size + " records!");
90
            LOGGER.debug("Checked " + (page + 1) * size + " records!");
86 91

  
87
            page+=1;
92
            page += 1;
88 93
            uriComponents = UriComponentsBuilder
89 94
                    .fromHttpUrl(baseAddress + "/ds/list/")
90 95
                    .path("/{page}/{size}/")
91
                    .build().expand(page,size).encode();
96
                    .build().expand(page, size).encode();
92 97
            rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
93 98
        }
94
        
99

  
95 100
        return ids;
96 101
    }
97 102

  
......
99 104

  
100 105
        JSONArray ids = new JSONArray(rs);
101 106
        List<String> agg_ids = new ArrayList<>();
102
        for(int i=0;i<ids.length();i++){
107
        for (int i = 0; i < ids.length(); i++) {
103 108
            String id = ids.getString(i);
104 109
            Aggregations aggregations = getRepositoryAggregations(id);
105
            if(aggregations.getAggregationHistory() != null)
110
            if (aggregations.getAggregationHistory() != null)
106 111
                agg_ids.add(id);
107 112
        }
108 113
        return agg_ids;
109 114
    }
110 115

  
111 116
    @Override
112
    public Country[] getCountries()  {
117
    public Country[] getCountries() {
113 118
        UriComponents uriComponents = UriComponentsBuilder
114 119
                .fromHttpUrl(baseAddress + "/ds/countries")
115 120
                .build().encode();
116
        return restTemplate.getForObject(uriComponents.toUri(),Country[].class);
121
        return restTemplate.getForObject(uriComponents.toUri(), Country[].class);
117 122
    }
118 123

  
119 124

  
......
127 132

  
128 133
        String countryCode = countriesMap.get(country);
129 134
        String filterKey = "UNKNOWN";
130
        if(mode.equalsIgnoreCase("opendoar")) {
135
        if (mode.equalsIgnoreCase("opendoar")) {
131 136
            filterKey = "openaire____::opendoar";
132
        } else if(mode.equalsIgnoreCase("re3data")) {
137
        } else if (mode.equalsIgnoreCase("re3data")) {
133 138
            filterKey = "openaire____::re3data";
134
        } else if(mode.equalsIgnoreCase("jour_aggr")) {
139
        } else if (mode.equalsIgnoreCase("jour_aggr")) {
135 140
            filterKey = "infrastruct_::openaire";
136 141
        }
137 142

  
138 143
        LOGGER.debug("Country code equals : " + countryCode);
139 144
        LOGGER.debug("Filter mode equals : " + filterKey);
140 145
        UriComponents uriComponents = UriComponentsBuilder
141
                                            .fromHttpUrl(baseAddress + "/ds/search/country/")
142
                                            .path("/{page}/{size}/")
143
                                            .queryParam("country",countryCode)
144
                                            .build().expand(page,size).encode();
146
                .fromHttpUrl(baseAddress + "/ds/search/country/")
147
                .path("/{page}/{size}/")
148
                .queryParam("country", countryCode)
149
                .build().expand(page, size).encode();
145 150

  
146 151
        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
147 152
        List<Repository> resultSet = new ArrayList<>();
148
        while(!rs.equals("[]")){
153
        while (!rs.equals("[]")) {
149 154
            List<Repository> rep = Converter.jsonToRepositoryList(new JSONArray(rs));
150 155

  
151
            Collection<Repository> repos = this.getRepositoriesByMode(filterKey,rep);
156
            Collection<Repository> repos = this.getRepositoriesByMode(filterKey, rep);
152 157
            resultSet.addAll(repos);
153 158

  
154
            page+=1;
159
            page += 1;
155 160
            uriComponents = UriComponentsBuilder
156 161
                    .fromHttpUrl(baseAddress + "/ds/search/country/")
157 162
                    .path("/{page}/{size}/")
158
                    .queryParam("country",countryCode)
159
                    .build().expand(page,size).encode();
163
                    .queryParam("country", countryCode)
164
                    .build().expand(page, size).encode();
160 165
            rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
161 166
        }
162
        for(Repository r : resultSet)
167
        for (Repository r : resultSet)
163 168
            r.setCountryName(getCountryName(r.getCountryCode()));
164 169
        return resultSet;
165 170
    }
......
168 173
    private Collection<Repository> getRepositoriesByMode(String mode, List<Repository> rs) {
169 174

  
170 175
        List<Repository> reps = new ArrayList<>();
171
        for(Repository r : rs){
172
            if(r.getCollectedFrom().equals(mode))
176
        for (Repository r : rs) {
177
            if (r.getCollectedFrom().equals(mode))
173 178
                reps.add(r);
174 179
        }
175 180
        return reps;
......
184 189
        UriComponents uriComponents = UriComponentsBuilder
185 190
                .fromHttpUrl(baseAddress + "/ds/search/email/")
186 191
                .path("/{page}/{size}/")
187
                .queryParam("contactemail",userEmail)
188
                .build().expand(page,size).encode();
189
        
192
                .queryParam("contactemail", userEmail)
193
                .build().expand(page, size).encode();
194

  
190 195
        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
191 196
        List<Repository> repos = Converter.jsonToRepositoryList(new JSONArray(rs));
192
        for(Repository r : repos)
197
        for (Repository r : repos)
193 198
            r.setCountryName(getCountryName(r.getCountryCode()));
194 199
        return repos;
195 200
    }
......
222 227

  
223 228
        Aggregations aggregations = new Aggregations();
224 229

  
225
        try{
230
        try {
226 231
            aggregations.setAggregationHistory(Converter.getAggregationHistoryFromJson(repository));
227 232
            aggregations.setLastCollection(Converter.getLastCollectionFromJson(repository));
228 233
            aggregations.setLastTransformation(Converter.getLastTransformationFromJson(repository));
229 234
            return aggregations;
230
        }catch (JSONException e){
231
            LOGGER.debug("JSON aggregation exception " ,e);
235
        } catch (JSONException e) {
236
            LOGGER.debug("JSON aggregation exception ", e);
232 237
            throw e;
233 238
        }
234 239

  
......
241 246
        UriComponents uriComponents = UriComponentsBuilder
242 247
                .fromHttpUrl(baseAddress + "/ds/search/name/")
243 248
                .path("/{page}/{size}")
244
                .queryParam("name",name)
245
                .build().expand(page,size).encode();
249
                .queryParam("name", name)
250
                .build().expand(page, size).encode();
246 251

  
247 252
        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
248 253
        List<Repository> repos = Converter.jsonToRepositoryList(new JSONArray(rs));
249
        for(Repository r : repos)
254
        for (Repository r : repos)
250 255
            r.setCountryName(getCountryName(r.getCountryCode()));
251 256
        return repos;
252 257
    }
......
259 264
                .path("/{id}/")
260 265
                .build().expand(id).encode();
261 266

  
262
        String rs = restTemplate.getForObject(uriComponents.toUri(),String.class);
263
        return  Converter.jsonToRepositoryInterfaceObject(new JSONArray(rs).getJSONObject(0));
267
        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);
268
        return Converter.jsonToRepositoryInterfaceObject(new JSONArray(rs).getJSONObject(0));
264 269
    }
265 270

  
266 271
    @Override
267
    public void addRepository(@RequestBody Repository repository) throws JSONException {
272
    public void addRepository(@RequestBody String params) throws Exception {
268 273

  
274
        JSONObject json_params = new JSONObject(params);
275
        String datatype = json_params.getString("datatype");
276
        String json_repository = json_params.getString("repository");
277
        Repository repository = null;
278
        ObjectMapper mapper = new ObjectMapper();
279
        try {
280
            repository = mapper.readValue(json_repository, Repository.class);
281
        } catch (Exception e) {
282
            LOGGER.debug("Error parsing repository ", e);
283
            e.printStackTrace();
284
        }
285

  
286

  
287
        LOGGER.debug("storing " + datatype + " repository with id: " + repository.getId());
288
        String retMessage;
289
        try {
290
            Date utilDate = new Date();
291
            Timestamp date = new Timestamp(utilDate.getTime());
292
            if (!datatype.equalsIgnoreCase("opendoar") && !datatype.equalsIgnoreCase("re3data")) {
293
                if (datatype.equalsIgnoreCase("journal") || datatype.equalsIgnoreCase("aggregator")) {
294
                    LOGGER.debug("looking if " + datatype + " " + repository.getOfficialName() + " is already in datasources");
295
                    if (getRepositoryById(repository.getId()) != null) {
296
                        retMessage = datatype + " '" + repository.getOfficialName() + "' is already in datasources.";
297
                        repository.getInterfaces().clear();
298
                        LOGGER.debug(retMessage);
299
                    } else {
300
                        LOGGER.debug(datatype + " " + repository.getOfficialName() + " is not in datasources. Inserting..");
301
                        repository.setDateOfCollection(date);
302
                        repository.setAggregator("OPENAIRE");
303
                        try {
304
                            storeRepository(repository);
305
                        } catch (JSONException e) {
306
                            LOGGER.debug("Error parsing repository", e);
307
                            throw e;
308
                        }
309
                    }
310
                }
311
            } else {
312
                // TODO call update method for register
313
            }
314
        } catch (Exception e) {
315
            LOGGER.debug("Error on storing repository", e);
316
            throw e;
317
        }
318

  
319
        storeInterfaces(datatype,repository);
320

  
321
    }
322

  
323
    private void storeInterfaces(String datatype,Repository repo) {
324

  
325
        LOGGER.debug("Inserting Interfaces");
326
        Iterator var11 = repo.getInterfaces().iterator();
327

  
328
        while (var11.hasNext()) {
329
            RepositoryInterface iFace = (RepositoryInterface) var11.next();
330
            if (!iFace.getBaseUrl().isEmpty() && !iFace.getDesiredCompatibilityLevel().isEmpty()) {
331
                if (iFace.getId() != null && !iFace.getId().isEmpty()) {
332
                    LOGGER.debug("updating iface..");
333
                    //TODO call update base url
334
                    //((DatasourceManagerService) this.dmService.getService()).updateBaseUrl(repo.getId(), iFace.getId(), iFace.getBaseUrl());
335

  
336
                    if (!iFace.getAccessSet().isEmpty()) {
337
                        LOGGER.debug("set not empty: " + iFace.getAccessSet());
338
                        //TODO call update method for access params
339
                       // ((DatasourceManagerService) this.dmService.getService()).updateAccessParam(repo.getId(), iFace.getId(), "set", iFace.getAccessSet(), false);
340
                    } else {
341
                        //TODO call deleteAccessParamOrExtraField
342
                        //((DatasourceManagerService) this.dmService.getService()).deleteAccessParamOrExtraField(repo.getId(), iFace.getId(), "set");
343
                    }
344
                    //TODO update content description
345
                    //((DatasourceManagerService) this.dmService.getService()).updateContentDescription(repo.getId(), iFace.getId(), "metadata");
346
                    if (datatype.equals("re3data")) {
347
                        //TODO call update access params
348
                      //  ((DatasourceManagerService) this.dmService.getService()).updateAccessParam(repo.getId(), iFace.getId(), "format", "oai_datacite", false);
349
                        iFace.setAccessFormat("oai_datacite");
350
                    } else {
351
                        //TODO call update access params
352
                        //((DatasourceManagerService) this.dmService.getService()).updateAccessParam(repo.getId(), iFace.getId(), "format", "oai_dc", false);
353
                        iFace.setAccessFormat("oai_dc");
354
                    }
355

  
356
                    LOGGER.debug("updated successfully");
357
                } else {
358
                    LOGGER.debug("adding new iface..");
359
                    iFace.setContentDescription("metadata");
360
                    iFace.setCompliance("UNKNOWN");
361
                    if (datatype.equals("re3data")) {
362
                        iFace.setAccessFormat("oai_datacite");
363
                    } else {
364
                        iFace.setAccessFormat("oai_dc");
365
                    }
366

  
367
                    if (repo.getDatasourceClass() != null && !repo.getDatasourceClass().isEmpty()) {
368
                        iFace.setTypology(repo.getDatasourceClass());
369
                    } else if (datatype.equalsIgnoreCase("journal")) {
370
                        iFace.setTypology("pubsrepository::journal");
371
                    } else if (datatype.equalsIgnoreCase("aggregator")) {
372
                        iFace.setTypology("aggregator::pubsrepository::unknown");
373
                    } else if (datatype.equalsIgnoreCase("opendoar")) {
374
                        iFace.setTypology("pubsrepository::unknown");
375
                    } else if (datatype.equalsIgnoreCase("re3data")) {
376
                        iFace.setTypology("datarepository::unknown");
377
                    }
378

  
379
                    iFace.setRemovable(true);
380
                    iFace.setAccessProtocol("oai");
381
                    iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']");
382
                    iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8));
383
                    if (iFace.getAccessSet().isEmpty()) {
384
                        LOGGER.debug("set is empty: " + iFace.getAccessSet());
385
                        iFace.removeAccessSet();
386
                    }
387

  
388
                    LOGGER.debug("new ifaceId :" + iFace.getId());
389
                    try {
390
                        addRepositoryInterface(repo,iFace);
391
                        LOGGER.debug("added successfully");
392
                    } catch (JSONException e) {
393
                        LOGGER.error("error while adding iface",e);
394
                        e.printStackTrace();
395
                    }
396
                }
397
            }
398
        }
399
    }
400

  
401
    private void storeRepository(Repository repository) throws JSONException {
402

  
269 403
        LOGGER.debug("Adding repository with name : " + repository.getOfficialName());
270

  
271 404
        UriComponents uriComponents = UriComponentsBuilder
272 405
                .fromHttpUrl(baseAddress + "/ds/add/")
273 406
                .build()
274 407
                .encode();
275

  
276 408
        try {
277 409
            LOGGER.debug("Repository equals : " + Converter.repositoryObjectToJson(repository));
278 410
        }catch (Exception e){
279 411
            LOGGER.debug("JSON exception ", e);
280 412
            throw e;
281 413
        }
282

  
283 414
        restTemplate.postForObject(uriComponents.toUri(),Converter.repositoryObjectToJson(repository),String.class);
284 415
    }
285 416

  
......
293 424
    }
294 425

  
295 426
    @Override
296
    public RepositoryInterface addRepositoryInterface(RepositoryInterface iFace) throws JSONException {
427
    public RepositoryInterface addRepositoryInterface(Repository repository,RepositoryInterface iFace) throws JSONException {
297 428

  
298 429
        try {
299
            LOGGER.debug("Repository iface equals : " + Converter.repositoryInterfaceObjectToJson(iFace));
430
            LOGGER.debug("Repository iface equals : " + Converter.repositoryInterfaceObjectToJson(repository,iFace));
300 431
        }catch (Exception e){
301 432
            LOGGER.debug("JSON exception ", e);
302 433
            throw e;
......
306 437
                .fromHttpUrl(baseAddress + "/ds/api/add/")
307 438
                .build()
308 439
                .encode();
309
        String rs =  restTemplate.postForObject(uriComponents.toUri(), Converter.repositoryInterfaceObjectToJson(iFace),String.class);
440
        String rs =  restTemplate.postForObject(uriComponents.toUri(), Converter.repositoryInterfaceObjectToJson(repository,iFace),String.class);
310 441
        return Converter.jsonToRepositoryInterfaceObject(new JSONArray(rs).getJSONObject(0));
311 442
    }
312 443

  

Also available in: Unified diff