Project

General

Profile

1 41200 katerina.i
package eu.dnetlib.data.claims.migration;
2
3 41791 argiro.kok
import eu.dnetlib.data.claims.migration.entity.*;
4 41213 katerina.i
import eu.dnetlib.data.claims.migration.handler.*;
5 52857 argiro.kok
import eu.dnetlib.data.claims.migration.parser.OafParser;
6 41200 katerina.i
import eu.dnetlib.data.claimsDemo.ClaimUtils;
7 47219 argiro.kok
import eu.dnetlib.data.claimsDemo.SQLStoreException;
8 49865 argiro.kok
import org.apache.log4j.BasicConfigurator;
9
import org.apache.log4j.Level;
10 47219 argiro.kok
import org.apache.log4j.Logger;
11 41200 katerina.i
import org.springframework.context.ApplicationContext;
12
import org.springframework.context.support.ClassPathXmlApplicationContext;
13 47008 argiro.kok
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
14 52857 argiro.kok
import org.xml.sax.SAXException;
15 41200 katerina.i
16 52857 argiro.kok
import javax.xml.parsers.ParserConfigurationException;
17
import javax.xml.transform.TransformerException;
18
import javax.xml.xpath.XPathExpressionException;
19 41350 argiro.kok
import java.io.*;
20 46977 argiro.kok
import java.math.BigInteger;
21
import java.security.MessageDigest;
22 41254 argiro.kok
import java.sql.Timestamp;
23
import java.util.ArrayList;
24
import java.util.Date;
25 41200 katerina.i
import java.util.List;
26
27
/**
28
 * Created by kiatrop on 4/2/2016.
29
 */
30
31
public class Migration {
32 47219 argiro.kok
    private static final Logger logger = Logger.getLogger(Migration.class);
33 41200 katerina.i
34 41350 argiro.kok
    RelationHandler relationHandler = null;
35
    ContextRelationHandler contextRelationHandler = null;
36
    ProjectHandler projectHandler = null;
37
    DMFContextHandler dmfContextHandler = null;
38
    DMFResultHandler dmfResultHandler = null;
39
    IndexResultHandler indexResultHandler = null;
40
    ExternalRecordHandler externalRecordHandler = null;
41
    ResultHandler resultHandler = null;
42 41200 katerina.i
43 41350 argiro.kok
    ClaimHandler claimHandler = null;
44
    ClaimValidation claimValidation = new ClaimValidation();
45 41254 argiro.kok
46
47 41350 argiro.kok
    public Migration() {
48 41213 katerina.i
        ApplicationContext context = new ClassPathXmlApplicationContext("eu/dnetlib/data/claims/migration/springContext-claimsDemo.xml");
49 41350 argiro.kok
        relationHandler = context.getBean(RelationHandler.class);
50
        contextRelationHandler = context.getBean(ContextRelationHandler.class);
51
        projectHandler = context.getBean(ProjectHandler.class);
52
        dmfContextHandler = context.getBean(DMFContextHandler.class);
53
        dmfResultHandler = context.getBean(DMFResultHandler.class);
54
        externalRecordHandler = context.getBean(ExternalRecordHandler.class);
55
        indexResultHandler = context.getBean(IndexResultHandler.class);
56
        resultHandler = context.getBean(ResultHandler.class);
57
        claimHandler = context.getBean(ClaimHandler.class);
58 47039 argiro.kok
        claimValidation = context.getBean(ClaimValidation.class);
59 41200 katerina.i
60
    }
61
62
63 47219 argiro.kok
    public List<Claim> createRelationsClaims() throws Exception, SQLStoreException {
64 41200 katerina.i
65 41254 argiro.kok
66 41200 katerina.i
        List<Relation> relations = relationHandler.fetchAllRelations();
67 41254 argiro.kok
        List<Claim> claims = new ArrayList<Claim>();
68 41213 katerina.i
        for (Relation relation : relations) {
69
70 41254 argiro.kok
            Claim claim= new Claim();
71
72
            claim.setUserMail(relation.getClaimedBy());
73
            claim.setDate(relation.getClaimDate());
74 41350 argiro.kok
            claim.setId(relation.getClaimId());
75 46977 argiro.kok
            System.out.println("Claim id:" + claim.getId());
76 41254 argiro.kok
            OpenaireEntity source = null;
77 41213 katerina.i
            //first work with the Source
78 41200 katerina.i
            if (relation.getSourceType().equals(ClaimUtils.PROJECT)) {
79 41621 argiro.kok
                source= buildProject(relation.getSourceId(),relation.getClaimId());
80 41254 argiro.kok
                claim.setSourceType(ClaimUtils.PROJECT);
81 41200 katerina.i
            } else if (relation.getSourceType().equals(ClaimUtils.CONTEXT)) {
82 53031 argiro.kok
                Context context = dmfContextHandler.fetchContextByIdFromDmf(relation.getSourceId(), false);
83 41254 argiro.kok
                source = context;
84
                claim.setSourceType(ClaimUtils.CONTEXT);
85 41200 katerina.i
            } else if (relation.getSourceType().equals(ClaimUtils.DATASET) || relation.getSourceType().equals(ClaimUtils.PUBLICATION)) {
86 41350 argiro.kok
                Result result = buildResult(relation.getDmf(), relation.getCollectedFrom(), relation.getSourceId(), relation.getClaimId());
87 41254 argiro.kok
                source = result;
88
                claim.setSourceType(result.getResultType());
89
            }
90
            claim.setSource(source);
91 41213 katerina.i
92 41254 argiro.kok
            Result target = null;
93
            if (relation.getTargetType().equals(ClaimUtils.DATASET) || relation.getTargetType().equals(ClaimUtils.PUBLICATION)) {
94 41350 argiro.kok
                target = buildResult(relation.getDmf(), relation.getCollectedFrom(), relation.getTargetId(), relation.getClaimId());
95 41213 katerina.i
96 41254 argiro.kok
            }
97 41350 argiro.kok
            if( target != null) {
98
                claim.setTarget(target);
99
                claim.setTargetType(target.getResultType());
100
            }
101
            if(claimValidation.validateClaim(claim)){ // is valid
102 41621 argiro.kok
//                System.out.println(claim);
103 41350 argiro.kok
                claims.add(claim);
104
            }
105 41213 katerina.i
106 41350 argiro.kok
        }
107
        return claims;
108
109
    }
110 49865 argiro.kok
    public void createAndSaveRelationsClaims(boolean save) throws Exception, SQLStoreException {
111
                    /*for stats*/
112
        List<Claim> claimsOp= new ArrayList<Claim>();
113
        List<Claim> claimsCr= new ArrayList<Claim>();
114
        List<Claim> claimsDat= new ArrayList<Claim>();
115
        List<Claim> claimsOrc= new ArrayList<Claim>();
116
        Integer targetOp=0;
117
        Integer targetCr=0;
118
        Integer targetDat=0;
119
        Integer targetOrc=0;
120
        Integer sourceRes=0;
121
        Integer sourceOp=0;
122
        Integer sourceCr=0;
123
        Integer sourceDat=0;
124
        Integer sourceOrc=0;
125
        Integer projects=0;
126
        Integer contexts=0;
127
        Integer totalClaims=0;
128
129
130
        List<Relation> relations = relationHandler.fetchAllRelations();
131
        List<Claim> claims = new ArrayList<Claim>();
132
        logger.info("Relation size: "+relations.size());
133
        for (Relation relation : relations) {
134
            try {
135
                Claim claim = new Claim();
136
137
                claim.setUserMail(relation.getClaimedBy());
138
                claim.setDate(relation.getClaimDate());
139
                claim.setId(relation.getClaimId());
140 51939 argiro.kok
                logger.info("Claim id:" + claim.getId()+" "+claim.getDate());
141 49865 argiro.kok
                OpenaireEntity source = null;
142
                //first work with the Source
143
                if (relation.getSourceType().equals(ClaimUtils.PROJECT)) {
144
                    source = buildProject(relation.getSourceId(), relation.getClaimId());
145
                    claim.setSourceType(ClaimUtils.PROJECT);
146
                } else if (relation.getSourceType().equals(ClaimUtils.CONTEXT)) {
147 53031 argiro.kok
                    Context context = dmfContextHandler.fetchContextByIdFromDmf(relation.getSourceId(), false);
148 49865 argiro.kok
                    source = context;
149
                    claim.setSourceType(ClaimUtils.CONTEXT);
150
                } else if (relation.getSourceType().equals(ClaimUtils.DATASET) || relation.getSourceType().equals(ClaimUtils.PUBLICATION)) {
151
                    Result result = buildResult(relation.getDmf(), relation.getCollectedFrom(), relation.getSourceId(), relation.getClaimId());
152
                    source = result;
153
                    claim.setSourceType(result.getResultType());
154
                }
155
156
                claim.setSource(source);
157
158
                Result target = null;
159
                if (relation.getTargetType().equals(ClaimUtils.DATASET) || relation.getTargetType().equals(ClaimUtils.PUBLICATION)) {
160
                    target = buildResult(relation.getDmf(), relation.getCollectedFrom(), relation.getTargetId(), relation.getClaimId());
161
162
                }
163
                if (target != null) {
164
                    claim.setTarget(target);
165
                    claim.setTargetType(target.getResultType());
166
                }
167
168
                if (claimValidation.validateClaim(claim)) { // is valid
169
                    if(save) {
170
                        claim = claimHandler.exportMedatataForClaim(claim);
171
                        claimHandler.saveClaim(claim);
172
                    }
173
                    /*for stats*/
174
                    if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_CROSSREF)){
175
                        targetCr++;
176
                    }else if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_DATACITE)){
177
                        targetDat++;
178
                    }else if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_ORCID)){
179
                        targetOrc++;
180
                    }else {
181
                        targetOp++;
182
                    }
183
                    if(claim.getSourceType().equals(ClaimUtils.PUBLICATION)||claim.getSourceType().equals(ClaimUtils.DATASET)){
184
                        sourceRes++;
185
                        if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_CROSSREF)){
186
                            sourceCr++;
187
                        }else if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_DATACITE)){
188
                            sourceDat++;
189
                        }else if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_ORCID)){
190
                            sourceOrc++;
191
                        }else {
192
                            sourceOp++;
193
                        }
194
                    }else if(claim.getSourceType().equals(ClaimUtils.PROJECT)) {
195
                        projects++;
196
                    }else if(claim.getSourceType().equals(ClaimUtils.CONTEXT)){
197
                        contexts++;
198
                    }
199
200 52857 argiro.kok
                }else{
201
                    claimHandler.saveOrphanClaimId(claim.getId());
202 49865 argiro.kok
                }
203
            }catch (Exception e){
204
                e.printStackTrace();
205
                logger.error("Exception: "+e.getMessage());
206
207
            }
208
        }
209
210
                    /*for stats*/
211
        System.out.println("\n\nTargets:\n\nCrossref : "+targetCr+ " DataCite :"+targetDat+" Orcid : "+targetOrc+" OpenAire : "+targetOp);
212
        System.out.println("\n\nSources:\n\nAll : "+sourceRes+" Crossref : "+sourceCr+ " DataCite :"+sourceDat+" Orcid : "+sourceOrc+" OpenAire : "+sourceOp);
213
        System.out.println("\n\nTotalClaims: "+totalClaims+" Projects: "+projects+ " Contexts :"+contexts);
214
215
216
}
217 47219 argiro.kok
    public List<Claim> createContextRelationClaims() throws Exception, SQLStoreException {
218 41350 argiro.kok
219
220
        List<ContextRelation> relations = contextRelationHandler.fetchAllConceptRelations();
221
        List<Claim> claims = new ArrayList<Claim>();
222
        for (ContextRelation relation : relations) {
223
224
            Claim claim= new Claim();
225
            claim.setId(relation.getClaimId());
226
            claim.setUserMail(relation.getClaimedBy());
227
            claim.setDate(relation.getClaimDate());
228 47008 argiro.kok
            System.out.println("Claim id:" + claim.getId());
229 41350 argiro.kok
230
            //first work with the Source
231 53031 argiro.kok
            Context context = dmfContextHandler.extractContextFromDMF(relation.getContextDmf(), false);
232 41350 argiro.kok
            claim.setSourceType(ClaimUtils.CONTEXT);
233
            claim.setSource(context);
234 41444 argiro.kok
235 41350 argiro.kok
            Result target =  buildResult(relation.getResultDmf(), relation.getCollectedFrom(), relation.getResultId(), relation.getClaimId());
236
            if( target != null){
237
                claim.setTarget(target);
238
                claim.setTargetType(target.getResultType());
239
            }
240
            if(claimValidation.validateClaim(claim)){ // is valid
241 41254 argiro.kok
                claims.add(claim);
242
            }
243 41213 katerina.i
244 41254 argiro.kok
        }
245 41350 argiro.kok
        return claims;
246 41213 katerina.i
247
248 41254 argiro.kok
    }
249 49865 argiro.kok
    public void createAndSaveContextRelationClaims(Boolean save) throws Exception, SQLStoreException {
250
                    /*for stats*/
251
        List<Claim> claimsOp= new ArrayList<Claim>();
252
        List<Claim> claimsCr= new ArrayList<Claim>();
253
        List<Claim> claimsDat= new ArrayList<Claim>();
254
        List<Claim> claimsOrc= new ArrayList<Claim>();
255
        Integer targetOp=0;
256
        Integer targetCr=0;
257
        Integer targetDat=0;
258
        Integer targetOrc=0;
259
        Integer sourceRes=0;
260
        Integer sourceOp=0;
261
        Integer sourceCr=0;
262
        Integer sourceDat=0;
263
        Integer sourceOrc=0;
264
        Integer projects=0;
265
        Integer contexts=0;
266
        Integer totalClaims=0;
267 41621 argiro.kok
268 49865 argiro.kok
        List<ContextRelation> relations = contextRelationHandler.fetchAllConceptRelations();
269
//        List<Claim> claims = new ArrayList<Claim>();
270
        for (ContextRelation relation : relations) {
271
272
            Claim claim= new Claim();
273
            claim.setId(relation.getClaimId());
274
            claim.setUserMail(relation.getClaimedBy());
275
            claim.setDate(relation.getClaimDate());
276
            logger.info("Claim id:" + claim.getId());
277
278
            //first work with the Source
279 53031 argiro.kok
            Context context = dmfContextHandler.extractContextFromDMF(relation.getContextDmf(),false);
280 49865 argiro.kok
            claim.setSourceType(ClaimUtils.CONTEXT);
281
            claim.setSource(context);
282
283
            Result target =  buildResult(relation.getResultDmf(), relation.getCollectedFrom(), relation.getResultId(), relation.getClaimId());
284
            if( target != null){
285
                claim.setTarget(target);
286
                claim.setTargetType(target.getResultType());
287
            }
288
            if(claimValidation.validateClaim(claim)){ // is valid
289
                if(save){
290
                    claim = claimHandler.exportMedatataForClaim(claim);
291
                    claimHandler.saveClaim(claim);
292
                }
293
294
                 /*for stats*/
295
                if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_CROSSREF)){
296
                    targetCr++;
297
                }else if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_DATACITE)){
298
                    targetDat++;
299
                }else if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_ORCID)){
300
                    targetOrc++;
301
                }else {
302
                    targetOp++;
303
                }
304
                if(claim.getSourceType().equals(ClaimUtils.PUBLICATION)||claim.getSourceType().equals(ClaimUtils.DATASET)){
305
                    sourceRes++;
306
                    if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_CROSSREF)){
307
                        sourceCr++;
308
                    }else if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_DATACITE)){
309
                        sourceDat++;
310
                    }else if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_ORCID)){
311
                        sourceOrc++;
312
                    }else {
313
                        sourceOp++;
314
                    }
315
                }else if(claim.getSourceType().equals(ClaimUtils.PROJECT)) {
316
                    projects++;
317
                }else if(claim.getSourceType().equals(ClaimUtils.CONTEXT)){
318
                    contexts++;
319
                }
320 52857 argiro.kok
            }else{
321
                if(save){
322
                    claimHandler.saveOrphanClaimId(claim.getId());
323
                }
324
325 49865 argiro.kok
            }
326
327
328
        }
329
330
             /*for stats*/
331
        System.out.println("\n\nTargets:\n\nCrossref : "+targetCr+ " DataCite :"+targetDat+" Orcid : "+targetOrc+" OpenAire : "+targetOp);
332
        System.out.println("\n\nSources:\n\nAll : "+sourceRes+" Crossref : "+sourceCr+ " DataCite :"+sourceDat+" Orcid : "+sourceOrc+" OpenAire : "+sourceOp);
333
        System.out.println("\n\nTotalClaims: "+totalClaims+" Projects: "+projects+ " Contexts :"+contexts);
334
335
    }
336
337 41254 argiro.kok
    /**
338 41621 argiro.kok
     *Returns a project from Openaire.
339
     * In case the project id has a "welcometrust" prefix changes it to "wt__________", before search for it in the index.
340
     *
341
     * @param projectId
342
     * @param claimId
343
     * @return Project
344
     * @throws Exception
345
     */
346 46918 argiro.kok
    public Project buildProject(String projectId, String claimId) throws Exception {
347 47219 argiro.kok
        String id = projectId;
348 41621 argiro.kok
        if(projectId != null && projectId.contains("welcometrust")){
349 47219 argiro.kok
            id=projectId.replace("welcometrust","wt__________");
350 41621 argiro.kok
        }
351 49865 argiro.kok
        Project project = projectHandler.fetchProjectByID(id, false);
352 41621 argiro.kok
        if (project == null) {
353 49865 argiro.kok
         project = projectHandler.fetchProjectByID(id, true);
354
        }
355
        if (project == null) {
356 41621 argiro.kok
            try{
357 49865 argiro.kok
                PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"missing_projects.txt", true)));
358 41621 argiro.kok
                Date date= new java.util.Date();
359 47219 argiro.kok
                out.println(new Timestamp(date.getTime())+"  ProjectId: "+id+ " ClaimId: "+claimId);
360 41621 argiro.kok
                out.close();
361
            }catch (IOException e) {
362 47219 argiro.kok
                logger.error("Couldn't write to file " + "missing_projects.txt",e);
363 41621 argiro.kok
            }
364
        }
365
        return project;
366
367
    }
368
    /**
369 41350 argiro.kok
     * *If there is a dmf, the result is extracted from the external sources (crossRef, Datacite, Orcid)
370 41254 argiro.kok
     * Otherwise Result is extracted from Openaire
371 41350 argiro.kok
     * @param dmf
372
     * @param collectedFrom
373
     * @param resultId
374
     * @param claimId
375 41254 argiro.kok
     * @return Result or null
376
     */
377 41350 argiro.kok
    private Result buildResult(String dmf, String collectedFrom, String resultId, String claimId){
378 41254 argiro.kok
        Result result = null;
379 41444 argiro.kok
        String external_id= null; // testing
380 41621 argiro.kok
        if((dmf !=null && collectedFrom != null)&& !collectedFrom.equals(ClaimUtils.COLLECTED_FROM_OPENAIRE)){
381 41213 katerina.i
382 41350 argiro.kok
            if (collectedFrom.equals(ClaimUtils.COLLECTED_FROM_CROSSREF)) {
383 41254 argiro.kok
                try {
384 41350 argiro.kok
                    String doi= dmfResultHandler.fetchDoiByDMF(dmf);
385 41444 argiro.kok
                    external_id=doi;
386 41254 argiro.kok
                    if(doi != null) {
387
                        result = externalRecordHandler.fetchResultfromCrossref(doi);
388
                    }
389
                } catch (Exception e) {
390 47219 argiro.kok
                    logger.error("Error fetching result from Crossref",e);
391 41444 argiro.kok
                }
392 41213 katerina.i
393 41350 argiro.kok
            } else if (collectedFrom.equals(ClaimUtils.COLLECTED_FROM_ORCID)) {
394 41254 argiro.kok
                try {
395 41350 argiro.kok
                    String orcidwork= dmfResultHandler.fetchOrcidWorkByDMF(dmf);
396 41444 argiro.kok
                    external_id=orcidwork;
397 41350 argiro.kok
                     if(orcidwork != null) {
398 41254 argiro.kok
                        result = externalRecordHandler.fetchResultfromOrcid(orcidwork);
399
                    }
400
                } catch (Exception e) {
401 47219 argiro.kok
                    logger.error("Error fetching result from Orcid",e);
402 41254 argiro.kok
                }
403 41350 argiro.kok
            } else if (collectedFrom.equals(ClaimUtils.COLLECTED_FROM_DATACITE)) {
404 41254 argiro.kok
                try {
405 41350 argiro.kok
                    String doi= dmfResultHandler.fetchDoiByDMF(dmf);
406 41444 argiro.kok
                    external_id=doi;
407 41254 argiro.kok
                    if(doi != null) {
408
                        result = externalRecordHandler.fetchResultfromDatacite(doi);
409
                    }
410
                } catch (Exception e) {
411 47219 argiro.kok
                    logger.error("Error fetching result from Datacite",e);
412 41254 argiro.kok
                }
413
            }
414 41449 argiro.kok
            if(result == null){
415
                //report missing external results
416 41350 argiro.kok
                try{
417 47008 argiro.kok
                    PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"missing_results_external_sources.txt", true)));
418 41350 argiro.kok
                    Date date= new java.util.Date();
419 41444 argiro.kok
                    out.println(new Timestamp(date.getTime())+"  OpenaireId: "+resultId+ " collectedFrom: "+collectedFrom +" externalId: "+external_id+" claimId: "+claimId);
420 41350 argiro.kok
                    out.close();
421
                }catch (IOException e) {
422 47219 argiro.kok
                    logger.error("Couldn't write to file " + "missing_results_external_sources.txt",e);
423 41350 argiro.kok
                }
424 41449 argiro.kok
                //give a second chance - search index
425
                result = buildOpenaireResult(resultId,claimId);
426 41450 argiro.kok
                if(result!=null) {
427 41449 argiro.kok
                    try {
428 47008 argiro.kok
                        PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"external_results_found_inOpenaire.txt", true)));
429 41449 argiro.kok
                        Date date = new java.util.Date();
430
                        out.println(new Timestamp(date.getTime()) + "  OpenaireId: " + resultId + " collectedFrom: " + collectedFrom + " externalId: " + external_id + " claimId: " + claimId);
431
                        out.close();
432
                    } catch (IOException e) {
433 47219 argiro.kok
                        logger.error("Couldn't write to file " + "external_results_found_inOpenaire.txt",e);
434 41449 argiro.kok
                    }
435
                }
436 41350 argiro.kok
437 41254 argiro.kok
            }
438 41449 argiro.kok
            if(result!=null){
439
                // if result found enriched with access rights/ embargo date from DMF
440
                try {
441
                    result.setAccessRights(dmfResultHandler.fetchAccessRights(dmf));
442
                    result.setEmbargoEndDate(dmfResultHandler.fetchEmbargoEndDateByDMF(dmf));
443
                } catch (Exception e) {
444 47219 argiro.kok
                    logger.error("Error fetching Access or Embargo end date from DMF",e);
445 41449 argiro.kok
                }
446
            }
447
        } else { //If dmf/ collecteFrom  is null or is collected from openaire search in the  index
448 41350 argiro.kok
            result = buildOpenaireResult(resultId,claimId);
449 41254 argiro.kok
        }
450 41200 katerina.i
451
452 41254 argiro.kok
        return result;
453
    }
454
455
    /**
456
     *
457 41350 argiro.kok
     * @param resultId
458
     * @param claimId
459 41254 argiro.kok
     * @return target Result from Openaire or null
460
     */
461 41350 argiro.kok
    public Result buildOpenaireResult(String resultId, String claimId  ){
462 41254 argiro.kok
463
        Result result = null;
464 41350 argiro.kok
        try {
465 49865 argiro.kok
           result = indexResultHandler.fetchPublicationById(resultId, false);
466 41350 argiro.kok
        } catch (Exception e) {
467 47219 argiro.kok
            logger.error("Error fetching result from Openaire",e);
468 49865 argiro.kok
            System.out.println("Error fetching result from Openaire");
469 41350 argiro.kok
        }
470
471 41254 argiro.kok
        //If it is not found in the index
472
        if (result == null) {
473 41450 argiro.kok
           //give a second chance as dedup
474 49865 argiro.kok
//            try {
475
//                result = indexResultHandler.fetchDedupResultById(resultId);
476
//            } catch (Exception e) {
477
//                logger.error("Error fetching dedup Result",e);
478
//            }
479
            //give a second chance in production
480 41450 argiro.kok
            try {
481 49865 argiro.kok
                result = indexResultHandler.fetchPublicationById(resultId, true);
482 41450 argiro.kok
            } catch (Exception e) {
483 49865 argiro.kok
                logger.error("Error fetching result from Openaire",e);
484
                System.out.println("Error fetching result from Openaire");
485 41254 argiro.kok
            }
486 49865 argiro.kok
            if (result == null) {
487
                try {
488
                    result = indexResultHandler.fetchDatasetById(resultId, false);
489
                } catch (Exception e) {
490
                    logger.error("Error fetching result from Openaire",e);
491
                    System.out.println("Error fetching result from Openaire");
492
                }
493
            }
494
            if (result == null) {
495
                try {
496
                    result = indexResultHandler.fetchDatasetById(resultId, true);
497
                } catch (Exception e) {
498
                    logger.error("Error fetching result from Openaire",e);
499
                    System.out.println("Error fetching result from Openaire");
500
                }
501
            }
502
            if (result == null) {
503
                try {
504
                    result = indexResultHandler.fetchDedupById(resultId, false);
505
                } catch (Exception e) {
506
                    logger.error("Error fetching result from Openaire",e);
507
                    System.out.println("Error fetching result from Openaire");
508
                }
509
            }
510 52857 argiro.kok
            if (result == null) {
511
                try {
512
                    result = indexResultHandler.fetchDedupById(resultId, true);
513
                } catch (Exception e) {
514
                    logger.error("Error fetching result from Openaire",e);
515
                    System.out.println("Error fetching result from Openaire");
516
                }
517
            }
518 41450 argiro.kok
            if (result != null) {
519
                try {
520 47008 argiro.kok
                    PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"dedup_results_found.txt", true)));
521 41450 argiro.kok
                    Date date = new java.util.Date();
522
                    out.println(new Timestamp(date.getTime()) + "  OpenaireId: " + resultId + " claimId: " + claimId);
523
                    out.close();
524
                } catch (IOException e) {
525 47219 argiro.kok
                    logger.error("Couldn't write to file " + "external_results_found_inOpenaire.txt",e);
526 41450 argiro.kok
                }
527
            }else {
528
                //report it as missing
529
                try {
530 47008 argiro.kok
                    PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"missing_results.txt", true)));
531 41450 argiro.kok
                    Date date = new java.util.Date();
532
                    out.println(new Timestamp(date.getTime()) + "  OpenaireId: " + resultId + " ClaimId: " + claimId);
533
                    out.close();
534
                } catch (IOException e) {
535 47219 argiro.kok
                    logger.error("Couldn't write to file " + "missing_results.txt",e);
536 41450 argiro.kok
                }
537
            }
538
539 41350 argiro.kok
        }else{
540
            //FOUND report results that came from datacite but are publications
541
            //TODO deal with them as externals?
542
543
            if(result.getProvenanceaction() != null && result.getProvenanceaction().equals("user:claim:datacite")&&result.getOai()!=null){
544
                try{
545 47008 argiro.kok
                    PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"datacite_claim_results.txt", true)));
546 41350 argiro.kok
                    Date date= new java.util.Date();
547 41444 argiro.kok
                    out.println(new Timestamp(date.getTime())+"  openaireId:"+resultId+" aoi: " +result.getOai() +" provenanceactions: "+result.getProvenanceaction()+" type: "+result.getResultType()+ " ClaimId: "+claimId);
548 41350 argiro.kok
                    out.close();
549
                    try {
550
                        result= externalRecordHandler.fetchResultfromDatacite(result.getOai());
551
                        result.setOpenaireId(resultId);
552
                    }catch(Exception e){
553 47219 argiro.kok
                        logger.error("Couldn't get Result from Datacite "+result.getOai(),e);
554 41350 argiro.kok
                    }
555
                }catch (IOException e) {
556 47219 argiro.kok
                    logger.error("Couldn't write to file " + "datacite_claim_results.txt",e);
557 41350 argiro.kok
                }
558
559
            }
560 41200 katerina.i
        }
561
562 41254 argiro.kok
        return result;
563 41200 katerina.i
    }
564
565 41350 argiro.kok
566
    /**
567
     * For each claim of the list
568
     * if there are results in the relation exports their metadaa file
569
     * Save the claim in DB
570
     * @param claims
571
     * @throws Exception
572
     */
573 47219 argiro.kok
    private void saveClaims(List<Claim> claims) throws Exception, SQLStoreException {
574 41350 argiro.kok
        for(Claim claim : claims){
575 41791 argiro.kok
           /* if(claim.getTargetType().equals(ClaimUtils.DATASET)||claim.getTargetType().equals(ClaimUtils.PUBLICATION)){
576 41350 argiro.kok
                String path = resultHandler.exportMetadataFileForResult((Result)claim.getTarget());
577 41408 argiro.kok
                ((Result) claim.getTarget()).setRecordPath(path);
578 41350 argiro.kok
            }
579
            if(claim.getSourceType().equals(ClaimUtils.DATASET)||claim.getSourceType().equals(ClaimUtils.PUBLICATION)){
580
                String path = resultHandler.exportMetadataFileForResult((Result)claim.getSource());
581 41408 argiro.kok
                ((Result) claim.getSource()).setRecordPath(path);
582 41791 argiro.kok
            }*/
583
            claim = claimHandler.exportMedatataForClaim(claim);
584 41350 argiro.kok
            claimHandler.saveClaim(claim);
585
        }
586
587
    }
588
589 41791 argiro.kok
590
591 47008 argiro.kok
    public static void main(String[] args) throws IOException {
592 49865 argiro.kok
        BasicConfigurator.configure();
593
        logger.setLevel(Level.DEBUG);
594 41350 argiro.kok
        Migration migration = new Migration();
595
        try {
596 49865 argiro.kok
//            migration.claimHandler.getQueryGenerator().setMigrationTable("claims_view");
597
//            migration.createAndSaveRelationsClaims(true);
598
//            migration.createAndSaveContextRelationClaims(true);
599 41444 argiro.kok
600 52857 argiro.kok
            migration.claimHandler.getQueryGenerator().setMigrationTable("claims_clean");
601 49865 argiro.kok
            migration.createAndSaveRelationsClaims(true);
602
            migration.createAndSaveContextRelationClaims(true);
603 41350 argiro.kok
604 49865 argiro.kok
605 41350 argiro.kok
        } catch (Exception e) {
606 47219 argiro.kok
            logger.error("Error in migration",e);
607
        } catch (SQLStoreException e) {
608 41350 argiro.kok
            e.printStackTrace();
609
        }
610 47008 argiro.kok
611 41200 katerina.i
    }
612 41350 argiro.kok
    public void printStatistics(List<Claim> claims){
613 41254 argiro.kok
614 41350 argiro.kok
        //testing
615
        List<Claim> claimsOp= new ArrayList<Claim>();
616
        List<Claim> claimsCr= new ArrayList<Claim>();
617
        List<Claim> claimsDat= new ArrayList<Claim>();
618
        List<Claim> claimsOrc= new ArrayList<Claim>();
619
        Integer targetOp=0;
620
        Integer targetCr=0;
621
        Integer targetDat=0;
622
        Integer targetOrc=0;
623
        Integer sourceRes=0;
624
        Integer sourceOp=0;
625
        Integer sourceCr=0;
626
        Integer sourceDat=0;
627
        Integer sourceOrc=0;
628
        Integer projects=0;
629
        Integer contexts=0;
630
        Integer totalClaims=0;
631
        for(Claim claim: claims){
632 41444 argiro.kok
//            System.out.println(claim.toString());
633 41350 argiro.kok
            if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_CROSSREF)){
634
                targetCr++;
635
            }else if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_DATACITE)){
636
                targetDat++;
637
            }else if(((Result)claim.getTarget()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_ORCID)){
638
                targetOrc++;
639
            }else {
640
                targetOp++;
641
            }
642
            if(claim.getSourceType().equals(ClaimUtils.PUBLICATION)||claim.getSourceType().equals(ClaimUtils.DATASET)){
643
                sourceRes++;
644
                if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_CROSSREF)){
645
                    sourceCr++;
646
                }else if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_DATACITE)){
647
                    sourceDat++;
648
                }else if(((Result)claim.getSource()).getCollectedFrom().equals(ClaimUtils.COLLECTED_FROM_ORCID)){
649
                    sourceOrc++;
650
                }else {
651
                    sourceOp++;
652
                }
653
            }else if(claim.getSourceType().equals(ClaimUtils.PROJECT)) {
654
                projects++;
655
            }else if(claim.getSourceType().equals(ClaimUtils.CONTEXT)){
656
                contexts++;
657
            }
658
        }
659
        totalClaims=claims.size();
660
        System.out.println("\n\nTargets:\n\nCrossref : "+targetCr+ " DataCite :"+targetDat+" Orcid : "+targetOrc+" OpenAire : "+targetOp);
661
        System.out.println("\n\nSources:\n\nAll : "+sourceRes+" Crossref : "+sourceCr+ " DataCite :"+sourceDat+" Orcid : "+sourceOrc+" OpenAire : "+sourceOp);
662
        System.out.println("\n\nTotalClaims: "+totalClaims+" Projects: "+projects+ " Contexts :"+contexts);
663 41254 argiro.kok
664 41350 argiro.kok
    }
665
666
667 41200 katerina.i
}