Project

General

Profile

« Previous | Next » 

Revision 52857

Enable claiming for ORPs
Set direct index url in properties
Migration:
save orphan Ids, claims
uncomment fetch dedup

View differences:

modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/migration/ClaimValidation.java
197 197
    }
198 198
    private boolean validateByType(OpenaireEntity entity,String type , String claimId ){
199 199
        //TODO add more types (e.g. patends) when they are available
200
        if(type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.SOFTWARE)){
200
        if(type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.SOFTWARE) ||type.equals(ClaimUtils.OTHER)){
201 201
            return validateResult((Result)entity,claimId);
202 202
        } else if(type.equals(ClaimUtils.CONTEXT)){
203 203
            return validateContext((Context)entity,claimId);
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/migration/Migration.java
2 2

  
3 3
import eu.dnetlib.data.claims.migration.entity.*;
4 4
import eu.dnetlib.data.claims.migration.handler.*;
5
import eu.dnetlib.data.claims.migration.parser.OafParser;
5 6
import eu.dnetlib.data.claimsDemo.ClaimUtils;
6 7
import eu.dnetlib.data.claimsDemo.SQLStoreException;
7 8
import org.apache.log4j.BasicConfigurator;
......
10 11
import org.springframework.context.ApplicationContext;
11 12
import org.springframework.context.support.ClassPathXmlApplicationContext;
12 13
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
14
import org.xml.sax.SAXException;
13 15

  
16
import javax.xml.parsers.ParserConfigurationException;
17
import javax.xml.transform.TransformerException;
18
import javax.xml.xpath.XPathExpressionException;
14 19
import java.io.*;
15 20
import java.math.BigInteger;
16 21
import java.security.MessageDigest;
......
192 197
                        contexts++;
193 198
                    }
194 199

  
200
                }else{
201
                    claimHandler.saveOrphanClaimId(claim.getId());
195 202
                }
196 203
            }catch (Exception e){
197 204
                e.printStackTrace();
......
310 317
                }else if(claim.getSourceType().equals(ClaimUtils.CONTEXT)){
311 318
                    contexts++;
312 319
                }
320
            }else{
321
                if(save){
322
                    claimHandler.saveOrphanClaimId(claim.getId());
323
                }
324

  
313 325
            }
314 326

  
315 327

  
......
495 507
                    System.out.println("Error fetching result from Openaire");
496 508
                }
497 509
            }
498
//            if (result == null) {
499
//                try {
500
//                    result = indexResultHandler.fetchDedupById(resultId, true);
501
//                } catch (Exception e) {
502
//                    logger.error("Error fetching result from Openaire",e);
503
//                    System.out.println("Error fetching result from Openaire");
504
//                }
505
//            }
510
            if (result == null) {
511
                try {
512
                    result = indexResultHandler.fetchDedupById(resultId, true);
513
                } catch (Exception e) {
514
                    logger.error("Error fetching result from Openaire",e);
515
                    System.out.println("Error fetching result from Openaire");
516
                }
517
            }
506 518
            if (result != null) {
507 519
                try {
508 520
                    PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(claimValidation.getPathToSaveReport()+"dedup_results_found.txt", true)));
......
585 597
//            migration.createAndSaveRelationsClaims(true);
586 598
//            migration.createAndSaveContextRelationClaims(true);
587 599

  
588
            migration.claimHandler.getQueryGenerator().setMigrationTable("last_claims");
600
            migration.claimHandler.getQueryGenerator().setMigrationTable("claims_clean");
589 601
            migration.createAndSaveRelationsClaims(true);
590 602
            migration.createAndSaveContextRelationClaims(true);
591 603

  
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/migration/handler/ContextRelationHandler.java
50 50
        }
51 51
        return relations;
52 52
    }
53
    public List<ContextRelation> fetchConceptRelationById() throws Exception, SQLStoreException {
53
    public List<ContextRelation> fetchConceptRelationById(String id) throws Exception, SQLStoreException {
54 54
        List<ContextRelation> relations = null;
55
        ResultSet rs = sqlDAO.executePreparedQuery(queryGenerator.generateSelectConceptClaimByIDEnrichedWithDMFClaimsQuery(-1,"169112"));
55
        ResultSet rs = sqlDAO.executePreparedQuery(queryGenerator.generateSelectConceptClaimByIDEnrichedWithDMFClaimsQuery(-1,id));
56 56
        relations = new ArrayList<ContextRelation>();
57 57

  
58 58
        while(rs.next()) {
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/migration/handler/ClaimHandler.java
79 79
        rs.close();
80 80
        return id;
81 81
    }
82
    // - used in Migration
83
    public void saveOrphanClaimId(String id) throws SQLStoreException, Exception {
84
        logger.info("Saving orphan claim id...");
82 85

  
86
        String query = queryGenerator.generateInsertQueryForClaimsOrphanIds(id);
87
        sqlDAO.executeUpdateQuery(query);
88
    }
89

  
83 90
    /**
84 91
     * Gets the main information for a claim relation - builds a claim object
85 92
     * exports metadata
......
191 198
            }
192 199

  
193 200
            return project;
194
        }else if (type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.SOFTWARE) ){
201
        }else if (type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.SOFTWARE) ||type.equals(ClaimUtils.OTHER) ){
195 202
            ExternalRecordHandler externalRecordHandler = new ExternalRecordHandler();
196 203
            if(collectedFrom == null){
197 204
                return null;
......
233 240
                    result = indexResultHandler.fetchDatasetById(id,useProductionIndex);
234 241
                }else if(type.equals(ClaimUtils.SOFTWARE)){
235 242
                    result = indexResultHandler.fetchSoftwareById(id,useProductionIndex);
243
                }else if(type.equals(ClaimUtils.OTHER)){
244
                    result = indexResultHandler.fetchOtherById(id,useProductionIndex);
236 245
                }
237 246
                if(result == null){
238 247
                    logger.error("Record with id:"+id + " and type " + type + " couldn't be fetched from openaire.");
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/migration/handler/IndexResultHandler.java
17 17
    private String fetchSoftwareResult(String id, boolean production) throws Exception {
18 18
        return  searchUtils.fetchSoftwareXmlFromIndex(id, production);
19 19
    }
20
    private String fetchOtherResult(String id, boolean production) throws Exception {
21
        return  searchUtils.fetchOtherXmlFromIndex(id, production);
22
    }
20 23

  
21 24
    private String fetchDatasetResult(String id, boolean production) throws Exception {
22 25
        return searchUtils.fetchDatasetXmlFromIndex(id, production);
......
82 85
        }
83 86
        return OafParser.oaf2Result(oaf);
84 87
    }
88
    public Result fetchOtherById(String id, boolean production) throws Exception {
89

  
90
        String oaf = fetchOtherResult(id, production);
91
        if (oaf == null) {
92
            return null;
93
        }
94
        return OafParser.oaf2Result(oaf);
95
    }
85 96
//    public Result fetchDedupResultById(String id) throws Exception {
86 97
//
87 98
//        String oaf = fetchDedupPublicationResult(id);
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claims/migration/handler/FetchClaimHandler.java
425 425
        OpenaireEntity openaireEntity = null;
426 426
        if(type == null){
427 427
            openaireEntity = null;
428
        }else if(type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.SOFTWARE)) {
428
        }else if(type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.SOFTWARE)||type.equals(ClaimUtils.OTHER)) {
429 429
            openaireEntity= buildResult(rs, index);
430 430
        }else if(type.equals(ClaimUtils.PROJECT)){
431 431
            openaireEntity = buildProject(rs,index);
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claimsDemo/ClaimUtils.java
11 11
    public static final String  PUBLICATION = "publication";
12 12
    public static final String  DATASET = "dataset";
13 13
    public static final String  SOFTWARE = "software";
14
    public static final String  OTHER = "other";
14 15
    public static final String  PROJECT = "project";
15 16
    public static final String  CONTEXT = "context";
16 17

  
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claimsDemo/SearchUtils.java
88 88
         return xml;
89 89
    }
90 90

  
91

  
91
    public static String fetchOtherXmlFromIndex(String id, boolean production) throws Exception{
92
        String xml=getRequest(getOtherApiUrl(id,production));
93
        return xml;
94
    }
92 95
    /**
93 96
     *Look up in API, if there is no result,
94 97
     * consider that id is a dedup id and search for the objIdentifier id.
......
135 138
        return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/publications?openairePublicationID="+id;
136 139
    }
137 140
    public static String  getSoftwareApiUrl(String id, boolean production)  {
138
        //TODO change to beta when it's ready
139 141
        return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/software?openaireSoftwareID="+id;
140
//        return "http://rudie.di.uoa.gr:8080/dnet-functionality-services-2.0.0-SNAPSHOT/api"+"/software?openaireSoftwareID="+id;
141 142
    }
143
    public static String  getOtherApiUrl(String id, boolean production)  {
144
        return ((production)?apiUrlForResultsProduction:apiUrlForResults)+"/other?openaireOtherID="+id;
145
    }
142 146
//    public static String getDedupPublicationApiUrl(String id)  {
143 147
//
144 148
//        return apiUrlForDedupResults+"/publications?openairePublicationID="+id;
modules/uoa-claims/trunk/src/main/java/eu/dnetlib/data/claimsDemo/QueryGenerator.java
3 3
import eu.dnetlib.data.claims.migration.entity.*;
4 4

  
5 5
import java.lang.reflect.Array;
6
import java.sql.Connection;
6 7
import java.text.SimpleDateFormat;
7 8
import java.util.ArrayList;
8 9
import java.util.Date;
......
26 27
    /**
27 28
  ** Insert Queries **
28 29
  **/
30
    public String generateInsertQueryForClaimsOrphanIds(String id){
31
        return "INSERT INTO claims_orphan_ids values (" + id + ");";
32
    }
29 33
    private String generateInsertResultQuery(String openaire_id, String result_type, String title, String collected_from , String external_url, String doi, String orcidworkid, String access_rights , String embargo_end_date, String best_license, String record_path, String record_format, ArrayList<Object> params) {
30 34
        if(title!=null&&title.contains("'")){
31 35
            title=title.replace("'","''");
......
261 265
        if(type==null){
262 266
            return null;
263 267
        }
264
        if(type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.SOFTWARE)){
268
        if(type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.SOFTWARE)||type.equals(ClaimUtils.OTHER)){
265 269
            Result result=(Result)openaireEntity;
266 270
            query=generateInsertResultQuery(result.getOpenaireId(),result.getResultType(),result.getTitle(),result.getCollectedFrom(),result.getExternalUrl(),result.getDoi(), result.getOrcidworkid(), result.getAccessRights(),result.getEmbargoEndDate(),result.getBestLicense(),result.getRecordPath(), result.getRecordFormat(), params);
267 271
        }else if(type.equals(ClaimUtils.PROJECT)){
......
346 350
//            semantics="resultResult_supplement_isSupplementTo";
347 351
//        }
348 352
        //add above more specific semantics for pub,data and software
349
        if((targetType.equals(ClaimUtils.PUBLICATION)||targetType.equals(ClaimUtils.DATASET)||targetType.equals(ClaimUtils.SOFTWARE))&&(sourceType.equals(ClaimUtils.PUBLICATION)||sourceType.equals(ClaimUtils.DATASET)||sourceType.equals(ClaimUtils.SOFTWARE))){
353
        if((targetType.equals(ClaimUtils.PUBLICATION)||targetType.equals(ClaimUtils.DATASET)||targetType.equals(ClaimUtils.SOFTWARE)||targetType.equals(ClaimUtils.OTHER))&&(sourceType.equals(ClaimUtils.PUBLICATION)||sourceType.equals(ClaimUtils.DATASET)||sourceType.equals(ClaimUtils.SOFTWARE)||sourceType.equals(ClaimUtils.OTHER))){
350 354
            semantics="resultResult_relationship_isRelatedTo";
351 355
        }
352
        else if((targetType.equals(ClaimUtils.PUBLICATION)||targetType.equals(ClaimUtils.DATASET)||targetType.equals(ClaimUtils.SOFTWARE))&&(sourceType.equals(ClaimUtils.PROJECT))){
356
        else if((targetType.equals(ClaimUtils.PUBLICATION)||targetType.equals(ClaimUtils.DATASET)||targetType.equals(ClaimUtils.SOFTWARE)||targetType.equals(ClaimUtils.OTHER))&&(sourceType.equals(ClaimUtils.PROJECT))){
353 357
            semantics= "resultProject_outcome_produces";  //"resultProject_outcome_isProducedBy";
354 358
        }
355
        else if((targetType.equals(ClaimUtils.PUBLICATION)||targetType.equals(ClaimUtils.DATASET)||targetType.equals(ClaimUtils.SOFTWARE))&&(sourceType.equals(ClaimUtils.CONTEXT))){
359
        else if((targetType.equals(ClaimUtils.PUBLICATION)||targetType.equals(ClaimUtils.DATASET)||targetType.equals(ClaimUtils.SOFTWARE)||targetType.equals(ClaimUtils.OTHER))&&(sourceType.equals(ClaimUtils.CONTEXT))){
356 360
            semantics= "isRelevantTo";
357 361
        }
358 362

  
......
421 425
                "drop table if exists project;\n" +
422 426
                "drop table if exists result;\n" +
423 427
                "drop table if exists context;\n" +
428
                "drop table if exists claims_orphan_ids;\n" +
424 429
                "\n ";
425 430

  
426 431
    }
......
496 501
                "CREATE TABLE has_target_result(\n" +
497 502
                "\tclaim_id int references claim(id) NOT NULL,\n" +
498 503
                "\topenaire_id varchar(60) references result(openaire_id) NOT NULL\n" +
504
                ");\n"+
505
                "CREATE TABLE claims_orphan_ids (\n" +
506
                "    id integer "+
499 507
                ");\n"
500 508

  
501 509
                ;
......
602 610
        logger.debug(types);
603 611
        if(types == null || types.isEmpty() || (types.size() == 1 && types.get(0).isEmpty())){
604 612
            return null;
605
        }else if (types.contains(ClaimUtils.PUBLICATION) && types.contains(ClaimUtils.DATASET) && types.contains(ClaimUtils.SOFTWARE) && types.contains(ClaimUtils.PROJECT) && types.contains(ClaimUtils.CONTEXT) ){
613
        }else if (types.contains(ClaimUtils.PUBLICATION) && types.contains(ClaimUtils.DATASET) && types.contains(ClaimUtils.SOFTWARE) && types.contains(ClaimUtils.PROJECT) && types.contains(ClaimUtils.CONTEXT)  && types.contains(ClaimUtils.OTHER) ){
606 614
            //it's all types - no need to filter
607 615
            return null;
608 616
        }
......
798 806
        ArrayList<Object> clauseParams = new ArrayList<>();
799 807
        clauseParams.add(claimId);
800 808
        String orderbyLimitClause= null;
801
        if(sourceType.equals(ClaimUtils.PUBLICATION)||sourceType.equals(ClaimUtils.DATASET)||sourceType.equals(ClaimUtils.SOFTWARE)){
809
        if(sourceType.equals(ClaimUtils.PUBLICATION)||sourceType.equals(ClaimUtils.DATASET)||sourceType.equals(ClaimUtils.SOFTWARE)||sourceType.equals(ClaimUtils.OTHER)){
802 810
            return generateSelectclaimQuery(sourceType,targetType,null,null,null,false, clause,clauseParams,keyword, params);
803 811
        }else if(sourceType.equals(ClaimUtils.PROJECT)){
804 812
            return generateSelectclaimQuery(sourceType,targetType,null,null,null,false, clause,clauseParams,keyword, params);
......
825 833
        String fields= null;
826 834

  
827 835
        if (type != null) {
828
            if (type.equals(ClaimUtils.PUBLICATION) || type.equals(ClaimUtils.DATASET) || type.equals(ClaimUtils.SOFTWARE)) {
836
            if (type.equals(ClaimUtils.PUBLICATION) || type.equals(ClaimUtils.DATASET) || type.equals(ClaimUtils.SOFTWARE)||type.equals(ClaimUtils.OTHER)) {
829 837
                fields = getResultFields(tableAlias);
830 838

  
831 839
            } else if (type.equals(ClaimUtils.PROJECT)) {
......
868 876
            return "";
869 877
        }
870 878
        keyword=keyword.toLowerCase();
871
        if (type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.SOFTWARE)){
879
        if (type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET)||type.equals(ClaimUtils.SOFTWARE)||type.equals(ClaimUtils.OTHER)){
872 880
            params.add("%" + keyword + "%");
873 881
            params.add("%" + keyword + "%");
874 882
            return "  (lower("+tableAlias+".title) like ? or lower("+tableAlias+".doi) like ?"+")";
......
902 910
                " ( " +generateSelectclaimQuery(ClaimUtils.PUBLICATION,ClaimUtils.PUBLICATION,null,null,null,false, specificWhereClause,whereParams,keyword, params)+" ) \nunion "+
903 911
                " ( " +generateSelectclaimQuery(ClaimUtils.DATASET,ClaimUtils.DATASET,null,null,null,false, specificWhereClause, whereParams,keyword, params)+" ) \nunion "+
904 912
                " ( " +generateSelectclaimQuery(ClaimUtils.SOFTWARE,ClaimUtils.SOFTWARE,null,null,null,false, specificWhereClause, whereParams,keyword, params)+" ) \nunion "+
913
                " ( " +generateSelectclaimQuery(ClaimUtils.OTHER,ClaimUtils.OTHER,null,null,null,false, specificWhereClause, whereParams,keyword, params)+" ) \nunion "+
905 914
                " ( " +generateSelectclaimQuery(ClaimUtils.PROJECT,ClaimUtils.PUBLICATION, null,null,null,false, specificWhereClause, whereParams,keyword, params)+" ) \nunion "+
906 915
                " ( " +generateSelectclaimQuery(ClaimUtils.CONTEXT,ClaimUtils.PUBLICATION,null,null,null,false, specificWhereClause, whereParams, keyword, params)+" )" +
907 916

  
......
917 926
        if(type == null){
918 927
            return null;
919 928
        }
920
        if(type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET) ||type.equals(ClaimUtils.SOFTWARE)){
929
        if(type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET) ||type.equals(ClaimUtils.SOFTWARE)||type.equals(ClaimUtils.OTHER)){
921 930
            return "result";
922 931
        }else{
923 932
            return type;
......
942 951
            return null;
943 952
        }
944 953
        String query = null;
945
        if(type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET) ||type.equals(ClaimUtils.SOFTWARE)){
954
        if(type.equals(ClaimUtils.PUBLICATION)||type.equals(ClaimUtils.DATASET) ||type.equals(ClaimUtils.SOFTWARE) ||type.equals(ClaimUtils.OTHER)){
946 955
            query = generateDeleteResultQuery(resultid,claimId, params);
947 956
        }else if(type.equals(ClaimUtils.PROJECT)){
948 957
            query = generateDeleteProjectQuery(resultid,claimId, params);
modules/uoa-claims/trunk/src/main/resources/eu/dnetlib/data/claims/migration/springContext-claimsDemo.xml
20 20
        <!--<property name="locations" value="classpath:eu/dnetlib/data/claims/migration/springContext-claimsDemo.properties"/>-->
21 21
    <!--</bean>-->
22 22
    <bean id="directIndexHandler" class="eu.dnetlib.data.claims.migration.handler.DirectIndexHandler">
23
        <property name="directClaimAPIUrl"  value="http://beta.services.openaire.eu:8280/is/mvc/api/publications"/>
23
        <property name="directClaimAPIUrl"  value="${services.claims.directClaimAPIUrl}"/>
24 24
    </bean>
25 25
    <bean id="fetchClaimHandler" class="eu.dnetlib.data.claims.migration.handler.FetchClaimHandler">
26 26
        <property name="sqlDAO" ref="sqlDao"/>
modules/uoa-claims/trunk/src/main/resources/eu/dnetlib/data/claims/migration/springContext-claimsDemo.properties
1 1

  
2 2
services.claimsDemo.db.driverClassName = org.postgresql.Driver
3
services.claimsDemo.db.url = jdbc:postgresql://scoobydoo.di.uoa.gr:5432/claims_beta_explore
3
services.claimsDemo.db.url = jdbc:postgresql://scoobydoo.di.uoa.gr:5432/claims_www_migration
4 4
services.claimsDemo.db.username = postgres
5 5
services.claimsDemo.db.password = snowflakes
6 6
services.claimsDemo.directClaim.APIURL = http://beta.services.openaire.eu:8280/is/mvc/api/publications
......
15 15
services.claims.mail.angularUrl = http://duffy.di.uoa.gr:3000/claims-project-manager?token=
16 16
services.claims.mail.specialRecipients = invalid_recipient@email.com
17 17

  
18
services.claimsDemo.results.pathToSaveRecord = /home/argirok/claims/records/
19
services.claimsDemo.reports.pathToSaveReport = /home/argirok/claims/reports/
18
services.claimsDemo.results.pathToSaveRecord = /home/argirok/claims_www/records/
19
services.claimsDemo.reports.pathToSaveReport = /home/argirok/claims_www/reports/
20

  
21
#old version
22
#services.claims.directClaimAPIUrl = http://beta.services.openaire.eu:8280/is/mvc/api/publications
23
services.claims.directClaimAPIUrl = http://beta.services.openaire.eu:8980/provision/mvc/api/results

Also available in: Unified diff