Project

General

Profile

« Previous | Next » 

Revision 58301

changes for cris validation

View differences:

modules/uoa-repository-manager-service/trunk/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java
2 2

  
3 3
import eu.dnetlib.api.functionality.ValidatorServiceException;
4 4
import eu.dnetlib.domain.data.RepositoryInterface;
5
import eu.dnetlib.domain.functionality.validator.JobForValidation;
6
import eu.dnetlib.domain.functionality.validator.Rule;
7
import eu.dnetlib.domain.functionality.validator.RuleSet;
8
import eu.dnetlib.domain.functionality.validator.StoredJob;
5
import eu.dnetlib.domain.functionality.validator.*;
9 6
import eu.dnetlib.repo.manager.domain.Constants;
10 7
import eu.dnetlib.repo.manager.domain.InterfaceInformation;
11 8
import eu.dnetlib.repo.manager.domain.ValidationServiceException;
9
import eu.dnetlib.repo.manager.utils.CrisValidatorUtils;
12 10
import eu.dnetlib.repo.manager.utils.OaiTools;
13 11
import gr.uoa.di.driver.util.ServiceLocator;
14 12
import org.apache.log4j.Logger;
......
167 165
        for (List<RuleSet> ruleSets : this.rulesetMap.values()) {
168 166
            for (RuleSet rSet : ruleSets) {
169 167
                if (rSet.getName().equalsIgnoreCase("CRIS Validation")) {
170

  
171
                }else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){
168
                    crisValidatorExecutor.submit(job.getBaseUrl(), job.getUserEmail());
169
                    return new ResponseEntity<>("OK",HttpStatus.OK);
170
                } else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){
172 171
                    ruleSet = rSet;
173 172
                    break;
174 173
                }
......
259 258
        }
260 259
        List<Job> cj = crisJobs.getJobs(user);
261 260
        for (Job job : cj) {
262
            StoredJob sj = new StoredJob();
263

  
264
            if (job.getDateFinished() != null) {
265
                sj.setEnded(job.getDateFinished().toString());
266
                sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds");
267
            } else {
268
                sj.setEnded("-");
269
                sj.setDuration("-");
270
            }
271
            sj.setStarted(job.getDateStarted().toString());
272

  
273
            sj.setUserEmail(job.getUser());
274
            sj.setCris(true);
275
            sj.setBaseUrl(job.getUrl());
276
            sj.setJobType("CRIS Validation");
277

  
278
            // TODO: status
279
            sj.setValidationStatus(job.getStatus());
280
            sj.setUsageJobStatus(job.getStatus());
281
            sj.setContentJobStatus(job.getStatus());
282

  
283
            // TODO: scores
284
//            sj.setFilteredScores();
285
            sj.setContentJobScore(job.getScore());
286
            sj.setUsageJobScore(job.getScore());
287

  
288
            jobs.add(sj);
261
            jobs.add(CrisValidatorUtils.convertJobToStoredJob(job));
289 262
        }
290 263
        /////////////////////////////////////////////////////////////////////////////////////////
291 264
        /////////////////////////////////////////////////////////////////////////////////////////
......
319 292
        return getValidationService().getJobSummary(repositoryService.getRepositoryInterface(repoId).stream().map(RepositoryInterface::getBaseUrl).collect(Collectors.toList()),limit);
320 293
    }
321 294

  
322

  
323 295
}
modules/uoa-repository-manager-service/trunk/src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java
5 5
import eu.dnetlib.domain.functionality.validator.StoredJob;
6 6
import eu.dnetlib.repo.manager.domain.Constants;
7 7
import eu.dnetlib.repo.manager.domain.JobsOfUser;
8
import eu.dnetlib.repo.manager.utils.CrisValidatorUtils;
8 9
import gr.uoa.di.driver.util.ServiceLocator;
9 10
import org.apache.log4j.Logger;
10 11
import org.eurocris.openaire.cris.validator.model.Job;
......
15 16

  
16 17
import javax.annotation.Resource;
17 18
import java.util.ArrayList;
18
import java.util.Date;
19 19
import java.util.List;
20 20
import java.util.stream.Collectors;
21 21

  
......
56 56

  
57 57
        LOGGER.debug("Getting jobs of user : " + user);
58 58
        LOGGER.debug(user + "/" + jobType + "/" + offset + "/" + dateFrom + "/" + dateTo + "/" + validationStatus + "/" + includeJobsTotal);
59

  
60
        /////////////////////////////////////////////////////////////////////////////////////////
61
        // FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
62
        /////////////////////////////////////////////////////////////////////////////////////////
63
        int crisOffset = 0;
64
        int limitWithCris = 0;
65
        if (offset.equals("0")) {
66
            limitWithCris = crisJobs.getJobs(user, validationStatus).size();
67
        } else {
68
            crisOffset = crisJobs.getJobs(user, validationStatus).size();
69
        }
70

  
71
        offset = String.valueOf(Math.max(Integer.parseInt(offset) - crisOffset, 0));
72
        limit = String.valueOf(Math.max(Integer.parseInt(limit) - limitWithCris, 0));
73
        /////////////////////////////////////////////////////////////////////////////////////////
74
        /////////////////////////////////////////////////////////////////////////////////////////
75

  
59 76
        JobsOfUser retJobs = new JobsOfUser();
60 77
        retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),
61 78
                Integer.parseInt(limit), dateFrom, dateTo, validationStatus));
......
64 81
            retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));
65 82
            retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));
66 83
            retJobs.setTotalJobsFailed(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_FAILED));
67
            retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType,Constants.VALIDATION_JOB_STATUS_ONGOING));
84
            retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_ONGOING));
68 85
        }
69 86

  
70 87
        //TODO fix status with new validator version
71
        if(retJobs.getJobs() != null){
72
            for(StoredJob job :retJobs.getJobs()){
88
        if (retJobs.getJobs() != null) {
89
            for (StoredJob job : retJobs.getJobs()) {
73 90
                if (job.getContentJobStatus().equals("ongoing") || job.getUsageJobStatus().equals("ongoing")) {
74 91
                    job.setValidationStatus("ongoing");
75 92
                } else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && job.getContentJobScore() > 50 && job.getUsageJobScore() > 50)
......
78 95
                    job.setValidationStatus("successful");
79 96
                } else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && (job.getContentJobScore() <= 50 || job.getUsageJobScore() <= 50))
80 97
                        || (job.getValidationType().equals("C") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("none") && job.getContentJobScore() <= 50)
81
                        || (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50) ) {
98
                        || (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50)) {
82 99
                    job.setValidationStatus("failed");
83 100
                }
84 101

  
85 102
            }
86 103
        }
104

  
105

  
87 106
        /////////////////////////////////////////////////////////////////////////////////////////
88 107
        // FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly //
89 108
        /////////////////////////////////////////////////////////////////////////////////////////
90
        List<StoredJob> jobs = new ArrayList<>();
109
        if (crisOffset == 0) {
110
            List<StoredJob> jobs = new ArrayList<>();
91 111

  
92
        List<Job> cj = crisJobs.getJobs(user);
93
        for (Job job : cj) {
112
            List<Job> cj = crisJobs.getJobs(user, validationStatus);
113
            for (Job job : cj) {
114
                StoredJob sj = CrisValidatorUtils.convertJobToStoredJob(job);
115
                jobs.add(sj);
94 116

  
95
            StoredJob sj = converJobToStoredJob(job);
117
//                // filter out entries based on 'validationStatus'
118
//                if ("all".equals(validationStatus)) {
119
//                    jobs.add(sj);
120
//                } else {
121
//                    if (sj.getValidationStatus().equals(validationStatus)) {
122
//                        jobs.add(sj);
123
//                    }
124
//                }
125
            }
96 126

  
97

  
98
            // filter out entries based on 'validationStatus'
99
            if ("all".equals(validationStatus)) {
100
                jobs.add(sj);
101
            } else {
102
                if (sj.getValidationStatus().equals(validationStatus)) {
103
                    jobs.add(sj);
104
                }
127
            // add to CRIS Jan jobs all other jobs
128
            if (retJobs.getJobs() != null) {
129
                jobs.addAll(retJobs.getJobs());
105 130
            }
131
            // set all jobs back to retJobs
132
            retJobs.setJobs(jobs);
106 133
        }
107 134

  
108
        // add to CRIS Jan jobs all other jobs
109
        if (retJobs.getJobs() != null) {
110
            jobs.addAll(retJobs.getJobs());
111
        }
112
        // set all jobs back to retJobs
113
        retJobs.setJobs(jobs);
114

  
115 135
        // fix number of jobs
116 136
        if (Boolean.parseBoolean(includeJobsTotal)) {
117 137
            retJobs.setTotalJobs(retJobs.getTotalJobs() + crisJobs.getJobs(user).size());
......
127 147
    }
128 148

  
129 149
    private int getJobsTotalNumberOfUser(String user, String jobType, String validationStatus) throws ValidatorServiceException {
130
        return  getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
150
        return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
131 151
    }
132 152

  
133 153
    @Override
......
135 155
                                                String jobType,
136 156
                                                String validationStatus) throws JSONException {
137 157
        LOGGER.debug("Getting job with validation status : " + validationStatus);
158

  
159
        if (jobType.equalsIgnoreCase("cris")) {
160
            return crisJobs.getJobs(user, validationStatus).size();
161
        }
162

  
138 163
        try {
139 164
            return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
140 165
        } catch (ValidatorServiceException e) {
......
160 185
            // not a good way to do it but Job id field is string
161 186
            List<Job> cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList());
162 187
            if (!cJobs.isEmpty()) {
163
                job = converJobToStoredJob(cJobs.get(0));
188
                job = CrisValidatorUtils.convertJobToStoredJob(cJobs.get(0));
164 189
            }
165 190
        }
166 191
        /////////////////////////////////////////////////////////////////////////////////////////
167 192
        /////////////////////////////////////////////////////////////////////////////////////////
168 193
        return job;
169 194
    }
170

  
171
    private StoredJob converJobToStoredJob(Job job) {
172
        StoredJob sj = new StoredJob();
173
        sj.setId(job.getId().hashCode());
174
        sj.setValidationStatus(job.getStatus());
175
        if (job.getDateFinished() != null) {
176
            sj.setEnded(job.getDateFinished().toString());
177
            sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString());
178
        } else {
179
            sj.setEnded("-");
180
            sj.setDuration("-");
181
        }
182
        sj.setStarted(job.getDateStarted().toString());
183

  
184
        sj.setUserEmail(job.getUser());
185
        sj.setCris(true);
186
        sj.setBaseUrl(job.getUrl());
187
        sj.setJobType("CRIS Validation");
188

  
189
        // TODO: status
190
        sj.setValidationStatus(job.getStatus());
191
        sj.setUsageJobStatus(job.getStatus());
192
        sj.setContentJobStatus(job.getStatus());
193

  
194
        // TODO: scores
195
//            sj.setFilteredScores();
196
        sj.setContentJobScore(job.getScore());
197
        sj.setUsageJobScore(job.getScore());
198

  
199
        sj.setValidationType("CRIS Validation");
200

  
201
        return sj;
202
    }
203

  
204 195
}
modules/uoa-repository-manager-service/trunk/src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java
1
package eu.dnetlib.repo.manager.utils;
2

  
3
import eu.dnetlib.domain.functionality.validator.JobResultEntry;
4
import eu.dnetlib.domain.functionality.validator.StoredJob;
5
import org.eurocris.openaire.cris.validator.model.Job;
6
import org.eurocris.openaire.cris.validator.model.RuleResults;
7

  
8
import java.text.DateFormat;
9
import java.text.SimpleDateFormat;
10
import java.util.ArrayList;
11
import java.util.Date;
12
import java.util.List;
13

  
14
public class CrisValidatorUtils {
15

  
16
    private static final DateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
17
    private static final DateFormat DURATION_FORMATTER = new SimpleDateFormat("HH 'hours' mm 'min' ss 'sec'");
18

  
19
    public static StoredJob convertJobToStoredJob(Job job) {
20
        StoredJob sj = new StoredJob();
21
        sj.setId(job.getId().hashCode());
22
        sj.setValidationStatus(job.getStatus());
23
        if (job.getDateFinished() != null) {
24
            sj.setEnded(DATE_FORMATTER.format(job.getDateFinished()));
25
            sj.setDuration(DURATION_FORMATTER.format(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime())));
26
        } else {
27
            sj.setEnded("-");
28
            sj.setDuration("-");
29
        }
30
        sj.setStarted(DATE_FORMATTER.format(job.getDateStarted()));
31

  
32
        sj.setUserEmail(job.getUser());
33
        sj.setCris(true);
34
        sj.setBaseUrl(job.getUrl());
35
        sj.setValidationType("CU");
36
        sj.setJobType("CRIS Validation");
37
        sj.setGuidelinesShortName("For CRIS Managers 1.1");
38
        sj.setValidationSet("-");
39

  
40
        // Set status
41
        sj.setValidationStatus(job.getStatus());
42
        sj.setUsageJobStatus(job.getUsageJobStatus());
43
        sj.setContentJobStatus(job.getContentJobStatus());
44

  
45
        // Set scores
46
//            sj.setFilteredScores(); // TODO what is this?
47
        sj.setContentJobScore(job.getContentScore());
48
        sj.setUsageJobScore(job.getUsageScore());
49

  
50
        sj.setResultEntries(CrisValidatorUtils.crisResultsToJobEntries(job));
51
        sj.setRecordsTested(job.getRecordsTested());
52

  
53
        return sj;
54
    }
55

  
56
    public static List<JobResultEntry> crisResultsToJobEntries(Job crisJob) {
57
        List<JobResultEntry> jobResultEntries = new ArrayList<>();
58
        for (RuleResults ruleResults : crisJob.getRuleResults()) {
59
            JobResultEntry jobResultEntry = new JobResultEntry();
60
            jobResultEntry.setName(ruleResults.getRuleMethodName());
61
            jobResultEntry.setRuleId(ruleResults.getRuleId());
62
            jobResultEntry.setMandatory(true);
63
            jobResultEntry.setWeight(Math.round(ruleResults.getWeight()));
64
            jobResultEntry.setType(ruleResults.getType());
65
            jobResultEntry.setHasErrors(false);
66
            if (ruleResults.getCount() == 0) {
67
                jobResultEntry.setSuccesses("-");
68
            } else {
69
                jobResultEntry.setSuccesses(ruleResults.getCount() - ruleResults.getFailed() + "/" + ruleResults.getCount());
70
            }
71
            if (ruleResults.getErrorMessages() != null && !ruleResults.getErrorMessages().isEmpty()) {
72
                jobResultEntry.setErrors(ruleResults.getErrorMessages());
73
                jobResultEntry.setHasErrors(true);
74
            }
75
            jobResultEntries.add(jobResultEntry);
76
        }
77
        return jobResultEntries;
78
    }
79

  
80
    private CrisValidatorUtils() {
81
    }
82
}

Also available in: Unified diff