Revision 58301
Added by Konstantinos Spyrou about 4 years ago
modules/uoa-repository-manager-service/trunk/src/main/java/eu/dnetlib/repo/manager/service/ValidatorServiceImpl.java | ||
---|---|---|
2 | 2 |
|
3 | 3 |
import eu.dnetlib.api.functionality.ValidatorServiceException; |
4 | 4 |
import eu.dnetlib.domain.data.RepositoryInterface; |
5 |
import eu.dnetlib.domain.functionality.validator.JobForValidation; |
|
6 |
import eu.dnetlib.domain.functionality.validator.Rule; |
|
7 |
import eu.dnetlib.domain.functionality.validator.RuleSet; |
|
8 |
import eu.dnetlib.domain.functionality.validator.StoredJob; |
|
5 |
import eu.dnetlib.domain.functionality.validator.*; |
|
9 | 6 |
import eu.dnetlib.repo.manager.domain.Constants; |
10 | 7 |
import eu.dnetlib.repo.manager.domain.InterfaceInformation; |
11 | 8 |
import eu.dnetlib.repo.manager.domain.ValidationServiceException; |
9 |
import eu.dnetlib.repo.manager.utils.CrisValidatorUtils; |
|
12 | 10 |
import eu.dnetlib.repo.manager.utils.OaiTools; |
13 | 11 |
import gr.uoa.di.driver.util.ServiceLocator; |
14 | 12 |
import org.apache.log4j.Logger; |
... | ... | |
167 | 165 |
for (List<RuleSet> ruleSets : this.rulesetMap.values()) { |
168 | 166 |
for (RuleSet rSet : ruleSets) { |
169 | 167 |
if (rSet.getName().equalsIgnoreCase("CRIS Validation")) { |
170 |
|
|
171 |
}else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){ |
|
168 |
crisValidatorExecutor.submit(job.getBaseUrl(), job.getUserEmail()); |
|
169 |
return new ResponseEntity<>("OK",HttpStatus.OK); |
|
170 |
} else if(rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())){ |
|
172 | 171 |
ruleSet = rSet; |
173 | 172 |
break; |
174 | 173 |
} |
... | ... | |
259 | 258 |
} |
260 | 259 |
List<Job> cj = crisJobs.getJobs(user); |
261 | 260 |
for (Job job : cj) { |
262 |
StoredJob sj = new StoredJob(); |
|
263 |
|
|
264 |
if (job.getDateFinished() != null) { |
|
265 |
sj.setEnded(job.getDateFinished().toString()); |
|
266 |
sj.setDuration(Long.toString(job.getDateFinished().getTime() - job.getDateStarted().getTime()) + " seconds"); |
|
267 |
} else { |
|
268 |
sj.setEnded("-"); |
|
269 |
sj.setDuration("-"); |
|
270 |
} |
|
271 |
sj.setStarted(job.getDateStarted().toString()); |
|
272 |
|
|
273 |
sj.setUserEmail(job.getUser()); |
|
274 |
sj.setCris(true); |
|
275 |
sj.setBaseUrl(job.getUrl()); |
|
276 |
sj.setJobType("CRIS Validation"); |
|
277 |
|
|
278 |
// TODO: status |
|
279 |
sj.setValidationStatus(job.getStatus()); |
|
280 |
sj.setUsageJobStatus(job.getStatus()); |
|
281 |
sj.setContentJobStatus(job.getStatus()); |
|
282 |
|
|
283 |
// TODO: scores |
|
284 |
// sj.setFilteredScores(); |
|
285 |
sj.setContentJobScore(job.getScore()); |
|
286 |
sj.setUsageJobScore(job.getScore()); |
|
287 |
|
|
288 |
jobs.add(sj); |
|
261 |
jobs.add(CrisValidatorUtils.convertJobToStoredJob(job)); |
|
289 | 262 |
} |
290 | 263 |
///////////////////////////////////////////////////////////////////////////////////////// |
291 | 264 |
///////////////////////////////////////////////////////////////////////////////////////// |
... | ... | |
319 | 292 |
return getValidationService().getJobSummary(repositoryService.getRepositoryInterface(repoId).stream().map(RepositoryInterface::getBaseUrl).collect(Collectors.toList()),limit); |
320 | 293 |
} |
321 | 294 |
|
322 |
|
|
323 | 295 |
} |
modules/uoa-repository-manager-service/trunk/src/main/java/eu/dnetlib/repo/manager/service/MonitorServiceImpl.java | ||
---|---|---|
5 | 5 |
import eu.dnetlib.domain.functionality.validator.StoredJob; |
6 | 6 |
import eu.dnetlib.repo.manager.domain.Constants; |
7 | 7 |
import eu.dnetlib.repo.manager.domain.JobsOfUser; |
8 |
import eu.dnetlib.repo.manager.utils.CrisValidatorUtils; |
|
8 | 9 |
import gr.uoa.di.driver.util.ServiceLocator; |
9 | 10 |
import org.apache.log4j.Logger; |
10 | 11 |
import org.eurocris.openaire.cris.validator.model.Job; |
... | ... | |
15 | 16 |
|
16 | 17 |
import javax.annotation.Resource; |
17 | 18 |
import java.util.ArrayList; |
18 |
import java.util.Date; |
|
19 | 19 |
import java.util.List; |
20 | 20 |
import java.util.stream.Collectors; |
21 | 21 |
|
... | ... | |
56 | 56 |
|
57 | 57 |
LOGGER.debug("Getting jobs of user : " + user); |
58 | 58 |
LOGGER.debug(user + "/" + jobType + "/" + offset + "/" + dateFrom + "/" + dateTo + "/" + validationStatus + "/" + includeJobsTotal); |
59 |
|
|
60 |
///////////////////////////////////////////////////////////////////////////////////////// |
|
61 |
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly // |
|
62 |
///////////////////////////////////////////////////////////////////////////////////////// |
|
63 |
int crisOffset = 0; |
|
64 |
int limitWithCris = 0; |
|
65 |
if (offset.equals("0")) { |
|
66 |
limitWithCris = crisJobs.getJobs(user, validationStatus).size(); |
|
67 |
} else { |
|
68 |
crisOffset = crisJobs.getJobs(user, validationStatus).size(); |
|
69 |
} |
|
70 |
|
|
71 |
offset = String.valueOf(Math.max(Integer.parseInt(offset) - crisOffset, 0)); |
|
72 |
limit = String.valueOf(Math.max(Integer.parseInt(limit) - limitWithCris, 0)); |
|
73 |
///////////////////////////////////////////////////////////////////////////////////////// |
|
74 |
///////////////////////////////////////////////////////////////////////////////////////// |
|
75 |
|
|
59 | 76 |
JobsOfUser retJobs = new JobsOfUser(); |
60 | 77 |
retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset), |
61 | 78 |
Integer.parseInt(limit), dateFrom, dateTo, validationStatus)); |
... | ... | |
64 | 81 |
retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null)); |
65 | 82 |
retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL)); |
66 | 83 |
retJobs.setTotalJobsFailed(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_FAILED)); |
67 |
retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType,Constants.VALIDATION_JOB_STATUS_ONGOING)); |
|
84 |
retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_ONGOING));
|
|
68 | 85 |
} |
69 | 86 |
|
70 | 87 |
//TODO fix status with new validator version |
71 |
if(retJobs.getJobs() != null){
|
|
72 |
for(StoredJob job :retJobs.getJobs()){
|
|
88 |
if (retJobs.getJobs() != null) {
|
|
89 |
for (StoredJob job : retJobs.getJobs()) {
|
|
73 | 90 |
if (job.getContentJobStatus().equals("ongoing") || job.getUsageJobStatus().equals("ongoing")) { |
74 | 91 |
job.setValidationStatus("ongoing"); |
75 | 92 |
} else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && job.getContentJobScore() > 50 && job.getUsageJobScore() > 50) |
... | ... | |
78 | 95 |
job.setValidationStatus("successful"); |
79 | 96 |
} else if ((job.getValidationType().equals("CU") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("finished") && (job.getContentJobScore() <= 50 || job.getUsageJobScore() <= 50)) |
80 | 97 |
|| (job.getValidationType().equals("C") && job.getContentJobStatus().equals("finished") && job.getUsageJobStatus().equals("none") && job.getContentJobScore() <= 50) |
81 |
|| (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50) ) {
|
|
98 |
|| (job.getValidationType().equals("U") && job.getContentJobStatus().equals("none") && job.getUsageJobStatus().equals("finished") && job.getUsageJobScore() <= 50)) { |
|
82 | 99 |
job.setValidationStatus("failed"); |
83 | 100 |
} |
84 | 101 |
|
85 | 102 |
} |
86 | 103 |
} |
104 |
|
|
105 |
|
|
87 | 106 |
///////////////////////////////////////////////////////////////////////////////////////// |
88 | 107 |
// FIXME: this is a hack for CRIS Jan Dvorak Validator, should be implemented properly // |
89 | 108 |
///////////////////////////////////////////////////////////////////////////////////////// |
90 |
List<StoredJob> jobs = new ArrayList<>(); |
|
109 |
if (crisOffset == 0) { |
|
110 |
List<StoredJob> jobs = new ArrayList<>(); |
|
91 | 111 |
|
92 |
List<Job> cj = crisJobs.getJobs(user); |
|
93 |
for (Job job : cj) { |
|
112 |
List<Job> cj = crisJobs.getJobs(user, validationStatus); |
|
113 |
for (Job job : cj) { |
|
114 |
StoredJob sj = CrisValidatorUtils.convertJobToStoredJob(job); |
|
115 |
jobs.add(sj); |
|
94 | 116 |
|
95 |
StoredJob sj = converJobToStoredJob(job); |
|
117 |
// // filter out entries based on 'validationStatus' |
|
118 |
// if ("all".equals(validationStatus)) { |
|
119 |
// jobs.add(sj); |
|
120 |
// } else { |
|
121 |
// if (sj.getValidationStatus().equals(validationStatus)) { |
|
122 |
// jobs.add(sj); |
|
123 |
// } |
|
124 |
// } |
|
125 |
} |
|
96 | 126 |
|
97 |
|
|
98 |
// filter out entries based on 'validationStatus' |
|
99 |
if ("all".equals(validationStatus)) { |
|
100 |
jobs.add(sj); |
|
101 |
} else { |
|
102 |
if (sj.getValidationStatus().equals(validationStatus)) { |
|
103 |
jobs.add(sj); |
|
104 |
} |
|
127 |
// add to CRIS Jan jobs all other jobs |
|
128 |
if (retJobs.getJobs() != null) { |
|
129 |
jobs.addAll(retJobs.getJobs()); |
|
105 | 130 |
} |
131 |
// set all jobs back to retJobs |
|
132 |
retJobs.setJobs(jobs); |
|
106 | 133 |
} |
107 | 134 |
|
108 |
// add to CRIS Jan jobs all other jobs |
|
109 |
if (retJobs.getJobs() != null) { |
|
110 |
jobs.addAll(retJobs.getJobs()); |
|
111 |
} |
|
112 |
// set all jobs back to retJobs |
|
113 |
retJobs.setJobs(jobs); |
|
114 |
|
|
115 | 135 |
// fix number of jobs |
116 | 136 |
if (Boolean.parseBoolean(includeJobsTotal)) { |
117 | 137 |
retJobs.setTotalJobs(retJobs.getTotalJobs() + crisJobs.getJobs(user).size()); |
... | ... | |
127 | 147 |
} |
128 | 148 |
|
129 | 149 |
private int getJobsTotalNumberOfUser(String user, String jobType, String validationStatus) throws ValidatorServiceException { |
130 |
return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);
|
|
150 |
return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus); |
|
131 | 151 |
} |
132 | 152 |
|
133 | 153 |
@Override |
... | ... | |
135 | 155 |
String jobType, |
136 | 156 |
String validationStatus) throws JSONException { |
137 | 157 |
LOGGER.debug("Getting job with validation status : " + validationStatus); |
158 |
|
|
159 |
if (jobType.equalsIgnoreCase("cris")) { |
|
160 |
return crisJobs.getJobs(user, validationStatus).size(); |
|
161 |
} |
|
162 |
|
|
138 | 163 |
try { |
139 | 164 |
return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus); |
140 | 165 |
} catch (ValidatorServiceException e) { |
... | ... | |
160 | 185 |
// not a good way to do it but Job id field is string |
161 | 186 |
List<Job> cJobs = crisJobs.getAll().stream().filter(j -> j.getId().hashCode() == Integer.parseInt(jobId)).collect(Collectors.toList()); |
162 | 187 |
if (!cJobs.isEmpty()) { |
163 |
job = converJobToStoredJob(cJobs.get(0));
|
|
188 |
job = CrisValidatorUtils.convertJobToStoredJob(cJobs.get(0));
|
|
164 | 189 |
} |
165 | 190 |
} |
166 | 191 |
///////////////////////////////////////////////////////////////////////////////////////// |
167 | 192 |
///////////////////////////////////////////////////////////////////////////////////////// |
168 | 193 |
return job; |
169 | 194 |
} |
170 |
|
|
171 |
private StoredJob converJobToStoredJob(Job job) { |
|
172 |
StoredJob sj = new StoredJob(); |
|
173 |
sj.setId(job.getId().hashCode()); |
|
174 |
sj.setValidationStatus(job.getStatus()); |
|
175 |
if (job.getDateFinished() != null) { |
|
176 |
sj.setEnded(job.getDateFinished().toString()); |
|
177 |
sj.setDuration(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()).toString()); |
|
178 |
} else { |
|
179 |
sj.setEnded("-"); |
|
180 |
sj.setDuration("-"); |
|
181 |
} |
|
182 |
sj.setStarted(job.getDateStarted().toString()); |
|
183 |
|
|
184 |
sj.setUserEmail(job.getUser()); |
|
185 |
sj.setCris(true); |
|
186 |
sj.setBaseUrl(job.getUrl()); |
|
187 |
sj.setJobType("CRIS Validation"); |
|
188 |
|
|
189 |
// TODO: status |
|
190 |
sj.setValidationStatus(job.getStatus()); |
|
191 |
sj.setUsageJobStatus(job.getStatus()); |
|
192 |
sj.setContentJobStatus(job.getStatus()); |
|
193 |
|
|
194 |
// TODO: scores |
|
195 |
// sj.setFilteredScores(); |
|
196 |
sj.setContentJobScore(job.getScore()); |
|
197 |
sj.setUsageJobScore(job.getScore()); |
|
198 |
|
|
199 |
sj.setValidationType("CRIS Validation"); |
|
200 |
|
|
201 |
return sj; |
|
202 |
} |
|
203 |
|
|
204 | 195 |
} |
modules/uoa-repository-manager-service/trunk/src/main/java/eu/dnetlib/repo/manager/utils/CrisValidatorUtils.java | ||
---|---|---|
1 |
package eu.dnetlib.repo.manager.utils; |
|
2 |
|
|
3 |
import eu.dnetlib.domain.functionality.validator.JobResultEntry; |
|
4 |
import eu.dnetlib.domain.functionality.validator.StoredJob; |
|
5 |
import org.eurocris.openaire.cris.validator.model.Job; |
|
6 |
import org.eurocris.openaire.cris.validator.model.RuleResults; |
|
7 |
|
|
8 |
import java.text.DateFormat; |
|
9 |
import java.text.SimpleDateFormat; |
|
10 |
import java.util.ArrayList; |
|
11 |
import java.util.Date; |
|
12 |
import java.util.List; |
|
13 |
|
|
14 |
public class CrisValidatorUtils { |
|
15 |
|
|
16 |
private static final DateFormat DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); |
|
17 |
private static final DateFormat DURATION_FORMATTER = new SimpleDateFormat("HH 'hours' mm 'min' ss 'sec'"); |
|
18 |
|
|
19 |
public static StoredJob convertJobToStoredJob(Job job) { |
|
20 |
StoredJob sj = new StoredJob(); |
|
21 |
sj.setId(job.getId().hashCode()); |
|
22 |
sj.setValidationStatus(job.getStatus()); |
|
23 |
if (job.getDateFinished() != null) { |
|
24 |
sj.setEnded(DATE_FORMATTER.format(job.getDateFinished())); |
|
25 |
sj.setDuration(DURATION_FORMATTER.format(new Date(job.getDateFinished().getTime() - job.getDateStarted().getTime()))); |
|
26 |
} else { |
|
27 |
sj.setEnded("-"); |
|
28 |
sj.setDuration("-"); |
|
29 |
} |
|
30 |
sj.setStarted(DATE_FORMATTER.format(job.getDateStarted())); |
|
31 |
|
|
32 |
sj.setUserEmail(job.getUser()); |
|
33 |
sj.setCris(true); |
|
34 |
sj.setBaseUrl(job.getUrl()); |
|
35 |
sj.setValidationType("CU"); |
|
36 |
sj.setJobType("CRIS Validation"); |
|
37 |
sj.setGuidelinesShortName("For CRIS Managers 1.1"); |
|
38 |
sj.setValidationSet("-"); |
|
39 |
|
|
40 |
// Set status |
|
41 |
sj.setValidationStatus(job.getStatus()); |
|
42 |
sj.setUsageJobStatus(job.getUsageJobStatus()); |
|
43 |
sj.setContentJobStatus(job.getContentJobStatus()); |
|
44 |
|
|
45 |
// Set scores |
|
46 |
// sj.setFilteredScores(); // TODO what is this? |
|
47 |
sj.setContentJobScore(job.getContentScore()); |
|
48 |
sj.setUsageJobScore(job.getUsageScore()); |
|
49 |
|
|
50 |
sj.setResultEntries(CrisValidatorUtils.crisResultsToJobEntries(job)); |
|
51 |
sj.setRecordsTested(job.getRecordsTested()); |
|
52 |
|
|
53 |
return sj; |
|
54 |
} |
|
55 |
|
|
56 |
public static List<JobResultEntry> crisResultsToJobEntries(Job crisJob) { |
|
57 |
List<JobResultEntry> jobResultEntries = new ArrayList<>(); |
|
58 |
for (RuleResults ruleResults : crisJob.getRuleResults()) { |
|
59 |
JobResultEntry jobResultEntry = new JobResultEntry(); |
|
60 |
jobResultEntry.setName(ruleResults.getRuleMethodName()); |
|
61 |
jobResultEntry.setRuleId(ruleResults.getRuleId()); |
|
62 |
jobResultEntry.setMandatory(true); |
|
63 |
jobResultEntry.setWeight(Math.round(ruleResults.getWeight())); |
|
64 |
jobResultEntry.setType(ruleResults.getType()); |
|
65 |
jobResultEntry.setHasErrors(false); |
|
66 |
if (ruleResults.getCount() == 0) { |
|
67 |
jobResultEntry.setSuccesses("-"); |
|
68 |
} else { |
|
69 |
jobResultEntry.setSuccesses(ruleResults.getCount() - ruleResults.getFailed() + "/" + ruleResults.getCount()); |
|
70 |
} |
|
71 |
if (ruleResults.getErrorMessages() != null && !ruleResults.getErrorMessages().isEmpty()) { |
|
72 |
jobResultEntry.setErrors(ruleResults.getErrorMessages()); |
|
73 |
jobResultEntry.setHasErrors(true); |
|
74 |
} |
|
75 |
jobResultEntries.add(jobResultEntry); |
|
76 |
} |
|
77 |
return jobResultEntries; |
|
78 |
} |
|
79 |
|
|
80 |
private CrisValidatorUtils() { |
|
81 |
} |
|
82 |
} |
Also available in: Unified diff
changes for cris validation