Revision 36623
Added by Claudio Atzori about 9 years ago
| DedupServiceInternalController.java | ||
|---|---|---|
| 10 | 10 |
|
| 11 | 11 |
import javax.annotation.Resource; |
| 12 | 12 |
|
| 13 |
import org.antlr.stringtemplate.StringTemplate; |
|
| 14 | 13 |
import org.apache.commons.codec.binary.Base64; |
| 15 | 14 |
import org.apache.commons.lang.StringUtils; |
| 16 | 15 |
import org.apache.commons.logging.Log; |
| ... | ... | |
| 53 | 52 |
import eu.dnetlib.functionality.index.solr.feed.InputDocumentFactory; |
| 54 | 53 |
import eu.dnetlib.functionality.modular.ui.AbstractAjaxController; |
| 55 | 54 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
| 56 |
import eu.dnetlib.pace.util.DedupConfig; |
|
| 57 |
import eu.dnetlib.pace.util.DedupConfigLoader; |
|
| 55 |
import eu.dnetlib.pace.config.DedupConfig; |
|
| 58 | 56 |
|
| 59 | 57 |
@Controller |
| 60 | 58 |
public class DedupServiceInternalController extends AbstractAjaxController {
|
| ... | ... | |
| 82 | 80 |
@Value("${dnet.dedup.index.collection}")
|
| 83 | 81 |
private String dedupIndexCollection; |
| 84 | 82 |
|
| 85 |
@Resource(name = "xqueryDedupWfTemplate") |
|
| 86 |
private StringTemplate xqueryWf; |
|
| 87 |
|
|
| 88 | 83 |
private IndexClient indexClient = null; |
| 89 | 84 |
|
| 90 | 85 |
public class OafResult {
|
| ... | ... | |
| 120 | 115 |
@RequestMapping(value = "/ui/dedup/search.do") |
| 121 | 116 |
public OafResult search(@RequestParam(value = "entityType", required = true) final String type, |
| 122 | 117 |
@RequestParam(value = "query", required = true) final String userQuery, |
| 118 |
@RequestParam(value = "actionSet", required = true) final String actionSet, |
|
| 123 | 119 |
@RequestParam(value = "start", required = true) final int start, |
| 124 | 120 |
@RequestParam(value = "rows", required = true) final int rows, |
| 125 | 121 |
@RequestParam(value = "fields", required = true) final String fields) throws Exception {
|
| 126 | 122 |
|
| 127 | 123 |
try {
|
| 128 |
final String cqlQuery = "(>s=SOLR s.q.op=AND) and oaftype = " + type + " and deletedbyinference = false and " + userQuery; |
|
| 124 |
final String cqlQuery = |
|
| 125 |
String.format("(>s=SOLR s.q.op=AND) and oaftype = %s and actionset = %s and deletedbyinference = false and %s", type, actionSet, userQuery);
|
|
| 129 | 126 |
|
| 130 | 127 |
final LookupResponse rsp = getIndexClient().lookup(cqlQuery, null, start, (start + rows) - 1); |
| 131 | 128 |
|
| ... | ... | |
| 178 | 175 |
// index update |
| 179 | 176 |
final CloudSolrServer solrServer = getSolrServer(); |
| 180 | 177 |
final ProtoDocumentMapper mapper = initProtoMapper(); |
| 181 |
final DedupConfig dedupConf = DedupConfigLoader.load(loadWfConf(group.getEntityType().getType(), "01")); |
|
| 182 | 178 |
|
| 179 |
final Map<String, String> config = Maps.newHashMap(); |
|
| 180 |
config.put("entityType", group.getEntityType().getType());
|
|
| 181 |
config.put("configurationId", group.getActionSet());
|
|
| 182 |
final DedupConfig dedupConf = DedupConfig.loadDefault(config); |
|
| 183 |
|
|
| 183 | 184 |
for (final String rootId : group.getRootIds()) {
|
| 184 | 185 |
solrServer.deleteById(rootId); |
| 185 | 186 |
} |
| ... | ... | |
| 189 | 190 |
final List<Oaf> groupDocs = Lists.newArrayList(markDeleted(asOafBuilder(parseBase64(queryIndex(group.getGroup()))))); |
| 190 | 191 |
|
| 191 | 192 |
buffer.addAll(Lists.newArrayList(asIndexDocs(oaf2solr, groupDocs))); |
| 192 |
buffer.add(oaf2solr.apply(OafEntityMerger.merge(dedupConf, newRootId(group), groupDocs).build())); |
|
| 193 |
final SolrInputDocument newRoot = oaf2solr.apply(OafEntityMerger.merge(dedupConf, newRootId(group), groupDocs).build()); |
|
| 194 |
newRoot.setField("actionset", dedupConf.getWf().getConfigurationId());
|
|
| 195 |
buffer.add(newRoot); |
|
| 193 | 196 |
|
| 194 | 197 |
final List<Oaf> dissimDocs = Lists.newArrayList(markUnDeleted(asOafBuilder(parseBase64(queryIndex(group.getDissimilar().keySet()))))); |
| 195 | 198 |
buffer.addAll(Lists.newArrayList(asIndexDocs(oaf2solr, dissimDocs))); |
| ... | ... | |
| 325 | 328 |
private ProtoDocumentMapper initProtoMapper() throws DocumentException, ISLookUpException, ISLookUpDocumentNotFoundException {
|
| 326 | 329 |
return new ProtoDocumentMapper( |
| 327 | 330 |
serviceLocator |
| 328 |
.getService(ISLookUpService.class)
|
|
| 329 |
.getResourceProfileByQuery(
|
|
| 330 |
"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + indexFormat
|
|
| 331 |
+ "']//LAYOUT[@name='index']/FIELDS"));
|
|
| 331 |
.getService(ISLookUpService.class) |
|
| 332 |
.getResourceProfileByQuery( |
|
| 333 |
"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + indexFormat
|
|
| 334 |
+ "']//LAYOUT[@name='index']/FIELDS")); |
|
| 332 | 335 |
} |
| 333 | 336 |
|
| 334 | 337 |
private Iterable<Oaf> markDeleted(final Iterable<Oaf.Builder> builders) {
|
| ... | ... | |
| 394 | 397 |
sql.append(String.format("DELETE FROM similarity_groups WHERE objidentifier = '%s'; ", id));
|
| 395 | 398 |
} |
| 396 | 399 |
final String type = group.getEntityType().getType(); |
| 397 |
sql.append(String.format("INSERT INTO groups(id, entitytype, date) VALUES('%s', '%s', '%s'); ", group.getId(), type, version));
|
|
| 400 |
sql.append(String.format("INSERT INTO groups(id, entitytype, date, actionsetid) VALUES('%s', '%s', '%s', '%s'); ", group.getId(), type, version,
|
|
| 401 |
group.getActionSet())); |
|
| 398 | 402 |
for (final String id : group.getGroup()) {
|
| 399 | 403 |
if (!dbService.contains(dbName, "similarity_groups", "objidentifier", id)) {
|
| 400 | 404 |
sql.append(String.format("INSERT INTO entities(id, entitytype) VALUES('%s', '%s'); ", id, type));
|
| ... | ... | |
| 457 | 461 |
return res; |
| 458 | 462 |
} |
| 459 | 463 |
|
| 460 |
private String getXquery(final String entityType, final String code, final StringTemplate template) {
|
|
| 461 |
final StringTemplate xquery = new StringTemplate(template.getTemplate()); |
|
| 462 |
xquery.setAttribute("entityType", entityType);
|
|
| 463 |
xquery.setAttribute("configCode", code);
|
|
| 464 |
return xquery.toString(); |
|
| 465 |
} |
|
| 466 |
|
|
| 467 |
private String loadWfConf(final String entityType, final String conf) throws ISLookUpException {
|
|
| 468 |
return getResourceProfileByQuery(getXquery(entityType, conf, xqueryWf)); |
|
| 469 |
} |
|
| 470 |
|
|
| 471 | 464 |
} |
Also available in: Unified diff
introduced support for action set ids