| 23 |
23 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
|
| 24 |
24 |
import eu.dnetlib.functionality.index.client.IndexClient;
|
| 25 |
25 |
import eu.dnetlib.functionality.index.client.IndexClientException;
|
| 26 |
|
import eu.dnetlib.functionality.index.client.ResolvingIndexClientFactory;
|
| 27 |
26 |
import eu.dnetlib.functionality.index.client.response.LookupResponse;
|
|
27 |
import eu.dnetlib.functionality.index.client.solr.SolrIndexClient;
|
|
28 |
import eu.dnetlib.functionality.index.client.solr.SolrIndexClientFactory;
|
| 28 |
29 |
import eu.dnetlib.functionality.modular.ui.dedup.SimilarityGroup;
|
| 29 |
30 |
import eu.dnetlib.pace.config.DedupConfig;
|
| 30 |
31 |
import org.apache.commons.codec.binary.Base64;
|
| 31 |
32 |
import org.apache.commons.lang.StringUtils;
|
| 32 |
33 |
import org.apache.commons.logging.Log;
|
| 33 |
34 |
import org.apache.commons.logging.LogFactory;
|
| 34 |
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
|
35 |
|
| 35 |
36 |
import org.apache.solr.common.SolrInputDocument;
|
| 36 |
37 |
import org.dom4j.DocumentException;
|
| 37 |
38 |
import org.springframework.beans.factory.annotation.Autowired;
|
| ... | ... | |
| 74 |
75 |
* The index client factory.
|
| 75 |
76 |
*/
|
| 76 |
77 |
@Autowired
|
| 77 |
|
private ResolvingIndexClientFactory indexClientFactory;
|
|
78 |
private SolrIndexClientFactory indexClientFactory;
|
| 78 |
79 |
|
| 79 |
80 |
private IndexClient indexClient = null;
|
| 80 |
81 |
|
| 81 |
|
@Value("${dnet.dedup.index.format}")
|
| 82 |
|
private String indexFormat;
|
| 83 |
|
|
| 84 |
82 |
@Value("${dnet.dedup.index.collection}")
|
| 85 |
83 |
private String dedupIndexCollection;
|
| 86 |
84 |
|
| ... | ... | |
| 119 |
117 |
|
| 120 |
118 |
log.info("starting index update");
|
| 121 |
119 |
|
| 122 |
|
try(final CloudSolrClient solrServer = getSolrServer()) {
|
|
120 |
try(final SolrIndexClient indexClient = (SolrIndexClient) getIndexClient()) {
|
| 123 |
121 |
final SolrProtoMapper mapper = initProtoMapper();
|
| 124 |
122 |
|
| 125 |
123 |
final Function<Oaf, SolrInputDocument> oaf2solr = oaf2solr(group, mapper);
|
| ... | ... | |
| 142 |
140 |
log.debug(String.format("adding %d documents to index %s", buffer.size(), dedupIndexCollection));
|
| 143 |
141 |
|
| 144 |
142 |
// add the changes to the server
|
| 145 |
|
addStatus = solrServer.add(buffer).getStatus();
|
|
143 |
addStatus = indexClient.feed(buffer);
|
| 146 |
144 |
log.debug("solr add status: " + addStatus);
|
| 147 |
145 |
|
| 148 |
146 |
// delete the old representatives, avoiding to remove the current one (if it didn't change)
|
| 149 |
147 |
log.debug(String.format("deleting %d documents from index %s", group.getRootIds().size(), dedupIndexCollection));
|
| 150 |
148 |
for (final String rootId : Iterables.filter(group.getRootIds(), rootId -> !rootId.equals(newRootId))) {
|
| 151 |
|
solrServer.deleteById(mapper.getRecordId(rootId, group.getActionSet()));
|
|
149 |
indexClient.remove(String.format("objidentifier:\"%s\"", mapper.getRecordId(rootId, group.getActionSet())));
|
| 152 |
150 |
}
|
| 153 |
151 |
|
| 154 |
|
commitStatus = solrServer.commit().getStatus();
|
|
152 |
commitStatus = indexClient.commit().getStatus();
|
| 155 |
153 |
|
| 156 |
154 |
log.debug("solr commit status: " + commitStatus);
|
| 157 |
155 |
}
|
| ... | ... | |
| 186 |
184 |
serviceLocator
|
| 187 |
185 |
.getService(ISLookUpService.class)
|
| 188 |
186 |
.getResourceProfileByQuery(
|
| 189 |
|
"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + indexFormat
|
| 190 |
|
+ "']//LAYOUT[@name='index']/FIELDS"));
|
|
187 |
"collection('')//RESOURCE_PROFILE["
|
|
188 |
+ ".//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and "
|
|
189 |
+ ".//NAME='OPENAIRE']//LAYOUT[@name='index']/FIELDS"));
|
| 191 |
190 |
}
|
| 192 |
191 |
|
| 193 |
|
private CloudSolrClient getSolrServer() {
|
| 194 |
|
final String zk = getIndexSolrUrlZk();
|
| 195 |
|
log.info(String.format("initializing solr client for collection %s, zk url: %s", dedupIndexCollection, zk));
|
| 196 |
|
final CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(zk).build();
|
| 197 |
|
solrServer.setDefaultCollection(dedupIndexCollection);
|
| 198 |
|
|
| 199 |
|
return solrServer;
|
| 200 |
|
}
|
| 201 |
|
|
| 202 |
|
private String getIndexSolrUrlZk() {
|
| 203 |
|
try {
|
| 204 |
|
return getResourceProfileByQuery(
|
| 205 |
|
"for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//PROTOCOL[./@name='solr']/@address/string()");
|
| 206 |
|
} catch (final ISLookUpException e) {
|
| 207 |
|
throw new IllegalStateException("unable to read solr ZK url from service profile", e);
|
| 208 |
|
}
|
| 209 |
|
}
|
| 210 |
|
|
| 211 |
192 |
private String getResourceProfileByQuery(final String xquery) throws ISLookUpException {
|
| 212 |
193 |
log.debug("quering for service property: " + xquery);
|
| 213 |
194 |
final String res = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
|
| ... | ... | |
| 247 |
228 |
|
| 248 |
229 |
private IndexClient getIndexClient() throws IndexClientException, ISLookUpDocumentNotFoundException, ISLookUpException {
|
| 249 |
230 |
if (indexClient == null) {
|
| 250 |
|
indexClient = indexClientFactory.getClient(indexFormat, "index", "dedup", "solr");
|
|
231 |
indexClient = indexClientFactory.getClient(dedupIndexCollection);
|
| 251 |
232 |
}
|
| 252 |
233 |
return indexClient;
|
| 253 |
234 |
}
|
Added by Claudio Atzori over 6 years ago
consolidated index modules and updated to solr 7.2.0