23 |
23 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
|
24 |
24 |
import eu.dnetlib.functionality.index.client.IndexClient;
|
25 |
25 |
import eu.dnetlib.functionality.index.client.IndexClientException;
|
26 |
|
import eu.dnetlib.functionality.index.client.ResolvingIndexClientFactory;
|
27 |
26 |
import eu.dnetlib.functionality.index.client.response.LookupResponse;
|
|
27 |
import eu.dnetlib.functionality.index.client.solr.SolrIndexClient;
|
|
28 |
import eu.dnetlib.functionality.index.client.solr.SolrIndexClientFactory;
|
28 |
29 |
import eu.dnetlib.functionality.modular.ui.dedup.SimilarityGroup;
|
29 |
30 |
import eu.dnetlib.pace.config.DedupConfig;
|
30 |
31 |
import org.apache.commons.codec.binary.Base64;
|
31 |
32 |
import org.apache.commons.lang.StringUtils;
|
32 |
33 |
import org.apache.commons.logging.Log;
|
33 |
34 |
import org.apache.commons.logging.LogFactory;
|
34 |
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
|
35 |
|
35 |
36 |
import org.apache.solr.common.SolrInputDocument;
|
36 |
37 |
import org.dom4j.DocumentException;
|
37 |
38 |
import org.springframework.beans.factory.annotation.Autowired;
|
... | ... | |
74 |
75 |
* The index client factory.
|
75 |
76 |
*/
|
76 |
77 |
@Autowired
|
77 |
|
private ResolvingIndexClientFactory indexClientFactory;
|
|
78 |
private SolrIndexClientFactory indexClientFactory;
|
78 |
79 |
|
79 |
80 |
private IndexClient indexClient = null;
|
80 |
81 |
|
81 |
|
@Value("${dnet.dedup.index.format}")
|
82 |
|
private String indexFormat;
|
83 |
|
|
84 |
82 |
@Value("${dnet.dedup.index.collection}")
|
85 |
83 |
private String dedupIndexCollection;
|
86 |
84 |
|
... | ... | |
119 |
117 |
|
120 |
118 |
log.info("starting index update");
|
121 |
119 |
|
122 |
|
try(final CloudSolrClient solrServer = getSolrServer()) {
|
|
120 |
try(final SolrIndexClient indexClient = (SolrIndexClient) getIndexClient()) {
|
123 |
121 |
final SolrProtoMapper mapper = initProtoMapper();
|
124 |
122 |
|
125 |
123 |
final Function<Oaf, SolrInputDocument> oaf2solr = oaf2solr(group, mapper);
|
... | ... | |
142 |
140 |
log.debug(String.format("adding %d documents to index %s", buffer.size(), dedupIndexCollection));
|
143 |
141 |
|
144 |
142 |
// add the changes to the server
|
145 |
|
addStatus = solrServer.add(buffer).getStatus();
|
|
143 |
addStatus = indexClient.feed(buffer);
|
146 |
144 |
log.debug("solr add status: " + addStatus);
|
147 |
145 |
|
148 |
146 |
// delete the old representatives, avoiding to remove the current one (if it didn't change)
|
149 |
147 |
log.debug(String.format("deleting %d documents from index %s", group.getRootIds().size(), dedupIndexCollection));
|
150 |
148 |
for (final String rootId : Iterables.filter(group.getRootIds(), rootId -> !rootId.equals(newRootId))) {
|
151 |
|
solrServer.deleteById(mapper.getRecordId(rootId, group.getActionSet()));
|
|
149 |
indexClient.remove(String.format("objidentifier:\"%s\"", mapper.getRecordId(rootId, group.getActionSet())));
|
152 |
150 |
}
|
153 |
151 |
|
154 |
|
commitStatus = solrServer.commit().getStatus();
|
|
152 |
commitStatus = indexClient.commit().getStatus();
|
155 |
153 |
|
156 |
154 |
log.debug("solr commit status: " + commitStatus);
|
157 |
155 |
}
|
... | ... | |
186 |
184 |
serviceLocator
|
187 |
185 |
.getService(ISLookUpService.class)
|
188 |
186 |
.getResourceProfileByQuery(
|
189 |
|
"collection('')//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and .//NAME='" + indexFormat
|
190 |
|
+ "']//LAYOUT[@name='index']/FIELDS"));
|
|
187 |
"collection('')//RESOURCE_PROFILE["
|
|
188 |
+ ".//RESOURCE_TYPE/@value = 'MDFormatDSResourceType' and "
|
|
189 |
+ ".//NAME='OPENAIRE']//LAYOUT[@name='index']/FIELDS"));
|
191 |
190 |
}
|
192 |
191 |
|
193 |
|
private CloudSolrClient getSolrServer() {
|
194 |
|
final String zk = getIndexSolrUrlZk();
|
195 |
|
log.info(String.format("initializing solr client for collection %s, zk url: %s", dedupIndexCollection, zk));
|
196 |
|
final CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(zk).build();
|
197 |
|
solrServer.setDefaultCollection(dedupIndexCollection);
|
198 |
|
|
199 |
|
return solrServer;
|
200 |
|
}
|
201 |
|
|
202 |
|
private String getIndexSolrUrlZk() {
|
203 |
|
try {
|
204 |
|
return getResourceProfileByQuery(
|
205 |
|
"for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='IndexServiceResourceType'] return $x//PROTOCOL[./@name='solr']/@address/string()");
|
206 |
|
} catch (final ISLookUpException e) {
|
207 |
|
throw new IllegalStateException("unable to read solr ZK url from service profile", e);
|
208 |
|
}
|
209 |
|
}
|
210 |
|
|
211 |
192 |
private String getResourceProfileByQuery(final String xquery) throws ISLookUpException {
|
212 |
193 |
log.debug("quering for service property: " + xquery);
|
213 |
194 |
final String res = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
|
... | ... | |
247 |
228 |
|
248 |
229 |
private IndexClient getIndexClient() throws IndexClientException, ISLookUpDocumentNotFoundException, ISLookUpException {
|
249 |
230 |
if (indexClient == null) {
|
250 |
|
indexClient = indexClientFactory.getClient(indexFormat, "index", "dedup", "solr");
|
|
231 |
indexClient = indexClientFactory.getClient(dedupIndexCollection);
|
251 |
232 |
}
|
252 |
233 |
return indexClient;
|
253 |
234 |
}
|
consolidated index modules and updated to solr 7.2.0