Revision 54265
Added by Claudio Atzori over 5 years ago
| DedupIndexDAO.java | ||
|---|---|---|
| 12 | 12 |
import com.google.common.collect.Lists; |
| 13 | 13 |
import com.google.common.collect.Maps; |
| 14 | 14 |
import com.google.common.collect.Sets; |
| 15 |
import com.google.protobuf.GeneratedMessage; |
|
| 15 | 16 |
import eu.dnetlib.data.mapreduce.util.OafDecoder; |
| 16 | 17 |
import eu.dnetlib.data.mapreduce.util.OafEntityDecoder; |
| 17 | 18 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
| 19 |
import eu.dnetlib.data.transform.AbstractProtoMapper; |
|
| 18 | 20 |
import eu.dnetlib.data.transform.OafEntityMerger; |
| 19 | 21 |
import eu.dnetlib.data.transform.SolrProtoMapper; |
| 20 | 22 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException; |
| ... | ... | |
| 28 | 30 |
import eu.dnetlib.functionality.index.client.solr.SolrIndexClientFactory; |
| 29 | 31 |
import eu.dnetlib.functionality.modular.ui.dedup.SimilarityGroup; |
| 30 | 32 |
import eu.dnetlib.pace.config.DedupConfig; |
| 33 |
import eu.dnetlib.pace.model.Field; |
|
| 34 |
import eu.dnetlib.pace.model.FieldDef; |
|
| 35 |
import eu.dnetlib.pace.model.FieldValueImpl; |
|
| 36 |
import eu.dnetlib.pace.model.ProtoDocumentBuilder; |
|
| 31 | 37 |
import org.apache.commons.codec.binary.Base64; |
| 32 | 38 |
import org.apache.commons.lang.StringUtils; |
| 33 | 39 |
import org.apache.commons.logging.Log; |
| ... | ... | |
| 215 | 221 |
res.put("groupSize", idList.isEmpty() ? "1" : idList.size() + "");
|
| 216 | 222 |
|
| 217 | 223 |
for (final String fieldName : fields) {
|
| 218 |
res.put(fieldName, Joiner.on("; ").skipNulls().join(ed.getFieldValues(paths.get(type).get(fieldName))));
|
|
| 224 |
|
|
| 225 |
res.put(fieldName, Joiner.on("; ").skipNulls().join(getFieldValues(ed.getEntity(), fieldName, paths.get(type).get(fieldName))));
|
|
| 219 | 226 |
} |
| 220 | 227 |
|
| 221 | 228 |
return res; |
| 222 | 229 |
}; |
| 223 | 230 |
} |
| 224 | 231 |
|
| 232 |
private List<String> getFieldValues(final GeneratedMessage m, final String fieldName, final String path) {
|
|
| 233 |
return new SolrDocumentMapper().processPath(m, fieldName, path).stream() |
|
| 234 |
.map(o -> o.toString()) |
|
| 235 |
.collect(Collectors.toCollection(LinkedList::new)); |
|
| 236 |
} |
|
| 237 |
|
|
| 238 |
class SolrDocumentMapper extends AbstractProtoMapper {
|
|
| 239 |
|
|
| 240 |
public List<Object> processPath(final GeneratedMessage m, final String fieldName, final String path) {
|
|
| 241 |
final FieldDef fd = new FieldDef(); |
|
| 242 |
fd.setName(fieldName); |
|
| 243 |
return processPath(m, fd, path); |
|
| 244 |
} |
|
| 245 |
} |
|
| 246 |
|
|
| 225 | 247 |
private String cleanId(final String id) {
|
| 226 | 248 |
return id.replaceFirst(ID_PREFIX_REGEX, ""); |
| 227 | 249 |
} |
Also available in: Unified diff
fixed mapping function, from protobuf to solrinputdocument