Revision 41267
Added by Eri Katsari over 8 years ago
| LodMapper.java | ||
|---|---|---|
| 83 | 83 |
final Oaf oaf = UpdateMerger.mergeBodyUpdates(context, result.getFamilyMap(Bytes.toBytes(type.toString()))); |
| 84 | 84 |
|
| 85 | 85 |
if (isValid(oaf)) {
|
| 86 |
// TODO set this only when the configuration file has include dups |
|
| 87 |
// to fals for Results |
|
| 88 |
// or else were gonna get deleted by inference entries |
|
| 89 |
// if (!deletedByInference(oaf)) |
|
| 86 |
|
|
| 90 | 87 |
if (deletedByInference(oaf)) {
|
| 91 | 88 |
context.getCounter(ENTITIES_COUNTER.DELETED_BY_INFERENCE).increment(1); |
| 92 | 89 |
} else {
|
| 93 | 90 |
context.getCounter(ENTITIES_COUNTER.NOT_DELETED_BY_INFERENCE).increment(1); |
| 94 | 91 |
} |
| 95 | 92 |
|
| 96 |
context.getCounter(ENTITIES_COUNTER.TOTAL_ENTITIES).increment(1); |
|
| 93 |
context.getCounter(ENTITIES_COUNTER.TOTAL_ENTITIES).increment(1); |
|
| 94 |
//TODO eri: here allow entities deletedbyinfer |
|
| 97 | 95 |
|
| 98 |
// entityConfigTable.includeDuplicates(type)) {
|
|
| 99 |
//TODO eri: here allow entities deletedbyinfer |
|
| 100 |
{
|
|
| 101 | 96 |
emitProtos(context, result, oaf); |
| 102 |
} |
|
| 103 | 97 |
} |
| 104 | 98 |
|
| 105 | 99 |
|
| ... | ... | |
| 118 | 112 |
} |
| 119 | 113 |
|
| 120 | 114 |
} catch (Exception e) {
|
| 121 |
// log.error("OAF NOT INITIALIZED ");
|
|
| 122 | 115 |
} |
| 123 | 116 |
|
| 124 | 117 |
return false; |
| ... | ... | |
| 141 | 134 |
if (de.isBefore(dt)) return true; |
| 142 | 135 |
|
| 143 | 136 |
} catch (Exception e) {
|
| 144 |
// log.error("OAF NOT INITIALIZED ");
|
|
| 137 |
|
|
| 145 | 138 |
} |
| 146 | 139 |
|
| 147 | 140 |
return false; |
| ... | ... | |
| 224 | 217 |
try {
|
| 225 | 218 |
Text TextKeyOut = new Text("relations");
|
| 226 | 219 |
if (oaf.getEntity().getId().contains("dedup")) {
|
| 220 |
|
|
| 227 | 221 |
if (rel.startsWith("dedup")) {
|
| 228 | 222 |
context.write((TextKeyOut), new ImmutableBytesWritable(rel.getBytes())); |
| 229 | 223 |
context.getCounter(ENTITIES_COUNTER.TOTAL_RELATIONS).increment(1); |
| ... | ... | |
| 256 | 250 |
OafRel.Builder relBuilder = OafRel.newBuilder(decodedOaf.getRel()); |
| 257 | 251 |
// skip dedups |
| 258 | 252 |
if (ld.getRelDescriptor().getIt().contains(SubRelType.dedup.toString()) && isDedupSelf(relBuilder)) {
|
| 259 |
// OafRel oafRel = relBuilder.setRelType(ld.getRelDescriptor().getRelType()).build(); |
|
| 260 |
// relBuilder.setCachedTarget(body.getEntity()); |
|
| 261 |
// rels.put("dedups", oafRel);
|
|
| 262 |
|
|
| 263 | 253 |
} else {
|
| 264 | 254 |
OafRel oafRel = relBuilder.setRelType(ld.getRelDescriptor().getRelType()).build(); |
| 265 | 255 |
relBuilder.setCachedTarget(body.getEntity()); |
| ... | ... | |
| 277 | 267 |
try {
|
| 278 | 268 |
return Oaf.parseFrom(body); |
| 279 | 269 |
} catch (InvalidProtocolBufferException e) {
|
| 280 |
// log.error("Corrupted proto ", e);
|
|
| 281 | 270 |
|
| 282 | 271 |
} |
| 283 | 272 |
return null; |
Also available in: Unified diff
'fix