Revision 40589
Added by Eri Katsari over 8 years ago
| LodMapper.java | ||
|---|---|---|
| 21 | 21 |
|
| 22 | 22 |
import java.io.IOException; |
| 23 | 23 |
import java.util.HashMap; |
| 24 |
import java.util.List; |
|
| 24 | 25 |
import java.util.Map; |
| 25 | 26 |
import java.util.Map.Entry; |
| 26 | 27 |
|
| ... | ... | |
| 120 | 121 |
private void emitEntity(Context context, Oaf oaf, Type type, Oaf.Builder oafBuilder) {
|
| 121 | 122 |
// ONLY EMIT ENTITIES THAT WE WANT TO IMPORT IN STATS DB |
| 122 | 123 |
|
| 124 |
String serialized = serializer.serialize(oafBuilder.build()); |
|
| 123 | 125 |
|
| 124 |
String serialized = serializer.serialize(oafBuilder.build()); |
|
| 125 |
if (serialized != null) {
|
|
| 126 |
|
|
| 127 |
try {
|
|
| 126 |
if (serialized != null) {
|
|
| 127 |
try {
|
|
| 128 | 128 |
Text TextKeyOut = new Text("entities" + "," + serializer.getId(oaf));
|
| 129 | 129 |
context.getCounter(STATS_COUNTER.OUTPUT_RECORDS).increment(1); |
| 130 | 130 |
context.getCounter(PROTOS_COUNTER.valueOf(type.toString().toUpperCase())).increment(1); |
| ... | ... | |
| 144 | 144 |
private void emitRelation(Context context, Result result, Oaf oaf, Type type, Oaf.Builder builder) {
|
| 145 | 145 |
|
| 146 | 146 |
|
| 147 |
|
|
| 148 |
try {
|
|
| 149 |
// derived relations such as ResultDatasources are |
|
| 150 |
// created here |
|
| 151 |
HashMap<String, List<String>> relMap = serializer.extractRelations(oaf); |
|
| 152 |
if (relMap != null && !relMap.isEmpty()) {
|
|
| 153 |
|
|
| 154 |
for (Entry<String, List<String>> rel : relMap.entrySet()) {
|
|
| 155 |
|
|
| 156 |
Text TextKeyOut = new Text("relations" + "," +rel.getKey() );
|
|
| 157 |
|
|
| 158 |
for (String relVals : rel.getValue()) {
|
|
| 159 |
|
|
| 160 |
if (relVals != null && !relVals.isEmpty()) {
|
|
| 161 |
|
|
| 162 |
context.write((TextKeyOut), new ImmutableBytesWritable(relVals.getBytes())); |
|
| 163 |
|
|
| 164 |
} |
|
| 165 |
} |
|
| 166 |
} |
|
| 167 |
|
|
| 168 |
} |
|
| 169 |
|
|
| 170 |
} catch (Exception e) {
|
|
| 171 |
log.error("Error writing relation to M/R output", e);
|
|
| 172 |
} |
|
| 173 |
|
|
| 174 |
|
|
| 175 |
|
|
| 147 | 176 |
// Existing Hbase relations are generated here |
| 148 | 177 |
if (entityConfigTable.getDescriptors(type) != null && !entityConfigTable.getDescriptors(type).isEmpty()) {
|
| 149 | 178 |
|
| ... | ... | |
| 162 | 191 |
|
| 163 | 192 |
try {
|
| 164 | 193 |
|
| 165 |
|
|
| 166 |
Text TextKeyOut = new Text(rel.getKey() + "," + serializer.getId(rel.getValue())); |
|
| 194 |
Text TextKeyOut = new Text("relations" + "," + rel.getValue().getRelType().toString());
|
|
| 167 | 195 |
String buff = serializer.serialize(rel.getValue()); |
| 168 | 196 |
context.getCounter(STATS_COUNTER.OUTPUT_RELATIONS).increment(1); |
| 169 | 197 |
context.write((TextKeyOut), new ImmutableBytesWritable(buff.getBytes())); |
| 170 |
|
|
| 171 |
|
|
| 172 |
} catch (Exception e) {
|
|
| 198 |
} catch (Exception e) {
|
|
| 173 | 199 |
log.error("Error while writing Relation Proto to M/R output", e);
|
| 174 | 200 |
} |
| 175 | 201 |
|
| ... | ... | |
| 201 | 227 |
// skip dedups |
| 202 | 228 |
|
| 203 | 229 |
if (ld.getRelDescriptor().getIt().contains(SubRelType.dedup.toString()) && isDedupSelf(relBuilder)) {
|
| 204 |
OafRel oafRel = relBuilder.setRelType(ld.getRelDescriptor().getRelType()).build(); |
|
| 205 |
relBuilder.setCachedTarget(body.getEntity()); |
|
| 206 |
rels.put("dedups", oafRel);
|
|
| 230 |
//OafRel oafRel = relBuilder.setRelType(ld.getRelDescriptor().getRelType()).build();
|
|
| 231 |
//relBuilder.setCachedTarget(body.getEntity());
|
|
| 232 |
// rels.put("dedups", oafRel);
|
|
| 207 | 233 |
|
| 208 | 234 |
} else {
|
| 209 | 235 |
|
| 210 | 236 |
OafRel oafRel = relBuilder.setRelType(ld.getRelDescriptor().getRelType()).build(); |
| 211 | 237 |
relBuilder.setCachedTarget(body.getEntity()); |
| 212 |
rels.put("entities", oafRel);
|
|
| 238 |
rels.put("relations", oafRel);
|
|
| 213 | 239 |
|
| 214 | 240 |
} |
| 215 | 241 |
} |
Also available in: Unified diff
''