Revision 49029
Added by Claudio Atzori over 6 years ago
DedupBuildRootsReducer.java | ||
---|---|---|
3 | 3 |
import java.io.IOException; |
4 | 4 |
import java.util.List; |
5 | 5 |
|
6 |
import org.apache.commons.lang.StringUtils; |
|
7 |
import org.apache.hadoop.hbase.client.Put; |
|
8 |
import org.apache.hadoop.hbase.io.ImmutableBytesWritable; |
|
9 |
import org.apache.hadoop.hbase.mapreduce.TableReducer; |
|
10 |
import org.apache.hadoop.hbase.util.Bytes; |
|
11 |
import org.apache.hadoop.io.Text; |
|
12 |
|
|
13 | 6 |
import com.google.common.collect.Iterables; |
14 | 7 |
import com.google.common.collect.Lists; |
15 |
import com.google.protobuf.InvalidProtocolBufferException; |
|
16 |
|
|
17 | 8 |
import eu.dnetlib.data.mapreduce.JobParams; |
18 | 9 |
import eu.dnetlib.data.mapreduce.hbase.index.config.RelClasses; |
19 | 10 |
import eu.dnetlib.data.mapreduce.util.DedupUtils; |
... | ... | |
21 | 12 |
import eu.dnetlib.data.mapreduce.util.OafHbaseUtils; |
22 | 13 |
import eu.dnetlib.data.mapreduce.util.OafRelDecoder; |
23 | 14 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
24 |
import eu.dnetlib.data.proto.RelTypeProtos.RelType; |
|
25 | 15 |
import eu.dnetlib.data.proto.TypeProtos.Type; |
26 | 16 |
import eu.dnetlib.data.transform.OafEntityMerger; |
27 | 17 |
import eu.dnetlib.pace.config.DedupConfig; |
18 |
import org.apache.commons.lang.StringUtils; |
|
19 |
import org.apache.hadoop.hbase.client.Put; |
|
20 |
import org.apache.hadoop.hbase.io.ImmutableBytesWritable; |
|
21 |
import org.apache.hadoop.hbase.mapreduce.TableReducer; |
|
22 |
import org.apache.hadoop.hbase.util.Bytes; |
|
23 |
import org.apache.hadoop.io.Text; |
|
28 | 24 |
|
29 | 25 |
public class DedupBuildRootsReducer extends TableReducer<Text, ImmutableBytesWritable, ImmutableBytesWritable> { |
30 | 26 |
|
... | ... | |
132 | 128 |
public boolean checkHack(final String root, final Oaf oaf) { |
133 | 129 |
|
134 | 130 |
boolean res; |
135 |
if (dedupConf.getWf().getEntityType().equals(Type.result.toString()) && oaf.getRel().getRelType().toString().equals(RelType.personResult.toString()) |
|
136 |
&& !md5matches(root, oaf.getRel().getSource())) { |
|
131 |
if (dedupConf.getWf().getEntityType().equals(Type.result.toString()) && !md5matches(root, oaf.getRel().getSource())) { |
|
137 | 132 |
|
138 | 133 |
res = true; |
139 | 134 |
} else { |
Also available in: Unified diff
getting rid of person entities