Revision 55644
Added by Antonis Lempesis almost 5 years ago
StatsMapper.java | ||
---|---|---|
2 | 2 |
|
3 | 3 |
import com.google.common.collect.ArrayListMultimap; |
4 | 4 |
import com.google.common.collect.Multimap; |
5 |
import com.google.protobuf.InvalidProtocolBufferException; |
|
6 | 5 |
import eu.dnetlib.data.mapreduce.hbase.index.config.EntityConfigTable; |
7 | 6 |
import eu.dnetlib.data.mapreduce.hbase.index.config.IndexConfig; |
8 | 7 |
import eu.dnetlib.data.mapreduce.hbase.index.config.LinkDescriptor; |
9 |
import eu.dnetlib.data.mapreduce.hbase.statsExport.utils.ContextTransformer; |
|
10 | 8 |
import eu.dnetlib.data.mapreduce.hbase.statsExport.utils.Serializer; |
11 |
import eu.dnetlib.data.mapreduce.util.OafDecoder; |
|
12 | 9 |
import eu.dnetlib.data.mapreduce.util.OafRowKeyDecoder; |
13 | 10 |
import eu.dnetlib.data.mapreduce.util.RelDescriptor; |
14 | 11 |
import eu.dnetlib.data.mapreduce.util.UpdateMerger; |
... | ... | |
16 | 13 |
import eu.dnetlib.data.proto.OafProtos.Oaf; |
17 | 14 |
import eu.dnetlib.data.proto.OafProtos.OafRel; |
18 | 15 |
import eu.dnetlib.data.proto.OafProtos.OafRelOrBuilder; |
19 |
import eu.dnetlib.data.proto.RelTypeProtos.SubRelType; |
|
20 |
import eu.dnetlib.data.proto.TypeProtos.Type; |
|
21 | 16 |
import org.apache.hadoop.hbase.client.Result; |
22 | 17 |
import org.apache.hadoop.hbase.io.ImmutableBytesWritable; |
23 | 18 |
import org.apache.hadoop.hbase.mapreduce.TableMapper; |
... | ... | |
25 | 20 |
import org.apache.hadoop.io.Text; |
26 | 21 |
import org.apache.log4j.Logger; |
27 | 22 |
|
28 |
import java.io.ByteArrayInputStream; |
|
29 | 23 |
import java.io.IOException; |
30 |
import java.io.InputStream; |
|
31 | 24 |
import java.util.ArrayList; |
32 | 25 |
import java.util.List; |
33 | 26 |
import java.util.Map; |
34 | 27 |
import java.util.Map.Entry; |
35 |
import java.util.Properties; |
|
36 | 28 |
|
37 | 29 |
/** |
38 | 30 |
* Mapper Class that reads HBASE contents and prepares them for the StatsDB |
... | ... | |
74 | 66 |
|
75 | 67 |
//Init class: Load Index config and mapping for DB tables. |
76 | 68 |
@Override |
77 |
protected void setup(Context context) throws IOException, InterruptedException {
|
|
69 |
protected void setup(Context context) { |
|
78 | 70 |
loadEntityConfig(context); |
79 | 71 |
} |
80 | 72 |
|
81 | 73 |
|
82 | 74 |
//Read HBASE table and decode Protos to OAF entities. |
83 | 75 |
@Override |
84 |
protected void map(final ImmutableBytesWritable keyIn, final Result result, final Context context) throws IOException {
|
|
76 |
protected void map(final ImmutableBytesWritable keyIn, final Result result, final Context context) { |
|
85 | 77 |
|
86 | 78 |
Oaf oaf = null; |
87 | 79 |
OafRowKeyDecoder keyDecoder = OafRowKeyDecoder.decode(keyIn.copyBytes()); |
... | ... | |
110 | 102 |
|
111 | 103 |
|
112 | 104 |
private boolean isValid(Oaf oaf) { |
113 |
if (oaf != null && oaf.isInitialized() && !deletedByInference(oaf) && !invisible(oaf)) return true;
|
|
105 |
return oaf != null && oaf.isInitialized() && !deletedByInference(oaf) && !invisible(oaf);
|
|
114 | 106 |
//if (oaf != null && oaf.isInitialized() && !deletedByInference(oaf)) return true; |
115 |
return false; |
|
116 | 107 |
} |
117 | 108 |
|
118 | 109 |
|
Also available in: Unified diff
numerous bug fixes and changes to schema