|
1
|
//package eu.dnetlib.data.mapreduce.hbase.dataimport;
|
|
2
|
//
|
|
3
|
//import java.io.IOException;
|
|
4
|
//import java.util.List;
|
|
5
|
//
|
|
6
|
//import com.google.common.base.Function;
|
|
7
|
//import com.google.common.collect.Iterables;
|
|
8
|
//import com.google.common.collect.Lists;
|
|
9
|
//import eu.dnetlib.data.mapreduce.JobParams;
|
|
10
|
//import eu.dnetlib.data.mapreduce.util.DNGFDecoder;
|
|
11
|
//import eu.dnetlib.data.proto.DNGFProtos.DNGF;
|
|
12
|
//import eu.dnetlib.data.proto.DNGFProtos.DNGF;
|
|
13
|
//import eu.dnetlib.data.proto.PersonProtos.Person;
|
|
14
|
//import eu.dnetlib.data.proto.PersonProtos.Person.CoAuthor;
|
|
15
|
//import eu.dnetlib.data.proto.TypeProtos.Type;
|
|
16
|
//import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions;
|
|
17
|
//import eu.dnetlib.miscutils.maps.ConcurrentSizedMap;
|
|
18
|
//import org.apache.commons.lang3.StringUtils;
|
|
19
|
//import org.apache.commons.logging.Log;
|
|
20
|
//import org.apache.commons.logging.LogFactory;
|
|
21
|
//import org.apache.hadoop.hbase.client.Put;
|
|
22
|
//import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
|
|
23
|
//import org.apache.hadoop.hbase.mapreduce.TableReducer;
|
|
24
|
//import org.apache.hadoop.hbase.util.Bytes;
|
|
25
|
//
|
|
26
|
//public class CoAuthorReducer extends TableReducer<ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable> {
|
|
27
|
//
|
|
28
|
// private static final Log log = LogFactory.getLog(CoAuthorReducer.class);
|
|
29
|
//
|
|
30
|
// private static int MAX_COAUTHORS = 50;
|
|
31
|
//
|
|
32
|
// private int max_coauthors = MAX_COAUTHORS;
|
|
33
|
//
|
|
34
|
// @Override
|
|
35
|
// protected void setup(final Context context) throws IOException, InterruptedException {
|
|
36
|
// max_coauthors = Integer.parseInt(context.getConfiguration().get("max.coauthors"));
|
|
37
|
// }
|
|
38
|
//
|
|
39
|
// @Override
|
|
40
|
// protected void reduce(final ImmutableBytesWritable key, final Iterable<ImmutableBytesWritable> values, final Context context) throws IOException,
|
|
41
|
// InterruptedException {
|
|
42
|
//
|
|
43
|
// final byte[] keyBytes = key.copyBytes();
|
|
44
|
// final String rowKey = new String(keyBytes);
|
|
45
|
//
|
|
46
|
// log.info("Reducing key: '" + rowKey + "'");
|
|
47
|
//
|
|
48
|
// final Function<Person, CoAuthor> personToCoAuthor = new Function<Person, CoAuthor>() {
|
|
49
|
//
|
|
50
|
// @Override
|
|
51
|
// public CoAuthor apply(final Person p) {
|
|
52
|
// final CoAuthor.Builder cb = CoAuthor.newBuilder();
|
|
53
|
// cb.setId(getId(rowKey, p.getMetadata().getFullname().getValue().trim()));
|
|
54
|
// cb.getMetadataBuilder().mergeFrom(p.getMetadata());
|
|
55
|
// return cb.build();
|
|
56
|
// }
|
|
57
|
// };
|
|
58
|
//
|
|
59
|
// final DNGF.Builder out = DNGF.newBuilder();
|
|
60
|
// final ConcurrentSizedMap<String, CoAuthor> coAuthors = new ConcurrentSizedMap<String, CoAuthor>();
|
|
61
|
// coAuthors.setQueueSize(max_coauthors);
|
|
62
|
//
|
|
63
|
// for (final ImmutableBytesWritable ibw : values) {
|
|
64
|
// final DNGFDecoder d = DNGFDecoder.decode(ibw.copyBytes());
|
|
65
|
//
|
|
66
|
// final Type type = d.getEntity().getType();
|
|
67
|
// switch (type) {
|
|
68
|
// case person:
|
|
69
|
// out.mergeFrom(d.getDNGF());
|
|
70
|
// out.getEntityBuilder().getPersonBuilder().clearCoauthor();
|
|
71
|
// context.getCounter("coauthor", "merge").increment(1);
|
|
72
|
// break;
|
|
73
|
// case publication:
|
|
74
|
// final List<Person> authorList = d.getEntity().getPublication().getAuthorList();
|
|
75
|
// for (final CoAuthor coauthor : Lists.newArrayList(Iterables.transform(authorList, personToCoAuthor))) {
|
|
76
|
// coAuthors.put(coauthor.getId(), incrementCount(coAuthors, coauthor));
|
|
77
|
// }
|
|
78
|
// break;
|
|
79
|
// case datasource:
|
|
80
|
// case organization:
|
|
81
|
// case project:
|
|
82
|
// default:
|
|
83
|
// context.getCounter("coauthor", "skipped entity type (reduce)").increment(1);
|
|
84
|
// break;
|
|
85
|
// }
|
|
86
|
// }
|
|
87
|
//
|
|
88
|
// coAuthors.remove(rowKey); // remove himself
|
|
89
|
// out.getEntityBuilder().getPersonBuilder().addAllCoauthor(coAuthors.values());
|
|
90
|
//
|
|
91
|
// final Put put = new Put(keyBytes).add(Bytes.toBytes(Type.person.toString()), Bytes.toBytes("body"), out.build().toByteArray());
|
|
92
|
// put.setWriteToWAL(JobParams.WRITE_TO_WAL);
|
|
93
|
// context.write(key, put);
|
|
94
|
// }
|
|
95
|
//
|
|
96
|
// private CoAuthor incrementCount(final ConcurrentSizedMap<String, CoAuthor> coAuthors, final CoAuthor coauthor) {
|
|
97
|
// if (coAuthors.containsKey(coauthor.getId())) {
|
|
98
|
// final CoAuthor.Builder c = CoAuthor.newBuilder(coAuthors.get(coauthor.getId()));
|
|
99
|
// c.setCoauthoredpapers(c.getCoauthoredpapers() + 1);
|
|
100
|
// return c.build();
|
|
101
|
// }
|
|
102
|
// return coauthor;
|
|
103
|
// }
|
|
104
|
//
|
|
105
|
// private String getId(final String resultId, final String name) {
|
|
106
|
// final String prefix = StringUtils.substringBefore(StringUtils.substringAfter(resultId, "|"), "::");
|
|
107
|
// // final String id = StringUtils.substringAfter(resultId, "|") + "::" + name.replaceAll("\\s+", " ").trim();
|
|
108
|
// final String id = name.replaceAll("\\s+", " ").trim();
|
|
109
|
// return AbstractDNetXsltFunctions.oafId(Type.person.toString(), prefix, id);
|
|
110
|
// }
|
|
111
|
//
|
|
112
|
//}
|