D-Net

import static eu.dnetlib.data.mapreduce.util.OafHbaseUtils.getKey;

import static eu.dnetlib.data.mapreduce.util.OafHbaseUtils.getPropertyValues;

import java.util.Map;

import java.util.Set;

import eu.dnetlib.data.mapreduce.Algorithms;

import eu.dnetlib.data.mapreduce.JobParams;

import eu.dnetlib.pace.config.DedupConfig;

import eu.dnetlib.pace.distance.PaceDocumentDistance;

import eu.dnetlib.pace.distance.eval.ScoreResult;

import eu.dnetlib.pace.model.MapDocument;

import eu.dnetlib.pace.model.ProtoDocumentBuilder;

import org.apache.commons.lang.math.RandomUtils;

import org.apache.commons.math.util.MathUtils;

import org.apache.hadoop.hbase.client.HTable;

import org.apache.hadoop.hbase.client.Result;

import org.apache.hadoop.hbase.client.ResultScanner;

import org.apache.hadoop.hbase.client.Scan;

import org.apache.hadoop.hbase.filter.FilterList;

import org.apache.hadoop.hbase.filter.FilterList.Operator;

import org.apache.hadoop.hbase.filter.PrefixFilter;

import org.apache.hadoop.hbase.util.Bytes;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import com.google.common.collect.Sets;

import com.google.protobuf.InvalidProtocolBufferException;

import eu.dnetlib.data.proto.OafProtos.Oaf;

public class EnrichmentReducer extends Reducer<ImmutableBytesWritable, ImmutableBytesWritable, Text, Text> {

	private static final int LIMIT = 1000;

	private static final int SCORE_DECIMALS = 2;

	private Map<String, String> dsTypeMap = Maps.newHashMap();

	private Set<String> dsWhitelist = Sets.newHashSet();

	private Set<String> dsBlacklist = Sets.newHashSet();

	// This is for EuropePMC. They expose OA abstracts, but we want to identify real OA publications. WTF.

	private Set<String> untrustedOaDsList = Sets.newHashSet();

	// White list for datasource typologies.

	private Set<String> dsTypeWhitelist = Sets.newHashSet();

	private DedupConfig dedupConf;

	/**

	 * lower bound for trust scaling

	 */

	private double scaleLB;

	protected void setup(final Context context) throws IOException, InterruptedException {

		super.setup(context);

		System.out.println("LIMIT: " + LIMIT);

		dsWhitelist.addAll(getPropertyValues(context, "broker.datasource.id.whitelist"));

		dsBlacklist.addAll(getPropertyValues(context, "broker.datasource.id.blacklist"));

		dsTypeWhitelist.addAll(getPropertyValues(context, "broker.datasource.type.whitelist"));

		untrustedOaDsList.addAll(getPropertyValues(context, "broker.datasource.untrusted.oa.list"));

		dsTypeMap = getDsTypeMap(context, dsTypeWhitelist);

		System.out.println("datasource whitelist: " + dsWhitelist);

		System.out.println("datasource blacklist: " + dsBlacklist);

		System.out.println("datasource OA list: " + untrustedOaDsList);

		System.out.println("datasource type whitelist: " + dsTypeWhitelist);

		final String dedupConfJson = context.getConfiguration().get(JobParams.DEDUP_CONF);

		System.out.println("got dedup conf: " + dedupConfJson);

		dedupConf = DedupConfig.load(dedupConfJson);

		System.out.println("parsed dedup conf: " + dedupConf.toString());

		scaleLB = dedupConf.getWf().getThreshold() - 0.01;

	private Map<String, String> getDsTypeMap(final Context context, final Set<String> dsTypeWhitelist) throws IOException {

		System.out.println("loading datasource typology mapping");

		final Map<String, String> dsTypeMap = Maps.newHashMap();

		final Scan scan = new Scan();

		final FilterList fl = new FilterList(Operator.MUST_PASS_ALL);

		fl.addFilter(new PrefixFilter(Bytes.toBytes("10")));

		scan.setFilter(fl);

		scan.addFamily(Bytes.toBytes("datasource"));

		final String tableName = context.getConfiguration().get("hbase.mapred.inputtable");

		System.out.println(String.format("table name: '%s'", tableName));

		try (final HTable table = new HTable(context.getConfiguration(), tableName);

				final ResultScanner res = table.getScanner(scan)) {

			for (final Result r : res) {

				final byte[] b = r.getValue(Bytes.toBytes("datasource"), Bytes.toBytes("body"));

				if (b != null) {

					final Oaf oaf = Oaf.parseFrom(b);

					final String dsId = StringUtils.substringAfter(oaf.getEntity().getId(), "|");

					final String dsType = oaf.getEntity().getDatasource().getMetadata().getDatasourcetype().getClassid();

					if (dsTypeWhitelist.contains(dsType)) {

						System.out.println(String.format("dsId '%s', dsType '%s'", dsId, dsType));

						dsTypeMap.put(dsId, dsType);

					}

				}

			}

		}

		System.out.println("datasource type map size: " + dsTypeMap.size());

		return dsTypeMap;

	}

	private float similarity(final Oaf oa, final Oaf ob) {

		final MapDocument a = ProtoDocumentBuilder.newInstance(oa.getEntity().getId(), oa.getEntity(), dedupConf.getPace().getModel());

		final MapDocument b = ProtoDocumentBuilder.newInstance(ob.getEntity().getId(), ob.getEntity(), dedupConf.getPace().getModel());

		final ScoreResult sr =  new PaceDocumentDistance().between(a, b, dedupConf);

		final float score = (float) Algorithms.scale(sr.getScore(), scaleLB, 1, 0, 1);

		return MathUtils.round(score, SCORE_DECIMALS);

	}

	private Function<ImmutableBytesWritable, Oaf> oafDeserialiser() {

		return new Function<ImmutableBytesWritable, Oaf>() {

			@Override

			public Oaf apply(final ImmutableBytesWritable input) {

				try {

					return Oaf.parseFrom(input.copyBytes());

				} catch (final InvalidProtocolBufferException e) {

					throw new IllegalArgumentException(e);

				}

			}

		};

	}