D-Net

package eu.dnetlib.data.mapreduce.hbase.broker.enrich;

import java.io.IOException;

import java.math.BigDecimal;

import java.util.Map;

import java.util.Set;

import java.util.Spliterator;

import java.util.Spliterators;

import java.util.stream.Stream;

import java.util.stream.StreamSupport;

import com.google.common.collect.Maps;

import com.google.common.collect.Sets;

import com.google.protobuf.InvalidProtocolBufferException;

import eu.dnetlib.data.mapreduce.Algorithms;

import eu.dnetlib.data.mapreduce.JobParams;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.pace.config.DedupConfig;

import eu.dnetlib.pace.distance.PaceDocumentDistance;

import eu.dnetlib.pace.distance.eval.ScoreResult;

import eu.dnetlib.pace.model.MapDocument;

import eu.dnetlib.pace.model.ProtoDocumentBuilder;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.math.util.MathUtils;

import org.apache.hadoop.hbase.client.HTable;

import org.apache.hadoop.hbase.client.Result;

import org.apache.hadoop.hbase.client.ResultScanner;

import org.apache.hadoop.hbase.client.Scan;

import org.apache.hadoop.hbase.filter.FilterList;

import org.apache.hadoop.hbase.filter.FilterList.Operator;

import org.apache.hadoop.hbase.filter.PrefixFilter;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;

import org.apache.hadoop.hbase.util.Bytes;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

import static eu.dnetlib.data.mapreduce.util.OafHbaseUtils.getPropertyValues;

/**

 * Created by claudio on 20/02/2017.

 */

public abstract class AbstractEnrichmentReducer extends Reducer<ImmutableBytesWritable, ImmutableBytesWritable, Text, Text> {

	protected DedupConfig dedupConf;

	protected static final int LIMIT = 1000;

	protected static final int SCORE_DECIMALS = 2;

	protected Map<String, String> dsTypeMap = Maps.newHashMap();

	protected Set<String> dsWhitelist = Sets.newHashSet();

	protected Set<String> dsBlacklist = Sets.newHashSet();

	// This is for EuropePMC. They expose OA abstracts, but we want to identify real OA publications. WTF.

	protected Set<String> untrustedOaDsList = Sets.newHashSet();

	// White list for datasource typologies.

	protected Set<String> dsTypeWhitelist = Sets.newHashSet();

	/**

	 * lower bound for trust scaling

	 */

	protected double scaleLB;

	protected abstract String counterGroup();

	@Override

	protected void setup(final Context context) throws IOException, InterruptedException {

		super.setup(context);

		System.out.println("LIMIT: " + LIMIT);

		dsWhitelist.addAll(getPropertyValues(context, "broker.datasource.id.whitelist"));

		dsBlacklist.addAll(getPropertyValues(context, "broker.datasource.id.blacklist"));

		dsTypeWhitelist.addAll(getPropertyValues(context, "broker.datasource.type.whitelist"));

		untrustedOaDsList.addAll(getPropertyValues(context, "broker.datasource.untrusted.oa.list"));

		dsTypeMap = getDsTypeMap(context, dsTypeWhitelist);

		System.out.println("datasource whitelist: " + dsWhitelist);

		System.out.println("datasource blacklist: " + dsBlacklist);

		System.out.println("datasource OA list: " + untrustedOaDsList);

		System.out.println("datasource type whitelist: " + dsTypeWhitelist);

		final String dedupConfJson = context.getConfiguration().get(JobParams.DEDUP_CONF);

		System.out.println("got dedup conf: " + dedupConfJson);

		dedupConf = DedupConfig.load(dedupConfJson);

		System.out.println("parsed dedup conf: " + dedupConf.toString());

		scaleLB = dedupConf.getWf().getThreshold() - 0.01;

	}

	protected Map<String, String> getDsTypeMap(final Context context, final Set<String> dsTypeWhitelist) throws IOException {

		System.out.println("loading datasource typology mapping");

		final Map<String, String> dsTypeMap = Maps.newHashMap();

		final Scan scan = new Scan();

		final FilterList fl = new FilterList(Operator.MUST_PASS_ALL);

		fl.addFilter(new PrefixFilter(Bytes.toBytes("10")));

		scan.setFilter(fl);

		scan.addFamily(Bytes.toBytes("datasource"));

		final String tableName = context.getConfiguration().get("hbase.mapred.inputtable");

		System.out.println(String.format("table name: '%s'", tableName));

		try (final HTable table = new HTable(context.getConfiguration(), tableName); final ResultScanner res = table.getScanner(scan)) {

			for (final Result r : res) {

				final byte[] b = r.getValue(Bytes.toBytes("datasource"), Bytes.toBytes("body"));

				if (b != null) {

					final Oaf oaf = Oaf.parseFrom(b);

					final String dsId = StringUtils.substringAfter(oaf.getEntity().getId(), "|");

					final String dsType = oaf.getEntity().getDatasource().getMetadata().getDatasourcetype().getClassid();

					if (dsTypeWhitelist.contains(dsType)) {

						//System.out.println(String.format("dsId '%s', dsType '%s'", dsId, dsType));

						dsTypeMap.put(dsId, dsType);

					}

				}

			}

		}

		System.out.println("datasource type map size: " + dsTypeMap.size());

		return dsTypeMap;

	}

	protected float similarity(final Oaf oa, final Oaf ob) {

		final MapDocument a = ProtoDocumentBuilder.newInstance(oa.getEntity().getId(), oa.getEntity(), dedupConf.getPace().getModel());

		final MapDocument b = ProtoDocumentBuilder.newInstance(ob.getEntity().getId(), ob.getEntity(), dedupConf.getPace().getModel());

		final ScoreResult sr =  new PaceDocumentDistance().between(a, b, dedupConf);

		final float score = (float) Algorithms.scale(sr.getScore(), scaleLB, 1, 0, 1);

		return MathUtils.round(score, SCORE_DECIMALS, BigDecimal.ROUND_HALF_DOWN);

	}

	protected static Oaf toOaf(ImmutableBytesWritable i) {

		try {

			return Oaf.parseFrom(i.copyBytes());

		} catch (final InvalidProtocolBufferException e) {

			throw new IllegalArgumentException(e);

		}

	}

	protected static <T> Stream<T> stream(Iterable<T> iterable) {

		return StreamSupport.stream(

				Spliterators.spliteratorUnknownSize(

						iterable.iterator(),

						Spliterator.ORDERED

				),

				false

		);

	}

}