D-Net

package eu.dnetlib.data.mapreduce.hbase.dedup.kv;

import java.io.DataInput;

import java.io.DataOutput;

import java.io.IOException;

import com.google.common.collect.ComparisonChain;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.io.WritableComparable;

/**

 * Created by claudio on 13/03/2017.

 */

public class DNGFKey implements WritableComparable<DNGFKey> {

	public final static int MERGES_REL = 0;

	public final static int OTHER_REL = 1;

	private IntWritable keyType;

	private Text id;

	public DNGFKey() {}

	public static DNGFKey create(final int keyType, final String id) {

		return new DNGFKey(keyType, id);

	}

	public static DNGFKey mergesRel(final String id) {

		return new DNGFKey(MERGES_REL, id);

	}

	public static DNGFKey otherRel(final String id) {

		return new DNGFKey(OTHER_REL, id);

	}

	public DNGFKey(final int keyType, final String id) {

		this.id = new Text(id);

		this.keyType = new IntWritable(keyType);

	}

	public void setKeyType(final IntWritable keyType) {

		this.keyType = keyType;

	}

	public void setId(final Text id) {

		this.id = id;

	}

	public Text getId() {

		return id;

	}

	public IntWritable getKeyType() {

		return keyType;

	}

	@Override

	public int compareTo(final DNGFKey o) {

		return ComparisonChain.start()

				.compare(getId(), o.getId())

				.compare(getKeyType(), o.getKeyType())

				.result();

	}

	@Override

	public void write(final DataOutput out) throws IOException {

		keyType.write(out);

		id.write(out);

	}

	@Override

	public void readFields(final DataInput in) throws IOException {

		keyType = new IntWritable();

		keyType.readFields(in);

		id = new Text();

		id.readFields(in);

	}

	@Override

	public String toString() {

		return (new StringBuilder())

				.append('{')

				.append(getKeyType().get())

				.append(',')

				.append(getId())

				.append('}')

				.toString();

	}

}

package eu.dnetlib.data.mapreduce.hbase.dedup.kv;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;

import org.apache.hadoop.mapreduce.Partitioner;

/**

 * Created by claudio on 13/03/2017.

 */

public class DNGFKeyPartitioner extends Partitioner<DNGFKey, ImmutableBytesWritable > {

	@Override

	public int getPartition(DNGFKey key, ImmutableBytesWritable  val, int numPartitions) {

		return Math.abs(key.getId().hashCode() % numPartitions);

	}

}

package eu.dnetlib.data.mapreduce.hbase.dedup.kv;

import org.apache.hadoop.io.WritableComparable;

import org.apache.hadoop.io.WritableComparator;

/**

 * Created by claudio on 13/03/2017.

 */

public class DNGFKeyGroupingComparator extends WritableComparator {

	protected DNGFKeyGroupingComparator() {

		super(DNGFKey.class, true);

	}

	@Override

	public int compare(WritableComparable w1, WritableComparable w2) {

		final DNGFKey k1 = (DNGFKey) w1;

		final DNGFKey k2 = (DNGFKey) w2;

		return k1.getId().compareTo(k2.getId());

	}

}

import eu.dnetlib.data.mapreduce.hbase.dedup.kv.DNGFKey;

public class DedupFixRelationMapper extends TableMapper<DNGFKey, ImmutableBytesWritable> {

        if ((!rowKeyDecoder.getType().equals(type)) || !isRoot(rowKey)) {

                if (dngf.getRel().getRelType().getClassid().equals("merges")) {

                    emit(context, DNGFKey.mergesRel(targetId), dngf.toByteArray());

                } else {

                    emit(context, DNGFKey.otherRel(targetId), dngf.toByteArray());

                }

    private void emit(final Context context, final DNGFKey key, final byte[] value) {

            context.write(key, ibw);

import java.util.Iterator;

        final Iterator<ImmutableBytesWritable> it = values.iterator();

        final DNGF first = DNGFDecoder.decode(it.next().copyBytes()).getDNGF();

        if (!first.getRel().getRelType().equals("merges")) {

            context.getCounter(COUNTER_GROUP, "Relation skipped").increment(1);

            return;

        context.getCounter(COUNTER_GROUP, "Item to fix").increment(1);

        final String dedupRoot = first.getRel().getSource();

        it.forEachRemaining(b -> {

            try {

                handleRels(context, DNGFDecoder.decode(b.copyBytes()).getDNGF(), dedupRoot);

            } catch (Exception e) {

               throw new RuntimeException(e);

        });

    private final static String SCHOLIX = "Scholix";

    private final static String DNGF = "DNGF";

                context.getCounter(SCHOLIX, "Source: " + source.getSource().getObjectType()).increment(1);

                        context.getCounter(SCHOLIX, "Rel type: " + target.getTarget().getObjectType()).increment(1);

                        context.getCounter(DNGF, "Rel deletedbyinference: " + type.name()).increment(1);

                context.getCounter(DNGF, "Deletedbyinference: " + type.name()).increment(1);

            context.getCounter(DNGF, "Invalid: " + type.name()).increment(1);

            context.getCounter(SCHOLIX, keyType == DliKey.ENTITY ? "ENTITY" : "REL").increment(1);

    private final static String SCHOLIX = "Scholix";

                incrementCounter(context, SCHOLIX, "Deletedbyinference: " + type.toString(), 1);

            incrementCounter(context, SCHOLIX, "Missing body: " + type.toString(), 1);

import java.util.Arrays;

import java.util.LinkedList;

import com.google.common.base.Splitter;

import com.google.common.collect.Lists;

		groupSizeCounter(context, groupSize.get(),

				"1,1",

				"1,10",

				"10,20",

				"20,100",

				"100,200",

				"200,500",

				"500,1000",

				"1000,2000",

				"2000,5000",

				"5000,10000",

				"10000,20000",

				"20000,*");

	private void groupSizeCounter(final Context context, final int groupSize, final String... groups) {

		Arrays.asList(groups).forEach(g -> {

			final LinkedList<String> i = Lists.newLinkedList(Splitter.on(",").split(g));

			int min = Integer.parseInt(i.getFirst());

			int max = i.getLast().equals("*") ? Integer.MAX_VALUE : Integer.parseInt(i.getLast());

			groupSizeCounter(context, groupSize, min, max);

		});

	}

	private void groupSizeCounter(final Context context, final int groupSize, Integer min, Integer max) {

		if (groupSize > min & groupSize <= max) {

			context.getCounter("scholix groups", String.format("group size (%s,%s)", min, max)).increment(1);

		}

	}