D-Net

package eu.dnetlib.openaire.hadoop.utils;

import java.util.Collection;

import java.util.Set;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import eu.dnetlib.data.proto.TypeProtos.Type;

/**

 * Common static utility methods to manage the hbase tables

 *

 * @author claudio

 */

public class HBaseTableUtils {

	private static final String SEPARATOR = "_";

	private static final Function<Type, String> typeName = type -> type.toString();

	public static Set<String> listAllColumns() {

		final Set<String> union = Sets.union(listEntities(), listRelationships());

		return Sets.union(union, listDedupRelationships());

	}

	public static Set<String> listDedupColumns(final Collection<Type> entityTypes) {

		final Set<String> entities = listEntities(Lists.newArrayList(Iterables.transform(entityTypes, typeName)));

		return Sets.union(entities, listDedupRelationships());

	}

	private static Set<String> listDedupRelationships() {

		final Set<String> cfs = Sets.newHashSet();

//		cfs.add(RelType.organizationOrganization + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.merges);

//		cfs.add(RelType.organizationOrganization + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.isMergedIn);

//		cfs.add(RelType.organizationOrganization + SEPARATOR + SubRelType.dedupSimilarity + SEPARATOR + DedupSimilarity.RelName.isSimilarTo);

//

//		cfs.add(RelType.personPerson + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.merges);

//		cfs.add(RelType.personPerson + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.isMergedIn);

//		cfs.add(RelType.personPerson + SEPARATOR + SubRelType.dedupSimilarity + SEPARATOR + DedupSimilarity.RelName.isSimilarTo);

//

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.merges);

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.isMergedIn);

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.dedupSimilarity + SEPARATOR + DedupSimilarity.RelName.isSimilarTo);

//

//		cfs.add(RelType.publicationDataset + SEPARATOR + SubRelType.dedupSimilarity + SEPARATOR + DedupSimilarity.RelName.isSimilarTo);

//		cfs.add(RelType.publicationDataset + SEPARATOR + SubRelType.isRelatedTo + SEPARATOR + IsRelatedTo.RelName.isRelatedTo);

//

//		cfs.add(RelType.datasetDataset + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.merges);

//		cfs.add(RelType.datasetDataset + SEPARATOR + SubRelType.dedup + SEPARATOR + Dedup.RelName.isMergedIn);

//		cfs.add(RelType.datasetDataset + SEPARATOR + SubRelType.dedupSimilarity + SEPARATOR + DedupSimilarity.RelName.isSimilarTo);

		return cfs;

	}

	private static Set<String> listEntities(final Collection<String> entityType) {

        return Sets.newHashSet(

                Iterables.filter(

                        Iterables.transform(Lists.newArrayList(Type.values()), typeName),

                        s -> entityType.contains(s)));

    }

	public static Set<String> listEntities() {

		return Sets.newHashSet(Iterables.transform(Lists.newArrayList(Type.values()), typeName));

	}

	public static Set<String> listRelationships() {

		final Set<String> cfs = Sets.newHashSet();

//		cfs.add(RelType.datasourceOrganization + SEPARATOR + SubRelType.provision + SEPARATOR + Provision.RelName.isProvidedBy);

//		cfs.add(RelType.datasourceOrganization + SEPARATOR + SubRelType.provision + SEPARATOR + Provision.RelName.provides);

//

//		cfs.add(RelType.personPerson + SEPARATOR + SubRelType.coauthorship + SEPARATOR + CoAuthorship.RelName.isCoauthorOf);

//

//		cfs.add(RelType.personPublication + SEPARATOR + SubRelType.authorship + SEPARATOR + Authorship.RelName.isAuthorOf);

//		cfs.add(RelType.personPublication + SEPARATOR + SubRelType.authorship + SEPARATOR + Authorship.RelName.hasAuthor);

//

//		cfs.add(RelType.personDataset + SEPARATOR + SubRelType.authorship + SEPARATOR + Authorship.RelName.isAuthorOf);

//		cfs.add(RelType.personDataset + SEPARATOR + SubRelType.authorship + SEPARATOR + Authorship.RelName.hasAuthor);

//

//		cfs.add(RelType.projectOrganization + SEPARATOR + SubRelType.participation + SEPARATOR + Participation.RelName.hasParticipant);

//		cfs.add(RelType.projectOrganization + SEPARATOR + SubRelType.participation + SEPARATOR + Participation.RelName.isParticipant);

//

//		cfs.add(RelType.projectPerson + SEPARATOR + SubRelType.contactPerson + SEPARATOR + ContactPerson.RelName.isContact);

//		cfs.add(RelType.projectPerson + SEPARATOR + SubRelType.contactPerson + SEPARATOR + ContactPerson.RelName.hasContact);

//

//		cfs.add(RelType.publicationProject + SEPARATOR + SubRelType.outcome + SEPARATOR + Outcome.RelName.isProducedBy);

//		cfs.add(RelType.publicationProject + SEPARATOR + SubRelType.outcome + SEPARATOR + Outcome.RelName.produces);

//

//		cfs.add(RelType.datasetProject + SEPARATOR + SubRelType.outcome + SEPARATOR + Outcome.RelName.isProducedBy);

//		cfs.add(RelType.datasetProject + SEPARATOR + SubRelType.outcome + SEPARATOR + Outcome.RelName.produces);

//

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.similarity + SEPARATOR + Similarity.RelName.hasAmongTopNSimilarDocuments);

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.similarity + SEPARATOR + Similarity.RelName.isAmongTopNSimilarDocuments);

//

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.supplement + SEPARATOR + Supplement.RelName.isSupplementedBy);

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.supplement + SEPARATOR + Supplement.RelName.isSupplementTo);

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.part + SEPARATOR + Part.RelName.isPartOf);

//		cfs.add(RelType.publicationPublication + SEPARATOR + SubRelType.part + SEPARATOR + Part.RelName.hasPart);

//

//		cfs.add(RelType.publicationDataset + SEPARATOR + SubRelType.isRelatedTo + SEPARATOR + IsRelatedTo.RelName.isRelatedTo);

//

//		cfs.add(RelType.datasetDataset + SEPARATOR + SubRelType.similarity + SEPARATOR + Similarity.RelName.hasAmongTopNSimilarDocuments);

//		cfs.add(RelType.datasetDataset + SEPARATOR + SubRelType.similarity + SEPARATOR + Similarity.RelName.isAmongTopNSimilarDocuments);

//

//		cfs.add(RelType.publicationOrganization + SEPARATOR + SubRelType.affiliation + SEPARATOR + Affiliation.RelName.isAffiliatedWith);

		return cfs;

	}

	public enum VolatileColumnFamily {

		dedup, dedupPerson; // instance is here to remove the old protos

		public static boolean isVolatile(final String columnName) {

			try {

				return VolatileColumnFamily.valueOf(columnName) != null;

			} catch (final Throwable e) {

				return false;

			}

		}

	}

}

package eu.dnetlib.data.mapreduce.util;

import eu.dnetlib.data.proto.TypeProtos.Type;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class DNGFRowKeyDecoder {

	public final static String ID_REGEX = "^[0-9][0-9]\\|.{12}::[a-zA-Z0-9]{32}$";

	/**

	 * logger.

	 */

	private static final Log log = LogFactory.getLog(DNGFRowKeyDecoder.class); // NOPMD by marko on 11/24/08 5:02 PM

	private static final String SEPARATOR = "|";

	private String key;

	private Type type = null;

	private String id = null;

	private DNGFRowKeyDecoder(final String key) throws IllegalArgumentException {

		this.key = key;

		if (!key.matches(ID_REGEX)) {

			String msg = "invalid key: '" + key + "'";

			log.error(msg);

			throw new IllegalArgumentException(msg);

		}

		int tag = Integer.parseInt(StringUtils.substringBefore(key, SEPARATOR));

		this.type = Type.valueOf(tag);

		this.id = StringUtils.substringAfter(key, SEPARATOR);

		// System.out.println(OafRowTypeDecoder.class.getName() +" decoded key: " + split);

	}

	public static DNGFRowKeyDecoder decode(final byte[] key) throws IllegalArgumentException {

		return new DNGFRowKeyDecoder(new String(key));

	}

	public static DNGFRowKeyDecoder decode(final String key) throws IllegalArgumentException {

		return new DNGFRowKeyDecoder(key);

	}

	public String getKey() {

		return key;

	}

	public Type getType() {

		return type;

	}

	public String getId() {

		return id;

	}

}

package eu.dnetlib.data.mapreduce.util;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity;

import eu.dnetlib.data.proto.DNGFProtos.DNGFRel;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

import eu.dnetlib.data.proto.TypeProtos.Type;

public class DNGFRelDecoder {

	private final DNGFRel dgnfRel;

	private DNGFRelDecoder(final DNGFRel DNGFRel) {

		this.dgnfRel = DNGFRel;

	}

	public static DNGFRelDecoder decode(final DNGFRel DNGFRel) {

		return new DNGFRelDecoder(DNGFRel);

	}

	public Qualifier getRelType() {

		return dgnfRel.getRelType();

	}

	public String getRelClass() {

		return dgnfRel.getRelType().getClassid();

	}

	public String getRelScheme() {

		return dgnfRel.getRelType().getSchemeid();

	}

	public DNGFRel.Builder setClassId(final String classId) {

		final DNGFRel.Builder relBuilder = DNGFRel.newBuilder(dgnfRel);

		relBuilder.getRelTypeBuilder().setClassid(classId);

		return relBuilder;

	}

	public Type getTargetType() {

		return dgnfRel.getTargetType();

	}

	public Type getSourceType() {

		return dgnfRel.getSourceType();

	}

//	protected FieldDescriptor fd(final MessageOrBuilder mb, final int fieldNumber) {

//		return mb.getDescriptorForType().findFieldByNumber(fieldNumber);

//	}

//

//	protected FieldDescriptor fd(final MessageOrBuilder mb, final String fieldName) {

//		return mb.getDescriptorForType().findFieldByName(fieldName);

//	}

	public String getCachedTargedId() {

		if (!dgnfRel.hasCachedTarget()) return null;

		final DNGFEntity entity = dgnfRel.getCachedTarget();

		return DNGFEntityDecoder.decode(entity).getId();

	}

	public String getRelSourceId() {

		return dgnfRel.getSource();

	}

	public String getRelTargetId() {

		return dgnfRel.getTarget();

	}

}

package eu.dnetlib.data.mapreduce.util;

import java.util.List;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.protobuf.Descriptors.FieldDescriptor;

import com.google.protobuf.GeneratedMessage;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.data.transform.AbstractProtoMapper;

public class DNGFEntityDecoder extends AbstractProtoMapper {

	private final DNGFEntity dngfEntity;

	private DNGFEntityDecoder(final DNGFEntity oafEntity) {

		this.dngfEntity = oafEntity;

	}

	public static DNGFEntityDecoder decode(final DNGFEntity oafEntity) {

		return new DNGFEntityDecoder(oafEntity);

	}

	public Type getType() {

		return dngfEntity.getType();

	}

	public String getId() {

		return dngfEntity.getId();

	}

	public GeneratedMessage getDNGFEntity() {

		return dngfEntity;

	}

	public GeneratedMessage getEntity() {

		final FieldDescriptor fd = dngfEntity.getDescriptorForType().findFieldByName(typeName());

		return (GeneratedMessage) dngfEntity.getField(fd);

	}

	public String typeName() {

		return dngfEntity.getType().toString();

	}

	public GeneratedMessage getMetadata() {

		final GeneratedMessage entity = getEntity();

		final FieldDescriptor fd = entity.getDescriptorForType().findFieldByName("metadata");

		final GeneratedMessage field = (GeneratedMessage) entity.getField(fd);

		return field;

	}

	public List<String> getFieldValues(final String path) {

		return Lists.newLinkedList(Iterables.transform(processPath(getDNGFEntity(), path, eu.dnetlib.pace.config.Type.String), new Function<Object, String>() {

			@Override

			public String apply(final Object o) {

				return o.toString();

			}

		}));

	}

	public String getDateOfCollection() {

		return dngfEntity.getDateofcollection();

	}

	public String getDateOfTransformation() {

		return dngfEntity.getDateoftransformation();

	}

}

package eu.dnetlib.data.mapreduce.util;

public class RelDescriptor {

    private final String code;

    private final String ontologyCode;

    private final String termCode;

	public RelDescriptor(final String value) {

		super();

        this.code = value;

		String[] s = value.split("_");

        this.ontologyCode = s[0];

        this.termCode = s[1];

    }

    public String getCode() {

        return code;

    }

    public String getOntologyCode() {

        return ontologyCode;

    }

    public String getTermCode() {

        return termCode;

    }

    @Override

	public String toString() {

        return getCode();

    }

	@Override

	public int hashCode() {

		final int prime = 31;

		int result = 1;

        result = (prime * result) + ((code == null) ? 0 : code.hashCode());

        return result;

	}

	@Override

	public boolean equals(final Object obj) {

		if (this == obj) return true;

		if (obj == null) return false;

		if (getClass() != obj.getClass()) return false;

		RelDescriptor other = (RelDescriptor) obj;

        if (code == null) {

            if (other.code != null) return false;

        } else if (!code.equals(other.code)) return false;

        return true;

	}

}

package eu.dnetlib.data.mapreduce.util;

import com.google.protobuf.Descriptors.EnumValueDescriptor;

import com.google.protobuf.Descriptors.FieldDescriptor;

import com.google.protobuf.ExtensionRegistry;

import com.google.protobuf.GeneratedMessage;

import com.google.protobuf.GeneratedMessage.GeneratedExtension;

import com.google.protobuf.InvalidProtocolBufferException;

import com.google.protobuf.Message;

import eu.dnetlib.data.proto.DNGFProtos.DNGF;

import eu.dnetlib.data.proto.DNGFProtos.DNGFEntity;

import eu.dnetlib.data.proto.DNGFProtos.DNGFRel;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

import eu.dnetlib.data.proto.KindProtos.Kind;

import java.util.List;

import java.util.Map.Entry;

/**

 * Helper class, to be used as accessor helper over the DNGF structure.

 *

 * @author claudio

 *

 */

public class DNGFDecoder {

	/**

	 * DNGF object

	 */

	private DNGF dngf;

	/**

	 * Cached sub decoder

	 */

	private DNGFEntityDecoder entityDecoder = null;

	/**

	 * Cached sub decoder

	 */

	private DNGFRelDecoder relDecoder = null;

	protected DNGFDecoder(final byte[] value) {

		this(value, null);

	}

	protected DNGFDecoder(final byte[] value, final GeneratedExtension... ge) {

		try {

			final ExtensionRegistry registry = ExtensionRegistry.newInstance();

			if (ge != null) {

				for(GeneratedExtension e : ge) {

					registry.add(e);

				}

			}

			this.dngf = DNGF.parseFrom(value, registry);

		} catch (InvalidProtocolBufferException e) {

			throw new RuntimeException("unable to deserialize proto: " + new String(value));

		}

	}

	private DNGFDecoder(final DNGF dngf) {

		this.dngf = dngf;

	}

    public static DNGFDecoder decode(final DNGF dngf) {

        return new DNGFDecoder(dngf);

    }

    public static DNGFDecoder decode(final byte[] b) {

        return new DNGFDecoder(b);

    }

    public static DNGFDecoder decode(final byte[] b, final GeneratedExtension... ge) {

        return new DNGFDecoder(b, ge);

    }

    private static String escapeXml(final String value) {

        return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");

    }

	public Kind getKind() {

		return dngf.getKind();

	}

    // / Entity

	public DNGF getDNGF() {

		return dngf;

	}

	public GeneratedMessage getMetadata() {

		return decodeEntity().getMetadata();

	}

	public GeneratedMessage getDNGFEntity() {

		return decodeEntity().getEntity();

	}

	public String getEntityId() {

		return decodeEntity().getId();

	}

	public DNGFEntity getEntity() {

		return dngf.getEntity();

	}

    // / Rel

	public DNGFEntityDecoder decodeEntity() {

		if (entityDecoder == null) {

			entityDecoder = DNGFEntityDecoder.decode(getEntity());

		}

		return entityDecoder;

	}

	public DNGFRel getDNGFRel() {

		return dngf.getRel();

	}

	public Qualifier relType() {

		return decodeRel().getRelType();

	}

	public String relTypeName() {

		return relType().toString();

	}

	public String relSourceId() {

		return decodeRel().getRelSourceId();

	}

	public String relTargetId() {

		return decodeRel().getRelTargetId();

	}

	private DNGFRelDecoder decodeRel() {

		if (relDecoder == null) {

			relDecoder = DNGFRelDecoder.decode(getDNGFRel());

		}

		return relDecoder;

	}

	public byte[] toByteArray() {

		return dngf.toByteArray();

	}

	public String asXml() {

		StringBuilder sb = new StringBuilder("<oaf>");

		for (Entry<FieldDescriptor, Object> e : dngf.getAllFields().entrySet()) {

			asXml(sb, e.getKey(), e.getValue());

		}

		sb.append("</oaf>");

		return sb.toString();

	}

	@SuppressWarnings("unchecked")

	private void asXml(final StringBuilder sb, final FieldDescriptor fd, final Object value) {

		if (fd.isRepeated() && (value instanceof List<?>)) {

			for (Object o : ((List<Object>) value)) {

				asXml(sb, fd, o);

			}

		} else if (fd.getType().equals(FieldDescriptor.Type.MESSAGE)) {

			sb.append("<" + fd.getName() + ">");

			for (Entry<FieldDescriptor, Object> e : ((Message) value).getAllFields().entrySet()) {

				asXml(sb, e.getKey(), e.getValue());

			}

			sb.append("</" + fd.getName() + ">");

		} else if (fd.getType().equals(FieldDescriptor.Type.ENUM)) {

			sb.append("<" + fd.getName() + ">");

			sb.append(((EnumValueDescriptor) value).getName());

			sb.append("</" + fd.getName() + ">");

		} else {

			sb.append("<" + fd.getName() + ">");

			sb.append(escapeXml(value.toString()));

			sb.append("</" + fd.getName() + ">");

		}

	}

}

import eu.dnetlib.data.mapreduce.util.DNGFDecoder;

import eu.dnetlib.data.mapreduce.util.DNGFRowKeyDecoder;

import eu.dnetlib.data.mapreduce.util.DNGFRowKeyDecoder;

import eu.dnetlib.data.mapreduce.util.DNGFRowKeyDecoder;

import eu.dnetlib.data.mapreduce.util.DNGFRowKeyDecoder;