D-Net

#!/bin/bash

VERSION=`xmlstarlet sel -t -m "/_:project/_:version" -v "." pom.xml`

echo "using version: $VERSION"

mvn clean install;

rm -rf ~/.m2/repository/eu/dnetlib/dnet-mapreduce-jobs-assembly;

mvn assembly:assembly -DskipTests=true && mvn install:install-file -Dfile=target/dnet-mapreduce-jobs-$VERSION-jar-with-dependencies.jar -DgroupId=eu.dnetlib -DartifactId=dnet-mapreduce-jobs-assembly -Dversion=$VERSION -Dpackaging=jar

{"type_source": "SVN", "goal": "package -U source:jar", "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-mapreduce-jobs/branches/beta", "deploy_repository": "dnet45-snapshots", "version": "4", "mail": "sandro.labruzzo@isti.cnr.it,michele.artini@isti.cnr.it, claudio.atzori@isti.cnr.it, alessia.bardi@isti.cnr.it", "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots", "name": "dnet-mapreduce-jobs-BETA"}

package eu.dnetlib.data.mapreduce.actions;

import com.google.common.collect.Lists;

import com.google.gson.JsonObject;

import com.google.gson.JsonParser;

import eu.dnetlib.actionmanager.actions.ActionFactory;

import eu.dnetlib.actionmanager.actions.AtomicAction;

import eu.dnetlib.actionmanager.common.Agent;

import eu.dnetlib.data.mapreduce.hbase.dataimport.DOIBoostToActions;

import eu.dnetlib.data.transform.Column;

import eu.dnetlib.data.transform.Row;

import org.apache.commons.codec.binary.Base64;

import org.apache.commons.lang3.StringUtils;

import org.junit.Before;

import org.junit.Test;

import java.io.*;

import java.util.List;

import java.util.zip.DataFormatException;

import java.util.zip.Inflater;

public class DOIBoostToActionsTest  {

    private String setName;

    private Agent agent;

    @Before

    public void setup() {

        setName = "DLI";

        agent= new Agent("agentId","agentName", Agent.AGENT_TYPE.service);

    }

    @Test

    public void testSingleDOIBoostAction() throws IOException {

        doTestSingleDOIBoostAction("/eu/dnetlib/data/mapreduce/actions/broken");

    }

    @Test

    public void testDOIBoostActionToXML() throws Exception {

        doTestSingleDOIBoostActionToXML("/eu/dnetlib/data/mapreduce/actions/broken");

    }

    @Test

    public void testMultipleDOIBoostAction() throws IOException {

        doTestAllDOIBoostAction("/eu/dnetlib/data/mapreduce/actions/part-00070");

    }

    @Test

    public void testDecompression() throws DataFormatException {

        final String s ="eJxtVM1u20YQfpW5pHAAWXCCIECORazKQYAEcHrpcbUckgvs7jA7S8vsye/QU4H25fwk/WYpKUbQ\n" +

                "mygOZ775fuaOXawjeVeY+HHi3IU6F1Ya3QNTCcqZvGQNHRd3iAuFTHVkmpxWen76i1gn9sHF9dWe\n" +

                "S3J5sZJlbeFiYdctaO6ZO+7ozc0rkr414VwD5g5FVKmTxFqDp6lIN/tKV193H2/p7c3N+9db+h3l\n" +

                "yWECuqlk6qWgRcBn/MBRpoRehEdHXXHJWZ+OkwzFTSN++9Hlga21Z9Ut7bKUJLNS4i54F+3NgLXR\n" +

                "IHe2cA15tveV/ZglytCqQkLdA9sw3VCVgbFGoWMAhY4m0VADdlbxQa4ZbSS17/D/DEySN2RERzAS\n" +

                "8oAGGEdusN+TTHM81RwBebRtOtDjK6gFtDlWqwMFzhg72kOe0wHzQSdHCITCBn8skm2uaRKxG9oa\n" +

                "4pVFU5O/z5w9r4R5wAGZkB7/LQYqCsgqNIpOoboY/my4Vm7Oo71oVZpQViFJhbDeKW/pTo5QpGxI\n" +

                "5tKIYAzXtnaWSvAQt71jlOOq4PPTP9bM7BfBoOTnp3+t5KQCen7KWkHZ5n83HyBIySYJiHa1cprq\n" +

                "Os8aBh+Mv178rPadGH1J7G3f46XZZq7hvOPFmY9BG91jy0eLBzTATvDPln6D+/jRpSny5qXtz4wp\n" +

                "HRlf1AKdrW2TonBIh7koqDoslHmQGhpxh7mDkVZ6e7ZAlevOLRvkw2Pi6g9HuoCF1dpwoWl0gfyz\n" +

                "Uk3pwgjJiu/Nhw/vV7K9s8L23dqPjk5Jx9AbkipHVzrIWvi64z7kk6wNWgtPh+PQQG7pWzALGVsH\n" +

                "BsTc6O0tsO8otDCu+cbqB252Ly5rz6Wga8hmgjMGK7qoc17mB2Et3AdnzKHiNrghwyhK9xwbgfsi\n" +

                "86R0dXu/19cowcSru92Xzzv6hb59vLvf/fF1v2/AcEk+GdMhGT5AuMx4eZBCvr6AOIVndXkDUhGv\n" +

                "ebDA9+GRm2S4Ne10YgWVaBvLeibVshDqcvbVqdvz098WbuUWmV/b2WpRPwX9pe3s4tj1XT1dBc6g\n" +

                "NOM+NBcnmOjBrdqdjNxWCeWHXwkKX9xuN0LtVGmwKJqwBgxWSVz+Aw5ePDg=";

        byte[] byteArray = Base64.decodeBase64(s.getBytes());

        Inflater decompresser = new Inflater();

        decompresser.setInput(byteArray);

        ByteArrayOutputStream bos = new ByteArrayOutputStream(byteArray.length);

        byte[] buffer = new byte[8192];

        while (!decompresser.finished()) {

            int size = decompresser.inflate(buffer);

            bos.write(buffer, 0, size);

        }

        byte[] unzippeddata = bos.toByteArray();

        decompresser.end();

        System.out.println(new String(unzippeddata));

    }

    private void doTestSingleDOIBoostActionToXML(final String filePath) throws Exception {

        final List<Row> rows = Lists.newArrayList();

        final InputStream is = this.getClass().getResourceAsStream(filePath);

        final BufferedReader in = new BufferedReader(new InputStreamReader(is));

        String line = in.readLine();

        final JsonParser parser = new JsonParser();

        JsonObject root = parser.parse(line).getAsJsonObject();

        List<AtomicAction> actions = DOIBoostToActions.generatePublicationActionsFromDump(root, new ActionFactory(), setName, agent, false, false);

        if (actions!= null) {

            actions.forEach(action-> {

                if (action.getTargetColumn().equals("body") && action.getTargetColumnFamily().equals("result"))

                {

                    Column<String, byte[]> col = new Column<>("body" , action.getTargetValue());

                    rows.add(new Row("result",action.getTargetRowKey() , Lists.newArrayList(col)));

                }

            });

        }

    }

    private void doTestSingleDOIBoostAction(final String filePath) throws IOException {

        final InputStream is = this.getClass().getResourceAsStream(filePath);

        final BufferedReader in = new BufferedReader(new InputStreamReader(is));

        String line = in.readLine();

        final JsonParser parser = new JsonParser();

        JsonObject root = parser.parse(line).getAsJsonObject();

        List<AtomicAction> actions = DOIBoostToActions.generatePublicationActionsFromDump(root, new ActionFactory(), setName, agent, false, false);

        if (actions!= null) {

            actions.forEach(it -> System.out.println(String.format(" RowKey:%s TargetColumnFamily:%s   TargetColumn: %s", it.getTargetRowKey(), it.getTargetColumnFamily(), it.getTargetColumn())));

        }

    }

    private void doTestAllDOIBoostAction(final String filePath) throws IOException {

        final InputStream is = this.getClass().getResourceAsStream(filePath);

        final BufferedReader in = new BufferedReader(new InputStreamReader(is));

        String line = in.readLine();

        int i = 0;

        int cnt = 0;

        while(StringUtils.isNotBlank(line)) {

            cnt ++;

            final JsonParser parser = new JsonParser();

            JsonObject root = parser.parse(line).getAsJsonObject();

            try {

                List<AtomicAction> atomicActions = DOIBoostToActions.generatePublicationActionsFromDump(root, new ActionFactory(), setName, agent, false, false);

                if (atomicActions!= null)

                {

                    i ++;

                }

//                    atomicActions.forEach(it -> System.out.println(String.format(" RowKey:%s TargetColumnFamily:%s   TargetColumn: %s", it.getTargetRowKey(), it.getTargetColumnFamily(), it.getTargetColumn())));

                else{

                    System.out.println("SKIPPED Type "+ root.get("type").getAsString());

                }

            } catch (Throwable e) {

                System.out.println(line);

                throw new RuntimeException(e);

            }

            line= in.readLine();

        }

        System.out.println("total "+i+" / "+cnt);

    }

}

package eu.dnetlib.data.mapreduce.actions;

import com.google.gson.JsonObject;

import com.google.gson.JsonParser;

import eu.dnetlib.actionmanager.actions.ActionFactory;

import eu.dnetlib.actionmanager.actions.AtomicAction;

import eu.dnetlib.actionmanager.common.Agent;

import eu.dnetlib.data.mapreduce.hbase.dataimport.CrossRefToActions;

import org.apache.commons.lang3.StringUtils;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

public class CrossRefToActionTest {

    private String setName;

    private Agent agent;

    @Before

    public void setup() {

        setName = "DLI";

        agent= new Agent("agentId","agentName", Agent.AGENT_TYPE.service);

    }

    @Ignore

    @Test

    public void testSingleScholixAction2() throws IOException {

        doTestSingleCrossRefAction("/eu/dnetlib/data/mapreduce/actions/broken");

    }

    @Test

    public void testAllScholixAction() throws IOException {

        doTestAllCrossRefAction("/eu/dnetlib/data/mapreduce/actions/part-06036");

    }

    private void doTestSingleCrossRefAction(final String filePath) throws IOException {

        final InputStream is = this.getClass().getResourceAsStream(filePath);

        final BufferedReader in = new BufferedReader(new InputStreamReader(is));

        String line = in.readLine();

        System.out.println(line);

        final JsonParser parser = new JsonParser();

        JsonObject root = parser.parse(line).getAsJsonObject();

        AtomicAction actions = CrossRefToActions.generateActionsFromDump(root, new ActionFactory(), setName, agent, false);

        System.out.println(actions.toJSON());

    }

    private void doTestAllCrossRefAction(final String filePath) throws IOException {

        final InputStream is = this.getClass().getResourceAsStream(filePath);

        final BufferedReader in = new BufferedReader(new InputStreamReader(is));

        String line = in.readLine();

        while(StringUtils.isNotBlank(line)) {

            final JsonParser parser = new JsonParser();

            JsonObject root = parser.parse(line).getAsJsonObject();

            try {

                CrossRefToActions.generateActionsFromDump(root, new ActionFactory(), setName, agent, false);

            } catch (Throwable e) {

                System.out.println(line);

                throw new RuntimeException(e);

            }

            line= in.readLine();

        }

    }

}

package eu.dnetlib.data.mapreduce.actions;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import com.google.gson.JsonObject;

import com.google.gson.JsonParser;

import eu.dnetlib.actionmanager.actions.ActionFactory;

import eu.dnetlib.actionmanager.actions.AtomicAction;

import eu.dnetlib.actionmanager.common.Agent;

import eu.dnetlib.data.mapreduce.hbase.dataimport.ScholExplorerConfiguration;

import eu.dnetlib.data.mapreduce.hbase.dataimport.ScholixToActions;

import eu.dnetlib.data.transform.xml.AbstractDNetXsltFunctions;

import eu.dnetlib.miscutils.datetime.DateUtils;

import org.apache.commons.lang3.StringUtils;

import org.junit.Assert;

import org.junit.Before;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

public class ScholexplorerActionMapperTest {

    private Map<String, ScholExplorerConfiguration> configurationMap;

    private String setName;

    private Agent agent;

    private String nsPrefix;

    private String dsName;

    private String dsId;

    @Before

    public void initializeCofiguration(){

        configurationMap = new HashMap<>();

        configurationMap.put("issn", new ScholExplorerConfiguration(null, false));

        configurationMap.put("openaire", new ScholExplorerConfiguration(null, false));

        configurationMap.put("pmid", new ScholExplorerConfiguration("pmid", true,"https://www.ncbi.nlm.nih.gov/pubmed/%s"));

        configurationMap.put("doi", new ScholExplorerConfiguration("doi", true,"http://dx.doi.org/%s"));

        configurationMap.put("pbmid", new ScholExplorerConfiguration("pmid", true,"https://www.ncbi.nlm.nih.gov/pubmed/%s"));

        configurationMap.put("pmcid", new ScholExplorerConfiguration("pmc", true,"https://europepmc.org/articles/%s"));

        configurationMap.put("pubmedid", new ScholExplorerConfiguration("pmid", true,"https://www.ncbi.nlm.nih.gov/pubmed/%s"));

        configurationMap.put("icpsr", new ScholExplorerConfiguration(null, false));

        configurationMap.put("dnet", new ScholExplorerConfiguration(null, false));

        configurationMap.put("url", new ScholExplorerConfiguration(null, true,"%s"));

        setName = "DLI";

        agent= new Agent("agentId","agentName", Agent.AGENT_TYPE.service);

        nsPrefix = "scholexplore";

        dsName = "ScholExplorer";

        dsId = "scholexplorer";

    }

    @Test

    public void testSubString () {

        final String dnetId ="50|dli_resolver::7b7b9a57a40818d10cf2532d71f012fa";

        assertEquals("7b7b9a57a40818d10cf2532d71f012fa", dnetId.substring(17));

        System.out.println(AbstractDNetXsltFunctions.md5("SNSF - Swiss National Science Foundation"));

    }

    @Test

    public void testSingleScholixAction() throws IOException {

        doTestSingleScholixAction("/eu/dnetlib/data/mapreduce/actions/part-00000");

    }

    @Test

    public void testSingleScholixAction2() throws IOException {

        doTestSingleScholixAction("/eu/dnetlib/data/mapreduce/actions/scholix.json");

    }

    private void doTestSingleScholixAction(final String filePath) throws IOException {

        final InputStream is = this.getClass().getResourceAsStream(filePath);

        final BufferedReader in = new BufferedReader(new InputStreamReader(is));

        //        in.readLine();

        //        in.readLine();

        String line = in.readLine();

        System.out.println(line);

        final JsonParser parser = new JsonParser();

        JsonObject root = parser.parse(line).getAsJsonObject();

        List<AtomicAction> actions = ScholixToActions

                .generateActionsFromScholix(root, configurationMap, setName, agent, new ActionFactory(), nsPrefix, dsName, dsId, DateUtils.now_ISO8601());

        actions.forEach(it-> System.out.println(String.format("%s    cf:%s    qualifier:%s", it.getTargetRowKey(), it.getTargetColumnFamily(), it.getTargetColumn())));

        System.out.println(actions.get(0).toJSON());

    }

    @Test

    public void testScholixAction() throws IOException {

        doTestMultipleScholixActions("/eu/dnetlib/data/mapreduce/actions/part-00000");

    }

    @Test

    public void testScholixAction2() throws IOException {

        doTestMultipleScholixActions("/eu/dnetlib/data/mapreduce/actions/scholix.json");

    }

    private void doTestMultipleScholixActions(final String filePath) throws IOException {

        final InputStream is = this.getClass().getResourceAsStream(filePath);

        final BufferedReader in = new BufferedReader(new InputStreamReader(is));

        String line = in.readLine();

        while (StringUtils.isNotEmpty(line)){

            final JsonParser parser = new JsonParser();

            JsonObject root = parser.parse(line).getAsJsonObject();

            try {

                List<AtomicAction> actions = ScholixToActions

                        .generateActionsFromScholix(root, configurationMap, setName, agent, new ActionFactory(), nsPrefix, dsName, dsId, DateUtils.now_ISO8601());

                Assert.assertNotNull(actions);

                Assert.assertTrue(actions.size() > 0);

            } catch (Throwable e) {

                System.out.println(line);

                throw (new RuntimeException(e));

            }

            line = in.readLine();

        }

    }

}

package eu.dnetlib.data.mapreduce.dedup;

import java.util.List;

import java.util.UUID;

import com.google.common.base.Predicate;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import eu.dnetlib.data.mapreduce.util.OafTest;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;

import eu.dnetlib.data.proto.KindProtos.Kind;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.data.proto.OafProtos.OafEntity.Builder;

import eu.dnetlib.data.proto.ResultProtos.Result;

import eu.dnetlib.data.proto.ResultProtos.Result.Instance;

import eu.dnetlib.data.proto.TypeProtos.Type;

import eu.dnetlib.data.transform.OafEntityMerger;

import org.junit.Before;

import org.junit.Test;

public class OafMergeTest {

	private List<Oaf> oafList;

	private OafEntityMerger merger;

	@Before

	public void setUp() throws Exception {

		merger = new OafEntityMerger();

		oafList = Lists.newArrayList();

		oafList.add(getOaf("0.1").setEntity(

				getEntity("id_1", null, "pid_1").setResult(

						Result.newBuilder().setMetadata(

								Result.Metadata.newBuilder().setDateofacceptance(OafTest.sf("2012-01-01"))

										.addTitle(OafTest.getStructuredproperty("vvvv Title", "main title", "dnet:dataCite_title"))

										.setResulttype(OafTest.getQualifier("publication", "dnet:result_typologies"))))).build());

		oafList.add(getOaf("0.1").setEntity(

				getEntity("id_2", "originalId_2", "pid_2").setResult(

						Result.newBuilder().setMetadata(

								Result.Metadata.newBuilder().setDateofacceptance(OafTest.sf(""))

										.addTitle(OafTest.getStructuredproperty("aaaa Title", "main title", "dnet:dataCite_title"))

										.setResulttype(OafTest.getQualifier("publication", "dnet:result_typologies"))))).build());

		oafList.add(getOaf("0.2").setEntity(

				getEntity("id_3", "originalId_2", "pid_2").setResult(

						Result.newBuilder().setMetadata(

								Result.Metadata.newBuilder().addTitle(OafTest.getStructuredproperty("cccc Title", "sub title", "dnet:dataCite_title"))

										.setResulttype(OafTest.getQualifier("publication", "dnet:result_typologies"))))).build());

		oafList.add(getOaf("0.3").setEntity(

				getEntity("id_$", null, "pid_3").setResult(

						Result.newBuilder().setMetadata(

								Result.Metadata.newBuilder().setPublisher(OafTest.sf("AMER CHEMICAL SOCXXXXXXXXXXXXXXXXX"))

										.setResulttype(OafTest.getQualifier("publication", "dnet:result_typologies"))))).build());

		oafList.add(getOaf("0.5").setEntity(

				getEntity("id_5", null, null).setResult(

						Result.newBuilder().setMetadata(

								Result.Metadata.newBuilder().addTitle(OafTest.getStructuredproperty("hhhh title", "main title", "dnet:dataCite_title"))

										.setPublisher(OafTest.sf("AMER CHEMICAL SOC X"))

										.setResulttype(OafTest.getQualifier("publication", "dnet:result_typologies")).setStoragedate(OafTest.sf("2012-11-18"))

										.setLanguage(OafTest.getQualifier("eng", "dnet:languages")).addDescription(OafTest.sf("original description")))))

				.build());

		oafList.add(getOaf("0.6").setEntity(

				getEntity("id_6", null, "pid_6").setResult(

						Result.newBuilder().setMetadata(

								Result.Metadata.newBuilder().setResulttype(OafTest.getQualifier("publication", "dnet:result_typologies"))

										.addDescription(OafTest.sf("new description"))).addInstance(OafTest.getInstance("id", "name")))).build());

	}

	@Test

	public void test_merge() {

		final Oaf.Builder builder = Oaf.newBuilder();

		for (final Oaf oaf : oafList) {

			builder.mergeFrom(oaf);

		}

		final Result.Metadata.Builder metadata = builder.getEntityBuilder().getResultBuilder().getMetadataBuilder();

		final Iterable<StructuredProperty> filter = Iterables.filter(metadata.getTitleList(), new Predicate<StructuredProperty>() {

			@Override

			public boolean apply(final StructuredProperty sp) {

				return (sp.getQualifier() != null) && sp.getQualifier().getClassname().equals("main title");

			}

		});

		final StructuredProperty last = Iterables.getLast(filter);

		metadata.clearTitle().addAllTitle(Lists.newArrayList(last));

		System.out.println(builder.build().toString());

	}

	@Test

	public void test_merger() {

		final Oaf merge = merger.mergeEntities(null, "id", oafList).build();

		System.out.println(merge.toString());

	}

	// @Test

	// public void test_sort() {

	// Queue<Result> q = new PriorityQueue<Result>(3, DedupReducer.cmp);

	// for (Oaf oaf : oafList) {

	// q.add(oaf.getEntity().getResult());

	// }

	//

	// while (!q.isEmpty()) {

	// Result r = q.remove();

	// List<StructuredProperty> titles = r.getMetadata().getTitleList();

	// if (!titles.isEmpty()) {

	// System.out.println(titles.get(0).getValue());

	// }

	// }

	// }

	private Oaf.Builder getOaf(final String trust) {

		return Oaf.newBuilder().setKind(Kind.entity).setDataInfo(OafTest.getDataInfo(trust)).setLastupdatetimestamp(System.currentTimeMillis());

	}

	private OafEntity.Builder getEntity(final String id, final String originalId, final String pid) {

		final Builder entity =

				OafEntity.newBuilder().setType(Type.result).setId(id).addOriginalId(originalId != null ? originalId : UUID.randomUUID().toString());

		if (pid != null) {

			entity.addPid(OafTest.getStructuredproperty(pid, "class", "scheme"));

		}

		return entity;

	}

}

package eu.dnetlib.data.mapreduce.dedup.cc;

import java.util.TreeSet;

import eu.dnetlib.data.mapreduce.hbase.dedup.cc.VertexWritable;

import org.apache.commons.lang.StringUtils;

import org.apache.hadoop.io.Text;

import org.junit.Test;

import static org.junit.Assert.assertTrue;

/**

 * Created by claudio on 15/10/15.

 */

public class VertexWritableTest {

	@Test

	public void testSerialise() {

		VertexWritable v = new VertexWritable();

		v.setActivated(true);

		v.setVertexId(new Text("a"));

		TreeSet<Text> edges = new TreeSet<Text>();

		for(int i=0; i<5; i++) {

			edges.add(new Text("" + i));

		}

		v.setEdges(edges);

		assertTrue(v.toString() != null);

		final String json = v.toJSON();

		assertTrue(StringUtils.isNotBlank(json));

		System.out.println(json);

		final VertexWritable v1 = VertexWritable.fromJSON(json);

		final String json1 = v1.toJSON();

		assertTrue(json.equals(json1));

		System.out.println(json1);

	}

}

package eu.dnetlib.data.mapreduce.dedup;

import java.util.LinkedList;

import java.util.List;

import java.util.Map;

import java.util.PriorityQueue;

import java.util.Queue;

import java.util.UUID;

import org.apache.commons.io.IOUtils;

import org.junit.Before;

import org.junit.Test;

import org.springframework.core.io.ClassPathResource;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.pace.clustering.NGramUtils;

import eu.dnetlib.pace.config.Type;

import eu.dnetlib.pace.model.Field;

import eu.dnetlib.pace.model.FieldListImpl;

import eu.dnetlib.pace.model.FieldValueImpl;

import eu.dnetlib.pace.model.MapDocument;

import eu.dnetlib.pace.model.MapDocumentComparator;

public class TitleOrderingTest {

	private List<MapDocument> results = Lists.newArrayList();

	@Before

	public void setUp() throws Exception {

		final List<String> lines = IOUtils.readLines(new ClassPathResource("eu/dnetlib/data/mapreduce/dedup/titles.txt").getInputStream());

		for (final String title : lines) {

			final Map<String, Field> fieldMap = Maps.newHashMap();

			final FieldListImpl list = new FieldListImpl();

			list.add(new FieldValueImpl(Type.String, "title", title));

			fieldMap.put("title", list);

			results.add(new MapDocument("id-" + UUID.randomUUID(), fieldMap));

		}

	}

	@Test

	public void test() {

		final Queue<MapDocument> queue = new PriorityQueue<MapDocument>(100, new MapDocumentComparator("title"));

		queue.addAll(results);

		final Queue<MapDocument> queue2 = simplifyQueue(queue);

		while (!queue2.isEmpty()) {

			final MapDocument doc = queue2.remove();

			System.out.println(doc.values("title").stringValue());

		}

	}

	private Queue<MapDocument> simplifyQueue(final Queue<MapDocument> queue) {

		final Queue<MapDocument> q = new LinkedList<MapDocument>();

		String fieldRef = "";

		final List<MapDocument> tempResults = Lists.newArrayList();

		while (!queue.isEmpty()) {

			final MapDocument result = queue.remove();

			if (!result.values("title").isEmpty()) {

				final String field = NGramUtils.cleanupForOrdering(result.values("title").stringValue());

				if (field.equals(fieldRef)) {

					tempResults.add(result);

				} else {

					if (tempResults.size() < 5) {

						q.addAll(tempResults);

					} else {

						System.out.println("Skipped field: " + fieldRef + " - size: " + tempResults.size());

					}

					tempResults.clear();

					tempResults.add(result);

					fieldRef = field;

				}

			}

		}

		if (tempResults.size() < 5) {

			q.addAll(tempResults);

		} else {

			System.out.println("Skipped field: " + fieldRef + " - size: " + tempResults.size());

		}

		return q;

	}

}

package eu.dnetlib.data.mapreduce.hbase.dedup.experiment;

import org.hsqldb.util.CSVWriter;

import org.junit.Test;

import org.junit.Before;

import org.junit.After;

import static org.junit.Assert.assertNotNull;

import static org.junit.Assert.assertTrue;

/**

 * CsvEntry Tester.

 *

 * @author <Authors name>

 * @version 1.0

 * @since <pre>Apr 20, 2016</pre>

 */

public class CsvEntryTest {

	private CsvEntry csvEntry;

	@Before

	public void before() throws Exception {

		csvEntry = new CsvEntry();

	}

	/**

	 * Method: addFeature(final String f)

	 */

	@Test

	public void testAddFeature() throws Exception {

		csvEntry.addFeature("a");

		assertTrue(csvEntry.getFeatures().contains("a"));

	}

	/**

	 * Method: getFeatures()

	 */

	@Test

	public void testGetFeatures() throws Exception {

		csvEntry.addFeature("a");

		assertNotNull(csvEntry.getFeatures());

		assertTrue(csvEntry.getFeatures().size() == 1);

	}

	/**

	 * Method: fromJson(final String json)

	 */

	@Test

	public void testFromJson() throws Exception {

		csvEntry.addFeature("a");

		csvEntry.addFeature("b");

		final String json = csvEntry.toString();

		final CsvEntry another = CsvEntry.fromJson(json);

		assertTrue(another.equals(csvEntry));

	}

}

package eu.dnetlib.data.mapreduce.hbase.dedup.experiment;

import static org.junit.Assert.assertEquals;

import java.util.Iterator;

import java.util.List;

import java.util.Set;

import java.util.UUID;

import com.google.common.base.Predicate;

import com.google.common.base.Splitter;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Sets;

import eu.dnetlib.pace.model.Person;

import org.apache.commons.lang.RandomStringUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.lang.math.RandomUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.junit.Before;

import org.junit.Test;

/**

 * Created by claudio on 26/04/16.

 */

public class CsvSerialiserTest {

	private static final Log log = LogFactory.getLog(CsvSerialiserTest.class);

	private CsvSerialiser csvSerialiser;

	private static List<String> ALL_FEATURES = Lists.newLinkedList();

	static {

		ALL_FEATURES.add("a");

		ALL_FEATURES.add("b");

		ALL_FEATURES.add("c");

		ALL_FEATURES.add("d");

		ALL_FEATURES.add("e");

		ALL_FEATURES.add("f");

	}

	@Before

	public void setUp() {

		csvSerialiser = new CsvSerialiser();

	}

	@Test

	public void testSerialiser() {

		final List<CsvEntry> list = Lists.newArrayList();

		final int nRows = RandomUtils.nextInt(10) + 3;

		for(int i = 0; i<nRows; i++) {

			list.add(getRandomCsvEntry());

		}

		final String csv = csvSerialiser.asCSV(list);

		log.info("\n" + csv);

		verifyLength(csv);

	}

	@Test

	public void testSerialiser2() {

		final List<CsvEntry> list = Lists.newArrayList();

		final int nRows = RandomUtils.nextInt(10) + 3;

		for(int i = 0; i<nRows; i++) {

			list.add(getCsvEntry(i));

		}

		final String csv = csvSerialiser.asCSV(list);

		log.info("\n" + csv);

		verifyLength(csv);

	}

	@Test

	public void testSerialiser3() {

		final String name = "Manghi, Paolo as ";

		final Person p = new Person(name, false);

		final String s = p.getSurnameString() + StringUtils.substring(p.getNameString(), 0, 1);

		log.info(String.format("'%s'", s.replaceAll("[^a-zA-Z ]", "").toLowerCase().trim()));

	}

	private void verifyLength(final String csv) {

		final Iterator<String> lines = Splitter.on("\n").split(csv).iterator();

		final List<String> header = Lists.newArrayList(Splitter.on(",").split(lines.next()));

		while(lines.hasNext()) {

			List<String> line = Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings().split(lines.next()));

			if (!line.isEmpty()) {

				assertEquals(header.size(), line.size());

			}

		}

	}

	private CsvEntry getCsvEntry(final int i) {

		final CsvEntry e = new CsvEntry();

		e.setKey(i+"");

		final Set<String> features = Sets.newLinkedHashSet(Iterables.filter(ALL_FEATURES, new Predicate<String>() {

			@Override

			public boolean apply(final String s) {

				return RandomUtils.nextBoolean();

			}

		}));

		e.setFeatures(features);

		log.info(String.format("%s - %s", i, features));

		//log.info(e.toString() + "\n");

		return e;

	}

	public CsvEntry getRandomCsvEntry() {

		final CsvEntry e = new CsvEntry();

		e.setKey(UUID.randomUUID().toString());

		final int nFeatures = RandomUtils.nextInt(4) + 1;

		for(int i = 0; i<nFeatures; i++) {

			e.addFeature("ft." + RandomStringUtils.randomAlphabetic(2) + " \"" + RandomStringUtils.randomAlphabetic(2) + "\" " + RandomStringUtils.randomAlphabetic(2));

		}

		return e;

	}

}

package eu.dnetlib.data.mapreduce.hbase.broker;

import java.io.IOException;

import java.io.InputStream;

import java.util.List;

import java.util.Map;

import java.util.Set;

import com.google.common.collect.Maps;

import com.google.common.collect.Sets;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.data.proto.OafProtos;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.transform.Column;

import eu.dnetlib.data.transform.Row;

import eu.dnetlib.data.transform.XsltRowTransformer;

import eu.dnetlib.data.transform.XsltRowTransformerFactory;

import org.apache.commons.io.IOUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.mapreduce.Counter;

import org.apache.hadoop.mapreduce.Reducer.Context;

import org.dom4j.Document;

import org.dom4j.DocumentException;

import org.dom4j.io.SAXReader;

import org.junit.Before;

import org.junit.Ignore;

import org.junit.Test;

import org.junit.runner.RunWith;

import org.mockito.Mock;

import org.mockito.junit.MockitoJUnitRunner;

import static org.junit.Assert.*;

import static org.mockito.ArgumentMatchers.anyString;

import static org.mockito.Mockito.when;

/**

 * Created by claudio on 22/07/16.

 */

@RunWith(MockitoJUnitRunner.Silent.class)

public class EventWrapperTest {

	private static final Log log = LogFactory.getLog(EventWrapperTest.class);

	private Oaf oaf1_1;

	private Oaf oaf1_2;

	private Oaf oaf2_1;

	private Oaf oaf2_2;

	private Oaf oaf3_1;

	private Oaf oaf3_2;

	private String xslt;

	@Mock

	private Context context;

	@Mock

	private Counter counter;

	private Map<String, String> baseUrlMap = Maps.newHashMap();

	private static final String basePathProfiles = "/eu/dnetlib/test/profiles/TransformationRuleDSResources/TransformationRuleDSResourceType/2hbase/";

	@Before

	public void setUp() throws Exception {

		xslt = loadFromTransformationProfile("oaf2hbase.xml");

		oaf1_1 = asOaf("recordOaf_1_1.xml");

		oaf1_2 = asOaf("recordOaf_1_2.xml");

		oaf2_1 = asOaf("recordOaf_2_1.xml");

		oaf2_2 = asOaf("recordOaf_2_2.xml");

		oaf3_1 = asOaf("recordOaf_3_1.xml");

		oaf3_2 = asOaf("recordOaf_3_2.xml");

		when(context.getCounter(anyString(), anyString())).thenReturn(counter);

		baseUrlMap.put("publication", "https://explore.openaire.eu/search/publication?articleId=%s");

		baseUrlMap.put("dataset", "https://explore.openaire.eu/search/dataset?datasetId=%s");

		baseUrlMap.put("software", "https://explore.openaire.eu/search/software?softwareId=%s");

		baseUrlMap.put("other", "https://explore.openaire.eu/search/other?orpId=%s");

	}

	@Test

	public void testAbstractEvent() throws Exception {

		final String id1 = AbstractEventFactory.process(oaf1_1, oaf1_2, 1.0f).get(0).asBrokerEvent().getEventId();

		final String id2 = AbstractEventFactory.process(oaf2_1, oaf2_2, 1.0f).get(0).asBrokerEvent().getEventId();

		final String id3 = AbstractEventFactory.process(oaf3_1, oaf3_2, 1.0f).get(0).asBrokerEvent().getEventId();

		verifyEquals(id1, id2);

		verifyDifferent(id1, id3);

	}

	@Test

	public void testOAVersionEvent() throws Exception {

		final Set<String> untrustedOaDs = Sets.newHashSet();

		final String id1 = OAVersionEventFactory.process(oaf1_1, oaf1_2, 1.0f, untrustedOaDs).get(0).asBrokerEvent().getEventId();

		final String id2 = OAVersionEventFactory.process(oaf2_1, oaf2_2, 1.0f, untrustedOaDs).get(0).asBrokerEvent().getEventId();

		verifyEquals(id1, id2);

	}

	@Test

	public void testPIDEvent() throws Exception {

		final String id1 = PIDEventFactory.process(oaf1_1, oaf1_2, 1.0f).get(0).asBrokerEvent().getEventId();

		final String id2 = PIDEventFactory.process(oaf2_1, oaf2_2, 1.0f).get(0).asBrokerEvent().getEventId();

		final String id3 = PIDEventFactory.process(oaf3_1, oaf3_2, 1.0f).get(0).asBrokerEvent().getEventId();

		verifyEquals(id1, id2);

		verifyDifferent(id1, id3);

	}

	@Test

	//@Ignore

	public void testProjectEvent() throws Exception {

		Oaf oaf1_1_c = enrichWithCachedOafRel(oaf1_1, "oafRel_isProducedBy_1.json", "project1.json");

		Oaf oaf1_2_c = enrichWithCachedOafRel(oaf1_2, "oafRel_isProducedBy_2.json", "project2.json");

		final String id1 = ProjectEventFactory.process(context, oaf1_1_c, oaf1_2_c, 1.0f, baseUrlMap).get(0).asBrokerEvent().getEventId();

		/*final String id2 = ProjectEventFactory.process(context, oaf2_1, oaf2_2, 1.0f, baseUrlMap).get(0).asBrokerEvent().getEventId();

		final String id3 = ProjectEventFactory.process(context, oaf3_1, oaf3_2, 1.0f, baseUrlMap).get(0).asBrokerEvent().getEventId();

		verifyEquals(id1, id2);

		verifyDifferent(id1, id3);

		*/

	}

	private Oaf enrichWithCachedOafRel(final Oaf oaf, final String oafRel, final String oafProject) throws IOException {

		final Oaf.Builder cachedRel = Oaf.newBuilder();

		JsonFormat.merge(IOUtils.toString(getClass().getResourceAsStream(oafRel)), cachedRel);

		final Oaf.Builder cachedTarget = Oaf.newBuilder();

		JsonFormat.merge(IOUtils.toString(getClass().getResourceAsStream(oafProject)), cachedTarget);

		cachedRel.getRelBuilder().setCachedOafTarget(cachedTarget);

		Oaf.Builder builder = Oaf.newBuilder(oaf);

		builder.getEntityBuilder().addCachedOafRel(cachedRel);

		return builder.build();

	}

	@Test

	public void testPublicationDateEvent() throws Exception {

		final String id1 = PublicationDateEventFactory.process(oaf1_1, oaf1_2, 1.0f).get(0).asBrokerEvent().getEventId();

		final String id2 = PublicationDateEventFactory.process(oaf2_1, oaf2_2, 1.0f).get(0).asBrokerEvent().getEventId();

		final String id3 = PublicationDateEventFactory.process(oaf3_1, oaf3_2, 1.0f).get(0).asBrokerEvent().getEventId();

		verifyEquals(id1, id2);

		verifyDifferent(id1, id3);

	}

	@Test

	@Ignore

	public void testSoftwareEventFactory() throws Exception {

		final String id1 = SoftwareEventFactory.process(context, oaf1_1, oaf1_2, 1.0f, baseUrlMap).get(0).asBrokerEvent().getEventId();

		final String id2 = SoftwareEventFactory.process(context, oaf2_1, oaf2_2, 1.0f, baseUrlMap).get(0).asBrokerEvent().getEventId();

		final String id3 = SoftwareEventFactory.process(context, oaf3_1, oaf3_2, 1.0f, baseUrlMap).get(0).asBrokerEvent().getEventId();

		verifyEquals(id1, id2);

		verifyDifferent(id1, id3);

	}

	@Test

	public void testSubjectEventFactory() throws Exception {

		final String id1 = SubjectEventFactory.process(context, oaf1_1, oaf1_2, 1.0f).get(0).asBrokerEvent().getEventId();

		final String id2 = SubjectEventFactory.process(context, oaf2_1, oaf2_2, 1.0f).get(0).asBrokerEvent().getEventId();

		final String id3 = SubjectEventFactory.process(context, oaf3_1, oaf3_2, 1.0f).get(0).asBrokerEvent().getEventId();

		verifyEquals(id1, id2);

		verifyDifferent(id1, id3);

	}

	private void verifyEquals(final String x, final String y) {

		assertNotNull(x);

		assertNotNull(y);

		assertTrue(x.length() > 10);

		assertTrue(y.length() > 10);

		assertEquals(x, y);

		log.info(String.format("%s = %s", x, y));

	}

	private void verifyDifferent(final String x, final String y) {

		log.info(String.format("%s != %s", x, y));

		assertNotNull(x);

		assertNotNull(y);

		assertTrue(x.length() > 10);

		assertTrue(y.length() > 10);

		assertFalse(x.equals(y));

	}

	private Oaf asOaf(final String xmlFile) throws Exception {

		for (final Row row : asRows(getClass().getResourceAsStream(xmlFile))) {

			for (final Column<String, byte[]> c : row.getColumns()) {

				final String cf = row.getColumnFamily();

				if (cf.equals("result") && c.getName().equals("body")) { return Oaf.parseFrom(c.getValue()); }

			}

		}

		throw new RuntimeException("Oaf not found");

	}

	private String loadFromTransformationProfile(final String profilePath) {

		log.info("Loading xslt from: " + basePathProfiles + profilePath);

		final InputStream profile = getClass().getResourceAsStream(basePathProfiles + profilePath);

		final SAXReader saxReader = new SAXReader();

		try {

			final Document doc = saxReader.read(profile);

			return doc.selectSingleNode("//SCRIPT/CODE/*[local-name()='stylesheet']").asXML();

		} catch (final DocumentException e) {

			e.printStackTrace();

			throw new RuntimeException(e);

		}

	}

	private List<Row> asRows(final InputStream recordStream)

			throws Exception {

		final XsltRowTransformer transformer = new XsltRowTransformerFactory().getTransformer(xslt);

		assertNotNull(transformer);

		final String record = IOUtils.toString(recordStream);

		final List<Row> rows = transformer.apply(record);

		assertNotNull(rows);

		assertFalse(rows.isEmpty());

		return rows;

	}

}

package eu.dnetlib.data.mapreduce.hbase.broker;

import java.util.UUID;

import org.apache.commons.lang.math.RandomUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.DocumentException;

import org.junit.Test;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.broker.objects.OpenAireEventPayload;

import eu.dnetlib.data.mapreduce.hbase.broker.mapping.EventFactory;

import eu.dnetlib.data.mapreduce.hbase.broker.mapping.HighlightFactory;

import eu.dnetlib.data.mapreduce.hbase.broker.mapping.OpenAireEventPayloadFactory;

import eu.dnetlib.data.mapreduce.hbase.broker.model.EventMessage;

import eu.dnetlib.data.mapreduce.hbase.broker.model.EventWrapper;

import eu.dnetlib.data.mapreduce.util.OafTest;

import eu.dnetlib.data.proto.FieldTypeProtos.KeyValue;

import eu.dnetlib.data.proto.FieldTypeProtos.Qualifier;

import eu.dnetlib.data.proto.FieldTypeProtos.StringField;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.data.proto.OafProtos.OafEntity.Builder;

import eu.dnetlib.data.proto.ResultProtos.Result;

import eu.dnetlib.data.proto.ResultProtos.Result.Instance;

import eu.dnetlib.data.proto.TypeProtos.Type;

/**

 * Created by claudio on 22/07/16.

 */

public class EventFactoryTest {

	private static final Log log = LogFactory.getLog(EventFactoryTest.class);

	@Test

	public void testEventFactory() throws DocumentException {

		final OafEntity source = getEntity("id_1", null, "pid_1").setResult(

				Result.newBuilder().setMetadata(

						Result.Metadata.newBuilder().setDateofacceptance(OafTest.sf("2012-01-01"))

								.addTitle(sp("vvvv Title", "main title", "dnet:dataCite_title"))

								.setResulttype(q("publication", "dnet:result_typologies")))

						.addInstance(Instance.newBuilder()

								.setHostedby(kv("456", "PubMed"))

								.setAccessright(q("OPEN", "dnet:licenses"))

								.addUrl("http://456")))

				.build();

		final OafEntity oaf = getEntity("id_2", "originalId_2", "pid_2").setResult(

				Result.newBuilder().setMetadata(

						Result.Metadata.newBuilder().setDateofacceptance(OafTest.sf(""))

								.addTitle(sp("aaaa Title", "main title", "dnet:dataCite_title"))

								.setDateofacceptance(sf("2017-07-13"))

								.setResulttype(q("publication", "dnet:result_typologies")))

						.addInstance(

								Instance.newBuilder()

										.setHostedby(kv("123", "Puma"))

										.setAccessright(q("CLOSED", "dnet:licenses"))

										.addUrl("http://123")))

				.build();

		log.info("oaf: " + JsonFormat.printToString(oaf));

		final float trust = RandomUtils.nextFloat();

		final OpenAireEventPayload openAireEventPayload = new OpenAireEventPayload();

		final EventMessage event = EventFactory.asEvent(oaf, Topic.ENRICH_MISSING_PUBLICATION_DATE, openAireEventPayload, source, trust);

		final OpenAireEventPayload p = OpenAireEventPayloadFactory.fromOAF(oaf, source, trust);

		final OpenAireEventPayload payload = HighlightFactory.highlightEnrichPublicationDate(p, OafTest.sf("2012-01-01"));

		final String payloadJson = payload.toJSON();

		log.info("payload: " + payloadJson);

		log.info("payload: " + OpenAireEventPayload.fromJSON(payloadJson).toJSON());

		event.setPayload(payload.toJSON());

		log.info("event message: " + event.toString());

		final EventWrapper wrapper = new EventWrapper(event, "2012-01-01", "test");

		log.info("broker event: " + wrapper.asBrokerEvent().toJson());

	}

	private Qualifier.Builder q(final String classid, final String scheme) {

		return OafTest.getQualifier(classid, scheme);

	}

	private KeyValue.Builder kv(final String k, final String v) {

		return KeyValue.newBuilder().setKey(k).setValue(v);

	}

	private StringField.Builder sf(final String value) {

		return StringField.newBuilder().setValue(value);

	}

	private StructuredProperty.Builder sp(final String v, final String classid, final String scheme) {

		return OafTest.getStructuredproperty(v, classid, scheme);

	}

	private OafEntity.Builder getEntity(final String id, final String originalId, final String pid) {

		final Builder entity =

				OafEntity.newBuilder().setType(Type.result).setId(id).addOriginalId(originalId != null ? originalId : UUID.randomUUID().toString());

		if (pid != null) {

			entity.addPid(OafTest.getStructuredproperty(pid, "doi", "scheme"));

		}

		return entity;

	}

}

package eu.dnetlib.data.mapreduce.hbase.dataimport;

import org.apache.commons.io.IOUtils;

import org.junit.Assert;

import org.junit.Test;

import java.io.IOException;

public class GetInvalidXmlRecordsMapperTest {

    @Test

    public void testParseDoi() throws IOException {

        final String xml = IOUtils.toString(getClass().getResourceAsStream("record.xml"));

        Assert.assertNotNull(xml);

        String doi = GetInvalidXmlRecordsMapper.getDoi(xml);

        Assert.assertNotNull(doi);

        System.out.println("doi = " + doi);

    }

}

package eu.dnetlib.data.mapreduce.hbase.index.config;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

public class RelClassesTest {

	public static final String relClassesJson =

			"[\"isProvidedBy:provides\","

					+ "\"provides:isProvidedBy\","

					+ "\"merges:isMergedIn\","

					+ "\"isMergedIn:merges\","

					+ "\"isSimilarTo:isSimilarTo\","

					+ "\"isCoAuthorOf:isCoAuthorOf\","

					+ "\"isAuthorOf:hasAuthor\","

					+ "\"hasAuthor:isAuthorOf\","

					+ "\"isParticipant:hasParticipant\","

					+ "\"hasParticipant:isParticipant\","

					+ "\"isProducedBy:produces\","

					+ "\"produces:isProducedBy\","

					+ "\"hasAmongTopNSimilarDocuments:isAmongTopNSimilarDocuments\","

					+ "\"isAmongTopNSimilarDocuments:hasAmongTopNSimilarDocuments\","

					+ "\"isRelatedTo:isRelatedTo\","

					+ "\"isContact:hasContact\","

					+ "\"hasContact:isContact\","

					+ "\"isContributorOf:hasContributor\","

					+ "\"hasContributor:isContributorOf\","

					+ "\"isPartOf:hasPart\","

					+ "\"hasPart:isPartOf\","

					+ "\"isSupplementedBy:isSupplementTo\","

					+ "\"isSupplementTo:isSupplementedBy\","

					+ "\"hasAuthorInstitution:isAuthorInstitutionOf\","

					+ "\"isAuthorInstitutionOf:hasAuthorInstitution\"]";

	@Test

	public void testRelClasses() {

		RelClasses relClasses = new RelClasses(relClassesJson);

		assertNotNull(relClasses);

		String providedBy = relClasses.getInverse("provides");

		assertEquals(providedBy, "isProvidedBy");

		String provides = relClasses.getInverse("isProvidedBy");

		assertEquals(provides, "provides");

	}

}

package eu.dnetlib.data.mapreduce.hbase.index.config;

public class Context {

	public final static String xml = "<ContextDSResources>"

			+ "<entry id=\"fam\" label=\"Fisheries and Acquacolture Management\" name=\"context\" type=\"community\"/>"

			+ "<entry id=\"clarin\" label=\"CLARIN\" name=\"context\" type=\"ri\"/>"

			+"<entry id=\"dh-ch\" label=\"DH-CH\" name=\"context\" type=\"community\"/>"

			+ "<entry id=\"dh-ch::subcommunity\" label=\"Subcommunitiess\" name=\"category\" type=\"\"/>"

			+ "<entry id=\"dh-ch::subcommunity::2\" label=\"CLARIN\" name=\"concept\" type=\"\"/>"

			+"<entry id=\"egi\" label=\"EGI\" name=\"context\" type=\"ri\"/>"

			+ "<entry id=\"egi::classification\" label=\"Scientific Disciplines\" name=\"category\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc\" label=\"Natural Sciences\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::math\" label=\"Mathematics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::math::applied\" label=\"Applied Mathematics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::math::pure\" label=\"Pure Mathematics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::math::stats\" label=\"Statistics and Probability\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::math::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc\" label=\"Computer sciences\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::algo\" label=\"Algorithms\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::artii\" label=\"Artificial Intelligence\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::carc\" label=\"Computer architecture\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::ccomm\" label=\"Computer communications\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::cgraph\" label=\"Computer graphics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::csecr\" label=\"Computer security and reliability\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::dstr\" label=\"Data structures\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::disc\" label=\"Distributed computing\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::hci\" label=\"Human-computer interaction\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::opes\" label=\"Operating systems\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::parc\" label=\"Parallel computing\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::plan\" label=\"Programming languages\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::qcom\" label=\"Quantum computing\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::softe\" label=\"software engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::theo\" label=\"Theory of computation\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::cosc::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::infos\" label=\"Information sciences\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::infos::dman\" label=\"Data management\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::infos::dmin\" label=\"Data mining\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::infos::iret\" label=\"Information retrieval\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::infos::iman\" label=\"Information management\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::infos::kman\" label=\"Knowledge management\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::infos::mult\" label=\"Multimedia, hypermedia\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::infos::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths\" label=\"Earth sciences\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::atms\" label=\"Atmospheric science\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::clim\" label=\"Climate research\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::geoc\" label=\"Geochemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::geol\" label=\"Geology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::geop\" label=\"Geophysics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::hydr\" label=\"Hydrology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::mine\" label=\"Mineralogy\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::ocea\" label=\"Oceanography\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::pala\" label=\"palaeontology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::pgeo\" label=\"Physical geography\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::seis\" label=\"Seismology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::volc\" label=\"Volcanology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::earths::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc\" label=\"Biological sciences\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::aerob\" label=\"Aerobiology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::bact\" label=\"Bacteriology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::behav\" label=\"Behavioural biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::bimol\" label=\"Biochemistry and molecular biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::bcons\" label=\"Biodiversity conservation\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::binfo\" label=\"Bioinformatics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::brhy\" label=\"Biological rhythm\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::biol\" label=\"Biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::bioph\" label=\"Biophysics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::bota\" label=\"Botany\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::cellb\" label=\"Cell biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::cobio\" label=\"Computational biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::cryo\" label=\"Cryobiology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::devb\" label=\"Developmental biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::ecol\" label=\"Ecology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::evolb\" label=\"Evolutionary biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::gehe\" label=\"Genetics and heredity\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::marin\" label=\"Marine and Freshwater biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::mbio\" label=\"Mathematical biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::mibio\" label=\"Microbiology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::myco\" label=\"Mycology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::plant\" label=\"Plant science\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::repr\" label=\"Reproductive biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::strub\" label=\"Structural biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::taxo\" label=\"Taxonomy\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::theob\" label=\"Theoretical biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::therm\" label=\"Thermal biology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::viro\" label=\"Virology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::zool\" label=\"Zoology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::biosc::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc\" label=\"Physical sciences\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::acce\" label=\"Acceleration physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::acou\" label=\"Acoustics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::aero\" label=\"Aerosol physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::astrb\" label=\"Astrobiology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::astrn\" label=\"Astronomy\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::astrp\" label=\"Astroparticle physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::astrph\" label=\"Astrophysics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::atmc\" label=\"Atomic\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::chph\" label=\"Chemical physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::comph\" label=\"Computational physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::conds\" label=\"Condensed matter physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::cryog\" label=\"Cryogenics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::fluid\" label=\"Fluid mechanics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::fusi\" label=\"Fusion\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::hene\" label=\"High energy physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::mathp\" label=\"Mathematical physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::medip\" label=\"Medical physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::molep\" label=\"Molecular physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::nuclp\" label=\"Nuclear physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::optc\" label=\"Optics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::partp\" label=\"Particle physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::phys\" label=\"Physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::plans\" label=\"Planetary science\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::plsm\" label=\"Plasma physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::spac\" label=\"Space physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::quant\" label=\"Quantum physics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::physc::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc\" label=\"Chemical sciences\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::analc\" label=\"Analytical chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::chem\" label=\"Chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::collc\" label=\"Colloid chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::compc\" label=\"Computational chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::elecc\" label=\"Electrochemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::innuc\" label=\"Inorganic and nuclear chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::mathc\" label=\"Mathematical chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::orgc\" label=\"Organic chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::phyc\" label=\"Physical chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::polc\" label=\"Polymer chemistry\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::natsc::chesc::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt\" label=\"Engineering and Technology\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::cieng\" label=\"Civil engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::civeng::arche\" label=\"Architecture engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::civeng::cieng\" label=\"Civil engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::civeng::cipro\" label=\"Civil protection\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::civeng::costr\" label=\"Construction/Structural engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::civeng::treng\" label=\"Transport engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::civeng::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::eeie\" label=\"Electrical, electronic and information engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::eeie::comes\" label=\"Communication engineering and systems\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::eeie::comha\" label=\"Computer hardware and architecture\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::eeie::elele\" label=\"Electrical and electronic engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::eeie::robaut\" label=\"Robotics, Automation and Control Systems\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::eeie::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::meng\" label=\"Mechanical engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::meng::apmec\" label=\"Applied mechanics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::meng::audm\" label=\"Audio engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::meng::nucl\" label=\"Nuclear related engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::meng::relia\" label=\"Reliability analysis\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::meng::therd\" label=\"Thermodynamics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::meng::other\" label=\"other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::aeros\" label=\"Aerospace engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::aeros::aeren\" label=\"Aeronautical engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::aeros::astren\" label=\"Astronautical engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::aeros::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::cheme\" label=\"Chemical engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::cheme::cheme\" label=\"Chemical engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::cheme::chempr\" label=\"Chemical process engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::cheme::other\" label=\"Other\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::maten\" label=\"Materials engineering\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::maten::ceram\" label=\"Ceramics\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::maten::coatf\" label=\"Coating and films\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::maten::compos\" label=\"Composites\" name=\"concept\" type=\"\"/>"

			+ "<entry id=\"egi::classification::engt::maten::paperw\" label=\"Paper and wood\" name=\"concept\" type=\"\"/>"