D-Net

[

{

  "type_source": "SVN", 

  "goal": "package -U -T 4C source:jar", 

  "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/icm-iis-collapsers/trunk/", 

  "deploy_repository": "dnet4-snapshots", 

  "version": "4",

  "mail": "m.horst@icm.edu.pl",

  "deploy_repository_url": "http://maven.research-infrastructures.eu/nexus/content/repositories/dnet4-snapshots", 

  "name": "icm-iis-collapsers"

},

{

  "type_source": "SVN",

  "goal": "clean verify -U -e -X",

  "url": "http://svn-public.driver.research-infrastructures.eu/driver/dnet40/modules/icm-iis-collapsers/trunk/",

  "nightly" : "true",

  "cron" : "H H * * *",

  "version": "4",

  "mail": "d.tkaczyk@icm.edu.pl,m.horst@icm.edu.pl",

  "name": "icm-iis-collapsers-embedded-integration-test"

}

]

package eu.dnetlib.iis.collapsers.basic;

import eu.dnetlib.iis.collapsers.SampleData;

import eu.dnetlib.iis.importer.schemas.DocumentMetadata;

import java.util.ArrayList;

import java.util.List;

import static org.junit.Assert.assertNull;

import org.junit.Test;

import org.python.google.common.collect.Lists;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

public class BestFilledCollapserTest {

    public static final List<DocumentMetadata> emptyList = 

            new ArrayList<DocumentMetadata>();

    public static final List<DocumentMetadata> list1 = 

            Lists.newArrayList(SampleData.metadataRecord11);

    public static final List<DocumentMetadata> list2 = 

            Lists.newArrayList(SampleData.metadataRecord12);

    public static final List<DocumentMetadata> list123 =

            Lists.newArrayList(SampleData.metadataRecord11, SampleData.metadataRecord12, SampleData.metadataRecord13);

    public static final List<DocumentMetadata> list321 =

            Lists.newArrayList(SampleData.metadataRecord13, SampleData.metadataRecord12, SampleData.metadataRecord11);

    @Test

	public void testBestFilledEmpty() throws Exception {

        BestFilledCollapser<DocumentMetadata> collapser = new BestFilledCollapser<DocumentMetadata>();

        assertNull(collapser.collapse(null));

        assertNull(collapser.collapse(emptyList));

    }

    @Test

	public void testBestFilledDefaultFieldSet() throws Exception {

        BestFilledCollapser<DocumentMetadata> collapser = new BestFilledCollapser<DocumentMetadata>();

        SampleData.assertEqualRecords(

                list1,

                collapser.collapse(list1));

        SampleData.assertEqualRecords(

                list1,

                collapser.collapse(list123));

        SampleData.assertEqualRecords(

                list1, 

                collapser.collapse(list321));

    }

    @Test

	public void testBestFilled() throws Exception {

        BestFilledCollapser<DocumentMetadata> collapser = new BestFilledCollapser<DocumentMetadata>();

        collapser.setFields(SampleData.significantFields);

        SampleData.assertEqualRecords(

                list1,

                collapser.collapse(list1));

        SampleData.assertEqualRecords(

                list2, 

                collapser.collapse(list123));

        SampleData.assertEqualRecords(

                list2, 

                collapser.collapse(list321));

    }

}

package eu.dnetlib.iis.collapsers.basic;

import eu.dnetlib.iis.collapsers.SampleData;

import eu.dnetlib.iis.metadataextraction.schemas.DocumentText;

import java.util.ArrayList;

import java.util.List;

import static org.junit.Assert.assertNull;

import org.junit.Test;

import org.python.google.common.collect.Lists;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

public class DocumentTextCollapserTest {

    public static final List<DocumentText> oneElementList = 

            Lists.newArrayList(

                DocumentText.newBuilder().setId("id").setText("This is text").build()

            );

    public static final List<DocumentText> list =

            Lists.newArrayList(

                DocumentText.newBuilder().setId("id").setText("This is text").build(),

                DocumentText.newBuilder().setId("id").setText("This is another text").build(),

                DocumentText.newBuilder().setId("id").setText("This is a duplicated text").build()

            );

    public static final List<DocumentText> collapsedList =

            Lists.newArrayList(

                DocumentText.newBuilder().setId("id").setText("This is text\n\nThis is another text\n\nThis is a duplicated text").build()

            );

    @Test

	public void testDocumentTextCollapserEmpty() throws Exception {

        DocumentTextCollapser collapser = new DocumentTextCollapser();

        assertNull(collapser.collapse(null));

        assertNull(collapser.collapse(new ArrayList<DocumentText>()));

    }

    @Test

	public void testDocumentTextCollapser() throws Exception {

        DocumentTextCollapser collapser = new DocumentTextCollapser();

        SampleData.assertEqualRecords(

                oneElementList,

                collapser.collapse(oneElementList));

        SampleData.assertEqualRecords(

                collapsedList,

                collapser.collapse(list));

    }

}

package eu.dnetlib.iis.collapsers.basic;

import eu.dnetlib.iis.IntegrationTest;

import eu.dnetlib.iis.core.AbstractWorkflowTestCase;

import org.junit.Test;

import org.junit.experimental.categories.Category;

/**

 * @author Dominika Tkaczyk

 * @author Michal Oniszczuk

 */

@Category(IntegrationTest.class)

public class WorkflowTest extends AbstractWorkflowTestCase {

    @Test

    public void testDefaultWorkflow() throws Exception {

        runWorkflow("eu/dnetlib/iis/collapsers/basic_collapser/default/oozie_app");

    }

    /*

    @Test

    public void testCitationWorkflow() throws Exception {

        runWorkflow("eu/dnetlib/iis/collapsers/collapser/citation/oozie_app");

    }

*/

    @Test

    public void testDocumentTextWorkflow() throws Exception {

        runWorkflow("eu/dnetlib/iis/collapsers/basic_collapser/documenttext/oozie_app");

    }

}

package eu.dnetlib.iis.collapsers.basic;

import eu.dnetlib.iis.collapsers.SampleData;

import eu.dnetlib.iis.importer.schemas.DocumentMetadata;

import java.util.ArrayList;

import java.util.List;

import static org.junit.Assert.assertNull;

import org.junit.Test;

import org.python.google.common.collect.Lists;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

public class BestFilledMergingCollapserTest {

    public static final List<DocumentMetadata> emptyList = 

            new ArrayList<DocumentMetadata>();

    public static final List<DocumentMetadata> list1 = 

            Lists.newArrayList(SampleData.metadataRecord11);

    public static final List<DocumentMetadata> mergedList12 = 

            Lists.newArrayList(SampleData.mergedRecord1112);

    public static final List<DocumentMetadata> mergedList21 = 

            Lists.newArrayList(SampleData.mergedRecord1211);

    public static final List<DocumentMetadata> list123 =

            Lists.newArrayList(SampleData.metadataRecord11, SampleData.metadataRecord12, SampleData.metadataRecord13);

    public static final List<DocumentMetadata> list321 =

            Lists.newArrayList(SampleData.metadataRecord13, SampleData.metadataRecord12, SampleData.metadataRecord11);

    @Test

	public void testBestFilledEmpty() throws Exception {

        BestFilledMergingCollapser<DocumentMetadata> collapser = new BestFilledMergingCollapser<DocumentMetadata>();

        assertNull(collapser.collapse(null));

        assertNull(collapser.collapse(emptyList));

    }

    @Test

	public void testBestFilledMergingDefaultFieldSet() throws Exception {

        BestFilledMergingCollapser<DocumentMetadata> collapser = new BestFilledMergingCollapser<DocumentMetadata>();

        SampleData.assertEqualRecords(

                list1,

                collapser.collapse(list1));

        SampleData.assertEqualRecords(

                mergedList12,

                collapser.collapse(list321));

    }

    @Test

	public void testBestFilledMerging() throws Exception {

        BestFilledMergingCollapser<DocumentMetadata> collapser = new BestFilledMergingCollapser<DocumentMetadata>();

        collapser.setFields(SampleData.significantFields);

        SampleData.assertEqualRecords(

                list1,

                collapser.collapse(list1));

        SampleData.assertEqualRecords(

                mergedList21,

                collapser.collapse(list321));

    }

}

package eu.dnetlib.iis.collapsers;

import java.util.ArrayList;

import java.util.List;

import org.apache.avro.generic.IndexedRecord;

import static org.junit.Assert.*;

import org.junit.Test;

import org.python.google.common.collect.Lists;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

public class CollapserUtilsTest {

    @Test

    public void testHaveEqualSchema() {

        assertTrue(CollapserUtils.haveEqualSchema(null));

        assertTrue(CollapserUtils.haveEqualSchema(new ArrayList<IndexedRecord>()));

        assertTrue(CollapserUtils.haveEqualSchema(

                Lists.newArrayList((IndexedRecord)SampleData.envMetadataRecord11, SampleData.envMetadataRecord12)));

        assertFalse(CollapserUtils.haveEqualSchema(

                Lists.newArrayList((IndexedRecord)SampleData.envMetadataRecord11, SampleData.envMetadataRecord12, SampleData.envTextRecord)));

    }

    @Test

    public void testIsOriginSchema() {

        assertTrue(CollapserUtils.isEnvelopeSchema(SampleData.envMetadataRecord11.getSchema()));

        assertFalse(CollapserUtils.isEnvelopeSchema(SampleData.textRecord.getSchema()));

    }

    @Test

    public void testGetOriginValue() {

        assertEquals("origin1", CollapserUtils.getOriginValue(SampleData.envMetadataRecord11));

        assertEquals("origin1", CollapserUtils.getOriginValue(SampleData.envTextRecord));

    }

    @Test

    public void testGetDataRecord() {

        SampleData.assertEqualRecords(

                SampleData.metadataRecord11, 

                CollapserUtils.getDataRecord(SampleData.envMetadataRecord11));

        SampleData.assertEqualRecords(

                SampleData.textRecord, 

                CollapserUtils.getDataRecord(SampleData.envTextRecord));

    }

    @Test

    public void testGetNumberOfFilledFields() {

        assertEquals(8, CollapserUtils.getNumberOfFilledFields(SampleData.metadataRecord11, null));

        assertEquals(2, CollapserUtils.getNumberOfFilledFields(SampleData.metadataRecord11, SampleData.significantFields));

    }

    @Test

    public void testSortByFilledFields() {

        List<IndexedRecord> empty = new ArrayList<IndexedRecord>();

        CollapserUtils.sortByFilledDataFields(empty, SampleData.significantFields);

        assertTrue(empty.isEmpty());

        List<IndexedRecord> oneElement = Lists.newArrayList((IndexedRecord)SampleData.metadataRecord13);

        CollapserUtils.sortByFilledDataFields(oneElement, SampleData.significantFields);

        assertEquals(Lists.newArrayList(SampleData.metadataRecord13),

                oneElement);

        List<IndexedRecord> list = Lists.newArrayList(

                (IndexedRecord)SampleData.metadataRecord11, SampleData.metadataRecord12, SampleData.metadataRecord13);

        CollapserUtils.sortByFilledDataFields(list, SampleData.significantFields);

        assertEquals(Lists.newArrayList(SampleData.metadataRecord12, SampleData.metadataRecord11, SampleData.metadataRecord13),

                list);

        CollapserUtils.sortByFilledDataFields(list, null);

        assertEquals(Lists.newArrayList(SampleData.metadataRecord11, SampleData.metadataRecord12, SampleData.metadataRecord13),

                list);

    }

    @Test

    public void testMerge() {

        SampleData.assertEqualRecords(

                SampleData.mergedRecord1112,

                CollapserUtils.merge(SampleData.metadataRecord11, SampleData.metadataRecord12));

        SampleData.assertEqualRecords(

                SampleData.mergedRecord2221,

                CollapserUtils.merge(SampleData.metadataRecord22, SampleData.metadataRecord21));

    }

    @Test

    public void testGetNestedFieldValue() {

        assertNull(CollapserUtils.getNestedFieldValue(null, null));

        assertNull(CollapserUtils.getNestedFieldValue(SampleData.metadataRecord11, null));

        assertNull(CollapserUtils.getNestedFieldValue(null, "notnull"));

        assertNull(CollapserUtils.getNestedFieldValue(SampleData.metadataRecord11, "field"));

        assertNull(CollapserUtils.getNestedFieldValue(SampleData.metadataRecord11, "id.field"));

        assertEquals("id-1", CollapserUtils.getNestedFieldValue(SampleData.metadataRecord11, "id"));

        assertEquals(1990, CollapserUtils.getNestedFieldValue(SampleData.metadataRecord11, "year"));

        assertEquals(true, CollapserUtils.getNestedFieldValue(SampleData.metadataRecord11, "publicationType.article"));

        assertEquals("id-1", CollapserUtils.getNestedFieldValue(SampleData.envMetadataRecord11, "data.id"));

        assertEquals(1990, CollapserUtils.getNestedFieldValue(SampleData.envMetadataRecord11, "data.year"));

        assertEquals(true, CollapserUtils.getNestedFieldValue(SampleData.envMetadataRecord11, "data.publicationType.article"));

    }

}

package eu.dnetlib.iis.collapsers.union;

import eu.dnetlib.iis.IntegrationTest;

import eu.dnetlib.iis.core.AbstractWorkflowTestCase;

import org.junit.Test;

import org.junit.experimental.categories.Category;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

@Category(IntegrationTest.class)

public class WorkflowTest extends AbstractWorkflowTestCase {

    @Test

	public void testWorkflow2Inputs() throws Exception {

    	runWorkflow("eu/dnetlib/iis/collapsers/union/input_2/oozie_app");

    }

    @Test

	public void testWorkflow3Inputs() throws Exception {

    	runWorkflow("eu/dnetlib/iis/collapsers/union/input_3/oozie_app");

    }

}

package eu.dnetlib.iis.collapsers.multiple_input;

import eu.dnetlib.iis.IntegrationTest;

import eu.dnetlib.iis.core.AbstractWorkflowTestCase;

import eu.dnetlib.iis.core.WorkflowConfiguration;

import org.junit.Test;

import org.junit.experimental.categories.Category;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

@Category(IntegrationTest.class)

public class WorkflowTest extends AbstractWorkflowTestCase {

    @Test

	public void testDefaultWorkflow() throws Exception {

        WorkflowConfiguration wc = new WorkflowConfiguration();

        wc.setTimeoutInSeconds(720);

    	runWorkflow("eu/dnetlib/iis/collapsers/multiple_input_collapser/default/oozie_app", wc);

    }

    @Test

	public void testDocumentTextWorkflow() throws Exception {

        WorkflowConfiguration wc = new WorkflowConfiguration();

        wc.setTimeoutInSeconds(720);

    	runWorkflow("eu/dnetlib/iis/collapsers/multiple_input_collapser/documenttext/oozie_app", wc);

    }

}

package eu.dnetlib.iis.collapsers.origins;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertNotNull;

import java.util.Arrays;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import org.junit.Test;

import eu.dnetlib.iis.common.citations.schemas.Citation;

import eu.dnetlib.iis.common.citations.schemas.CitationEntry;

/**

 * {@link PMCCitationCollapser} test class.

 * @author mhorst

 *

 */

public class PMCCitationCollapserTest {

	@Test

	public void testCollapsingWhenPmcTargetIdSet() throws Exception {

		PMCCitationCollapser collapser = new PMCCitationCollapser();

		Map<String,List<Citation>> objects = new HashMap<String, List<Citation>>();

		String sourceId = "sourceId";

		String pmcTargetId = "pmcTargetId";

		String cermineTargetId = "cermineTargetId";

		String text = "citation text";

		objects.put("ingested", Arrays.asList(new Citation[] {

				Citation

						.newBuilder()

						.setSourceDocumentId(sourceId)

						.setEntry(

								CitationEntry.newBuilder()

										.setConfidenceLevel(1f)

										.setDestinationDocumentId(pmcTargetId)

										.setExternalDestinationDocumentIds(new HashMap<CharSequence, CharSequence>())

										.setRawText(text).build()).build()		

		}));

		objects.put("matched", Arrays.asList(new Citation[] {

				Citation

						.newBuilder()

						.setSourceDocumentId(sourceId)

						.setEntry(

								CitationEntry

										.newBuilder()

										.setConfidenceLevel(0.1f)

										.setDestinationDocumentId(cermineTargetId)

										.setExternalDestinationDocumentIds(new HashMap<CharSequence, CharSequence>())

										.setRawText(text).build()).build()

		}));

		List<Citation> results = collapser.collapseBetweenOrigins(objects);

		assertNotNull(results);

		assertEquals(1, results.size());

		assertEquals(pmcTargetId,results.get(0).getEntry().getDestinationDocumentId());

		assertEquals(new Float(1f),results.get(0).getEntry().getConfidenceLevel());

	}

	@Test

	public void testCollapsingWhenPmcTargetIdNotSet() throws Exception {

		PMCCitationCollapser collapser = new PMCCitationCollapser();

		Map<String,List<Citation>> objects = new HashMap<String, List<Citation>>();

		String sourceId = "sourceId";

		String cermineTargetId = "cermineTargetId";

		String text = "citation text";

		objects.put("ingested", Arrays.asList(new Citation[] {

				Citation

						.newBuilder()

						.setSourceDocumentId(sourceId)

						.setEntry(

								CitationEntry.newBuilder()

										.setExternalDestinationDocumentIds(new HashMap<CharSequence, CharSequence>())

										.setRawText(text).build()).build()		

		}));

		objects.put("matched", Arrays.asList(new Citation[] {

				Citation

						.newBuilder()

						.setSourceDocumentId(sourceId)

						.setEntry(

								CitationEntry

										.newBuilder()

										.setConfidenceLevel(0.1f)

										.setDestinationDocumentId(cermineTargetId)

										.setExternalDestinationDocumentIds(new HashMap<CharSequence, CharSequence>())

										.setRawText(text).build()).build()

		}));

		List<Citation> results = collapser.collapseBetweenOrigins(objects);

		assertNotNull(results);

		assertEquals(1, results.size());

		assertEquals(cermineTargetId,results.get(0).getEntry().getDestinationDocumentId());

		assertEquals(new Float(0.1f),results.get(0).getEntry().getConfidenceLevel());

	}

	@Test

	public void testCollapsingWithDifferentText() throws Exception {

		PMCCitationCollapser collapser = new PMCCitationCollapser();

		Map<String,List<Citation>> objects = new HashMap<String, List<Citation>>();

		String sourceId = "sourceId";

		String pmcTargetId = "pmcTargetId";

		String cermineTargetId = "cermineTargetId";

		String textPmc = "pmc citation text";

		String textCermine = "cermine citation text";

		objects.put("ingested", Arrays.asList(new Citation[] {

				Citation

						.newBuilder()

						.setSourceDocumentId(sourceId)

						.setEntry(

								CitationEntry.newBuilder()

										.setConfidenceLevel(1f)

										.setDestinationDocumentId(pmcTargetId)

										.setExternalDestinationDocumentIds(new HashMap<CharSequence, CharSequence>())

										.setRawText(textPmc).build()).build()		

		}));

		objects.put("matched", Arrays.asList(new Citation[] {

				Citation

						.newBuilder()

						.setSourceDocumentId(sourceId)

						.setEntry(

								CitationEntry

										.newBuilder()

										.setConfidenceLevel(0.1f)

										.setDestinationDocumentId(cermineTargetId)

										.setExternalDestinationDocumentIds(new HashMap<CharSequence, CharSequence>())

										.setRawText(textCermine).build()).build()

		}));

		List<Citation> results = collapser.collapseBetweenOrigins(objects);

		assertNotNull(results);

		assertEquals(2, results.size());

		assertEquals(pmcTargetId,results.get(0).getEntry().getDestinationDocumentId());

		assertEquals(new Float(1f),results.get(0).getEntry().getConfidenceLevel());

		assertEquals(textPmc,results.get(0).getEntry().getRawText());

		assertEquals(cermineTargetId,results.get(1).getEntry().getDestinationDocumentId());

		assertEquals(new Float(0.1f),results.get(1).getEntry().getConfidenceLevel());

		assertEquals(textCermine,results.get(1).getEntry().getRawText());

	}

}

package eu.dnetlib.iis.collapsers.origins;

import com.google.common.collect.Lists;

import eu.dnetlib.iis.collapsers.SampleData;

import eu.dnetlib.iis.collapsers.schemas.DocumentMetadataEnvelope;

import eu.dnetlib.iis.importer.schemas.DocumentMetadata;

import java.util.ArrayList;

import java.util.List;

import static org.junit.Assert.assertNull;

import org.junit.Test;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

public class OriginConfidenceMergingCollapserTest {

    public static final List<DocumentMetadataEnvelope> emptyList = new ArrayList<DocumentMetadataEnvelope>();

    public static final List<DocumentMetadataEnvelope> oneElementList = 

            Lists.newArrayList(SampleData.envMetadataRecord11);

    public static final List<DocumentMetadata> mergedOneElementList = 

            Lists.newArrayList(SampleData.metadataRecord11);

    public static final List<DocumentMetadataEnvelope> list = 

            Lists.newArrayList(SampleData.envMetadataRecord11, SampleData.envMetadataRecord21);

    public static final List<DocumentMetadata> mergedList = 

            Lists.newArrayList(SampleData.mergedRecord1121);

    @Test

	public void testOriginConfidenceMerging() throws Exception {

        OriginConfidenceMergingCollapser<DocumentMetadataEnvelope, DocumentMetadata> collapser = 

                new OriginConfidenceMergingCollapser<DocumentMetadataEnvelope, DocumentMetadata>();

        collapser.setOrigins(SampleData.origins);

        assertNull(collapser.collapse(null));

        assertNull(collapser.collapse(emptyList));

        SampleData.assertEqualRecords(

                mergedOneElementList,

                collapser.collapse(oneElementList));

        SampleData.assertEqualRecords(

                mergedList,

                collapser.collapse(list));

    }

}

package eu.dnetlib.iis.collapsers.origins;

import eu.dnetlib.iis.collapsers.SampleData;

import eu.dnetlib.iis.collapsers.schemas.DocumentTextEnvelope;

import eu.dnetlib.iis.metadataextraction.schemas.DocumentText;

import java.util.ArrayList;

import java.util.List;

import static org.junit.Assert.assertNull;

import org.junit.Test;

import org.python.google.common.collect.Lists;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

public class DocumentTextCollapserTest {

    public static final List<String> origins = Lists.newArrayList("origin1", "origin2");

    public static final DocumentTextEnvelope record1 = DocumentTextEnvelope.newBuilder()

            .setOrigin("origin1")

            .setData(DocumentText.newBuilder().setId("id").setText("This is text").build())

            .build();

    public static final DocumentTextEnvelope record2 = DocumentTextEnvelope.newBuilder()

            .setOrigin("origin2")

            .setData(DocumentText.newBuilder().setId("id").setText("This is text").build())

            .build();

    public static final DocumentTextEnvelope record3 = DocumentTextEnvelope.newBuilder()

            .setOrigin("origin2")

            .setData(DocumentText.newBuilder().setId("id").setText("This is another text").build())

            .build();

    public static final DocumentTextEnvelope record4 = DocumentTextEnvelope.newBuilder()

            .setOrigin("origin2")

            .setData(DocumentText.newBuilder().setId("id").setText("This is a duplicated text").build())

            .build();

    public static final DocumentText collapsed1 = DocumentText.newBuilder()

            .setId("id")

            .setText("This is text").build();

    public static final DocumentText collapsed2 = DocumentText.newBuilder()

            .setId("id")

            .setText("This is text\n\nThis is text\n\nThis is another text\n\nThis is a duplicated text").build();

    public static final List<DocumentTextEnvelope> oneElementList = 

            Lists.newArrayList(record1);

    public static final List<DocumentText> collapsedOneElementList = 

            Lists.newArrayList(collapsed1);

    public static final List<DocumentTextEnvelope> list =

            Lists.newArrayList(record1, record2, record3, record4);

    public static final List<DocumentText> collapsedList =

            Lists.newArrayList(collapsed2);

    @Test

	public void testDocumentTextCollapserEmpty() throws Exception {

        DocumentTextCollapser collapser = new DocumentTextCollapser();

        collapser.setOrigins(origins);

        assertNull(collapser.collapse(null));

        assertNull(collapser.collapse(new ArrayList<DocumentTextEnvelope>()));

    }

    @Test

	public void testDocumentTextCollapser() throws Exception {

        DocumentTextCollapser collapser = new DocumentTextCollapser();

        collapser.setOrigins(origins);

        SampleData.assertEqualRecords(

                collapsedOneElementList,

                collapser.collapse(oneElementList));

        SampleData.assertEqualRecords(

                collapsedList,

                collapser.collapse(list));

    }

}

package eu.dnetlib.iis.collapsers.origins;

import eu.dnetlib.iis.collapsers.SampleData;

import eu.dnetlib.iis.collapsers.schemas.DocumentMetadataEnvelope;

import eu.dnetlib.iis.importer.schemas.DocumentMetadata;

import java.util.ArrayList;

import java.util.List;

import static org.junit.Assert.assertNull;

import org.junit.Test;

import org.python.google.common.collect.Lists;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

public class OriginConfidenceCollapserTest {

    public static final List<DocumentMetadataEnvelope> emptyList = new ArrayList<DocumentMetadataEnvelope>();

    public static final List<DocumentMetadataEnvelope> oneElementList = 

            Lists.newArrayList(SampleData.envMetadataRecord11);

    public static final List<DocumentMetadata> mergedOneElementList = 

            Lists.newArrayList(SampleData.metadataRecord11);

    public static final List<DocumentMetadataEnvelope> list = 

            Lists.newArrayList(SampleData.envMetadataRecord11, SampleData.envMetadataRecord21);

    @Test

	public void testOriginConfidence() throws Exception {

        OriginConfidenceCollapser<DocumentMetadataEnvelope, DocumentMetadata> collapser = 

                new OriginConfidenceCollapser<DocumentMetadataEnvelope, DocumentMetadata>();

        collapser.setOrigins(SampleData.origins);

        assertNull(collapser.collapse(null));

        assertNull(collapser.collapse(emptyList));

        SampleData.assertEqualRecords(

                mergedOneElementList,

                collapser.collapse(oneElementList));

        SampleData.assertEqualRecords(

                mergedOneElementList,

                collapser.collapse(list));

    }

}

package eu.dnetlib.iis.collapsers.origins;

import eu.dnetlib.iis.IntegrationTest;

import eu.dnetlib.iis.core.AbstractWorkflowTestCase;

import org.junit.Test;

import org.junit.experimental.categories.Category;

/**

 * @author Dominika Tkaczyk

 * @author Michal Oniszczuk

 */

@Category(IntegrationTest.class)

public class WorkflowTest extends AbstractWorkflowTestCase {

    @Test

    public void testDefaultWorkflow() throws Exception {

        runWorkflow("eu/dnetlib/iis/collapsers/origins_collapser/default/oozie_app");

    }

/*

    @Test

    public void testCitationWorkflow() throws Exception {

        runWorkflow("eu/dnetlib/iis/collapsers/collapser/citation/oozie_app");

    }

*/

    @Test

    public void testDocumentTextWorkflow() throws Exception {

        runWorkflow("eu/dnetlib/iis/collapsers/origins_collapser/documenttext/oozie_app");

    }

}

package eu.dnetlib.iis.collapsers;

import eu.dnetlib.iis.collapsers.schemas.DocumentMetadataEnvelope;

import eu.dnetlib.iis.collapsers.schemas.DocumentTextEnvelope;

import eu.dnetlib.iis.importer.schemas.DocumentMetadata;

import eu.dnetlib.iis.importer.schemas.PublicationType;

import eu.dnetlib.iis.metadataextraction.schemas.DocumentText;

import java.util.List;

import org.apache.avro.generic.GenericData;

import org.apache.avro.generic.IndexedRecord;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertTrue;

import org.python.google.common.collect.Lists;

/**

 *

 * @author Dominika Tkaczyk

 */

public class SampleData {

    /* parameters */

    public static final List<String> origins = Lists.newArrayList("origin1", "origin2");

    public static final List<String> significantFields = Lists.newArrayList("title", "authorIds", "abstract", "journal", "year");

    /* input records */

    public static final DocumentMetadata metadataRecord11 = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 1")

                .setLanguage("en")

                .setYear(1990)

                .setKeywords(Lists.newArrayList((CharSequence)"kwd 1", "kwd 2"))

                .setDatasourceIds(Lists.newArrayList((CharSequence)"d 1", "d 2"))

                .setPublisher("publisher 1")

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .build();

    public static final DocumentMetadataEnvelope envMetadataRecord11 = DocumentMetadataEnvelope.newBuilder()

            .setOrigin("origin1")

            .setData(metadataRecord11).build();

    public static final DocumentMetadata metadataRecord12 = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 2")

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 1", "aut 2"))

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .setTitle("title 2")

                .setYear(1991)

                .build();

    public static final DocumentMetadataEnvelope envMetadataRecord12 = DocumentMetadataEnvelope.newBuilder()

            .setOrigin("origin1")

            .setData(metadataRecord12).build();

    public static final DocumentMetadata metadataRecord13 = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .build();

    public static final DocumentMetadataEnvelope envMetadataRecord13 = DocumentMetadataEnvelope.newBuilder()

            .setOrigin("origin1")

            .setData(metadataRecord13).build();

    public static final DocumentMetadata metadataRecord21 = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 3")

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 13", "aut 23"))

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .setTitle("title 3")

                .setYear(1999)

                .build();

    public static final DocumentMetadataEnvelope envMetadataRecord21 = DocumentMetadataEnvelope.newBuilder()

            .setOrigin("origin2")

            .setData(metadataRecord21).build();

    public static final DocumentMetadata metadataRecord22 = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .build();

    public static final DocumentMetadataEnvelope envMetadataRecord22 = DocumentMetadataEnvelope.newBuilder()

            .setOrigin("origin2")

            .setData(metadataRecord22).build();

    public static final DocumentText textRecord = DocumentText.newBuilder()

                 .setId("text-1")

                 .setText("text text")

                 .build();

    public static final DocumentTextEnvelope envTextRecord = DocumentTextEnvelope.newBuilder()

            .setOrigin("origin1")

            .setData(textRecord).build();

    /* merged records */

    public static final DocumentMetadata mergedRecord1112 = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 1")

                .setLanguage("en")

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 1", "aut 2"))

                .setTitle("title 2")

                .setYear(1990)

                .setKeywords(Lists.newArrayList((CharSequence)"kwd 1", "kwd 2"))

                .setDatasourceIds(Lists.newArrayList((CharSequence)"d 1", "d 2"))

                .setPublisher("publisher 1")

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .build();

    public static final DocumentMetadata mergedRecord1211 = DocumentMetadata.newBuilder()

                 .setId("id-1")

                 .setAbstract$("abstract 2")

                 .setAuthorIds(Lists.newArrayList((CharSequence)"aut 1", "aut 2"))

                 .setTitle("title 2")

                 .setLanguage("en")

                 .setYear(1991)

                 .setKeywords(Lists.newArrayList((CharSequence)"kwd 1", "kwd 2"))

                 .setDatasourceIds(Lists.newArrayList((CharSequence)"d 1", "d 2"))

                 .setPublisher("publisher 1")

                 .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                 .build();

    public static final DocumentMetadata mergedRecord1121 = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 1")

                .setLanguage("en")

                .setYear(1990)

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 13", "aut 23"))

                .setTitle("title 3")

                .setKeywords(Lists.newArrayList((CharSequence)"kwd 1", "kwd 2"))

                .setDatasourceIds(Lists.newArrayList((CharSequence)"d 1", "d 2"))

                .setPublisher("publisher 1")

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .build();

    public static final DocumentMetadata mergedRecord2221 = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 3")

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 13", "aut 23"))

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .setTitle("title 3")

                .setYear(1999)

                .build();

    /* collapsed records */

    // within no merge, between no merge

    public static final DocumentMetadata recordWNoMergeBNoMerge = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 2")

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 1", "aut 2"))

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .setTitle("title 2")

                .setYear(1991)

                .build();

    // within merge, between no merge

    public static final DocumentMetadata recordWMergeBNoMerge = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 2")

                .setLanguage("en")

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 1", "aut 2"))

                .setTitle("title 2")

                .setYear(1991)

                .setKeywords(Lists.newArrayList((CharSequence)"kwd 1", "kwd 2"))

                .setDatasourceIds(Lists.newArrayList((CharSequence)"d 1", "d 2"))

                .setPublisher("publisher 1")

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .build();

    // within no merge, between merge

    public static final DocumentMetadata recordWNoMergeBMerge = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 2")

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 1", "aut 2"))

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .setTitle("title 2")

                .setYear(1991)

                .build();

    // within merge, between merge

    public static final DocumentMetadata recordWMergeBMerge = DocumentMetadata.newBuilder()

                .setId("id-1")

                .setAbstract$("abstract 2")

                .setLanguage("en")

                .setAuthorIds(Lists.newArrayList((CharSequence)"aut 1", "aut 2"))

                .setTitle("title 2")

                .setYear(1991)

                .setKeywords(Lists.newArrayList((CharSequence)"kwd 1", "kwd 2"))

                .setDatasourceIds(Lists.newArrayList((CharSequence)"d 1", "d 2"))

                .setPublisher("publisher 1")

                .setPublicationType(PublicationType.newBuilder().setArticle(true).build())

                .build();

    public static void assertEqualRecords(IndexedRecord expected, IndexedRecord actual) {

        assertEquals("Records are not equal: \nExpected: " + expected + "\nActual: " + actual + "\n", 

                0, GenericData.get().compare(expected, actual, expected.getSchema()));

    }

    public static <T extends IndexedRecord> void assertEqualRecords(List<T> expected, List<T> actual) {

        assertEquals("Records lists have different sizes: " + expected.size() + " and " + actual.size() + "\n",

                expected.size(), actual.size());

        List<T> actualCopy = Lists.newArrayList(actual);

        for (T exp : expected) {

            T found = null;

            for (T act : actualCopy) {

                if (0 == GenericData.get().compare(exp, act, exp.getSchema())) {

                    found = act;

                }

            }

            assertTrue(

                    "Expected record " + exp.toString() + " was not found among the actual records\n", 

                    found != null);

            actualCopy.remove(found);

        }

    }

}

## This is a classpath-based import file (this header is required)

basic_collapser classpath eu/dnetlib/iis/collapsers/basic_collapser/oozie_app

<workflow-app xmlns="uri:oozie:workflow:0.2" name="test-basic_collapser_documenttext">

    <start to="producer"/>

    <action name="producer">

        <java>

            <job-tracker>${jobTracker}</job-tracker>

            <name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/producer" />

				<mkdir path="${nameNode}${workingDir}/producer" />

			</prepare>

            <configuration>

                <property>

                    <name>mapred.job.queue.name</name>

                    <value>${queueName}</value>

                </property>

            </configuration>

            <!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.java.jsonworkflownodes.Producer</arg>

			<!-- Specification of the output ports -->

			<arg>-C{input,

				eu.dnetlib.iis.metadataextraction.schemas.DocumentText,

				eu/dnetlib/iis/collapsers/basic_collapser/documenttext/data/texts.json}</arg>

			<!-- All input and output ports have to be bound to paths in HDFS, working 

				directory has to be specified as well -->

            <arg>-SworkingDir=${workingDir}/producer/working_dir</arg>

            <arg>-Oinput=${workingDir}/producer/input</arg>

        </java>

        <ok to="basic_collapser"/>

        <error to="fail"/>

    </action>

    <action name="basic_collapser">

        <sub-workflow>

            <app-path>${wf:appPath()}/basic_collapser</app-path>

            <configuration>

                <property>

                    <name>jobTracker</name>

                    <value>${jobTracker}</value>

                </property>

                <property>

                    <name>nameNode</name>

                    <value>${nameNode}</value>

                </property>

                <property>

                    <name>queueName</name>

                    <value>${queueName}</value>

                </property>

                <!-- Working directory of the subworkflow -->                

                <property>

                    <name>workingDir</name>

                    <value>${workingDir}/basic_collapser/working_dir</value>

                </property>

                <property>

                    <name>blocking_field</name>

                    <value>id</value>

                </property>

                <property>

                    <name>record_collapser</name>

                    <value>eu.dnetlib.iis.collapsers.basic.DocumentTextCollapser</value>

        		</property>

                <property>

                    <name>schema</name>

                    <value>eu.dnetlib.iis.metadataextraction.schemas.DocumentText</value>

                </property>

                <!-- Input ports. -->

                <property>

                    <name>input</name>

                    <value>${workingDir}/producer/input</value>

                </property>

                <!-- Output port bound to given path -->

                <property>

                    <name>output</name>

                    <value>${workingDir}/basic_collapser/output</value>

                </property>

            </configuration>

        </sub-workflow>

        <ok to="consumer"/>

        <error to="fail"/>

    </action>

    <action name="consumer">

		<java>

			<job-tracker>${jobTracker}</job-tracker>

			<name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/consumer" />

				<mkdir path="${nameNode}${workingDir}/consumer" />

			</prepare>

			<configuration>

				<property>

					<name>mapred.job.queue.name</name>

					<value>${queueName}</value>

				</property>

			</configuration>

			<!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.java.jsonworkflownodes.TestingConsumer</arg>

			<!-- Specification of the input ports -->

			<arg>-C{output,

				eu.dnetlib.iis.metadataextraction.schemas.DocumentText,

				eu/dnetlib/iis/collapsers/basic_collapser/documenttext/data/output.json}</arg>

    		<!-- All input and output ports have to be bound to paths in HDFS, working 

				directory has to be specified as well -->

			<arg>-SworkingDir=${workingDir}/consumer/working_dir</arg>

			<arg>-Ioutput=${workingDir}/basic_collapser/output</arg>

		</java>

		<ok to="end" />

		<error to="fail" />

	</action>

    <kill name="fail">

		<message>Unfortunately, the workflow failed -- error message:

			[${wf:errorMessage(wf:lastErrorNode())}]</message>

    </kill>

    <end name="end"/>

</workflow-app>

{"id": "id-6", "text": "This is a different text"}

{"id": "id-3", "text": "This is another text"}

{"id": "id-3", "text": "This is yet another text"}

{"id": "id-6", "text": "This is a totally different text"}

{"id": "id-7", "text": "This is a text"}

{"id": "id-4", "text": "This is not a text"}

{"id": "id-6", "text": "This is a duplicated text"}

{"id": "id-3", "text": "This is another text\n\nThis is yet another text"}

{"id": "id-6", "text": "This is a different text\n\nThis is a totally different text\n\nThis is a duplicated text"}

{"id": "id-7", "text": "This is a text"}

{"id": "id-4", "text": "This is not a text"}

## This is a classpath-based import file (this header is required)

basic_collapser classpath eu/dnetlib/iis/collapsers/basic_collapser/oozie_app

<workflow-app xmlns="uri:oozie:workflow:0.2" name="test-basic_collapser_default">

    <start to="producer"/>

    <action name="producer">

        <java>

            <job-tracker>${jobTracker}</job-tracker>

            <name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/producer" />

				<mkdir path="${nameNode}${workingDir}/producer" />

			</prepare>

            <configuration>

                <property>

                    <name>mapred.job.queue.name</name>

                    <value>${queueName}</value>

                </property>

            </configuration>

            <!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.java.jsonworkflownodes.Producer</arg>

			<!-- Specification of the output ports -->

			<arg>-C{input,

				eu.dnetlib.iis.importer.schemas.DocumentMetadata,

				eu/dnetlib/iis/collapsers/basic_collapser/default/data/metadata.json}</arg>

			<!-- All input and output ports have to be bound to paths in HDFS, working 

				directory has to be specified as well -->

            <arg>-SworkingDir=${workingDir}/producer/working_dir</arg>

            <arg>-Oinput=${workingDir}/producer/input</arg>

        </java>

        <ok to="basic_collapser"/>

        <error to="fail"/>

    </action>

    <action name="basic_collapser">

        <sub-workflow>

            <app-path>${wf:appPath()}/basic_collapser</app-path>

            <configuration>

                <property>

                    <name>jobTracker</name>

                    <value>${jobTracker}</value>

                </property>

                <property>

                    <name>nameNode</name>

                    <value>${nameNode}</value>

                </property>

                <property>

                    <name>queueName</name>

                    <value>${queueName}</value>

                </property>

                <!-- Working directory of the subworkflow -->                

                <property>

                    <name>workingDir</name>

                    <value>${workingDir}/basic_collapser/working_dir</value>

                </property>

                <property>

                    <name>blocking_field</name>

                    <value>id</value>

                </property>

                <property>

                    <name>schema</name>

                    <value>eu.dnetlib.iis.importer.schemas.DocumentMetadata</value>

                </property>

                <property>

                    <name>significant_fields</name>

                    <value>title,authorIds,journal,year</value>

                </property>

                <!-- Input ports. -->

                <property>

                    <name>input</name>

                    <value>${workingDir}/producer/input</value>

                </property>

                <!-- Output port bound to given path -->

                <property>

                    <name>output</name>

                    <value>${workingDir}/basic_collapser/output</value>

                </property>

            </configuration>

        </sub-workflow>

        <ok to="consumer"/>

        <error to="fail"/>

    </action>

    <action name="consumer">

		<java>

			<job-tracker>${jobTracker}</job-tracker>

			<name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/consumer" />

				<mkdir path="${nameNode}${workingDir}/consumer" />

			</prepare>

			<configuration>

				<property>

					<name>mapred.job.queue.name</name>

					<value>${queueName}</value>

				</property>

			</configuration>

			<!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.java.jsonworkflownodes.TestingConsumer</arg>

			<!-- Specification of the input ports -->

			<arg>-C{output,

				eu.dnetlib.iis.importer.schemas.DocumentMetadata,

				eu/dnetlib/iis/collapsers/basic_collapser/default/data/output.json}</arg>

    		<!-- All input and output ports have to be bound to paths in HDFS, working 

				directory has to be specified as well -->

			<arg>-SworkingDir=${workingDir}/consumer/working_dir</arg>

			<arg>-Ioutput=${workingDir}/basic_collapser/output</arg>

		</java>

		<ok to="end" />

		<error to="fail" />

	</action>

    <kill name="fail">

		<message>Unfortunately, the workflow failed -- error message:

			[${wf:errorMessage(wf:lastErrorNode())}]</message>

    </kill>

    <end name="end"/>

</workflow-app>

{"id": "id-1", "title": "Ender's Game", "abstract": null, "language": null, "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": "Journal-1", "year": 2010, "publisher": "", "publicationType": {"article": false, "dataset": false}, "authorIds": ["id-1", "id-2", "id-3"], "datasourceIds": null}

{"id": "id-2", "title": "Seventh Son (Tales of Alvin Maker)", "abstract": "The tales of Alvin Maker", "language": null, "keywords": null, "externalIdentifiers": {"id-1": "val-1", "id-2": "val-2"}, "journal": "Journal", "year": 1993, "publisher": null, "publicationType": {"article": true, "dataset": false}, "authorIds": ["id"], "datasourceIds": null}

{"id": "id-1", "title": "Enders game", "abstract": null, "language": null, "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": "Journal-1", "year": null, "publisher": "", "publicationType": {"article": false, "dataset": false}, "authorIds": ["id-1", "id-256", "id-3"], "datasourceIds": null}

{"id": "id-2", "title": "Seventh Son (Tales of Alvin Maker) 67", "abstract": "The tales of Alvin Maker", "language": null, "keywords": null, "externalIdentifiers": {"id-1": "val-1", "id-2": "val-2"}, "journal": "Journal", "year": 1993, "publisher": null, "publicationType": {"article": true, "dataset": false}, "authorIds": null, "datasourceIds": null}

{"id": "id-3", "title": "A Disk", "abstract": "A flat circular world carried on the back of a giant turtle - Discworld", "language": "eng", "keywords": null, "externalIdentifiers": {"id-1": "val-1", "id-2": "val-2"}, "journal": null, "year": 2003, "publisher": "HarperTorch", "publicationType": {"article": true, "dataset": false}, "authorIds": ["id-1", "id-2"], "datasourceIds": null}

{"id": "id-5", "title": "HArry Potter and the Sorcerer's Stone", "abstract": null, "language": null, "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "publicationType": {"article": false, "dataset": true}, "authorIds": null, "datasourceIds": null}

{"id": "id-4", "title": "null", "abstract": null, "language": "pl", "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": "Ballantine Books", "publicationType": {"article": true, "dataset": false}, "authorIds": null, "datasourceIds": null}

{"id": "id-3", "title": null, "abstract": "A flat, circular world carried on the back of a giant turtle - Discworld", "language": "eng", "keywords": null, "externalIdentifiers": {"id-1": "val-1", "id-2": "val-2"}, "journal": null, "year": 2003, "publisher": "HarperTorch", "publicationType": {"article": true, "dataset": false}, "authorIds": ["id-1", "id-2"], "datasourceIds": null}

{"id": "id-5", "title": "Harry Potter and the Sorcerer's Stone", "abstract": null, "language": null, "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "publicationType": {"article": false, "dataset": true}, "authorIds": ["id"], "datasourceIds": null}

{"id": "id-4", "title": null, "abstract": null, "language": "eng", "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": "Ballantine Books", "publicationType": {"article": true, "dataset": false}, "authorIds": null, "datasourceIds": null}

{"id": "id-8", "title": "Harry Potter and the Sorcerer's Stone", "abstract": null, "language": null, "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "publicationType": {"article": false, "dataset": true}, "authorIds": null, "datasourceIds": null}

{"id": "id-6", "title": null, "abstract": null, "language": "eng", "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": "Ballantine Books", "publicationType": {"article": true, "dataset": false}, "authorIds": null, "datasourceIds": null}

{"id": "id-1", "title": "Ender's Game", "abstract": null, "language": null, "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": "Journal-1", "year": 2010, "publisher": "", "publicationType": {"article": false, "dataset": false}, "authorIds": ["id-1", "id-2", "id-3"], "datasourceIds": null}

{"id": "id-2", "title": "Seventh Son (Tales of Alvin Maker)", "abstract": "The tales of Alvin Maker", "language": null, "keywords": null, "externalIdentifiers": {"id-1": "val-1", "id-2": "val-2"}, "journal": "Journal", "year": 1993, "publisher": null, "publicationType": {"article": true, "dataset": false}, "authorIds": ["id"], "datasourceIds": null}

{"id": "id-3", "title": "A Disk", "abstract": "A flat circular world carried on the back of a giant turtle - Discworld", "language": "eng", "keywords": null, "externalIdentifiers": {"id-1": "val-1", "id-2": "val-2"}, "journal": null, "year": 2003, "publisher": "HarperTorch", "publicationType": {"article": true, "dataset": false}, "authorIds": ["id-1", "id-2"], "datasourceIds": null}

{"id": "id-4", "title": "null", "abstract": null, "language": "pl", "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": "Ballantine Books", "publicationType": {"article": true, "dataset": false}, "authorIds": null, "datasourceIds": null}

{"id": "id-5", "title": "Harry Potter and the Sorcerer's Stone", "abstract": null, "language": null, "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "publicationType": {"article": false, "dataset": true}, "authorIds": ["id"], "datasourceIds": null}

{"id": "id-6", "title": null, "abstract": null, "language": "eng", "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": "Ballantine Books", "publicationType": {"article": true, "dataset": false}, "authorIds": null, "datasourceIds": null}

{"id": "id-8", "title": "Harry Potter and the Sorcerer's Stone", "abstract": null, "language": null, "keywords": ["kwd_1", "kwd_2", "kwd_3", "kwd_4"], "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "publicationType": {"article": false, "dataset": true}, "authorIds": null, "datasourceIds": null}

## This is a classpath-based import file (this header is required)

basic_collapser classpath eu/dnetlib/iis/collapsers/basic_collapser/oozie_app

<workflow-app xmlns="uri:oozie:workflow:0.2" name="test-collapser_citation">

    <start to="producer"/>

    <action name="producer">

        <java>

            <job-tracker>${jobTracker}</job-tracker>

            <name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/producer" />

				<mkdir path="${nameNode}${workingDir}/producer" />

			</prepare>

            <configuration>

                <property>

                    <name>mapred.job.queue.name</name>

                    <value>${queueName}</value>

                </property>

            </configuration>

            <!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.java.jsonworkflownodes.Producer</arg>

			<!-- Specification of the output ports -->

			<arg>-C{input,

                eu.dnetlib.iis.collapsers.schemas.CitationEnvelope,

                eu/dnetlib/iis/collapsers/collapser/citation/data/citation.json}</arg>

			<!-- All input and output ports have to be bound to paths in HDFS, working

				directory has to be specified as well -->

            <arg>-SworkingDir=${workingDir}/producer/working_dir</arg>

            <arg>-Oinput=${workingDir}/producer/input</arg>

        </java>

        <ok to="collapser"/>

        <error to="fail"/>

    </action>

    <action name="collapser">

        <sub-workflow>

            <app-path>${wf:appPath()}/collapser</app-path>

            <configuration>

                <property>

                    <name>jobTracker</name>

                    <value>${jobTracker}</value>

                </property>

                <property>

                    <name>nameNode</name>

                    <value>${nameNode}</value>

                </property>

                <property>

                    <name>queueName</name>

                    <value>${queueName}</value>

                </property>

                <!-- Working directory of the subworkflow -->                

                <property>

                    <name>workingDir</name>

                    <value>${workingDir}/collapser/working_dir</value>

                </property>

                <!-- Input ports & parameters. -->

                <property>

                    <name>blocking_field</name>

                    <value>sourceDocumentId</value>

                </property>

                <property>

                    <name>origins</name>

                    <value>orig,cermine</value>

                </property>

                <property>

                    <name>schema_input</name>

                    <value>eu.dnetlib.iis.collapsers.schemas.CitationEnvelope</value>

                </property>

                <property>

                    <name>schema_output</name>

                    <value>eu.dnetlib.iis.citationmatching.schemas.Citation</value>

                </property>

                <property>

                    <name>input</name>

                    <value>${workingDir}/producer/input</value>

                </property>

                <!-- Output port bound to given path -->

                <property>

                    <name>output</name>

                    <value>${workingDir}/collapser_collapser/output</value>

                </property>

            </configuration>

        </sub-workflow>

        <ok to="consumer"/>

        <error to="fail"/>

    </action>

    <action name="consumer">

		<java>

			<job-tracker>${jobTracker}</job-tracker>

			<name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/consumer" />

				<mkdir path="${nameNode}${workingDir}/consumer" />

			</prepare>

			<configuration>

				<property>

					<name>mapred.job.queue.name</name>

					<value>${queueName}</value>

				</property>