D-Net

package eu.dnetlib.iis.transformers.export.document;

import eu.dnetlib.iis.IntegrationTest;

import eu.dnetlib.iis.core.AbstractWorkflowTestCase;

import eu.dnetlib.iis.core.WorkflowConfiguration;

import org.junit.Test;

import org.junit.experimental.categories.Category;

/**

 * 

 * @author Dominika Tkaczyk

 *

 */

@Category(IntegrationTest.class)

public class WorkflowTest extends AbstractWorkflowTestCase {

    @Test

	public void testWorkflow() throws Exception {

        WorkflowConfiguration wf = new WorkflowConfiguration();

        wf.setTimeoutInSeconds(720);

        runWorkflow("eu/dnetlib/iis/transformers/export/document/sampledataproducer/oozie_app", wf);

    }

}

## This is a classpath-based import file (this header is required)

transformer_export_document classpath eu/dnetlib/iis/transformers/export/document/oozie_app

<workflow-app xmlns="uri:oozie:workflow:0.2" name="test-transformers_export_document_sampledataproducer">

    <start to="producer"/>

    <action name="producer">

        <java>

            <job-tracker>${jobTracker}</job-tracker>

            <name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/producer" />

				<mkdir path="${nameNode}${workingDir}/producer" />

			</prepare>

            <configuration>

                <property>

                    <name>mapred.job.queue.name</name>

                    <value>${queueName}</value>

                </property>

            </configuration>

            <!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.java.jsonworkflownodes.Producer</arg>

			<!-- Specification of the output ports -->

            <arg>-C{extracted_document_metadata,

				eu.dnetlib.iis.metadataextraction.schemas.ExtractedDocumentMetadata,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/extracted_document_metadata.json}</arg>

            <arg>-C{citation,

				eu.dnetlib.iis.citationmatching.schemas.Citation,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/citations.json}</arg>

            <arg>-C{document_to_project,

				eu.dnetlib.iis.referenceextraction.project.schemas.DocumentToProject,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/document_to_project.json}</arg>

            <arg>-C{document_to_dataset,

				eu.dnetlib.iis.referenceextraction.dataset.schemas.DocumentToDataSet,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/document_to_dataset.json}</arg>

            <arg>-C{document_to_research_initiative,

				eu.dnetlib.iis.referenceextraction.researchinitiative.schemas.DocumentToResearchInitiative,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/document_to_research_initiative.json}</arg>

            <arg>-C{document_to_document_clusters,

				eu.dnetlib.iis.documentsclustering.schemas.DocumentToDocumentClusters,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/document_to_document_clusters.json}</arg>    

            <arg>-C{document_to_document_classes,

				eu.dnetlib.iis.documentsclassification.schemas.DocumentToDocumentClasses,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/document_to_document_classes.json}</arg>    

            <arg>-C{document_to_document_statistics,

				eu.dnetlib.iis.statistics.schemas.DocumentToDocumentStatistics,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/document_to_document_statistics.json}</arg>

            <arg>-C{document_with_website_usage_similarities,

				eu.dnetlib.iis.websiteusage.schemas.DocumentsWithWebsiteUsageSimilarities,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/document_with_website_usage_similarities.json}</arg>

			<!-- All input and output ports have to be bound to paths in HDFS, working 

				directory has to be specified as well -->

            <arg>-SworkingDir=${workingDir}/producer/working_dir</arg>

            <arg>-Oextracted_document_metadata=${workingDir}/producer/extracted_document_metadata</arg>

            <arg>-Ocitation=${workingDir}/producer/citation</arg>

            <arg>-Odocument_to_project=${workingDir}/producer/document_to_project</arg>

            <arg>-Odocument_to_dataset=${workingDir}/producer/document_to_dataset</arg>

            <arg>-Odocument_to_research_initiative=${workingDir}/producer/document_to_research_initiative</arg>

            <arg>-Odocument_to_document_clusters=${workingDir}/producer/document_to_document_clusters</arg>

            <arg>-Odocument_to_document_classes=${workingDir}/producer/document_to_document_classes</arg>

            <arg>-Odocument_to_document_statistics=${workingDir}/producer/document_to_document_statistics</arg>

            <arg>-Odocument_with_website_usage_similarities=${workingDir}/producer/document_with_website_usage_similarities</arg>    

        </java>

        <ok to="transformer_export_document"/>

        <error to="fail"/>

    </action>

    <action name="transformer_export_document">

        <sub-workflow>

            <app-path>${wf:appPath()}/transformer_export_document</app-path>

            <configuration>

                <property>

                    <name>jobTracker</name>

                    <value>${jobTracker}</value>

                </property>

                <property>

                    <name>nameNode</name>

                    <value>${nameNode}</value>

                </property>

                <property>

                    <name>queueName</name>

                    <value>${queueName}</value>

                </property>

                <!-- Working directory of the subworkflow -->

                <property>

                    <name>workingDir</name>

                    <value>${workingDir}/transformer_export_document/working_dir</value>

                </property>

                <!-- Input ports. -->

                <property>

                    <name>input_extracted_document_metadata</name>

                    <value>${workingDir}/producer/extracted_document_metadata</value>

                </property>

                <property>

                    <name>input_citation</name>

                    <value>${workingDir}/producer/citation</value>

                </property>

                <property>

                    <name>input_document_to_project</name>

                    <value>${workingDir}/producer/document_to_project</value>

                </property>

                <property>

                    <name>input_document_to_dataset</name>

                    <value>${workingDir}/producer/document_to_dataset</value>

                </property>

                <property>

                    <name>input_document_to_research_initiative</name>

                    <value>${workingDir}/producer/document_to_research_initiative</value>

                </property>

                <property>

                    <name>input_document_to_document_clusters</name>

                    <value>${workingDir}/producer/document_to_document_clusters</value>

                </property>

                <property>

                    <name>input_document_to_document_classes</name>

                    <value>${workingDir}/producer/document_to_document_classes</value>

                </property>

                <property>

                    <name>input_document_to_document_statistics</name>

                    <value>${workingDir}/producer/document_to_document_statistics</value>

                </property>

                <property>

                    <name>input_document_with_website_usage_similarities</name>

                    <value>${workingDir}/producer/document_with_website_usage_similarities</value>

                </property>

                <!-- Output port bound to given path -->

                <property>

                    <name>output_document_with_inferenced_data</name>

                    <value>${workingDir}/transformer_export_document/document_with_inferenced_data</value>

                </property>

            </configuration>

        </sub-workflow>

        <ok to="consumer"/>

        <error to="fail"/>

    </action>

    <action name="consumer">

		<java>

			<job-tracker>${jobTracker}</job-tracker>

			<name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/consumer" />

				<mkdir path="${nameNode}${workingDir}/consumer" />

			</prepare>

			<configuration>

				<property>

					<name>mapred.job.queue.name</name>

					<value>${queueName}</value>

				</property>

			</configuration>

			<!-- This is simple wrapper for the Java code -->

			<main-class>eu.dnetlib.iis.core.java.ProcessWrapper</main-class>

			<!-- The business Java code that gets to be executed -->

			<arg>eu.dnetlib.iis.core.java.jsonworkflownodes.TestingConsumer</arg>

			<!-- Specification of the input ports -->

			<arg>-C{document_with_inferenced_data,

				eu.dnetlib.iis.export.schemas.DocumentWithInferencedData,

				eu/dnetlib/iis/transformers/export/document/sampledataproducer/data/document_with_inferenced_data.json}</arg>

			<!-- All input and output ports have to be bound to paths in HDFS, working 

				directory has to be specified as well -->

			<arg>-SworkingDir=${workingDir}/consumer/working_dir</arg>

			<arg>-Idocument_with_inferenced_data=${workingDir}/transformer_export_document/document_with_inferenced_data</arg>

		</java>

		<ok to="end" />

		<error to="fail" />

	</action>    

    <kill name="fail">

		<message>Unfortunately, the workflow failed -- error message:

			[${wf:errorMessage(wf:lastErrorNode())}]</message>

    </kill>

    <end name="end"/>

</workflow-app>

{"documentId": "id-10", "classes": {"arXivClasses": null, "WoSClasses": [{"classLabels": ["TOXICOLOGY"], "confidenceLevel": 0.112}, {"classLabels": ["ONCOLOGY"], "confidenceLevel": 0.098}], "DDCClasses":[{"classLabels": ["Technology", "Agriculture"], "confidenceLevel": 0.403}, {"classLabels": ["Science", "Biology"], "confidenceLevel": 0.286}], "meshEuroPMCClasses": null}}

{"documentId": "id-2", "classes": {"arXivClasses": [{"classLabels": ["Quantitative Biology", "Cell Behavior"], "confidenceLevel": 0.102}, {"classLabels": ["Quantitative Biology", "Tissues and Organs"], "confidenceLevel": 0.101}], "WoSClasses": null, "DDCClasses":[{"classLabels": ["Science", "Science"], "confidenceLevel": 0.285}, {"classLabels": ["Technology", "Medicine & health"], "confidenceLevel": 0.198}], "meshEuroPMCClasses": null}}

{"documentId": "id-3", "classes": {"arXivClasses": null, "WoSClasses": [{"classLabels": ["SUBSTANCE ABUSE"], "confidenceLevel": 0.096}, {"classLabels": ["COMPUTER SCIENCE, HARDWARE & ARCHITECTURE"], "confidenceLevel": 0.089}], "DDCClasses": null, "meshEuroPMCClasses": null}}

{"documentId": "id-9", "classes": null}

{"documentId": "id-2", "otherDocumentId": "id-5", "covisitedSimilarity": 0.45}

{"documentId": "id-9", "otherDocumentId": "id-5", "covisitedSimilarity": 0.58}

{"documentId": "id-2", "otherDocumentId": "id-9", "covisitedSimilarity": 0.97}

{"documentId": "id-3", "otherDocumentId": "id-1", "covisitedSimilarity": null}

{"documentId": "id-4", "otherDocumentId": "id-10", "covisitedSimilarity": null}

{"documentId": "id-5", "otherDocumentId": "id-9", "covisitedSimilarity": 0.58}

{"sourceDocumentId": "id-1", "rawText": "abc", "destinationDocumentId": "id-2", "confidenceLevel": null}

{"sourceDocumentId": "id-1", "rawText": "def", "destinationDocumentId": "id-4", "confidenceLevel": null}

{"sourceDocumentId": "id-3", "rawText": "abc", "destinationDocumentId": "id-4", "confidenceLevel": null}

{"sourceDocumentId": "id-4", "rawText": "xxx", "destinationDocumentId": "id-1", "confidenceLevel": null}

{"sourceDocumentId": "id-4", "rawText": "xyz", "destinationDocumentId": "id-2", "confidenceLevel": null}

{"sourceDocumentId": "id-7", "rawText": "xyz", "destinationDocumentId": "id-4", "confidenceLevel": null}

{"sourceDocumentId": "id-7", "rawText": "qwerty", "destinationDocumentId": "id-1", "confidenceLevel": null}

{"sourceDocumentId": "id-7", "rawText": "bnmm", "destinationDocumentId": "id-2", "confidenceLevel": null}

{"documentId":"id-1","egiConceptId":"egi-8095","confidenceLevel":1.5766148184367308}

{"documentId":"id-8","egiConceptId":"egi-0820","confidenceLevel":0.8652002245558127}

{"documentId":"id-1","egiConceptId":"egi-5103","confidenceLevel":1.3005493344846906}

{"documentId":"id-4","egiConceptId":"egi-0763","confidenceLevel":1.6733200530681511}

{"documentId":"id-4","egiConceptId":"egi-3463","confidenceLevel":1.6733200530681511}

{"documentId":"id-9","egiConceptId":"egi-5840","confidenceLevel":1.6053482043291596}

{"publisher": "Tor Science Fiction", "affiliations": null, "authors": [{"authorFullName": "author-1", "affiliationPositions": null}, {"authorFullName": "author-2", "affiliationPositions": null}, {"authorFullName": "author-3", "affiliationPositions": null}], "language": "eng", "title": "Enders Game", "externalIdentifiers": null, "journal": "Journal-2", "id": "id-1", "pages": {"start": "123", "end": "128"}, "volume": null, "references": null, "year": null, "keywords": null, "issue": null, "abstract": null}

{"publisher": null, "affiliations": null, "authors": [{"authorFullName": "author-1", "affiliationPositions": null}, {"authorFullName": "author-2", "affiliationPositions": null}], "language": null, "title": null, "externalIdentifiers": {"id-1": "val-extr-1", "id-3": "val-extr-3"}, "journal": null, "id": "id-2", "pages": null, "volume": "124", "references": [{"position": 1, "basicMetadata": {"publisher": null, "title": "The Lord of the Rings", "url": null, "series": null, "authors": null, "volume": null, "edition": null, "source": null, "year": "2012", "issue": null, "pages": {"start": "1", "end": "236"}, "location": null}, "text": "J.R.R. Tolkien, The Lord of the Rings, 2012"}, {"position": 2, "basicMetadata": {"publisher": null, "title": "The Other Wind", "url": null, "series": null, "authors": null, "volume": "vol.23", "edition": null, "source": null, "year": "2003", "issue": null, "pages": null, "location": null}, "text": "Ursula K. Le Guin, The Other Wind, 2003"}], "year": 1970, "keywords": ["kwd_1", "kwd_3", "kwd_5"], "issue": null, "abstract": "The tales"}

{"publisher": "Harp3r T0rch", "affiliations": null, "authors": null, "language": "en", "title": "Small Gods", "externalIdentifiers": null, "journal": "Journal", "id": "id-3", "pages": null, "volume": "32", "references": [{"position": 1, "basicMetadata": {"publisher": null, "title": "The Lord of the Rings", "url": null, "series": null, "authors": null, "volume": null, "edition": null, "source": null, "year": "2012", "issue": null, "pages": {"start": "1", "end": "236"}, "location": null}, "text": "J.R.R. Tolkien, The Lord of the Rings, 2012"}], "year": null, "keywords": null, "issue": "4", "abstract": null}

{"publisher": null, "affiliations": null, "authors": null, "language": null, "title": null, "externalIdentifiers": {"id-1": "val-extr-1", "id-3": "val-extr-3"}, "journal": null, "id": "id-6", "pages": null, "volume": "vol3", "references": null, "year": 2011, "keywords": null, "issue": "6", "abstract": "A Game of Thrones"}

{"publisher": null, "affiliations": null, "authors": null, "language": null, "title": null, "externalIdentifiers": {"id-1": "val-extr-1", "id-3": "val-extr-3"}, "journal": null, "id": "id-4", "pages": {"start": "3", "end": "503"}, "volume": null, "references": null, "year": 1997, "keywords": ["kwd_1", "kwd_3", "kwd_5"], "issue": "2", "abstract": "Interview with the Vampire"}

{"documentId": "id-4", "clusters": {"clusteringMethod1ClusterId": 123, "clusteringMethod2ClusterId": 90}}

{"documentId": "id-10", "clusters": {"clusteringMethod1ClusterId": 123, "clusteringMethod2ClusterId": 90}}

{"documentId": "id-7", "clusters": {"clusteringMethod1ClusterId": 45, "clusteringMethod2ClusterId": 90}}

{"documentId": "id-8", "clusters": {"clusteringMethod1ClusterId": 69, "clusteringMethod2ClusterId": null}}

{"documentId": "id-1", "clusters": null}

{"id": "id-1", "title": "Enders Game", "abstract": null, "language": "eng", "keywords": null, "externalIdentifiers": null, "journal": "Journal-2", "year": null, "publisher": "Tor Science Fiction", "text": null, "projectIds": null, "authorIds": null, "matchedCitationDocumentIds": [{"id" : "id-2", "text" : "abc"},{"id" : "id-4", "text" : "def"}], "referencedDataSetIds": ["8095", "5103"], "researchInitiativeConceptIds" : ["egi-8095", "egi-5103"], "clusters": null, "classes": null, "statistics": {"citationsFromAllPapers": {"numberOfCitations": 0, "numberOfCitationsPerYear": {}}, "citationsFromPublishedPapers": {"numberOfCitations": 0, "numberOfCitationsPerYear": {}}}, "websiteUsageSimilarities": null}

{"id": "id-2", "title": null, "abstract": "The tales", "language": null, "keywords": ["kwd_1", "kwd_3", "kwd_5"], "externalIdentifiers": {"id-1": "val-extr-1", "id-3": "val-extr-3"}, "journal": null, "year": 1970, "publisher": null, "text": null, "projectIds": ["248095"], "authorIds": null, "matchedCitationDocumentIds": null, "referencedDataSetIds": null, "researchInitiativeConceptIds" : null, "clusters": null, "classes": {"arXivClasses": [{"classLabels": ["Quantitative Biology", "Cell Behavior"], "confidenceLevel": 0.102}, {"classLabels": ["Quantitative Biology", "Tissues and Organs"], "confidenceLevel": 0.101}], "WoSClasses": null, "DDCClasses":[{"classLabels": ["Science", "Science"], "confidenceLevel": 0.285}, {"classLabels": ["Technology", "Medicine & health"], "confidenceLevel": 0.198}], "meshEuroPMCClasses": null}, "statistics": {"citationsFromAllPapers": {"numberOfCitations": 2, "numberOfCitationsPerYear": {"2001": 2}}, "citationsFromPublishedPapers": {"numberOfCitations": 1, "numberOfCitationsPerYear": {"2001": 1}}}, "websiteUsageSimilarities": [{"documentId": "id-5", "covisitedSimilarity": 0.45}, {"documentId": "id-9", "covisitedSimilarity": 0.97}]}

{"id": "id-3", "title": "Small Gods", "abstract": null, "language": "en", "keywords": null, "externalIdentifiers": null, "journal": "Journal", "year": null, "publisher": "Harp3r T0rch", "text": null, "projectIds": ["300820", "275103"], "authorIds": null, "matchedCitationDocumentIds": [{"id" : "id-4", "text" : "abc"}], "referencedDataSetIds": null, "researchInitiativeConceptIds" : null, "clusters": null, "classes": {"arXivClasses": null, "WoSClasses": [{"classLabels": ["SUBSTANCE ABUSE"], "confidenceLevel": 0.096}, {"classLabels": ["COMPUTER SCIENCE, HARDWARE & ARCHITECTURE"], "confidenceLevel": 0.089}], "DDCClasses": null, "meshEuroPMCClasses": null}, "statistics": {"citationsFromAllPapers": {"numberOfCitations": 2, "numberOfCitationsPerYear": {"2001": 2}}, "citationsFromPublishedPapers": {"numberOfCitations": 1, "numberOfCitationsPerYear": {"2001": 1}}}, "websiteUsageSimilarities": [{"documentId": "id-1", "covisitedSimilarity": null}]}

{"id": "id-4", "title": null, "abstract": "Interview with the Vampire", "language": null, "keywords": ["kwd_1", "kwd_3", "kwd_5"], "externalIdentifiers": {"id-1": "val-extr-1", "id-3": "val-extr-3"}, "journal": null, "year": 1997, "publisher": null, "text": null, "projectIds": null, "authorIds": null, "matchedCitationDocumentIds": [{"id" : "id-1", "text" : "xxx"},{"id" : "id-2", "text" : "xyz"}], "referencedDataSetIds": ["0763"], "researchInitiativeConceptIds" : ["egi-0763", "egi-3463"], "clusters": {"clusteringMethod1ClusterId": 123, "clusteringMethod2ClusterId": 90}, "classes": null, "statistics": {"citationsFromAllPapers": {"numberOfCitations": 3, "numberOfCitationsPerYear": {"2010": 1, "2001": 2}}, "citationsFromPublishedPapers": {"numberOfCitations": 2, "numberOfCitationsPerYear": {"2010": 1, "2001": 1}}}, "websiteUsageSimilarities": [{"documentId": "id-10", "covisitedSimilarity": null}]}

{"id": "id-5", "title": null, "abstract": null, "language": null, "keywords": null, "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "text": null, "projectIds": null, "authorIds": null, "matchedCitationDocumentIds": null, "referencedDataSetIds": null, "researchInitiativeConceptIds" : null, "clusters": null, "classes": null, "statistics": null, "websiteUsageSimilarities": [{"documentId": "id-9", "covisitedSimilarity": 0.58}]}

{"id": "id-6", "title": null, "abstract": "A Game of Thrones", "language": null, "keywords": null, "externalIdentifiers": {"id-1": "val-extr-1", "id-3": "val-extr-3"}, "journal": null, "year": 2011, "publisher": null, "text": null, "projectIds": null, "authorIds": null, "matchedCitationDocumentIds": null, "referencedDataSetIds": null, "researchInitiativeConceptIds" : null, "clusters": null, "classes": null, "statistics": null, "websiteUsageSimilarities": null}

{"id": "id-7", "title": null, "abstract": null, "language": null, "keywords": null, "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "text": null, "projectIds": null, "authorIds": null, "matchedCitationDocumentIds": [{"id" : "id-4", "text" : "xyz"},{"id" : "id-1", "text" : "qwerty"},{"id" : "id-2", "text" : "bnmm"}], "referencedDataSetIds": null, "researchInitiativeConceptIds" : null, "clusters": {"clusteringMethod1ClusterId": 45, "clusteringMethod2ClusterId": 90}, "classes": null, "statistics": null, "websiteUsageSimilarities": null}

{"id": "id-8", "title": null, "abstract": null, "language": null, "keywords": null, "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "text": null, "projectIds": ["240763", "275840"], "authorIds": null, "matchedCitationDocumentIds": null, "referencedDataSetIds": ["0820"], "researchInitiativeConceptIds" : ["egi-0820"], "clusters": {"clusteringMethod1ClusterId": 69, "clusteringMethod2ClusterId": null}, "classes": null, "statistics": null, "websiteUsageSimilarities": null}

{"id": "id-9", "title": null, "abstract": null, "language": null, "keywords": null, "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "text": null, "projectIds": null, "authorIds": null, "matchedCitationDocumentIds": null, "referencedDataSetIds": ["5840"], "researchInitiativeConceptIds" : ["egi-5840"], "clusters": null, "classes": null, "statistics": null, "websiteUsageSimilarities": [{"documentId": "id-5", "covisitedSimilarity": 0.58}]}

{"id": "id-10", "title": null, "abstract": null, "language": null, "keywords": null, "externalIdentifiers": null, "journal": null, "year": null, "publisher": null, "text": null, "projectIds": null, "authorIds": null, "matchedCitationDocumentIds": null, "referencedDataSetIds": null, "researchInitiativeConceptIds" : null, "clusters": {"clusteringMethod1ClusterId": 123, "clusteringMethod2ClusterId": 90}, "classes": {"arXivClasses": null, "WoSClasses": [{"classLabels": ["TOXICOLOGY"], "confidenceLevel": 0.112}, {"classLabels": ["ONCOLOGY"], "confidenceLevel": 0.098}], "DDCClasses":[{"classLabels": ["Technology", "Agriculture"], "confidenceLevel": 0.403}, {"classLabels": ["Science", "Biology"], "confidenceLevel": 0.286}], "meshEuroPMCClasses": null}, "statistics": null, "websiteUsageSimilarities": null}

{"documentId":"id-1","datasetId":"8095","confidenceLevel":1.5766148184367308}

{"documentId":"id-8","datasetId":"0820","confidenceLevel":0.8652002245558127}

{"documentId":"id-1","datasetId":"5103","confidenceLevel":1.3005493344846906}

{"documentId":"id-4","datasetId":"0763","confidenceLevel":1.6733200530681511}

{"documentId":"id-9","datasetId":"5840","confidenceLevel":1.6053482043291596}

{"documentId":"id-2","projectId":"248095","confidenceLevel":1.5766148184367308}

{"documentId":"id-3","projectId":"300820","confidenceLevel":0.8652002245558127}

{"documentId":"id-3","projectId":"275103","confidenceLevel":1.3005493344846906}

{"documentId":"id-8","projectId":"240763","confidenceLevel":1.6733200530681511}

{"documentId":"id-8","projectId":"275840","confidenceLevel":1.6053482043291596}

{"documentId": "id-1", "statistics": {"citationsFromAllPapers": {"numberOfCitations": 0, "numberOfCitationsPerYear": {}}, "citationsFromPublishedPapers": {"numberOfCitations": 0, "numberOfCitationsPerYear": {}}}}

{"documentId": "id-2", "statistics": {"citationsFromAllPapers": {"numberOfCitations": 2, "numberOfCitationsPerYear": {"2001": 2}}, "citationsFromPublishedPapers": {"numberOfCitations": 1, "numberOfCitationsPerYear": {"2001": 1}}}}

{"documentId": "id-3", "statistics": {"citationsFromAllPapers": {"numberOfCitations": 2, "numberOfCitationsPerYear": {"2001": 2}}, "citationsFromPublishedPapers": {"numberOfCitations": 1, "numberOfCitationsPerYear": {"2001": 1}}}}

{"documentId": "id-4", "statistics": {"citationsFromAllPapers": {"numberOfCitations": 3, "numberOfCitationsPerYear": {"2010": 1, "2001": 2}}, "citationsFromPublishedPapers": {"numberOfCitations": 2, "numberOfCitationsPerYear": {"2010": 1, "2001": 1}}}}

define avro_load_extracted_document_metadata

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_extracted_document_metadata');

define avro_load_citation

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_citation');

define avro_load_document_to_project

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_document_to_project');

define avro_load_document_to_dataset

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_document_to_dataset');

define avro_load_document_to_research_initiative

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_document_to_research_initiative');

define avro_load_document_to_document_clusters

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_document_to_document_clusters');

define avro_load_document_to_document_classes

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_document_to_document_classes');

define avro_load_document_to_document_statistics

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_document_to_document_statistics');

define avro_load_document_with_website_usage_similarities

org.apache.pig.piggybank.storage.avro.AvroStorage(

'input_schema_class', '$schema_input_document_with_website_usage_similarities');

define avro_store_document_with_inferenced_data

org.apache.pig.piggybank.storage.avro.AvroStorage(

'index', '0',

'output_schema_class', '$schema_output_document_with_inferenced_data');

define FIRST_NOT_NULL_STR eu.dnetlib.iis.transformers.udfs.StringFirstNotEmpty;

define NULL_EMPTY eu.dnetlib.iis.transformers.udfs.EmptyBagToNull;

define NULL_EMPTY_TUPLE_FIELDS eu.dnetlib.iis.transformers.udfs.NullTupleFieldsToNull;

define CREATE_ARRAY eu.dnetlib.iis.transformers.udfs.NullToEmptyBag;

extractedDocument = load '$input_extracted_document_metadata' using avro_load_extracted_document_metadata;

citation = load '$input_citation' using avro_load_citation;

documentToProject = load '$input_document_to_project' using avro_load_document_to_project;

documentToDataset = load '$input_document_to_dataset' using avro_load_document_to_dataset;

documentToResearchInitiative = load '$input_document_to_research_initiative' using avro_load_document_to_research_initiative;

documentToDocumentClusters = load '$input_document_to_document_clusters' using avro_load_document_to_document_clusters;

documentToDocumentClasses = load '$input_document_to_document_classes' using avro_load_document_to_document_classes;

documentToDocumentStatistics = load '$input_document_to_document_statistics' using avro_load_document_to_document_statistics;

documentWithWebsiteUsageSimilarities = load '$input_document_with_website_usage_similarities' using avro_load_document_with_website_usage_similarities;

documentToProjectGroupped = group documentToProject by documentId;

documentToProjectWithArrays = foreach documentToProjectGroupped {

    projectIds = foreach documentToProject generate projectId;

    generate group as id, projectIds;

}

documentToDatasetGroupped = group documentToDataset by documentId;

documentToDatasetWithArrays = foreach documentToDatasetGroupped {

    datasetIds = foreach documentToDataset generate datasetId;

    generate group as id, datasetIds;

}

citationGroupped = group citation by sourceDocumentId;

citationGrouppedWithText = foreach citationGroupped {

    idWithText = foreach citation generate destinationDocumentId as id, rawText as text;

    generate group as id, idWithText as citations;

}

joined1 = join documentToProjectWithArrays by id full, citationGrouppedWithText by id;

joined1Cleaned = foreach joined1 generate

    FIRST_NOT_NULL_STR(documentToProjectWithArrays::id, citationGrouppedWithText::id) as id,

    NULL_EMPTY(null) as authorIds,

    citationGrouppedWithText::citations as matchedCitationDocumentIds,

    NULL_EMPTY(documentToProjectWithArrays::projectIds) as projectIds;

joined2 = join joined1Cleaned by id full, documentToDatasetWithArrays by id;

joined2Cleaned = foreach joined2 generate

    FIRST_NOT_NULL_STR(joined1Cleaned::id, documentToDatasetWithArrays::id) as id,

    joined1Cleaned::authorIds as authorIds,

    joined1Cleaned::matchedCitationDocumentIds as matchedCitationDocumentIds,

    joined1Cleaned::projectIds as projectIds,

    NULL_EMPTY(documentToDatasetWithArrays::datasetIds) as datasetIds;

joined3 = join joined2Cleaned by id full, documentToDocumentClusters by documentId;

joined3Cleaned = foreach joined3 generate

    FIRST_NOT_NULL_STR(joined2Cleaned::id, documentToDocumentClusters::documentId) as id,

    joined2Cleaned::authorIds as authorIds,

    joined2Cleaned::matchedCitationDocumentIds as matchedCitationDocumentIds,

    joined2Cleaned::projectIds as projectIds,

    joined2Cleaned::datasetIds as datasetIds,

    documentToDocumentClusters::clusters as clusters;

joined4 = join joined3Cleaned by id full, documentToDocumentClasses by documentId;

joined4Cleaned = foreach joined4 generate

    FIRST_NOT_NULL_STR(joined3Cleaned::id, documentToDocumentClasses::documentId) as id,

    joined3Cleaned::authorIds as authorIds,

    joined3Cleaned::matchedCitationDocumentIds as matchedCitationDocumentIds,

    joined3Cleaned::projectIds as projectIds,

    joined3Cleaned::datasetIds as datasetIds,

    joined3Cleaned::clusters as clusters,

    documentToDocumentClasses::classes as classes;

joined5 = join joined4Cleaned by id full, documentToDocumentStatistics by documentId;

joined5Cleaned = foreach joined5 generate

    FIRST_NOT_NULL_STR(joined4Cleaned::id, documentToDocumentStatistics::documentId) as id,

    joined4Cleaned::authorIds as authorIds,

    joined4Cleaned::matchedCitationDocumentIds as matchedCitationDocumentIds,

    joined4Cleaned::projectIds as projectIds,

    joined4Cleaned::datasetIds as datasetIds,

    joined4Cleaned::clusters as clusters,

    joined4Cleaned::classes as classes,

    documentToDocumentStatistics::statistics as statistics;

documentWithWebsiteUsageSimilaritiesGroupped = group documentWithWebsiteUsageSimilarities by documentId;

outputSimilarities = foreach documentWithWebsiteUsageSimilaritiesGroupped {

    websiteUsageSimilarities = foreach documentWithWebsiteUsageSimilarities generate otherDocumentId as documentId, covisitedSimilarity as covisitedSimilarity;

    generate group as id, websiteUsageSimilarities;

}

joined6 = join joined5Cleaned by id full, outputSimilarities by id;

joined6Cleaned = foreach joined6 generate

    FIRST_NOT_NULL_STR(joined5Cleaned::id, outputSimilarities::id) as id,

    joined5Cleaned::authorIds as authorIds,

    joined5Cleaned::matchedCitationDocumentIds as matchedCitationDocumentIds,

    joined5Cleaned::projectIds as projectIds,

    joined5Cleaned::datasetIds as datasetIds,

    joined5Cleaned::clusters as clusters,

    joined5Cleaned::classes as classes,

    joined5Cleaned::statistics as statistics,

    outputSimilarities::websiteUsageSimilarities as websiteUsageSimilarities;

researchInitiativeGroupped = group documentToResearchInitiative by documentId;

researchInitiative = foreach researchInitiativeGroupped {

    ids = foreach documentToResearchInitiative generate egiConceptId;

    generate group as id, ids as researchInitiativeConceptIds;

}

joined7 = join joined6Cleaned by id full, researchInitiative by id;

joined7Cleaned = foreach joined7 generate

    FIRST_NOT_NULL_STR(joined6Cleaned::id, researchInitiative::id) as id,

    joined6Cleaned::authorIds as authorIds,

    joined6Cleaned::matchedCitationDocumentIds as matchedCitationDocumentIds,

    joined6Cleaned::projectIds as projectIds,

    joined6Cleaned::datasetIds as datasetIds,

    researchInitiative::researchInitiativeConceptIds as researchInitiativeConceptIds,

    joined6Cleaned::clusters as clusters,

    joined6Cleaned::classes as classes,

    joined6Cleaned::statistics as statistics,

    joined6Cleaned::websiteUsageSimilarities as websiteUsageSimilarities;

joinedFull = join extractedDocument by id full, joined7Cleaned by id;

joinedFullCleaned = foreach joinedFull generate 

    FIRST_NOT_NULL_STR(extractedDocument::id, joined7Cleaned::id) as id, 

    extractedDocument::title as title, extractedDocument::abstract as abstract, 

    extractedDocument::language as language, extractedDocument::keywords as keywords, 

    extractedDocument::externalIdentifiers as externalIdentifiers, 

    extractedDocument::journal as journal, extractedDocument::year as year, 

    extractedDocument::publisher as publisher, (chararray)null as text,

    joined7Cleaned::projectIds as projectIds,

    joined7Cleaned::authorIds as authorIds,

    joined7Cleaned::matchedCitationDocumentIds as matchedCitationDocumentIds,

    joined7Cleaned::datasetIds as referencedDataSetIds,

    joined7Cleaned::researchInitiativeConceptIds as researchInitiativeConceptIds,

    joined7Cleaned::clusters as clusters,

    joined7Cleaned::classes as classes,

    joined7Cleaned::statistics as statistics,

    joined7Cleaned::websiteUsageSimilarities as websiteUsageSimilarities;

store joinedFullCleaned into '$output_document_with_inferenced_data' using avro_store_document_with_inferenced_data;

<workflow-app xmlns="uri:oozie:workflow:0.4" name="transformers_export_document">

	<parameters>

		<property>

			<name>input_extracted_document_metadata</name>

		</property>

		<property>

			<name>input_citation</name>

		</property>

		<property>

			<name>input_document_to_project</name>

		</property>

		<property>

			<name>input_document_to_dataset</name>

		</property>

        <property>

			<name>input_document_to_research_initiative</name>

		</property>

		<property>

			<name>input_document_to_document_clusters</name>

		</property>

		<property>

			<name>input_document_to_document_classes</name>

		</property>

		<property>

			<name>input_document_to_document_statistics</name>

		</property>

		<property>

			<name>input_document_with_website_usage_similarities</name>

		</property>

		<property>

			<name>output_document_with_inferenced_data</name>

		</property>

	</parameters>

    <start to="transformer"/>

    <action name="transformer">

        <pig>

            <job-tracker>${jobTracker}</job-tracker>

            <name-node>${nameNode}</name-node>

			<!-- The data generated by this node is deleted in this section -->

			<prepare>

				<delete path="${nameNode}${workingDir}/transformer" />

				<delete path="${nameNode}${output_document_with_inferenced_data}" />

				<mkdir path="${nameNode}${workingDir}/transformer" />

			</prepare>

            <configuration>

                <property>

                    <name>mapred.job.queue.name</name>

                    <value>${queueName}</value>

                </property>

            </configuration>

            <!-- Path to PIG script the workflow executes. -->

            <script>lib/scripts/transformer.pig</script>

            <!-- The working directory of the workflow node. -->

            <param>workingDir=${workingDir}/transformer/working_dir</param>

            <param>input_extracted_document_metadata=${input_extracted_document_metadata}</param>

            <param>schema_input_extracted_document_metadata=eu.dnetlib.iis.metadataextraction.schemas.ExtractedDocumentMetadata</param>

            <param>input_citation=${input_citation}</param>

            <param>schema_input_citation=eu.dnetlib.iis.citationmatching.schemas.Citation</param>

            <param>input_document_to_project=${input_document_to_project}</param>

            <param>schema_input_document_to_project=eu.dnetlib.iis.referenceextraction.project.schemas.DocumentToProject</param>

            <param>input_document_to_dataset=${input_document_to_dataset}</param>

            <param>schema_input_document_to_dataset=eu.dnetlib.iis.referenceextraction.dataset.schemas.DocumentToDataSet</param>

            <param>input_document_to_research_initiative=${input_document_to_research_initiative}</param>

            <param>schema_input_document_to_research_initiative=eu.dnetlib.iis.referenceextraction.researchinitiative.schemas.DocumentToResearchInitiative</param>

            <param>input_document_to_document_clusters=${input_document_to_document_clusters}</param>

            <param>schema_input_document_to_document_clusters=eu.dnetlib.iis.documentsclustering.schemas.DocumentToDocumentClusters</param>

            <param>input_document_to_document_classes=${input_document_to_document_classes}</param>

            <param>schema_input_document_to_document_classes=eu.dnetlib.iis.documentsclassification.schemas.DocumentToDocumentClasses</param>

            <param>input_document_to_document_statistics=${input_document_to_document_statistics}</param>

            <param>schema_input_document_to_document_statistics=eu.dnetlib.iis.statistics.schemas.DocumentToDocumentStatistics</param>

            <param>input_document_with_website_usage_similarities=${input_document_with_website_usage_similarities}</param>

            <param>schema_input_document_with_website_usage_similarities=eu.dnetlib.iis.websiteusage.schemas.DocumentsWithWebsiteUsageSimilarities</param>

            <param>output_document_with_inferenced_data=${output_document_with_inferenced_data}</param>

            <param>schema_output_document_with_inferenced_data=eu.dnetlib.iis.export.schemas.DocumentWithInferencedData</param>

        </pig>

        <ok to="end"/>

        <error to="fail"/>

    </action>

    <kill name="fail">

		<message>Unfortunately, the workflow failed -- error message:

			[${wf:errorMessage(wf:lastErrorNode())}]</message>

    </kill>

    <end name="end"/>

</workflow-app>