D-Net

import com.google.common.collect.Lists;

import eu.dnetlib.data.mapreduce.util.XmlRecordFactory;

import eu.dnetlib.data.transform.Column;

import eu.dnetlib.data.transform.Row;

import eu.dnetlib.data.transform.XsltRowTransformerFactoryTest;

import org.dom4j.Document;

import org.dom4j.io.DocumentResult;

import org.dom4j.io.DocumentSource;

import org.dom4j.io.SAXReader;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerFactory;

import java.io.*;

import java.util.Map;

public class DOIBoostToActionsTest extends XsltRowTransformerFactoryTest {

    private final static String xslt =

            "<xsl:stylesheet version=\"1.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n"

                    + "  <xsl:template match=\"@*|node()\">\n"

                    + "    <xsl:copy>\n"

                    + "      <xsl:apply-templates select=\"@*|node()\"/>\n"

                    + "    </xsl:copy>\n"

                    + "  </xsl:template>\n"

                    + "</xsl:stylesheet>";

    private Transformer transformer;

        doTestSingleDOIBoostAction("/eu/dnetlib/data/mapreduce/actions/broken");

    public void testDOIBoostActionToXML() throws Exception {

        doTestSingleDOIBoostActionToXML("/eu/dnetlib/data/mapreduce/actions/broken");

    }

    @Test

    private void doTestSingleDOIBoostActionToXML(final String filePath) throws Exception {

        final List<Row> rows = Lists.newArrayList();

        final InputStream is = this.getClass().getResourceAsStream(filePath);

        final BufferedReader in = new BufferedReader(new InputStreamReader(is));

        String line = in.readLine();

        final JsonParser parser = new JsonParser();

        JsonObject root = parser.parse(line).getAsJsonObject();

        List<AtomicAction> actions = DOIBoostToActions.generatePublicationActionsFromDump(root, new ActionFactory(), setName, agent, false, false);

        if (actions!= null) {

            actions.forEach(action-> {

                if (action.getTargetColumn().equals("body") && action.getTargetColumnFamily().equals("result"))

                {

                    Column<String, byte[]> col = new Column<>("body" , action.getTargetValue());

                    rows.add(new Row("result",action.getTargetRowKey() , Lists.newArrayList(col)));

                }

            });

        }

        Map<String, XmlRecordFactory> stringXmlRecordFactoryMap = mapAll(buildTable(rows));

        transformer = TransformerFactory.newInstance().newTransformer(new DocumentSource((new SAXReader()).read(new StringReader(xslt))));

        final SAXReader saxReader = new SAXReader();

        stringXmlRecordFactoryMap.values().forEach(it -> {

            try {

                final DocumentResult result = new DocumentResult();

                final Document document = saxReader.read(new StringReader(it.build()));

                transformer.transform(new DocumentSource(document), result);

                System.out.println(result.getDocument().asXML());

            }

            catch (Exception e) {

                e.printStackTrace();

            }

            });

    }

        List<AtomicAction> actions = DOIBoostToActions.generatePublicationActionsFromDump(root, new ActionFactory(), setName, agent, false, false);

	protected Map<String, Map<String, Map<String, byte[]>>> buildTable(final List<Row> rows) throws UnsupportedEncodingException {

	protected Map<String, XmlRecordFactory> mapAll(final Map<String, Map<String, Map<String, byte[]>>> table) throws Exception {

{"publisher": "Elsevier BV", "doi": "10.1016/j.dib.2015.09.011", "license": [{"url": "http://www.elsevier.com/tdm/userlicense/1.0/", "content-version": "tdm", "\"delay-in-days": null, "date-time": "2015-12-01T00:00:00Z"}, {"url": "http://creativecommons.org/licenses/by/4.0/", "content-version": "vor", "\"delay-in-days": null, "date-time": "2015-09-15T00:00:00Z"}], "title": ["Data on individual PCR efficiency values as quality control for circulating miRNAs"], "issued": "2015-12-1", "abstract": [{"provenance": "MAG", "value": "This data article contains data related to the research article entitled “Variability in microRNA recovery from plasma: Comparison of five commercial kits, doi:10.1016/j.ab.2015.07.018” Brunet-Vega (2015) [1]. PCR efficiency, along with RNA and cDNA quality, are the most important factors affecting the quality of qPCR results. Constant amplification efficiency in all compared samples is indispensable when relative quantification is used to measure changes in gene expression. An easy way to measure PCR efficiency, without the need of a standard curve, is LinRegPCR software. Individual PCR efficiency can be determined as a part of qPCR quality control. This is especially important when the initial RNA quantity is so low that cannot be accurately quantified, such as in circulating RNA extractions. This data article reports the Cqs and PCR efficiencies of 5 miRNAs quantified in RNA isolated from 4 patients with colorectal cancer (CRC) and 4 healthy donors using five commercially available kits."}], "issn": [{"type": "print", "value": "2352-3409"}], "doi-url": "http://dx.doi.org/10.1016/j.dib.2015.09.011", "instances": [{"url": "http://api.elsevier.com/content/article/PII:S235234091500205X?httpAccept=text/xml", "provenance": "CrossRef", "access-rights": "UNKNOWN"}, {"url": "http://api.elsevier.com/content/article/PII:S235234091500205X?httpAccept=text/plain", "provenance": "CrossRef", "access-rights": "UNKNOWN"}, {"url": "http://doi.org/10.1016/j.dib.2015.09.011", "provenance": "UnpayWall", "access-rights": "OPEN"}, {"url": "https://academic.microsoft.com/#/detail/2115687396", "provenance": "MAG", "access-rights": "UNKNOWN"}], "published-online": null, "authors": [{"affiliations": [{"official-page": "http://www.uab.edu/", "provenance": "MAG", "value": "University of Alabama at Birmingham", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Alabama_at_Birmingham", "schema": "wikpedia"}, {"value": "grid.265892.2", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/32389192", "schema": "URL"}]}], "given": "Anna", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2335602571", "schema": "URL"}], "fullname": "Anna Brunet-Vega", "family": "Brunet-Vega"}, {"affiliations": [{"official-page": "http://www.uab.edu/", "provenance": "MAG", "value": "University of Alabama at Birmingham", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Alabama_at_Birmingham", "schema": "wikpedia"}, {"value": "grid.265892.2", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/32389192", "schema": "URL"}]}], "given": "Carles", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2692345643", "schema": "URL"}], "fullname": "Carles Pericay", "family": "Pericay"}, {"affiliations": [{"official-page": "http://www.uab.edu/", "provenance": "MAG", "value": "University of Alabama at Birmingham", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Alabama_at_Birmingham", "schema": "wikpedia"}, {"value": "grid.265892.2", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/32389192", "schema": "URL"}]}], "given": "María Elisa", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2160096374", "schema": "URL"}], "fullname": "María Elisa Quílez", "family": "Quílez"}, {"affiliations": [{"official-page": "http://www.uab.edu/", "provenance": "MAG", "value": "University of Alabama at Birmingham", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Alabama_at_Birmingham", "schema": "wikpedia"}, {"value": "grid.265892.2", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/32389192", "schema": "URL"}]}], "given": "María José", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2751674309", "schema": "URL"}], "fullname": "María José Ramírez-Lázaro", "family": "Ramírez-Lázaro"}, {"affiliations": [{"official-page": "http://www.uab.edu/", "provenance": "MAG", "value": "University of Alabama at Birmingham", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Alabama_at_Birmingham", "schema": "wikpedia"}, {"value": "grid.265892.2", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/32389192", "schema": "URL"}]}], "given": "Xavier", "identifiers": [{"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2683017426", "schema": "URL"}], "fullname": "Xavier Calvet", "family": "Calvet"}, {"affiliations": [{"official-page": "http://www.uab.edu/", "provenance": "MAG", "value": "University of Alabama at Birmingham", "identifiers": [{"value": "http://en.wikipedia.org/wiki/University_of_Alabama_at_Birmingham", "schema": "wikpedia"}, {"value": "grid.265892.2", "schema": "grid.ac"}, {"value": "https://academic.microsoft.com/#/detail/32389192", "schema": "URL"}]}], "given": "Sergio", "identifiers": [{"provenance": "CrossRef", "value": "http://orcid.org/0000-0001-9294-1585", "schema": null}, {"provenance": "MAG", "value": "https://academic.microsoft.com/#/detail/2752779186", "schema": "URL"}, {"provenance": "ORCID", "value": "https://orcid.org/0000-0001-9294-1585", "schema": null}], "fullname": "Sergio Lario", "family": "Lario"}], "collectedFrom": ["CrossRef", "MAG", "ORCID", "UnpayWall"], "accepted": null, "type": "journal-article", "published-print": "2015-12-1", "subject": ["Multidisciplinary"]}

import java.util.function.Function;

        System.out.println(JsonFormat.printToString(oaf.build()));

            }).collect(

                    Collectors.toMap(

                            FieldTypeProtos.KeyValue::getKey,

                            Function.identity(),

                            (a,b) -> a

                    )).values().forEach(abuilder::addPid);

package eu.dnetlib.data.mapreduce.hbase.dataexport;

import com.googlecode.protobuf.format.JsonFormat;

import eu.dnetlib.data.mapreduce.util.DedupUtils;

import eu.dnetlib.data.proto.OafProtos;

import eu.dnetlib.data.proto.TypeProtos;

import org.apache.hadoop.hbase.client.Result;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;

import org.apache.hadoop.hbase.mapreduce.TableMapper;

import org.apache.hadoop.io.Text;

import java.io.IOException;

import java.util.Map;

public class ExportDuplicatesMapper extends TableMapper<Text, Text> {

    private Text keyOut;

    private Text valueOut;

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

        keyOut = new Text("");

        valueOut = new Text();

    }

    @Override

    protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {

        final Map<byte[], byte[]> mergedInMap = value.getFamilyMap(DedupUtils.getDedupCF_mergedInBytes(TypeProtos.Type.result));

        if (mergedInMap != null && !mergedInMap.isEmpty()) {

            final byte[] body = value.getValue("result".getBytes(), DedupUtils.BODY_B);

            if (body != null) {

                OafProtos.Oaf oaf = OafProtos.Oaf.parseFrom(body);

                valueOut.set(JsonFormat.printToString(oaf));

                context.write(keyOut, valueOut);

            }

        }

    }

}