D-Net

import org.joda.time.format.DateTimeFormat;

import org.joda.time.format.DateTimeFormatter;

import org.jsoup.Jsoup;

import java.io.*;

import java.text.DateFormat;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.List;

/**

 * Created by eri on 1/14/16.

 */

public class Test {

    public static void main(String[] args) throws ParseException, IOException {

        File fs = new File("C:\\Users\\eri_k\\Downloads\\relation_proj");

        StringBuilder sb = new StringBuilder();

        try (BufferedReader br = new BufferedReader(new FileReader(fs))) {

            String line;

            while ((line = br.readLine()) != null) {

                String[] split = line.split("!");

                sb.append(split[0] + "!" + split[1] + "!" + split[4] + "!" + split[3] + "!" + split[2] + "\n");

//                resultProject!result!corda__h2020::c0210cb8b2256f4ca08d18391ae137c0!project!od______1108::18ddc87d7409554b58d0a230b533e516!

            }

        }

        System.out.println(sb.toString());

        File fs2 = new File("C:\\Users\\eri_k\\Downloads\\relations_fixed");

        FileWriter fw = new FileWriter(fs2);

        fw.write(sb.toString());

        fw.close();

    /*    String date = "2015-05-26";

        DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd");

        System.out.println(formatter.parseDateTime(date));

        DateFormat df = new SimpleDateFormat("yyyy-mm-dd");

        System.out.println(df.parse(date));

        String value = " \"lala\" /lo {}li?   la ";

        value = "i";

        System.out.println("value " + value);

        value = value.replace(" ", "");

        System.out.println("value " + value);

        value = value.replace(" ", "");

        value="Gary  Richard,Lewin ,Lewin % Gary > Richard>";

        value = value.replaceAll("[\"\\r\\\\;]", "");

        value = value.replaceAll("[^a-zA-Z0-9 .-_:/&!@+=]+", " ");

        System.out.println("VALUE IS" + value);

        value="http://dx.doi.org/10.17014/ijog.vol1no2.20063-Papandayan-is-an-A-type-active-strato-volcano-located-at-some-20-km-SW-of-Garut-or-about-70-km-SE-of-Bandung-the-capital-city-of-West-Java-Province.-Geographically-the-summit-of-this-volcano-lies-at-the-intersection-between-07--19--42--S-and-107--44--E.-The-2002-Papandayan-eruption-was-preceded-by-two-felt-earthquakes-8-times-of-A-type-volcanic-earthquakes-and-150-times-of-B--type-volcanic-earthquake.-These-events-were-followed-by-a-phreatic-eruption-that-took-place-on-11-November-at-16.02-local-time.-Field-observation-shows-that-the-summit-region-mainly-around-the-craters-consists-of-rocks-that-have-hydrothermally-altered-to-yield-clay-rich-material.-This-clay-rich-material-covers-most-of-the-crater-fl-oors-and-the-crater-rim.-Mount-Nangklak-that-forms-part-of-the-rim-also-contains-a-lava-plug-from-the-Old-Papandayan-volcano.-This-mountain-is-covered-by-fi-ne-grained-unconsolidated-material-and-altered-rocks.-Much-of-this-altered-rocks-coincides-with-solfataric-and-fumarolic-activities-of-80-to->-300-C.-The-summit-area-also-contains-high-discharge-of-water-either-originating-from-the-springs-or-surface-water.-The-increase-in-seismicity-the-fi-ne-grained-hydrothermal-altered-rocks-and-the-existence-of-some-faults-that-pass-through-the-summit-region-might-have-weaken-the-stability-of-the-summit-area.-As-the-result-a-landslide-occurred-on-the-north-fl-ank-of-Mount-Nangklak-where-the-landslide-material-blocked-the-upper-course-of-Cibeureum-Gede-River.-This-landslide-material-had-formed-big-mudfl-ows-that-caused-several-houses-of-fi-ve-villages-were-partly-burried-some-bridges-were-devastated-and-several-hectares-of-cultivated-land-were-damaged.\n" +

                "\tat java.net.URI$Parser.fail(URI.java:2829)";

        System.out.println("char is " +value.charAt(1057) + value.charAt(1058));

        value="http://dx.doi.org/10.5902/1984686X5486-O-objetivo-deste-estudo-foi-descrever-o-processo-de-valida-o-de-conte-do-de-um-instrumento-de-avalia-o-do-esquema-corporal-para-crian-as-comcegueira-cong-nita-com-idade-entre-6-e-9-anos.-A-popula-o-foi-constitu-dapor-sete-professores-universit-rios-os-quais-receberam-uma-primeiravers-o-do-instrumento-eles-analisaram-cada-item-nos-quesitos-clareza-delinguagem-pertin-ncia-te-rica-viabilidade-de-aplica-o-e-adequa-o-aoconstructo.-A-prova-1--obteve-80%-de-concord-ncia-a-Prova-2-recebeu-70%-a-prova-3---60%-e-a-prova-4--75%.-Adotou-se-como-crit-rio-de-perman-nciada-prova-o-m-nimo-de-80%-de-concord-ncia.-A-prova-1-permaneceu-como-naprimeira-vers-o-a-prova-2-foi-exclu-da-e-as-provas-3-e-4-foram-modificadas.-Com-base-nas-sugest-es-dos-ju-zes-e-no--ndice-de-concord-ncia-elaborou-se-uma-segunda-vers-o-do-instrumento-composta-por-tr-s-itens.-Com-isso-foi-poss-vel-considerar-que-as-adapta-es-sugeridas-pelos-ju-zes-foram-importantes-para-garantir-a-validade-de-conte-do-do-instrumento.-Palavras-chave:-Educa-o-Especial-Educa-o-F-sica-adaptada-Cegueira-Avalia-o-Esquema-corporal.\n" +

                "java.net.URISyntaxException: Malformed escape pair at index 489: http://dx.doi.org/10.5902/1984686X5486-O-objetivo-deste-estudo-foi-descrever-o-processo-de-valida-o-de-conte-do-de-um-instrumento-de-avalia-o-do-esquema-corporal-para-crian-as-comcegueira-cong-nita-com-idade-entre-6-e-9-anos.-A-popula-o-foi-constitu-dapor-sete-professores-universit-rios-os-quais-receberam-uma-primeiravers-o-do-instrumento-eles-analisaram-cada-item-nos-quesitos-clareza-delinguagem-pertin-ncia-te-rica-viabilidade-de-aplica-o-e-adequa-o-aoconstructo.-A-prova-1--obteve-80%-de-concord-ncia-a-Prova-2-recebeu-70%-a-prova-3---60%-e-a-prova-4--75%.-Adotou-se-como-crit-rio-de-perman-nciada-prova-o-m-nimo-de-80%-de-concord-ncia.-A-prova-1-permaneceu-como-naprimeira-vers-o-a-prova-2-foi-exclu-da-e-as-provas-3-e-4-foram-modificadas.-Com-base-nas-sugest-es-dos-ju-zes-e-no--ndice-de-concord-ncia-elaborou-se-uma-segunda-vers-o-do-instrumento-composta-por-tr-s-itens.-Com-isso-foi-poss-vel-considerar-que-as-adapta-es-sugeridas-pelos-ju-zes-foram-importantes-para-garantir-a-validade-de-conte-do-do-instrumento.-Palavras-chave:-Educa-o-Especial-Educa-o-F-sica-adaptada-Cegueira-Avalia-o-Esquema-corporal.\n" +

                "\tat java.net.URI$Parser.fail(URI.java:2829)";

        System.out.println("malformed escaped pair char is " +value.charAt(489) +value.charAt(490) );

        *//*value = value.replace("\\r", " ");

        value = value.replace("\\", "");

		*//*

		 *//*value = value.replace("\\r", " ");

         value = value.replace("\\", "");

*//*

        value = "http://dx.doi.org/10.5902/198050985739<br /><br />O presente trabalho objetivou avaliar a capacidade de adesão da madeira serrada de eucalipto proveniente de três sistemas de manejo, empregando-se os adesivos: resorcina formaldeído, e dois adesivos em emulsão aquosa à base de poliacetato de vinila. Os sistemas de manejo foram caracterizados por três estratos, sendo o estrato um (E1) caracterizado por madeira proveniente de talhadia e idade de 70 meses; o estrato dois (E2) caracterizado por madeira de reforma e idade de 166 meses e o estrato três (E3) caracterizado também por reforma aos 70 meses de idade.";

        System.out.println(Jsoup.parse(value).text());

        List<String> s = new ArrayList<String>();

        s.add("A");

        s.add("B");

        s.add("C");

        s.add("D");

        int listSize= s.size();

        List<String> newList= new ArrayList<String>();

String DELIM=";";

        String valueEntity="result";

        for(int i=0; i< s.size()-1;i++)

        {

            System.out.println(i);

            for(int j=i+1; j<s.size();j++)

            {System.out.println("j is " +j);

                newList.add("dedup" + DELIM +

                        valueEntity + DELIM +

                        s.get(i) + DELIM +

                        valueEntity+ DELIM+

                        s.get(j) + DELIM);

            }

        }

        System.out.println(newList);

*/

        //  MyString.formStrings("");

    }

    public static List<List<String>> generatePerm(List<String> original) {

        if (original.size() == 0) {

            List<List<String>> result = new ArrayList<List<String>>();

            result.add(new ArrayList<String>());

            return result;

        }

        String firstElement = original.remove(0);

        List<List<String>> returnValue = new ArrayList<List<String>>();

        List<List<String>> permutations = generatePerm(original);

        for (List<String> smallerPermutated : permutations) {

            for (int index = 0; index <= smallerPermutated.size(); index++) {

                List<String> temp = new ArrayList<String>(smallerPermutated);

                temp.add(index, firstElement);

                returnValue.add(temp);

            }

        }

        return returnValue;

    }

    public static class MyString {

        static final String[] chars = {"a", "b", "c"};

        static final int reqLen = 2;

        public static void formStrings(String crtStr) {

            if (crtStr.length() == reqLen) {

                System.out.println(crtStr);

                return;

            } else

                for (int i = 1; i < chars.length; i++) {

                    if (!crtStr.contains(chars[i] + crtStr))

                        formStrings(crtStr + chars[i]);

                    else {

                        formStrings(crtStr);

                    }

                }

        }

    }

    static void fillList(ArrayList list) {

        list.add("lala");

    }

}

import eu.dnetlib.data.mapreduce.hbase.lodExport.LodReducer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;

import org.junit.Before;

import org.junit.Test;

import java.io.IOException;

import java.util.ArrayList;

public class LodReducerTest {

    ReduceDriver<Text, Text, Text, Text> reduceDriver;

    @Before

    public void setUp() {

        Configuration configuration = new Configuration();

        configuration.set("lod.delim", ",");

        configuration.set("mapreduce.multipleoutputs.namedOutput.result.key", "org.apache.hadoop.io.Text");

        configuration.set("mapreduce.multipleoutputs.namedOutput.result.value", "org.apache.hadoop.io.Text");

        configuration.set("mapreduce.multipleoutputs.namedOutput.result.format", "org.apache.hadoop.mapreduce.lib.output.TextOutputFormat");

        configuration.set("mapreduce.multipleoutputs", "result");

        LodReducer reducer = new LodReducer();

        //String sourceMappings = "{\"type\":\"result\", \"fields\":[\"<http://purl.org/dc/terms/identifier>\",\"<http://www.eurocris.org/ontologies/cerif/1.3#name>\",\"<http://lod.openaire.eu/vocab/year>\"]}";

        reduceDriver = ReduceDriver.newReduceDriver(reducer).withConfiguration(configuration);

    }

    @Test

    public void testReducer() throws IOException {

        ArrayList list = new ArrayList();

        String value = "doajarticles::9aacfbb6168b5346aa63a49c79a7a88d,2017-01-14T12:40:47.751Z,2016-10-12T12:23:03.305Z,oai:doaj.org/article:85b1bd2dbf344a6cab93f36c7bdfff69;,Zika virus in Brazil and the danger of infestation by Aedes  tegomyia) mosquitoes,2015-01-01, ociedade Brasileira de Medicina Tropical  BMT), 10.1590/0037-8682-0220-2015;,eng, , Aedes aegypti;Aedes albopictus;Arbovirus; tegomyia;Culicidae;Medicine (General);R5-920;Medicine;R;Arctic medicine. Tropical medicine;RC955-962;, , Revista da  ociedade Brasileira de Medicina Tropical, Iss 0 (2015);, , , ,Open Access, Abstract Zika virus, already widely distributed in Africa and Asia, was recently reported in two Northeastern Brazilian:  tate of Bahia and  tate of Rio Grande do Norte, and one  outheastern:  tate of  ão Paulo. This finding adds a potentially noxious virus to a list of several other viruses that are widely transmitted by Aedes  tegomyia) aegypti and Aedes  tegomyia) albopictus in Brazil. The pathology and epidemiology, including the distribution and vectors associated with Zika virus, are reviewed. This review is focused on viruses transmitted by Aedes  tegomyia) mosquitoes, including dengue, Chikungunya, Zika, Mayaro, and yellow fever virus, to emphasize the risks of occurrence for these arboviruses in Brazil and neighboring countries. Other species of Aedes  tegomyia) are discussed, emphasizing their involvement in arbovirus transmission and the possibility of adaptation to environments modified by human activities and introduction in Brazil.,Revista da  ociedade Brasileira de Medicina Tropical,,,,,,,2015,publication, ,";

        list.add(new Text(value));

        reduceDriver.withInput(new Text("result"), list);

        reduceDriver.withOutput(new Text("result"), new Text(value));

        reduceDriver.run();

    }

}

package eu.dnetlib.data.mapreduce.hbase.lodExport.utils;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import javax.xml.transform.stream.StreamSource;

import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.io.InputStream;

public class ContextTransformer {

    public String transformXSL(String xml) throws Exception {

        Transformer transformer;

        TransformerFactory tFactory = TransformerFactory.newInstance();

        if (xml == null) {

            throw new Exception("NULL XML ENTITY CONFIGURATION   ");

        }

        InputStream inputStream = null;

        ByteArrayInputStream readerStream = null;

        ByteArrayOutputStream writerStream = null;

        try {

            inputStream = ClassLoader.getSystemResourceAsStream("eu/dnetlib/data/mapreduce/hbase/statsExport/" + "context.xsl");

            transformer = tFactory.newTransformer(new StreamSource(inputStream));

            readerStream = new ByteArrayInputStream(xml.getBytes("UTF-8"));

            writerStream = new ByteArrayOutputStream();

            transformer.transform(new StreamSource(readerStream), new StreamResult(writerStream));

            return writerStream.toString("UTF8");

        } catch (Exception e) {

            throw new Exception(e.getMessage(), e);

        } finally {

            inputStream.close();

            readerStream.close();

            writerStream.close();

        }

    }

}

package eu.dnetlib.data.mapreduce.hbase.lodExport.utils;

import org.apache.log4j.Logger;

public class FundingParser {

	private Logger log = Logger.getLogger(this.getClass());

	public static String getFundingLevel(String funding_level, int level,String delim) {

		String NULL_STRING = " ";

		if (funding_level.isEmpty()) {

			return NULL_STRING + delim;

		}

		if (!funding_level.contains("<funding_level_" + level + ">")) {

			return NULL_STRING + delim;

		}

		String[] split = funding_level.split("<funding_level_" + level + ">");

		funding_level = split[1];

		split = funding_level.split("<name>");

		funding_level = split[1];

		funding_level = funding_level.substring(0, funding_level.indexOf("</name>"));

		funding_level = funding_level.replaceAll("\"", "");

		funding_level = funding_level.replaceAll("/>", "");

		funding_level = funding_level.replaceAll("<", "");

		funding_level = funding_level.replaceAll("&", "");

		if (level == 1) {

			if (funding_level.equalsIgnoreCase("SP1")) {

				funding_level = "SP1-Cooperation";

			} else if (funding_level.equalsIgnoreCase("SP2")) {

				funding_level = "SP2-Ideas";

			}

			if (funding_level.equalsIgnoreCase("SP3")) {

				funding_level = "SP3-People";

			} else if (funding_level.equalsIgnoreCase("SP4")) {

				funding_level = "SP4-Capacities";

			} else if (funding_level.equalsIgnoreCase("SP5")) {

				funding_level = "SP5-Euratom";

			}

		}

		funding_level = funding_level.replaceAll(">", "");

		funding_level = funding_level.replaceAll("</", "");

		return funding_level + delim;

	}

	public  static String getFundingInfo(String buff,String delim) {

		return FundingParser.getFunder(buff,delim) + getFundingLevel(buff, 0,delim) + (getFundingLevel(buff, 1,delim) + getFundingLevel(buff, 2,delim)

                + getFundingLevel(buff, 3,delim));

	}

	public static String getFunder(String buff,String delim) {

		String NULL_STRING = " ";

		if (buff.isEmpty()) {

				return NULL_STRING +delim;

		}

		if (!buff.contains("<funder>")) {

			return NULL_STRING +delim;

		}

		String[] split = buff.split("<funder>");

		String funder = split[1];

		split = funder.split("<name>");

		funder = split[1];

		funder = funder.substring(0, funder.indexOf("</name>"));

		funder = funder.replaceAll(">", "");

		funder = funder.replaceAll("</", "");

		funder = funder.replaceAll("\"", "");

		funder = funder.replaceAll("&", "");

		return funder + delim;

	}

}

package eu.dnetlib.data.mapreduce.hbase.lodExport.utils;

import eu.dnetlib.data.mapreduce.util.LicenseComparator;

import eu.dnetlib.data.proto.DatasourceProtos.Datasource.Metadata;

import eu.dnetlib.data.proto.FieldTypeProtos;

import eu.dnetlib.data.proto.FieldTypeProtos.StringField;

import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.data.proto.OafProtos.OafRel;

import eu.dnetlib.data.proto.OrganizationProtos.Organization;

//import eu.dnetlib.data.proto.PersonProtos;

import eu.dnetlib.data.proto.ProjectProtos.Project;

import eu.dnetlib.data.proto.ResultProtos.Result;

import eu.dnetlib.data.proto.ResultProtos.Result.Instance;

import java.util.ArrayList;

import java.util.List;

import java.util.Set;

/**

 * @author eri Simple serializer that parses input Oaf Protos and prepares them

 *         for sqoop

 */

public class Serializer {

    private static final String TEXT_PATTERN = "[^a-zA-Z0-9 .-_:/@+=-]";

    private static final String ID_PATTERN = "[^a-zA-Z0-9.:/_-]";

    private static final String DOI_PATTERN = ".*\\/(10.)";

    public static String serialize(Oaf oaf, String DELIM) {

        switch (oaf.getKind()) {

            case entity:

                OafEntity valueEntity = oaf.getEntity();

                switch (valueEntity.getType()) {

                    case datasource:

                        return buildDatasource(valueEntity, DELIM);

                    case organization:

                        return buildOrganization(valueEntity, DELIM);

                    case project:

                        return buildProject(valueEntity, DELIM);

                    case result:

                        return buildResult(valueEntity, DELIM);

                        /*

                    case person:

                        return buildPerson(valueEntity, DELIM);

                        */

                    default:

                        break;

                }

                break;

            case relation:

                OafRel valueRel = oaf.getRel();

                return serialize(valueRel, DELIM);

        }

        return null;

    }

    public static void extractRelations(Oaf oaf, String DELIM, Set<String> relations) {

        OafEntity valueEntity = oaf.getEntity();

        switch (valueEntity.getType()) {

            case result:

                getResultDatasources(valueEntity, DELIM, relations);

                getDedups(valueEntity, DELIM, relations);

            case datasource:

                getDedups(valueEntity, DELIM, relations);

                /*

            case person:

                getDedups(valueEntity, DELIM, relations);

                */

            case organization:

                getDedups(valueEntity, DELIM, relations);

            default:

        }

    }

    public static String serialize(OafRel Rel, String DELIM) {

        StringBuilder buff;

        switch (Rel.getRelType()) {

            case datasourceOrganization:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("datasource").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM)

                        .append("organization").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM).append("\n");

                return buff.toString();

            case resultResult:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("result").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM)

                        .append("result").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM);

                return buff.toString();

                /*

            case personPerson:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("person").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM)

                        .append("person").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM);

                return buff.toString();

                */

            case organizationOrganization:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("organization").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM)

                        .append("organization").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM);

                return buff.toString();

                /*

            case personResult:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("person").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM)

                        .append("result").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM);

                return buff.toString();

                */

            case projectOrganization:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("project").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM)

                        .append("organization").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM);

                return buff.toString();

                /*

            case projectPerson:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("project").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM)

                        .append("person").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM);

                return buff.toString();

                */

            case resultOrganization:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("result").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM)

                        .append("organization").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM);

                return buff.toString();

//TODO - SOS RELATION RESULT PROJECT IS INVERTED! SOURCE IS PROJECT, TARGET IS RESULT

            case resultProject:

                buff = new StringBuilder();

                buff.append(Rel.getRelType().name()).append(DELIM).append("result").append(DELIM).append(cleanId(Rel.getTarget())).append(DELIM)

                        .append("project").append(DELIM).append(cleanId(Rel.getSource())).append(DELIM);

                return buff.toString();

            default:

        }

        return "";

    }

    private static String getHeader(OafEntity data, String DELIM) {

        String SEPERATOR = ";";

        StringBuilder buff = new StringBuilder();

        //  EntityType

        buff.append(data.getType().name()).append(DELIM);

        // OpenaireID

        buff.append(cleanId(data.getId())).append(DELIM);

        //  dateOfTransformation

        // TODO  CRITERIA FOR IDENTIFYING UPDATED RECORDS

        buff.append(cleanId(data.getDateoftransformation())).append(DELIM);

        //    dateOfCollection

        buff.append(clean(data.getDateofcollection())).append(DELIM);

        String dataStr = new String();

        // originalId

        for (String oid : data.getOriginalIdList()) {

            dataStr += cleanDoi(oid) + SEPERATOR; //oid is not openaireid, clean instead of cleanid

        }

        buff.append(dataStr).append(DELIM);

        return buff.toString();

    }

    private static String buildDatasource(OafEntity data, String DELIM) {

        String SEPERATOR = ";";

        StringBuilder buff = new StringBuilder();

        buff.append(getHeader(data, DELIM));

        Metadata metadata = data.getDatasource().getMetadata();

        //Datasourcetype

        buff.append(clean(metadata.getDatasourcetype().getClassname())).append(DELIM);

        //Openairecompatibility

        buff.append(clean(metadata.getOpenairecompatibility().getClassname())).append(DELIM);

        //OfficialName

        buff.append(clean(metadata.getOfficialname().getValue())).append(DELIM);

        //  Englishname

        buff.append(clean(metadata.getEnglishname().getValue())).append(DELIM);

        //Websiteurl

        buff.append(clean(metadata.getWebsiteurl().getValue())).append(DELIM);

        //LogoURL

        buff.append(clean(metadata.getLogourl().getValue())).append(DELIM);

        //Contactemail

        buff.append(clean(metadata.getContactemail().getValue())).append(DELIM);

        //Namespaceprefix

        buff.append(clean(metadata.getNamespaceprefix().getValue())).append(DELIM);

        // latitude

        buff.append(clean(metadata.getLatitude().getValue())).append(DELIM);

        // longtitude

        buff.append(clean(metadata.getLongitude().getValue())).append(DELIM);

        // dateofvalidation,

        buff.append(clean(metadata.getDateofvalidation().getValue())).append(DELIM);

        //Description

        buff.append(clean(metadata.getDescription().getValue())).append(DELIM);

        //subjects

        String subj = new String();

        for (StructuredProperty s : metadata.getSubjectsList()) {

            subj += clean(s.getValue()) + SEPERATOR;

        }

        //subjectList

        buff.append(clean(subj)).append(DELIM);

        //Number of items

        buff.append(clean(metadata.getOdnumberofitems().getValue())).append(DELIM);

        //Date of number of items

        buff.append(clean(metadata.getOdnumberofitemsdate().getValue())).append(DELIM);

        // Policies

        buff.append(clean(metadata.getOdpolicies().getValue())).append(DELIM);

        //languages

        String dataStr = new String();

        for (StringField lang : metadata.getOdlanguagesList()) {

            dataStr += clean(lang.getValue()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        ;

        // Content type

        dataStr = " ";

        for (StringField c : metadata.getOdcontenttypesList()) {

            dataStr += clean(c.getValue()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        //Access info package

        dataStr = " ";

        for (StringField c : metadata.getAccessinfopackageList()) {

            dataStr += clean(c.getValue()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        //Release start date

        buff.append(clean(metadata.getReleasestartdate().getValue())).append(DELIM);

        //Release end date

        buff.append(clean(metadata.getReleaseenddate().getValue())).append(DELIM);

        //Mission statement url

        buff.append(clean(metadata.getMissionstatementurl().getValue())).append(DELIM);

        //Data provider

        buff.append(clean(String.valueOf(metadata.getDataprovider().getValue()))).append(DELIM);

        //Service provider

        buff.append(clean(String.valueOf(metadata.getServiceprovider().getValue()))).append(DELIM);

        //Database access type

        buff.append(clean(metadata.getDatabaseaccessrestriction().getValue())).append(DELIM);

        //Data upload type

        buff.append(clean(metadata.getDatauploadtype().getValue())).append(DELIM);

        //Data upload restrictions

        buff.append(clean(metadata.getDatauploadrestriction().getValue())).append(DELIM);

        //Versioning

        buff.append(clean(String.valueOf(metadata.getVersioning().getValue()))).append(DELIM);

        //Citation guideline url

        buff.append(clean(metadata.getCitationguidelineurl().getValue())).append(DELIM);

        //Quality management kind

        buff.append(clean(metadata.getQualitymanagementkind().getValue())).append(DELIM);

        //PID systems

        buff.append(clean(metadata.getPidsystems().getValue())).append(DELIM);

        //Certificates

        buff.append(clean(metadata.getCertificates().getValue())).append(DELIM);

        //Policies

        dataStr = " ";

        for (FieldTypeProtos.KeyValue property : metadata.getPoliciesList()) {

            dataStr += clean(property.getValue()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        buff.append(getTrust(data)).append(DELIM);

        return buff.toString();

    }

    private static String buildOrganization(OafEntity data, String DELIM) {

        String SEPERATOR = ";";

        StringBuilder buff = new StringBuilder();

        buff.append(getHeader(data, DELIM));

        Organization organization = data.getOrganization();

        Organization.Metadata metadata = organization.getMetadata();

        //getLegalshortname

        buff.append(clean(metadata.getLegalshortname().getValue())).append(DELIM);

        // `name`,

        buff.append(clean(metadata.getLegalname().getValue())).append(DELIM);

        //website URL

        String[] split = metadata.getWebsiteurl().getValue().split(",");

        String dataStr = new String();

        for (String s : split) {

            dataStr += s.replace(DELIM, " ") + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        //logourl

        buff.append(clean(metadata.getLogourl().getValue())).append(DELIM);

        // `country`,

        buff.append(clean(metadata.getCountry().getClassid())).append(DELIM);

        buff.append(getTrust(data)).append(DELIM);

        return buff.toString();

    }

    static String getTrust(OafEntity data) {

        for (FieldTypeProtos.ExtraInfo info : data.getExtraInfoList()) {

            return (info.getTrust());

        }

        return " ";

    }

    private static String buildResult(OafEntity data, String DELIM) {

        String SEPERATOR = ";";

        Result.Metadata metadata = data.getResult().getMetadata();

        StringBuilder buff = new StringBuilder();

        buff.append(getHeader(data, DELIM));

        //   titleString

        String dataStr = new String();

        for (int i = 0; i < metadata.getTitleList().size(); i++) {

            StructuredProperty title = metadata.getTitleList().get(i);

            dataStr = clean(title.getValue());

            break;

        }

        //  pubtitle

        buff.append(clean(dataStr)).append(DELIM);

        // date of acceptance CHANGED THIS TO DATE FORMAT

        buff.append(clean(metadata.getDateofacceptance().getValue())).append(DELIM);

        // publisher

        buff.append(clean(metadata.getPublisher().getValue())).append(DELIM);

        //PID

        dataStr = " ";

        for (StructuredProperty p : data.getPidList()) {

            dataStr += clean(p.getValue()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        //language

        buff.append(clean(metadata.getLanguage().getClassid())).append(DELIM);

        // RelevantDate

        dataStr = " ";

        for (StructuredProperty p : metadata.getRelevantdateList()) {

            dataStr += clean(p.getValue());

            break;

        }

        buff.append(dataStr).append(DELIM);

        //Subject

        dataStr = " ";

        for (StructuredProperty subj : metadata.getSubjectList()) {

            if (subj.getValue() != null && !subj.getValue().isEmpty()) {

                dataStr += clean(subj.getValue()) + SEPERATOR;

            }

        }

        buff.append(dataStr).append(DELIM);

        //TODO ExternalReference

        buff.append(" ").append(DELIM);

        //Source

        dataStr = " ";

        for (StringField s : metadata.getSourceList()) {

            dataStr += clean(s.getValue()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        //TODO Format     

        buff.append(" ").append(DELIM);

        //Context

        dataStr = " ";

        for (Result.Context s : metadata.getContextList()) {

            dataStr += clean(s.getId()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        //country

        String country = " ";

        for (FieldTypeProtos.Qualifier c : metadata.getCountryList()) {

            country += clean(c.getClassid()) + SEPERATOR;

        }

        buff.append(country).append(DELIM);

        //Best License

        buff.append(getBestLicense(data.getResult())).append(DELIM);

        //Description

        dataStr = " ";

        for (StringField desc : metadata.getDescriptionList()) {

            dataStr += clean(desc.getValue());

            break;

        }

        buff.append(dataStr).append(DELIM);

        //Journal  

        buff.append(clean(metadata.getJournal().getName())).append(DELIM);  //#null#!

        // TODO ERI SOS : HERE IN GET JOUTNAL. GET DATA INFO I CAN FIND PROVENANCE AND SIMILARITY

        // TODO isRelatedTo

        //   resource type

        buff.append(clean(metadata.getResourcetype().getClassname())).append(DELIM);

        //   device

        buff.append(clean(metadata.getDevice().getValue())).append(DELIM);

        //   size

        buff.append(clean(metadata.getSize().getValue())).append(DELIM);

        //     version

        buff.append(clean(metadata.getVersion().getValue())).append(DELIM);

        //   metadata update

        buff.append(clean(metadata.getLastmetadataupdate().getValue())).append(DELIM);

        //   metadata version

        buff.append(clean(metadata.getMetadataversionnumber().getValue())).append(DELIM);

        // year

        buff.append(clean(getYearInt(metadata.getDateofacceptance().getValue()))).append(DELIM);

        // type

        buff.append(clean(metadata.getResulttype().getClassname())).append(DELIM);

        buff.append(getTrust(data)).append(DELIM);

        //Authors

        /*

        dataStr = " ";

        for (StringField author : metadata.getContributorList()) {

            dataStr += clean(author.getValue()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        */

        return buff.toString();

    }

    private static String buildProject(OafEntity data, String DELIM) {

        String SEPERATOR = ";";

        StringBuilder buff = new StringBuilder();

        buff.append(getHeader(data, DELIM));

        Project.Metadata metadata = data.getProject().getMetadata();

        //Code

        buff.append(metadata.getCode().getValue()).append(DELIM);

        // `Websiteurl`,

        buff.append(clean(metadata.getWebsiteurl().getValue())).append(DELIM);

        //TODO here

        // `Acronym`,

        buff.append(clean(metadata.getAcronym().getValue())).append(DELIM);

        //Title

        buff.append(clean(metadata.getTitle().getValue())).append(DELIM);

        // Startdate

        buff.append(clean(metadata.getStartdate().getValue())).append(DELIM);

        // Enddate

        buff.append(clean(metadata.getEnddate().getValue())).append(DELIM);

        //`Call identifer`

        buff.append(clean(metadata.getCallidentifier().getValue())).append(DELIM);

        //`KeyWords`

        buff.append(clean(metadata.getKeywords().getValue())).append(DELIM);

        //`Duration`

        buff.append(clean(metadata.getDuration().getValue())).append(DELIM);

        //esc39

        buff.append(clean(metadata.getEcsc39().getValue())).append(DELIM);

        //`Contracttype`

        buff.append(clean(metadata.getContracttype().getClassname())).append(DELIM);

        //`OA mandate pubs`  TODO DOES NOT EXIST

        buff.append(clean(metadata.getOamandatepublications().getValue())).append(DELIM);

        //`Subjects`

        String dataStr = new String();

        for (StructuredProperty s : metadata.getSubjectsList()) {

            dataStr += clean(s.getValue()) + SEPERATOR;

        }

        buff.append(dataStr).append(DELIM);

        //`EC293`

        buff.append(clean(metadata.getEcarticle293().getValue())).append(DELIM);

        List<StringField> fundList = metadata.getFundingtreeList();

        if (!fundList.isEmpty()) // `funding_lvl0`,

        {//TODO funder + 3 funding levels

           /* funder text,

            funding_lvl0 text,

	        funding_lvl1 text,

	        funding_lvl2 text,

	        funding_lvl3 text,*/

            buff.append(FundingParser.getFundingInfo(fundList.get(0).getValue(), DELIM));

        } else {

            buff.append(FundingParser.getFundingInfo("", DELIM));

        }

        buff.append(getTrust(data)).append(DELIM);

        return buff.toString();

    }

    /*

    private static String buildPerson(OafEntity data, String DELIM) {

        String SEPERATOR = ";";

        PersonProtos.Person person = data.getPerson();

        PersonProtos.Person.Metadata metadata = person.getMetadata();

        StringBuilder buff = new StringBuilder();

        buff.append(getHeader(data, DELIM));

        // `firstname`,

        buff.append(clean(metadata.getFirstname().getValue())).append(DELIM);

        // `secondNames`,

        String dataStr = new String();

        for (StringField s : metadata.getSecondnamesList()) {

            dataStr += clean(s.getValue()) + ' ';

        }

        buff.append(dataStr).append(DELIM);

        // `fullname`,

        buff.append(clean(metadata.getFullname().getValue())).append(DELIM);

        // `Fax`,

        buff.append(clean(metadata.getFax().getValue())).append(DELIM);

        // `Email`,

        buff.append(clean(metadata.getEmail().getValue())).append(DELIM);

        // `Phone`,

        buff.append(clean(metadata.getPhone().getValue())).append(DELIM);

        // `Nationality`,

        buff.append(clean(metadata.getNationality().getClassid())).append(DELIM);

        // `PIDS`,

        dataStr = " ";

        for (StructuredProperty s : data.getPidList()) {

            dataStr += clean(s.getValue()) + ";";

        }

        buff.append(dataStr).append(DELIM);

        buff.append(getTrust(data)).append(DELIM);

        return buff.toString();

    }

    */

    private static void getResultDatasources(OafEntity valueEntity, String DELIM, Set<String> returnList) {

        String SEPERATOR = ";";

        Result result = valueEntity.getResult();

        if (valueEntity.getId().contains("dedup")) return;

        //TODO hosted by

        for (Instance instance : (result.getInstanceList())) {

            String hostedBy = instance.getHostedby().getKey();

            if (hostedBy != null && !hostedBy.isEmpty()) {

                returnList.add("resultDatasource" + DELIM + "result" + DELIM +

                        cleanId(valueEntity.getId()) + DELIM + "datasource" + DELIM + cleanId(hostedBy) + DELIM);

            }

        }

        //TODO  collected froms

        for (FieldTypeProtos.KeyValue collectedFromValue : (valueEntity.getCollectedfromList())) {

            String collectedFrom = collectedFromValue.getKey();

            if (collectedFrom != null && !collectedFrom.isEmpty())

                returnList.add(("resultDatasource" + DELIM

                        + "result" + DELIM + cleanId(valueEntity.getId()) + DELIM

                        + "datasource" + DELIM + cleanId(collectedFrom) + DELIM));

        }

    }

    public static String cleanId(String value) {

        if (value == null) {

            return " ";

        }

        //   DO NOT CHANGE THIS: IT REMOVES ID PREFIX ( 5|datacite____::

        // to datacite____:: )

        // AND REPLACES OCCURENCES OF DELIM CHARS IN DATA

        value = value.replaceFirst(".*\\|", "");

        value = value.replaceAll(ID_PATTERN, "");

        value = value.replace("#", " ");

        value = value.replace("\n", " ");

        return value;

    }

    private static String clean(String value) {

        if (value == null) {

            return " ";

        }

        // TODO DO NOT CHANGE THIS: IT REMOVES ID PREFIX ( 5|datacite____::

        value = value.replaceAll("[\"\\r\\\\;]", "");

        value = value.replace("\\", "");

        value = value.replaceAll(TEXT_PATTERN, " ");

        value = value.replace(">", " ");

        value = value.replace("<", " ");

        value = value.replace("\"", " ");

        value = value.replace("\\", " ");

        value = value.replace("'", " ");

        value = value.replace("«", " ");

        value = value.replace("»", " ");

        value = value.replace("#", " ");

        value = value.replace("\\", " ");

        value = value.replace("\n", " ");

        value = value.replace(",", "");

        value = value.replace("#", "");

        value = value.replace(";", "");

        return value;

    }

    private static String cleanDoi(String value) {

        value = value.replaceAll(DOI_PATTERN, "");

        value = value.replace("#", " ");

        value = value.replace("\n", " ");

        return value;

    }

    //TODO make them in pairs

    private static void getDedups(OafEntity valueEntity, String DELIM, Set<String> returnList) {

        if (!valueEntity.getChildrenList().isEmpty() && valueEntity.getId().contains("dedup")) {

            ArrayList<String> entries = new ArrayList<String>();

            for (OafEntity child : valueEntity.getChildrenList()) {

                if (child.getType() == valueEntity.getType() && !child.getId().contains("dedup")) {

                    // if it is result, then its the deduplication

                    entries.add(cleanId(child.getId()));

                }

            }

            for (int i = 0; i < entries.size() - 1; i++) {

                for (int j = i + 1; j < entries.size(); j++) {

                    returnList.add("dedup" + DELIM +

                            valueEntity.getType().name() + DELIM +

                            entries.get(i) + DELIM +

                            valueEntity.getType().name() + DELIM +

                            entries.get(j) + DELIM);

                }

            }

        }

    }

    private static String getYearInt(String data) {

        if (data == null || data.isEmpty() || data.equals("-1")) {

            return " ";

        }

        String[] split = data.split("-");

        if (split == null || split.length == 0) {

            return " ";

        }

        return split[0];

    }

    private static String getBestLicense(Result result) {

        FieldTypeProtos.Qualifier bestLicense = null;

        LicenseComparator lc = new LicenseComparator();

        for (Instance instance : (result.getInstanceList())) {

            if (lc.compare(bestLicense, instance.getLicence()) > 0) {

                bestLicense = instance.getLicence();

            }

        }

        if (bestLicense != null) {

            return bestLicense.getClassname();

        } else {

            return null;

        }

    }

}

package eu.dnetlib.data.mapreduce.hbase.lodExport.utils;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.log4j.Logger;

import java.util.ArrayList;

import java.util.List;

public class ContextExporter {

    private ContextTransformer contextTransformer = new ContextTransformer();

    private String outputPath;

    private Logger log = Logger.getLogger(this.getClass());

    private ArrayList<String> context = new ArrayList<String>();

    private ArrayList<String> category = new ArrayList<String>();

    private ArrayList<String> concept = new ArrayList<String>();

    public ContextExporter(String outputPath, String contextMap, boolean readFromUrl) throws Exception {

        if (!outputPath.endsWith("/")) {

            outputPath += "/";

        }

        this.outputPath = outputPath;

        if (readFromUrl) {

            readFromUrl(contextMap);

        } else {

            readFromBuffer(contextMap);

        }

    }

    public void readFromUrl(String url) throws Exception {

        List<String> concepts = getContextResouces(url);

        log.info("Returned concept  " + concepts.size()

        );

        for (String data : concepts) {

            log.info("++++++++++++++ Transforming concept data ");

            String res = contextTransformer.transformXSL(data);

            processData(res);

        }

        writeData(this.context, "context");

        writeData(this.category, "category");

        writeData(this.concept, "concept");

    }

    private void readFromBuffer(String contextMap) throws Exception {

        if (contextMap == null || contextMap.isEmpty()) {

            log.error("Context Resources file is empty.");

            throw new Exception("Context Resources file is empty.");

        }

        String data = contextTransformer.transformXSL(contextMap);

        log.info(data);

        processData(data);

    }

    private List<String> getContextResouces(String url) throws ISLookUpException {

        ISLookUpService lookUpService;

        JaxWsProxyFactoryBean factory = new JaxWsProxyFactoryBean();

        factory.setServiceClass(ISLookUpService.class);

        factory.setAddress(url);

        lookUpService = (ISLookUpService) factory.create();

//		for $x in //RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ContextDSResourceType']

//		[.//RESOURCE_KIND/@value='ContextDSResources'] return  $x

        return lookUpService.quickSearchProfile("//RESOURCE_PROFILE[.//RESOURCE_TYPE/@value='ContextDSResourceType'][.//RESOURCE_KIND/@value='ContextDSResources']");

    }

    private void writeData(ArrayList<String> dataList, String listName) throws Exception {

        if (dataList.isEmpty()) return;

        log.info(listName + "  size " + dataList.size());

        String data = new String();

        for (int i = 0; i < dataList.size(); i++) {

            data += dataList.get(i);

        }

        data = data.substring(0, data.lastIndexOf("\n"));

        flushString(data, outputPath + listName);

    }

    private void processData(String data) throws Exception {

        try {

            String[] split = data.split("COPY\n");

            if (split.length > 0) {

                context.add(split[0]);

            }

            if (split.length > 1) {

                category.add(split[1]);

            }

            if (split.length > 2) {

                concept.add(split[2]);

            }

        } catch (Exception e) {

            String msg = " Unable to create file with context, " + "concept and category values in output path " + outputPath + ". Reason: ";

            log.error(msg);

            throw new Exception(msg, e);

        }

    }

    private void flushString(String data, String destination) throws Exception {

        FSDataOutputStream fin = null;

        try {

            log.info("***********************Writing data:***********************\n" + data);

            log.info("***********************  data:***********************\n");

            FileSystem fs = FileSystem.get(new Configuration());

            fin = fs.create(new Path(destination), true);

            fin.write(data.getBytes());

        } catch (Exception e) {

            log.error("Failed  to write exported data to a file : ", e);

            throw new Exception("Failed  to write exported data to a file : " + e.toString(), e);

        } finally {

            fin.close();

        }

    }

    public String getOutputPath() {

        return outputPath;

    }

    public void setOutputPath(String outputPath) {

        this.outputPath = outputPath;

    }

}

package eu.dnetlib.data.mapreduce.hbase.lodExport;

import com.google.protobuf.InvalidProtocolBufferException;

import eu.dnetlib.data.mapreduce.hbase.index.config.EntityConfigTable;

import eu.dnetlib.data.mapreduce.hbase.index.config.IndexConfig;

import eu.dnetlib.data.mapreduce.hbase.index.config.LinkDescriptor;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.Serializer;

import eu.dnetlib.data.mapreduce.util.OafRowKeyDecoder;

import eu.dnetlib.data.mapreduce.util.UpdateMerger;

import eu.dnetlib.data.proto.OafProtos.Oaf;

import eu.dnetlib.data.proto.OafProtos.OafRel;

import eu.dnetlib.data.proto.OafProtos.OafRelOrBuilder;

import eu.dnetlib.data.proto.RelTypeProtos.SubRelType;

import eu.dnetlib.data.proto.TypeProtos.Type;

import org.apache.hadoop.hbase.client.Result;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;

import org.apache.hadoop.hbase.mapreduce.TableMapper;

import org.apache.hadoop.hbase.util.Bytes;

import org.apache.hadoop.io.Text;

import org.apache.log4j.Logger;

import java.io.IOException;

import java.text.SimpleDateFormat;

import java.util.*;

import java.util.Map.Entry;

/**

 * Mapper Class that reads HBASE contents and prepares them for the StatsDB

 * export

 */

public class LodMapper extends TableMapper<Text, Text> {

    private static final String DATE_OF_TRANSFORMATION_PATTERN = "yyyy-MM-dd'T'HH:mm:ss";

    private static final String LAST_EXECUTION_DATE_PATTERN = "yyyy-MM-dd";

    private Logger log = Logger.getLogger(this.getClass());

    private EntityConfigTable entityConfigTable;

    private String lastExecutionDate = "";

    public static enum ENTITIES_COUNTER {

        RESULT,

        PROJECT,

        DATASOURCE,

        PERSON,

        ORGANIZATION,

        DELETED_BY_INFERENCE,

        NOT_DELETED_BY_INFERENCE,

        TOTAL_ENTITIES,

        TOTAL_RELATIONS,

        UPDATED,

        NOT_UPDATED

    }

    ;

    private String DELIM;

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

        loadEntityConfig(context);

        DELIM = context.getConfiguration().get("lod.delim");

        lastExecutionDate = context.getConfiguration().get("lod.lastExecutionDate");

    }

    @Override

    protected void map(final ImmutableBytesWritable keyIn, final Result result, final Context context) throws IOException {

        final OafRowKeyDecoder decoder = OafRowKeyDecoder.decode(keyIn.copyBytes());

        final Type type = decoder.getType();

        final Oaf oaf = UpdateMerger.mergeBodyUpdates(context, result.getFamilyMap(Bytes.toBytes(type.toString())));

        if (isValid(oaf)) {

            if (deletedByInference(oaf)) {

                context.getCounter(ENTITIES_COUNTER.DELETED_BY_INFERENCE).increment(1);

            } else {

                context.getCounter(ENTITIES_COUNTER.NOT_DELETED_BY_INFERENCE).increment(1);

            }

            context.getCounter(ENTITIES_COUNTER.TOTAL_ENTITIES).increment(1);

            emitProtos(context, result, oaf);

        }

    }

    private boolean isValid(Oaf oaf) {

        try {

            if (oaf != null && oaf.isInitialized()) {

                return true;

            }

        } catch (Exception e) {

            log.error("invalid proto", e);

        }

        return false;

    }

    private boolean isUpdated(Oaf oaf) throws IOException {

        String dateOfTransformationString = "";

        try {

            SimpleDateFormat simpleDateFormat = new SimpleDateFormat(DATE_OF_TRANSFORMATION_PATTERN, Locale.getDefault());

            simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));

            dateOfTransformationString = oaf.getEntity().getDateoftransformation();

            if (dateOfTransformationString == null || dateOfTransformationString.isEmpty() || dateOfTransformationString.equals(" ")) {

                return true;

            }

            Date dateOfTransformation = simpleDateFormat.parse(dateOfTransformationString);

            SimpleDateFormat lastExecDateFormatter = new SimpleDateFormat(LAST_EXECUTION_DATE_PATTERN);

            simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));

            Date lastExecDate = lastExecDateFormatter.parse(lastExecutionDate);

            if (lastExecDate.before(dateOfTransformation)) {

                return true;

            }

        } catch (Exception e) {

            log.error("invalid date " + dateOfTransformationString, e);

            throw new IOException(e);

        }

        return false;

    }

    private void emitProtos(Context context, Result result, Oaf oaf) throws IOException {

        Oaf.Builder oafBuilder = Oaf.newBuilder().setKind(oaf.getKind()).setDataInfo(oaf.getDataInfo());

        Type type = oaf.getEntity().getType();

        oafBuilder.setEntity(oaf.getEntity());

        // emit relation first so we can cache them to entity protos

        emitRelation(context, result, oaf, type, oafBuilder);

        if (isUpdated(oaf)) {