D-Net

	<bean id="StatsJobDriver" class="eu.dnetlib.data.mapreduce.hbase.statsExport.drivers..StatsJobDriver">

	<bean id="SqoopDriver" class="eu.dnetlib.data.mapreduce.hbase.statsExport.drivers.SqoopDriver">

	<bean id="DBDriver" class="eu.dnetlib.data.mapreduce.hbase.statsExport.drivers.DBDriver">

	<bean id="statsJob" class="eu.dnetlib.data.mapreduce.hbase.statsExport.StatsJob"> </bean>

	<bean id="statsDao" class="eu.dnetlib.data.mapreduce.hbase.statsExport.daos.SqlDAO">

	<bean id="statsStore" class="eu.dnetlib.data.mapreduce.hbase.statsExport.daos.SqlStore">

<?xml version="1.0" encoding="UTF-8"?>

<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">

  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">

    <output url="file://$MODULE_DIR$/target/classes" />

    <output-test url="file://$MODULE_DIR$/target/test-classes" />

    <content url="file://$MODULE_DIR$">

      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />

      <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />

      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />

      <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />

      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />

      <excludeFolder url="file://$MODULE_DIR$/target" />

    </content>

    <orderEntry type="inheritedJdk" />

    <orderEntry type="sourceFolder" forTests="false" />

    <orderEntry type="library" name="Maven: javax.xml:jaxp-api:1.4.2" level="project" />

    <orderEntry type="library" name="Maven: net.sf.opencsv:opencsv:2.3" level="project" />

    <orderEntry type="library" name="Maven: org.apache.poi:poi:3.8" level="project" />

    <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.5" level="project" />

    <orderEntry type="module-library">

      <library name="Maven: jdk.tools:jdk.tools:1.7.0_05">

        <CLASSES>

          <root url="jar:///usr/lib/jvm/java-7-oracle/lib/tools.jar!/" />

        </CLASSES>

        <JAVADOC />

        <SOURCES />

      </library>

    </orderEntry>

    <orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.0-rc2" level="project" />

    <orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-core:2.0-rc2" level="project" />

    <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />

    <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.6" level="project" />

    <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.6" level="project" />

    <orderEntry type="library" name="Maven: com.google.guava:guava-collections:r03" level="project" />

    <orderEntry type="library" name="Maven: com.google.guava:guava-annotations:r03" level="project" />

    <orderEntry type="library" name="Maven: com.google.guava:guava-primitives:r03" level="project" />

    <orderEntry type="library" name="Maven: com.google.guava:guava-base:r03" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:dnet-mapreduce-jobs:0.0.5" level="project" />

    <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />

    <orderEntry type="library" name="Maven: com.google.code.gson:gson:2.2.2" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:dnet-index-solr-common:1.1.1" level="project" />

    <orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:4.7.0" level="project" />

    <orderEntry type="library" name="Maven: commons-io:commons-io:2.1" level="project" />

    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.0-alpha4" level="project" />

    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.0-beta1" level="project" />

    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpmime:4.3.1" level="project" />

    <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.5" level="project" />

    <orderEntry type="library" name="Maven: org.noggit:noggit:0.5" level="project" />

    <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />

    <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />

    <orderEntry type="library" name="Maven: com.google.guava:guava:14.0" level="project" />

    <orderEntry type="library" name="Maven: com.mycila:xmltool:3.3" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:cnr-misc-utils:1.0.1" level="project" />

    <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />

    <orderEntry type="library" name="Maven: jaxen:jaxen:1.1.6" level="project" />

    <orderEntry type="library" name="Maven: saxonica:saxon:9.1.0.8" level="project" />

    <orderEntry type="library" name="Maven: saxonica:saxon-dom:9.1.0.8" level="project" />

    <orderEntry type="library" name="Maven: jgrapht:jgrapht:0.7.2" level="project" />

    <orderEntry type="library" name="Maven: net.sf.ehcache:ehcache:2.8.0" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-test:4.0.0.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-core:4.0.0.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:dnet-mapreduce-submitter:2.0.1" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:dnet-hadoop-commons:1.0.4" level="project" />

    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-common:2.0.0-cdh4.3.0" level="project" />

    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-annotations:2.0.0-cdh4.3.0" level="project" />

    <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />

    <orderEntry type="library" name="Maven: org.apache.commons:commons-math:2.1" level="project" />

    <orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" />

    <orderEntry type="library" name="Maven: commons-net:commons-net:3.1" level="project" />

    <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty:6.1.26.cloudera.2" level="project" />

    <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26.cloudera.2" level="project" />

    <orderEntry type="library" name="Maven: com.sun.jersey:jersey-core:1.8" level="project" />

    <orderEntry type="library" name="Maven: com.sun.jersey:jersey-json:1.8" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.jettison:jettison:1.1" level="project" />

    <orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />

    <orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-impl:2.2.6" level="project" />

    <orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.1" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.8" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.8" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.8.8" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-xc:1.8.8" level="project" />

    <orderEntry type="library" name="Maven: com.sun.jersey:jersey-server:1.8" level="project" />

    <orderEntry type="library" name="Maven: asm:asm:3.3.1" level="project" />

    <orderEntry type="library" scope="RUNTIME" name="Maven: tomcat:jasper-compiler:5.5.23" level="project" />

    <orderEntry type="library" name="Maven: tomcat:jasper-runtime:5.5.23" level="project" />

    <orderEntry type="library" name="Maven: commons-el:commons-el:1.0" level="project" />

    <orderEntry type="library" name="Maven: net.java.dev.jets3t:jets3t:0.6.1" level="project" />

    <orderEntry type="library" name="Maven: junit:junit:4.8.2" level="project" />

    <orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />

    <orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.1" level="project" />

    <orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" />

    <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" />

    <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />

    <orderEntry type="library" name="Maven: org.mockito:mockito-all:1.8.5" level="project" />

    <orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.4" level="project" />

    <orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" level="project" />

    <orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" />

    <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />

    <orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />

    <orderEntry type="library" name="Maven: net.sf.kosmosfs:kfs:0.3" level="project" />

    <orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />

    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.0.0-cdh4.3.0" level="project" />

    <orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.42" level="project" />

    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-core:2.0.0-mr1-cdh4.3.0" level="project" />

    <orderEntry type="library" name="Maven: hsqldb:hsqldb:1.8.0.10" level="project" />

    <orderEntry type="library" name="Maven: org.eclipse.jdt:core:3.1.1" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-beans:4.0.0.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-hdfs:2.0.0-cdh4.3.0" level="project" />

    <orderEntry type="library" name="Maven: commons-daemon:commons-daemon:1.0.3" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-context-support:4.0.0.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-context:4.0.0.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-aop:4.0.0.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />

    <orderEntry type="library" name="Maven: org.springframework:spring-expression:4.0.0.RELEASE" level="project" />

    <orderEntry type="library" name="Maven: org.apache.hbase:hbase:0.94.6-cdh4.3.0" level="project" />

    <orderEntry type="library" name="Maven: com.yammer.metrics:metrics-core:2.1.2" level="project" />

    <orderEntry type="library" name="Maven: com.github.stephenc.high-scale-lib:high-scale-lib:1.1.1" level="project" />

    <orderEntry type="library" name="Maven: org.apache.thrift:libthrift:0.9.0" level="project" />

    <orderEntry type="library" name="Maven: org.jruby:jruby-complete:1.6.5" level="project" />

    <orderEntry type="library" name="Maven: org.mortbay.jetty:jsp-2.1:6.1.14" level="project" />

    <orderEntry type="library" name="Maven: org.mortbay.jetty:jsp-api-2.1:6.1.14" level="project" />

    <orderEntry type="library" name="Maven: org.mortbay.jetty:servlet-api-2.5:6.1.14" level="project" />

    <orderEntry type="library" name="Maven: org.jamon:jamon-runtime:2.3.1" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:dnet-openaire-data-protos:3.3.0" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:dnet-openaireplus-mapping-utils:2.2.2" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:dnet-pace-core:1.3.1" level="project" />

    <orderEntry type="library" name="Maven: edu.cmu:secondstring:1.0.0" level="project" />

    <orderEntry type="library" name="Maven: com.typesafe:config:0.3.1" level="project" />

    <orderEntry type="library" name="Maven: org.antlr:stringtemplate:3.2" level="project" />

    <orderEntry type="library" name="Maven: org.antlr:antlr:2.7.7" level="project" />

    <orderEntry type="library" name="Maven: org.json:json:20140107" level="project" />

    <orderEntry type="library" name="Maven: org.mongodb:mongo-java-driver:2.9.3" level="project" />

    <orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />

    <orderEntry type="library" name="Maven: org.apache.sqoop:sqoop:1.4.4" level="project" />

    <orderEntry type="library" name="Maven: postgresql:postgresql:9.1-901.jdbc4" level="project" />

    <orderEntry type="library" name="Maven: eu.dnetlib:cnr-rmi-api:2.0.0" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-api:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.woodstox:woodstox-core-asl:4.2.0" level="project" />

    <orderEntry type="library" name="Maven: org.codehaus.woodstox:stax2-api:3.1.1" level="project" />

    <orderEntry type="library" name="Maven: org.apache.ws.xmlschema:xmlschema-core:2.0.3" level="project" />

    <orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-javamail_1.4_spec:1.7.1" level="project" />

    <orderEntry type="library" name="Maven: wsdl4j:wsdl4j:1.6.3" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-frontend-jaxws:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: xml-resolver:xml-resolver:1.2" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-core:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-bindings-soap:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-databinding-jaxb:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-bindings-xml:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-frontend-simple:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-transports-http:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-ws-addr:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.apache.cxf:cxf-rt-ws-policy:2.7.8" level="project" />

    <orderEntry type="library" name="Maven: org.apache.neethi:neethi:3.0.2" level="project" />

  </component>

</module>

    private static String DELIM;

    private Logger log = Logger.getLogger(this.getClass());

    private String NULL_STRING;

    private String NULL_NUM;

    //TODO no longer used

    private static String ENCLOSED;

    private FundingParser fundingParser = new FundingParser();

    public Serializer() {

    }

    public String serialize(Oaf oaf) {

        switch (oaf.getKind()) {

            case entity:

                OafEntity valueEntity = oaf.getEntity();

                switch (valueEntity.getType()) {

                    case datasource:

                        return buildDatasource(valueEntity);

                    case organization:

                        return buildOrganization(valueEntity);

                    case project:

                        return buildProject(valueEntity);

                    case result:

                        return buildResult(valueEntity);

                    case person:

                        return buildPerson(valueEntity);

                    default:

                        log.error("wrong type");

                        break;

                }

                break;

            case relation:

                OafRel valueRel = oaf.getRel();

                return buildRel(valueRel);

        }

        return null;

    }

    public String serialize(OafRel oaf) {

        switch (oaf.getRelType()) {

            case resultProject:

                return getResultProject(oaf);

            default:

                return buildRel(oaf);

        }

    }

    private String buildRel(OafRel Rel) {

        return getStringField(Rel.getTarget());

    }

    public HashMap<String, List<String>> extractRelations(Oaf oaf) {

        OafEntity valueEntity = oaf.getEntity();

        switch (valueEntity.getType()) {

            case datasource:

                return getDatasourceLanguages(valueEntity);

            case result:

                HashMap<String, List<String>> relations = new HashMap<String, List<String>>();

                relations.putAll(getResultLanguages(valueEntity));

                //TODO no more resultTopics needed!!!

                // relations.putAll(getResultTopics(valueEntity));

                relations.putAll(getResultClassifications(valueEntity));

                relations.putAll(getResultDatasources(valueEntity));

                relations.putAll(getResultConcepts(valueEntity));

                return relations;

            default:

                return null;

        }

    }

    private String getResultProject(OafRel oaf) {

        String buff = new String();

        String result = oaf.getTarget();

        buff += getStringField(result);

        // TODO is declared as int!!!

        long diff = DATEDIFF(oaf.getResultProject().getOutcome().getRelMetadata().getEnddate(), oaf.getResultProject().getOutcome().getRelMetadata().getStartdate());

        if (diff < 0) {

            diff = 0;

        }

        buff += getNumericField(String.valueOf(diff));

        return buff;

    }

    private HashMap<String, List<String>> getDatasourceLanguages(OafEntity valueEntity) {

        HashMap<String, List<String>> rels = new HashMap<String, List<String>>();

        List<String> buffs = new ArrayList<String>();

        Datasource d = valueEntity.getDatasource();

        Metadata metadata = d.getMetadata();

        for (StringField lang : metadata.getOdlanguagesList()) {

            buffs.add(getStringField(lang.getValue()));

        }

        rels.put("datasourceLanguage", buffs);

        return rels;

    }

    private HashMap<String, List<String>> getResultLanguages(OafEntity valueEntity) {

        HashMap<String, List<String>> rels = new HashMap<String, List<String>>();

        List<String> buffs = new ArrayList<String>();

        Result d = valueEntity.getResult();

        eu.dnetlib.data.proto.ResultProtos.Result.Metadata metadata = d.getMetadata();

        if (metadata.getLanguage().getClassname() != null && !metadata.getLanguage().getClassname().isEmpty()) {

            buffs.add(getStringField(metadata.getLanguage().getClassname()));

        }

        rels.put("resultLanguage", buffs);

        return rels;

    }

    private HashMap<String, List<String>> getResultClassifications(OafEntity valueEntity) {

        HashMap<String, List<String>> rels = new HashMap<String, List<String>>();

        List<String> buffs = new ArrayList<String>();

        Result result = valueEntity.getResult();

        String classification = new String();

        for (Instance instance : (result.getInstanceList())) {

            if (instance.getInstancetype().getClassname() != null && !instance.getInstancetype().getClassname().isEmpty()) {

                classification += instance.getInstancetype().getClassname() + ';';

            }

        }

        buffs.add(getStringField(classification));

        rels.put("resultClassification", buffs);

        return rels;

    }

    private HashMap<String, List<String>> getResultConcepts(OafEntity valueEntity) {

        HashMap<String, List<String>> rels = new HashMap<String, List<String>>();

        List<String> buffs = new ArrayList<String>();

        Result result = valueEntity.getResult();

        for (eu.dnetlib.data.proto.ResultProtos.Result.Context context : result.getMetadata().getContextList()) {

            buffs.add(getStringField(context.getId()));

        }

        rels.put("resultConcept", buffs);

        return rels;

    }

    private HashMap<String, List<String>> getResultDatasources(OafEntity valueEntity) {

        HashMap<String, List<String>> rels = new HashMap<String, List<String>>();

        List<String> buffs = new ArrayList<String>();

        Result result = valueEntity.getResult();

        for (Instance instance : (result.getInstanceList())) {

            String hostedBy = instance.getHostedby().getKey();

            if (hostedBy != null && !hostedBy.isEmpty()) {

                buffs.add((getStringField(hostedBy)));

            }

        }

        for (FieldTypeProtos.KeyValue collectedFromValue : (valueEntity.getCollectedfromList())) {

            String collectedFrom = collectedFromValue.getKey();

            if (collectedFrom != null && !collectedFrom.isEmpty()) buffs.add((getStringField(collectedFrom)));

        }

        rels.put("resultDatasource", buffs);

        return rels;

    }

    public static boolean isNumeric(String str) {

        str = str.replaceAll("[^A-Za-z0-9 ]", "");

        str = str.replaceAll(" ", "");

        return str.matches("-?\\d+(\\.\\d+)?"); // match a number with optional

        // '-' and decimal.

    }

    // TODO there are topics with "null" as value -> replace them

    private boolean isValidTopic(String t) {

        if (t == null || t.isEmpty()) {

            return false;

        }

        if (t.equals("") || t.equals(" ")) {

            return false;

        }

        if (t.equals("null") || t.equals("Null") || t.equals("NULL")) {

            return false;

        }

        if (t.equals(ENCLOSED + ENCLOSED + DELIM) || t.equals(ENCLOSED + NULL_STRING + ENCLOSED + DELIM)) {

            return false;

        }

        // skip dedups

        if (t.contains("ddc:")) {

            return false;

        }

        return true;

    }

    private HashMap<String, List<String>> getResultTopics(OafEntity valueEntity) {

        HashMap<String, List<String>> rels = new HashMap<String, List<String>>();

        List<String> buffs = new ArrayList<String>();

        Result d = valueEntity.getResult();

        eu.dnetlib.data.proto.ResultProtos.Result.Metadata metadata = d.getMetadata();

        List<StructuredProperty> Topics = metadata.getSubjectList();

        String buff = new String();

        for (StructuredProperty topic : Topics) {

            // TODOs

            if (isValidTopic(topic.getValue())) {

                if (!isNumeric(topic.getValue())) {

                    String t = getStringField(topic.getValue());

                    if (isValidTopic(t)) {

                        buff += t + " ";

                    }

                }

            }

        }

        if (!buff.isEmpty()) {

            buff = getStringField(buff);

            buffs.add(buff);

        }

        rels.put("resultTopic", buffs);

        return rels;

    }

    private String buildDatasource(OafEntity data) {

        String buff = new String();

        Datasource d = data.getDatasource();

        Metadata metadata = d.getMetadata();

      /*  String full_id = getStringField(data.getId());*/

       /* buff += full_id;

        buff += full_id;

        buff += full_id;

        buff += full_id;

*/

        //`original Id`

        String originalId = new String();

        for (String oid : data.getOriginalIdList()) {

            originalId += oid;

            break;

        }

        buff += getStringField(originalId);

        //dateOfCollection

        buff += getStringDateField(data.getDateofcollection());

        // name

        if (metadata.getOfficialname().getValue().equalsIgnoreCase("unknown")) {

            buff += getStringField("Unknown Repository");

        } else {

            buff += getStringField(metadata.getOfficialname().getValue());

        }

        // type

        if (metadata.hasDatasourcetype())

        {

            buff += getStringField(metadata.getDatasourcetype().getClassname().replaceFirst(".*::", ""));

        } else {

            buff += getStringField(null);

        }

        // compatibility,

        buff += getStringField(metadata.getOpenairecompatibility().getClassname());

        // latitude

        buff += getStringField(metadata.getLatitude().getValue());

        // longtitude

        buff += getStringField(metadata.getLongitude().getValue());

        // dateofvalidation,

        buff += getStringField(metadata.getDateofvalidation().getValue());

        // yearofvalidation,

        // parse year of validation

        buff += getYearInt(metadata.getDateofvalidation().getValue());

        // Website URL

        buff += getStringField(metadata.getWebsiteurl().getValue());

        // certificates

        buff += getStringField(metadata.getCertificates().getValue());

        // /citation guideline

        buff += getStringField(metadata.getCitationguidelineurl().getValue());

        // contact mail

        buff += getStringField(metadata.getContactemail().getValue());

        // isDataProvider

        buff += getStringField(String.valueOf(metadata.getDataprovider().getValue()));

        //Description

        buff += getStringField(metadata.getDescription().getValue());

        //EnglishName

        buff += getStringField(metadata.getEnglishname().getValue());

        //access restriction

        buff += getStringField(metadata.getDatabaseaccessrestriction().getValue());

        //logoURL

        buff += getStringField(metadata.getLogourl().getValue());

        //mission statament

        buff += getStringField(metadata.getMissionstatementurl().getValue());

        //namespace prefix

        buff += getStringField(metadata.getNamespaceprefix().getValue());

        //number of items

        buff += getStringField(metadata.getOdnumberofitems().getValue());

        //OD policies

        buff += getStringField(metadata.getOdpolicies().getValue());

        //subjects

        //	buff += getStringField(metadata.getSubjectsList());

        // number??

        buff += getStringField("1");

        return buff;

    }

    private String buildOrganization(OafEntity data) {

        String buff = new String();

        Organization organization = data.getOrganization();

        eu.dnetlib.data.proto.OrganizationProtos.Organization.Metadata metadata = organization.getMetadata();

        //'organization_datasources`,

     /*   String full_id = getStringField(data.getId());

        buff += full_id;*/

        // organization_projects

      /*  buff += full_id;*/

        //`original Id`

        String originalId = new String();

        for (String oid : data.getOriginalIdList()) {

            originalId += oid;

            break;

        }

        buff += getStringField(originalId);

        //dateOfCollection

        buff += getStringDateField(data.getDateofcollection());

        // `name`,

        buff += getStringField(metadata.getLegalname().getValue());

        // `country`,

        buff += getStringField(metadata.getCountry().getClassname());

        //logourl

        buff += getStringField(metadata.getLogourl().getValue());

        buff += getStringField(metadata.getEcenterprise().getValue());

        buff += getStringField(metadata.getEchighereducation().getValue());

        //legal org

        buff += getStringField(metadata.getEcinternationalorganization().getValue());

        buff += getStringField(metadata.getEcnonprofit().getValue());

        buff += getStringField(metadata.getEcinternationalorganizationeurinterests().getValue());

        buff += getStringField(metadata.getEclegalbody().getValue());

        buff += getStringField(metadata.getEclegalperson().getValue());

        //nuts code

        buff += getStringField(metadata.getEcnutscode().getValue());

        //research org

        buff += getStringField(metadata.getEcresearchorganization().getValue());

        //getEcsmevalidated

        buff += getStringField(metadata.getEcsmevalidated().getValue());

        //getLegalshortname

        buff += getStringField(metadata.getLegalshortname().getValue());

        //website URL

        buff += getStringField(metadata.getWebsiteurl().getValue());

        // `number`,

        buff += getStringField("1");

        return buff;

    }

    private String buildResult(OafEntity data) {

        String buff = new String();

        Result result = data.getResult();

        eu.dnetlib.data.proto.ResultProtos.Result.Metadata metadata = result.getMetadata();

        // result_topics/

        String full_id = getStringField(data.getId());

      /*  buff += full_id;

        // result_languages

        buff += full_id;

        // `result_projects`,

        buff += full_id;

        // `result_datasources`,

        buff += full_id;

        // `result_classifications`,

        buff += full_id;

        // / `result_infrastructures`,

        buff += full_id;

        // `result_claims`,

        buff += full_id;

        // `result_results`,

        buff += full_id;*/

        //`original Id`

        String originalId = new String();

        for (String oid : data.getOriginalIdList()) {

            originalId += oid;

            break;

        }

        buff += getStringField(originalId);

        //dateOfCollection

        buff += getStringDateField(data.getDateofcollection());

        String titleString = new String();

        String alternativeTitles = new String();

        for (int i = 0; i < metadata.getTitleList().size(); i++) {

            StructuredProperty title = metadata.getTitleList().get(i);

            if (i == 0) {

                titleString = title.getValue().replaceAll("\\s+", " ");

                titleString = titleString.replaceAll("\n", " ");

            } else {

                alternativeTitles += title.getValue().replaceAll("\\s+", " ") + " ; ";

                alternativeTitles = alternativeTitles.replaceAll("\n", " ");

            }

            break;

        }

        //TODO pubtitle

        buff += getStringField(titleString);  //!#Osvaldo Raineri: Gli Atti etiopici del martire egiziano Giorgio il nuovo (β  978)#!#

        //TODO alternative titles

        //buff += getStringField(alternativeTitles);    //  null#!

        // TODO format

        String formatString = new String();

        for (StringField format : metadata.getFormatList()) {

            formatString = format.getValue();

            break;

        }

        buff += getStringField(formatString);

        // publisher

        buff += getStringField(metadata.getPublisher().getValue());

        // journal

        buff += getStringField(metadata.getJournal().getName().replaceAll("\n", " "));  //#null#!

        // year

        buff += getYearInt(metadata.getDateofacceptance().getValue());

        // date CHANGED THIS TO DATE FORMAT

        buff += getStringDateField(metadata.getDateofacceptance().getValue());

        // access_mode,

        buff += getStringField(getAccessMode(result));

        // bestlicense

        buff += getStringField(getBestLicense(result));

        // type

        buff += getStringField(metadata.getResulttype().getClassname());

        // embargo_end_date

        buff += getStringField(metadata.getEmbargoenddate().getValue());

        // `authors`,

        int authors = 0;

        String delayed = "no";

        for (OafRel rel : data.getCachedRelList()) {

            if (rel.getRelType().equals(RelType.personResult)) {

                authors++;

            } else if (rel.getRelType().equals(RelType.resultProject))

            // TODO remember : in result Project, first id is project, second is  result.

            {

                String daysfromend = getYearDifferenceInteger(rel.getResultProject().getOutcome().getRelMetadata().getEnddate(), rel.getResultProject().getOutcome().getRelMetadata().getStartdate());

                if (Integer.parseInt(daysfromend) > 0) {

                    delayed = "yes";

                }

            }

        }

        // `delayed`,

        buff += getStringField(delayed);

        // log.info("Result " + full_id +"Author count : " + authors );

        buff += getNumericField(String.valueOf(authors));

        // `context `,

        String context = new String();

        for (Result.Context s : metadata.getContextList()) {

            context += s.getId() + ";";

        }

        buff += getStringField(context);

        String country = new String();

        for (Qualifier s : metadata.getCountryList()) {

            country += s.getClassname() + ";";

        }

        buff += getStringField(country);

        String desc = new String();

        for (StringField s : metadata.getDescriptionList()) {

            desc += s.getValue() + ";";

        }

        buff += getStringField(desc);

        // number??

        buff += getStringField("1");

        if (isValid(buff, full_id)) {

            return buff;

        } else {

            return null;

        }

    }

    //TODO here see if check is ok

    private boolean isValid(String buff, String id) {

        if (buff.endsWith(ENCLOSED)) {

            log.error("Empty Result with  " + id + " with body: \n" + buff);

            return false;

        }

        return true;

    }

    private String getBestLicense(Result result) {

        Qualifier bestLicense = null;

        LicenseComparator lc = new LicenseComparator();

        for (Instance instance : (result.getInstanceList())) {

            if (lc.compare(bestLicense, instance.getLicence()) > 0) {

                bestLicense = instance.getLicence();

            }

        }

        if (bestLicense != null) {

            return bestLicense.getClassname();

        } else {

            return null;

        }

    }

    // TODO here iterate over all values

    private String getAccessMode(Result result) {

        String accessMode = NULL_STRING;

        for (Instance instance : (result.getInstanceList())) {

            if (instance.getLicence().getClassname() != null && !instance.getLicence().getClassname().isEmpty()) {

                accessMode = instance.getLicence().getClassname();

                break;

            }

        }

        return accessMode;

    }

    private String buildProject(OafEntity data) {

        String buff = new String();

        Project project = data.getProject();

        eu.dnetlib.data.proto.ProjectProtos.Project.Metadata metadata = project.getMetadata();

        // project_organizations

        //  String full_id = getStringField(data.getId());

        // buff += full_id;

        // project_results

        // buff += full_id;

        //`original Id`

        String originalId = new String();

        for (String oid : data.getOriginalIdList()) {

            originalId += (oid);

            break;

        }

        buff += getStringField(originalId);

        //dateOfCollection

        buff += getStringDateField(data.getDateofcollection());

        // `acronym`,

        String acronym = metadata.getAcronym().getValue();

        if (acronym.equalsIgnoreCase("UNKNOWN")) {