D-Net

package eu.dnetlib.data.mapreduce.hbase.statsExport.utils;

import javax.xml.transform.Transformer;

import javax.xml.transform.TransformerFactory;

import javax.xml.transform.stream.StreamResult;

import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.io.InputStream;

public class ContextTransformer {

    public String transformXSL(String xml) throws Exception {

        Transformer transformer;

        TransformerFactory tFactory = TransformerFactory.newInstance();

        if (xml == null) {

        }

        InputStream inputStream = null;

        ByteArrayInputStream readerStream = null;

        ByteArrayOutputStream writerStream = null;

        try {

            inputStream = ClassLoader.getSystemResourceAsStream("eu/dnetlib/data/mapreduce/hbase/statsExport/" + "context.xsl");

            transformer = tFactory.newTransformer(new StreamSource(inputStream));

            writerStream = new ByteArrayOutputStream();

            transformer.transform(new StreamSource(readerStream), new StreamResult(writerStream));

            return writerStream.toString("UTF8");

        } finally {

        }

    }

}

public class FundingParser {

		if (funding_level.isEmpty()) {

			return ENCLOSING + " " + ENCLOSING + DELIM;

		}

			} else if (funding_level.equalsIgnoreCase("SP2")) {

				funding_level = "SP2-Ideas";

			}

			if (funding_level.equalsIgnoreCase("SP3")) {

				funding_level = "SP3-People";

			} else if (funding_level.equalsIgnoreCase("SP4")) {

				funding_level = "SP4-Capacities";

			} else if (funding_level.equalsIgnoreCase("SP5")) {

				funding_level = "SP5-Euratom";

			}

		}

		funding_level = funding_level.replaceAll(">", "");

		funding_level = funding_level.replaceAll("</", "");

		funding_level = funding_level.replace(DELIM, " ");

		funding_level = funding_level.replace(ENCLOSING, " ");

		return ENCLOSING + funding_level + ENCLOSING + DELIM;

	}

	}

		if (buff.isEmpty()) {

			return ENCLOSING + " " + ENCLOSING + DELIM;

		split = funder.split("<name>");

		funder = split[1];

		funder = funder.substring(0, funder.indexOf("</name>"));

		funder = funder.replaceAll(">", "");

		funder = funder.replaceAll("</", "");

		funder = funder.replaceAll("\"", "");

		funder = funder.replaceAll("&", "");

		funder = funder.replace(ENCLOSING, " ");

		return ENCLOSING + funder + ENCLOSING + DELIM;

	}

}

import com.google.common.collect.Multimap;

import eu.dnetlib.data.mapreduce.util.LicenseComparator;

import eu.dnetlib.data.proto.DatasourceProtos.Datasource;

import eu.dnetlib.data.proto.DatasourceProtos.Datasource.Metadata;

import eu.dnetlib.data.proto.OafProtos.OafEntity;

import eu.dnetlib.data.proto.OafProtos.OafRel;

import eu.dnetlib.data.proto.OrganizationProtos.Organization;

import eu.dnetlib.data.proto.ProjectProtos.Project;

import eu.dnetlib.data.proto.RelTypeProtos.RelType;

import eu.dnetlib.data.proto.ResultProtos.Result;

import java.text.DateFormat;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.List;

import org.w3c.dom.Element;

public class Serializer {

    private static Logger logger = Logger.getLogger(Serializer.class);

        switch (oaf.getKind()) {

            case entity:

                OafEntity valueEntity = oaf.getEntity();

                switch (valueEntity.getType()) {

                    case datasource:

                    case organization:

                    case project:

                    case result:

                    /*

                    case person:

                        return buildPerson(oaf, DELIM, ENCLOSING);

                break;

            case relation:

        }

        return null;

    }

        switch (oaf.getRelType()) {

            case resultProject:

            default:

        }

    }

    }

        OafEntity valueEntity = oaf.getEntity();

        switch (valueEntity.getType()) {

            case datasource:

            case result:

//                getResultDescriptions(valueEntity, relations, DELIM, ENCLOSING);

//                getResultExtra(valueEntity, relations, DELIM, ENCLOSING);

            case project:

            default:

        }

    }

        Datasource d = valueEntity.getDatasource();

        Metadata metadata = d.getMetadata();

    }

        Result result = valueEntity.getResult();

        Result.Metadata metadata = result.getMetadata();

        }

        //  pubtitle

        String sources = "";

        for (Instance instance : (result.getInstanceList())) {

            List<String> urls = instance.getUrlList();

            for (String url : urls) {

            }

        }

        relations.put("resultExtra", buff.toString());

    }

        String relName = oafEntity.getType().toString().toLowerCase() + "Oid";

        for (String oid : oafEntity.getOriginalIdList()) {

        }

    }

    }

        for (StructuredProperty subj : oafEntity.getProject().getMetadata().getSubjectsList()) {

        }

    }

        StringBuilder buff = new StringBuilder();

        // TODO is declared as int!!!

        long diff = DATEDIFF(oaf.getResultProject().getOutcome().getRelMetadata().getEnddate(), oaf.getResultProject().getOutcome().getRelMetadata().getStartdate());

        if (diff < 0) {

            diff = 0;

        }

        return buff.toString();

    }

        Datasource d = valueEntity.getDatasource();

        Metadata metadata = d.getMetadata();

        for (StringField lang : metadata.getOdlanguagesList()) {

        }

    }

        Result d = valueEntity.getResult();

        Result.Metadata metadata = d.getMetadata();

        if (metadata.getLanguage().getClassname() != null && !metadata.getLanguage().getClassname().isEmpty()) {

        }

    }

        for (StructuredProperty pid : valueEntity.getPidList()) {

        }

    }

        Result result = valueEntity.getResult();

        for (Instance instance : (result.getInstanceList())) {

            String classification = instance.getInstancetype().getClassname();

            if (classification != null && !classification.isEmpty()) {

            }

        }

    }

        Result result = valueEntity.getResult();

        for (StringField s : result.getMetadata().getDescriptionList()) {

        }

    }

        Result result = valueEntity.getResult();

        for (Result.Context context : result.getMetadata().getContextList()) {

        }

    }

        Result result = valueEntity.getResult();

//TODO hosted by

        for (Instance instance : (result.getInstanceList())) {

            String hostedBy = instance.getHostedby().getKey();

            if (hostedBy != null && !hostedBy.isEmpty()) {

            }

        }

//TODO  collected froms

        for (FieldTypeProtos.KeyValue collectedFromValue : (valueEntity.getCollectedfromList())) {

            if (collectedFrom != null && !collectedFrom.isEmpty()) {

            }

        }

    }

        Result d = valueEntity.getResult();

        Result.Metadata metadata = d.getMetadata();

        List<StructuredProperty> Topics = metadata.getSubjectList();

        for (StructuredProperty topic : Topics) {

        }

    }

        for (FieldTypeProtos.ExtraInfo extraInfo : oafEntity.getExtraInfoList()) {

            if (extraInfo.getName().equals("result citations")) {

                DOMParser parser = new DOMParser();

        }

    }

        Metadata metadata = oaf.getEntity().getDatasource().getMetadata();

        StringBuilder buff = new StringBuilder();

        // name

        if (metadata.getOfficialname().getValue().equalsIgnoreCase("unknown")) {

        } else {

        }

        // type

        if (metadata.hasDatasourcetype()) {

        }

        // compatibility,

        // latitude

        // longtitude

        // dateofvalidation,

        // yearofvalidation,

        //harvested

        //piwik_id

        String piwik_id = "";

                break;

            }

        }

        return buff.toString();

    }

        StringBuilder buff = new StringBuilder();

        Organization.Metadata metadata = oaf.getEntity().getOrganization().getMetadata();

        // `name`,

        // `country`,

        return buff.toString();

    }

        StringBuilder buff = new StringBuilder();

        Result.Metadata metadata = oaf.getEntity().getResult().getMetadata();

        }

        //  pubtitle

        //  publisher

        //  journal

        // year

        // date

        // bestlicense

        // type

        // embargo_end_date

        // `authors`,

        int authors = metadata.getAuthorCount();

            // result.

            {

                String daysfromend = getYearDifferenceInteger(rel.getResultProject().getOutcome().getRelMetadata().getEnddate(),

                if (Integer.parseInt(daysfromend) > 0) {

                    delayed = "yes";

                }

        }

        // `delayed`,

        //authors

        String sources = new String();

        for (Instance instance : (oaf.getEntity().getResult().getInstanceList())) {

            List<String> urls = instance.getUrlList();

            for (String url : urls) {

            }

        }

    }

    */

        Qualifier bestLicense = null;

        LicenseComparator lc = new LicenseComparator();

        for (Instance instance : (result.getInstanceList())) {

    }

    // TODO here iterate over all values

        String accessMode = " ";

        for (Instance instance : (result.getInstanceList())) {

            if (instance.getAccessright().getClassname() != null && !instance.getAccessright().getClassname().isEmpty()) {

        return accessMode;

    }

        StringBuilder buff = new StringBuilder();

        Project.Metadata metadata = oaf.getEntity().getProject().getMetadata();

        if (acronym.equalsIgnoreCase("UNKNOWN")) {

            acronym = metadata.getTitle().getValue();

        }

        //title

        //funding_lvl

        List<StringField> fundList = metadata.getFundingtreeList();

	        funding_lvl1 text,

	        funding_lvl2 text,

	        funding_lvl3 text,*/

        } else {

        }

        //sc39

        } else if (sc39.equalsIgnoreCase("false") || sc39.equalsIgnoreCase("f") || sc39.contains("no")) {

            sc39 = "no";

        }

        //project_type

        // start_year

        // end_year

        // duration enddate-startdate

        // haspubs

        // numpubs

        // enddate

        // startdate

        // `daysforlastpub`,

        // `delayedpubs`,

        //call identifier

        //code

        return buff.toString();

    }

    /*

    */

        if (enddate != null && !enddate.isEmpty() && startdate != null && !startdate.isEmpty()) {

        return ENCLOSING + "0" + ENCLOSING + DELIM;

    }

        if (data == null || data.isEmpty() || data.equals("-1")) {

            return ENCLOSING + "0" + ENCLOSING + DELIM;

        }

    }

        number = number.replaceAll("[^A-Za-z0-9:,____]", "");

        return number;

    }

        return ENCLOSING + data.replaceAll("[^-0-9.]+", "")  + ENCLOSING + DELIM;

    }

    }

        if (data == null || data.isEmpty() || data.equals("") || data.equals("-1")) {

            return ENCLOSING + "0" + ENCLOSING + DELIM;

        } else {

        }

    }

        if (data == null || data.isEmpty() || data.equals("")) {

            return ENCLOSING + "0" + ENCLOSING + DELIM;

        } else {

        }

    }

        switch (oaf.getKind()) {

            case entity:

            case relation:

        }

        return null;

    }

    }

        if (value != null) {

            value = value.replaceAll("[\"\\r\\\\;]", "");

            //value = value.replaceAll("[^a-zA-Z0-9 .-_:/!@+=]+", " ");

            return value;

        } else {

            return "";

        }

    }

        if (value != null) {

            // TODO DO NOT CHANGE THIS: IT REMOVES ID PREFIX ( 5|datacite____::

    }

        value = value.replace(DELIM, " ");

        value = value.replace(ENCLOSING, " ");

        value = value.replace(" ", "");

    }

        long MILLISECS_PER_DAY = 24 * 60 * 60 * 1000;

        long days = 0l;

        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); // "dd/MM/yyyy HH:mm:ss");

        return days;

    }

}

        project

    }

    private enum REL_COUNTERS {

        resultProject,

        datasourceOrganization,

        rottenRelations

    }

    //Init class: Load Index config and mapping for DB tables.

    @Override

    protected void setup(Context context) {

        loadEntityConfig(context);

    }

    private void emitEntity(Context context, Oaf oaf) {

        if (serialized != null) {

                try {

                    context.write((TextKeyOut), new ImmutableBytesWritable(serialized.getBytes()));

                    context.getCounter(STATS_COUNTERS.valueOf(oaf.getEntity().getType().toString())).increment(1);

            Multimap<String, String> relMap = ArrayListMultimap.create();

            if (!relMap.isEmpty()) {

                for (Entry<String, String> rel : relMap.entries()) {

                    //TODO here output

                    context.write((TextKeyOut), new ImmutableBytesWritable(rel.getValue().getBytes()));

                }

            }

        } catch (Throwable e) {

            log.error("Error writing relation to M/R output", e);

        }

        // Existing Hbase relations are generated here

        if (entityConfigTable.getDescriptors(oaf.getEntity().getType()) != null) {

                        //  builder.getEntityBuilder().addCachedRel(rel);

                            context.write((TextKeyOut), new ImmutableBytesWritable(buff.getBytes()));

                            context.getCounter(REL_COUNTERS.valueOf(rel.getRelType().toString())).increment(1);

                        }

                    }

                } catch (Throwable e) {

                    log.error("Error for record ", e);

                    context.getCounter(ERROR_COUNTERS.rottenRelations).increment(1);

                }

            }

        }

    }

    private void decodeRelation(final Oaf body, final Context context, Result result, final LinkDescriptor ld, List<OafRel> rels) {

                for (Entry<byte[], byte[]> e : columnMap.entrySet()) {

                    Oaf decodedOaf = decodeProto(e.getValue(), context);

                }

            }

        } catch (Throwable throwable) {

            log.error("Error Decoding relation for: " + body.getRel().getRelType() + " " + body.getEntity().getId() + " ", throwable);

            context.getCounter(ERROR_COUNTERS.rottenRelations).increment(1);

        }

    }

    private Oaf decodeProto(final byte[] body, Context context) {

        try {

            return Oaf.parseFrom(body);

        } catch (Exception e) {

            log.error(e);

            context.getCounter(ERROR_COUNTERS.rottenRecords).increment(1);

        }

        return null;

    }

        return columnMap != null && !columnMap.isEmpty();

    }

    public EntityConfigTable getEntityConfigTable() {

        return entityConfigTable;

    }

    public void setEntityConfigTable(EntityConfigTable entityConfigTable) {

        this.entityConfigTable = entityConfigTable;

    }

}

			String value = new String(it.next().copyBytes(),Charset.forName("UTF-8"));

            value = value.trim();

			MultipleOutputWriter.write(type, new Text(id.getBytes(Charset.forName("UTF-8"))), new Text(value.getBytes(Charset.forName("UTF-8"))), type.toString());

		}

	}

	@Override