D-Net

package eu.dnetlib.data.mapreduce.hbase.lodExport.preprocessing;

import java.io.IOException;

import org.apache.commons.lang.StringUtils;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.log4j.Logger;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.configuration.LodConfiguration;

/**

 * Mapper Class that reads HBASE contents and prepares them for the StatsDB

 * export

 */

/*

-> Parse LOD dump files

        Process lod input files and divide by entity type (both source and target)

        Transform to id, array of [ properties] format

        Store to HDFS

        For -> Multiple outputs and inputs

        Multiple inputs: all source and target datasets and their corresponding mappings

        M/O: separate output files for each dataset: mark records so that they are written to the correct one

*/

/*

<http://lod.openaire.eu/data/result/doajarticles::89217af00809a91acc15a416e56b3782> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.eurocris.org/ontologies/cerif/1.3#ResultEntity> .

<http://lod.openaire.eu/data/result/doajarticles::89217af00809a91acc15a416e56b3782> <http://www.eurocris.org/ontologies/cerif/1.3#name> "Une nouvelle anomalie a allure h r ditaire chez des agneaux  it khouzistans /it" .

*/

public class SourceMapper extends Mapper<LongWritable, Text, Text, Text> {

    private Logger log = Logger.getLogger(SourceMapper.class);

    private LodConfiguration lodConfiguration;

    public static enum SOURCE_COUNTERS {

        SOURCE_ENTITIES, TOTAL_ENTITIES

    }

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

        lodConfiguration = new LodConfiguration();

        lodConfiguration.load(context.getConfiguration().get("lod.sourceMappings"));

        log.info("file loaded!" + context.getConfiguration().get("lod.sourceMappings"));

    }

    @Override

    protected void map(final LongWritable keyIn, final Text result, final Context context) throws IOException {

        try {

            context.getCounter(SOURCE_COUNTERS.TOTAL_ENTITIES).increment(1);

            //get ID - output source_recordID so we can group by id and get all props of a record

            StringBuilder value = new StringBuilder();

            String[] inputParts = result.toString().split("\"");

            int i = 0;

            while (i < inputParts.length) {

                inputParts[i] = inputParts[i].replaceAll("\\s", "\t");

                i += 2;

            }

            String output = StringUtils.join(inputParts, "\"");

            String[] Fields = output.split("\t");

            if (Fields.length >= 2) {

                //here addd all  fields as array props and append "\t"

                // betweeen them so we can write directly to output

                //DO NOT enter id- we'll get it from key output

                i = 1;

                //extract entity type from subject

                String[] tmp = Fields[0].split("/");

                if (tmp.length >= 5) {

                    String type = tmp[4];

                    String subject = Fields[0];

                    if (lodConfiguration.entityExists(type)) {

                        while (i < Fields.length - 1) {

                            String field = Fields[i];

                            String fieldValue = Fields[i + 1];

                            if (lodConfiguration.isValidField(field)) {

                                value.append(subject).append("\t").append(field).append("\t").append(fieldValue).append("\t.\t");

                            }

                            i += 2;

                        }

                        // write out type,source_ID as key, and rest of props as value

                        Text key = new Text("OA" + "," + type + "," + subject);

                       if(value.toString().length()>0){

                        context.write(key, new Text(value.toString()));

                        context.getCounter(SOURCE_COUNTERS.SOURCE_ENTITIES).increment(1);

                    }}

                }

            }

        } catch (Exception e) {

            log.error("Error writing entity to M/R output", e);

            // throw new InterruptedIOException(e.toString());

        }

    }

    private static String cleanInput(Text result) {

        String resulString = result.toString().replace("<", "").replace(">", "");

        int ind = resulString.lastIndexOf(".");

        if (ind >= 0) {

            resulString = resulString.substring(0, ind);

        }

        return resulString;

    }

    @Override

    protected void cleanup(Context context) throws IOException, InterruptedException {

        super.cleanup(context);

    }

}