D-Net

package eu.dnetlib.data.mapreduce.hbase.lodExport.wordFrequencies;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.log4j.Logger;

import java.io.IOException;

public class WordFrequencyMapper extends Mapper<LongWritable, Text, Text, Text> {

    private Logger log = Logger.getLogger(this.getClass());

    private final String SOURCE_LABEL = "http://www.eurocris.org/ontologies/cerif/1.3#name";

    private final String TARGET_LABEL = "http://www.w3.org/2000/01/rdf-schema#label";

    public static enum STOPWORD_COUNTERS {

        STOPWORD_RECORDS, WORDS

    }

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

    }

    @Override

    protected void map(final LongWritable keyIn, final Text result, final Context context) throws IOException {

        try {

            context.getCounter(STOPWORD_COUNTERS.STOPWORD_RECORDS).increment(1);

            String recordTitle = getRecordTitle(result.toString());

            if (recordTitle != null) {

                for (String titleWord : recordTitle.split(" ")) {

                    context.write(new Text(titleWord), new Text("1"));

                    context.getCounter(STOPWORD_COUNTERS.WORDS).increment(1);

                }

            }

        } catch (Exception e) {

            log.error("Error writing entity to M/R output", e);

            log.error("result error    " + result.toString());

            throw new RuntimeException(e);

        }

    }

    private String getRecordTitle(String record) throws Exception {

        String[] recordFields = record.split(",");

        for (String recordField : recordFields) {

            String[] entryPair = recordField.split("\t");

            if (entryPair[0].equals(SOURCE_LABEL) || entryPair[0].equals(TARGET_LABEL)) {

                return entryPair[1];

            }

        }

        return null;

    }

    @Override

    protected void cleanup(Context context) throws IOException, InterruptedException {

        super.cleanup(context);

        log.info("Cleaning up mapper...");

    }

}

package eu.dnetlib.data.mapreduce.hbase.lodExport.wordFrequencies ;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.log4j.Logger;

import java.io.IOException;

import java.util.Iterator;

/* Class that counts word frequencies in title fields so we can create a stop word list*/

public class WordFrequencyReducer extends Reducer<Text, Text, Text, Text> {

    public static enum STOPWORDS_COUNTER {

        STOPWORDS

    }

    private Logger log = Logger.getLogger(WordFrequencyReducer.class);

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

    }

    @Override

    protected void reduce(final Text key, final Iterable<Text> values, final Context context) throws IOException, InterruptedException {

        Iterator<Text> it = values.iterator();

        try {

            int stopwordFrequency = 0;

            while (it.hasNext()) {

                stopwordFrequency++;

            }

            context.write(key, new Text(String.valueOf(stopwordFrequency)));

            context.getCounter(STOPWORDS_COUNTER.STOPWORDS).increment(1);

        } catch (Exception e) {

            log.error(e);

            throw new InterruptedException(e.getMessage());

        }

    }

    protected void cleanup(Context context) throws IOException, InterruptedException {

        log.info("Cleaning up reducer...");

    }

}