D-Net

    protected void reduce(final Text key, final Iterable<Text> values, final Context context) throws IOException, InterruptedException {

        Iterator<Text> it = values.iterator();

        StringBuilder field = new StringBuilder();

        try {

            boolean hasSource = false;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.blocking.Blocking;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.configuration.LodConfiguration;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.log4j.Logger;

import java.io.IOException;

/**

 * Mapper Class that reads HBASE contents and prepares them for the StatsDB

    private Logger log = Logger.getLogger(this.getClass());

    private LodConfiguration lodConfiguration = new LodConfiguration();

    private String lastExecutionDate;

    private HashSet<String> stopWordsMap = new HashSet<>();

    private boolean createCompositekey;

    private Set<String> usedProperties = new HashSet<>();

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

        try {

            usedProperties.add("<http://www.eurocris.org/ontologies/cerif/1.3#name>");

            usedProperties.add("<http://lod.openaire.eu/vocab/year>");

        } catch (Exception ex) {

            log.error("An error occured during Mapper Setup " + ex.toString(), ex);

            System.out.println(ex.getCause().toString());

            StringBuilder id = new StringBuilder();

            context.getCounter(SOURCE_BUILD_COUNTERS.INPUT_SOURCE_RECORDS).increment(1);

            List<String> tokenList = parseFieldsAndGenerateTokens(id, triples);

        List<String> tokenList = new ArrayList<>();

        for (String triple : triples) {

            if (fields.length == 3) {

                if (id.length() < 1) {

                }

                String property = fields[1];

                String value = fields[2];

    private void writeSingleKeysAndValues(Text result, Context context, List<String> tokenList, StringBuilder id) throws IOException, InterruptedException {

        for (String token : tokenList) {

            //Write BlockingKey, RecordID to output

            context.getCounter(SOURCE_BUILD_COUNTERS.BLOCKING_KEYS).increment(1);

        }

    }

    private void writeCompositeKeyAndValue(Text result, Context context, List<String> tokenList, StringBuilder id) throws IOException, InterruptedException {

        String compositeKey = createCompositeKey(tokenList);

        context.getCounter(SOURCE_BUILD_COUNTERS.BLOCKING_KEYS).increment(1);

    }

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.blocking.Blocking;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.configuration.LodConfiguration;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.log4j.Logger;

import java.io.IOException;

/*-> Parse LOD dump files

*/

public class TargetBuildNoCacheMapper extends Mapper<LongWritable, Text, Text, Text> {

    private Logger log = Logger.getLogger(this.getClass());

    private LodConfiguration lodConfiguration = new LodConfiguration();

    private HashSet<String> stopWordsMap = new HashSet<>();

    private boolean createCompositekey;

    private Set<String> usedProperties = new HashSet<>();

    public static enum TARGET_BUILD_COUNTERS {

        BLOCKING_KEYS,

        INPUT_TARGET_RECORDS

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

        try {

            usedProperties.add("<http://purl.org/dc/terms/issued>");

            usedProperties.add("<http://www.w3.org/2000/01/rdf-schema#label>");

        } catch (Exception ex) {

            log.error("An error occured during Mapper Setup " + ex.toString(), ex);

        }

            StringBuilder id = new StringBuilder();

            context.getCounter(TARGET_BUILD_COUNTERS.INPUT_TARGET_RECORDS).increment(1);

            List<String> tokenList = parseFieldsAndGenerateTokens(id, triples);

        List<String> tokenList = new ArrayList<>();

        for (String triple : triples) {

            if (fields.length == 3) {

                if (id.length() < 1) {

                }

                String property = fields[1];

                String value = fields[2];

    private void writeSingleKeysAndValues(Text result, Context context, List<String> tokenList, StringBuilder id) throws IOException, InterruptedException {

        for (String token : tokenList) {

            //Write BlockingKey, RecordID to output

            context.getCounter(TARGET_BUILD_COUNTERS.BLOCKING_KEYS).increment(1);

        }

    }

    private void writeCompositeKeyAndValue(Text result, Context context, List<String> tokenList, StringBuilder id) throws IOException, InterruptedException {

        String compositeKey = createCompositeKey(tokenList);

        context.getCounter(TARGET_BUILD_COUNTERS.BLOCKING_KEYS).increment(1);

    }

        }

        return tokenString.toString();

    }

    @Override

    protected void cleanup(Context context) throws IOException, InterruptedException {

        super.cleanup(context);

/*Found in https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#Java*/

public class DistanceCalculator {

    public static int getLevenshteinDistance(String s1, String s2) {

            return 0;

        }

        if (sourceValue != null && targetValue != null) {

            sourceValue = sourceValue.replaceAll("[^A-Za-z0-9]", "").toLowerCase();

            targetValue = targetValue.replaceAll("[^A-Za-z0-9]", "").toLowerCase();

            int end = sourceValue.length() <= targetValue.length() ? sourceValue.length() : targetValue.length();

            for (int i = 0; i < end; ++i) {

                if (sourceValue.charAt(i) == targetValue.charAt(i)) {

                }

            }

        } else {

            return 0.0D;

        }

package eu.dnetlib.data.mapreduce.hbase.lodExport.utils.compators;

import java.util.HashMap;

import java.util.Map;

public class MyComparator {

    private static final String LINE_SEPERATOR = "\t.\t";

    private static final String FIELD_DELIM = "\t";

    private static Map<String, String> sourceRecordMappings = new HashMap<>();

        Map<String, String> sourceRecordMap = getRecordsFiledMap(source);

        Map<String, String> targetRecordMap = getRecordsFiledMap(target);

        double recordSimilarity = 0.0D;

        int maxFieldLength = 1;

        for (Map.Entry<String, String> sourceField : sourceRecordMap.entrySet()) {

            String correspondingTargetField = sourceRecordMappings.get(sourceField.getKey());

            String targetFieldValue = targetRecordMap.get(correspondingTargetField);

            }

            }

        }

        return recordSimilarity / totalFields;

    }

    private static Map<String, String> getRecordsFiledMap(String source) {

        String sourceRecord = source.substring(source.indexOf(FIELD_DELIM) + 1, source.length()).trim();

        String[] sourceTriples = sourceRecord.split(LINE_SEPERATOR);

        for (int i = 0; i < sourceTriples.length - 1; i++) {

        String target = "target_conf/clef/LarsonNJ08\tconf/clef/LarsonNJ08\t<http://www.w3.org/2000/01/rdf-schema#label>\tOverview of VideoCLEF 2008: Automatic Generation of Topic-Based Feeds for Dual Language Audio-Visual Content.\t.\t,";

        String source = "source_<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://purl.org/dc/terms/identifier>\t\"od_______119::60f21cae791a925a78d0844ad00cea5a\"\t.\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://purl.org/dc/terms/identifier>\t\"oai:doras.dcu.ie:16187\"\t.\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://www.eurocris.org/ontologies/cerif/1.3#name>\t\"Overview of VideoCLEF 2008: Automatic generation of topic based feeds for dual language audio visual content\"\t.\t,";

    }

}

        return tokens.toString();

    }

}

public class DatasetReducer extends Reducer<Text, Text, Text, Text> {

    public static enum ENTITIES_COUNTER {

        TOTAL_ENTITIES,

    }

    protected void reduce(final Text key, final Iterable<Text> values, final Context context) throws IOException, InterruptedException {

        Iterator<Text> it = values.iterator();

        StringBuffer value = new StringBuffer();

        String[] tmp = key.toString().split(",");

        String dataset = tmp[0].replaceAll("[^a-zA-Z0-9]", "");

        String entityType = tmp[1].replaceAll("[^a-zA-Z0-9]", "");

package eu.dnetlib.data.mapreduce.hbase.lodExport.preprocessing;

import org.apache.commons.lang.StringUtils;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.log4j.Logger;

/**

 * Mapper Class that reads HBASE contents and prepares them for the StatsDB

<http://lod.openaire.eu/data/result/doajarticles::89217af00809a91acc15a416e56b3782> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.eurocris.org/ontologies/cerif/1.3#ResultEntity> .

<http://lod.openaire.eu/data/result/doajarticles::89217af00809a91acc15a416e56b3782> <http://www.eurocris.org/ontologies/cerif/1.3#name> "Une nouvelle anomalie a allure h r ditaire chez des agneaux  it khouzistans /it" .

*/

public class SourceMapper extends Mapper<LongWritable, Text, Text, Text> {

    private Logger log = Logger.getLogger(SourceMapper.class);

    private LodConfiguration lodConfiguration;

    public static enum SOURCE_COUNTERS {

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

        lodConfiguration = new LodConfiguration();

    }

    @Override

    protected void map(final LongWritable keyIn, final Text result, final Context context) throws IOException {

        try {

            context.getCounter(SOURCE_COUNTERS.TOTAL_ENTITIES).increment(1);

            //get ID - output source_recordID so we can group by id and get all props of a record

            StringBuilder value = new StringBuilder();

        super.cleanup(context);

    }

}

package eu.dnetlib.data.mapreduce.hbase.lodExport.preprocessing;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.configuration.LodConfiguration;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.parsing.DblpCsvParser;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

 * export

 */

public class TargetMapper extends Mapper<LongWritable, Text, Text, Text> {

    private Logger log = Logger.getLogger(TargetMapper.class);

    private LodConfiguration lodConfiguration;

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

        lodConfiguration = new LodConfiguration();

    }

        try {

            context.getCounter(TARGET_COUNTERS.TOTAL_ENTITIES).increment(1);

            Text[] pair = DblpCsvParser.parseTriples(result, lodConfiguration);

            context.write(pair[0], pair[1]);

            context.getCounter(TARGET_COUNTERS.TARGET_ENTITIES).increment(1);

package eu.dnetlib.data.mapreduce.hbase.lodExport.linkage;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.caching.RedisUtils;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.compators.MyComparator;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.log4j.Logger;

    private Logger log = Logger.getLogger(LinkCustomReducer.class);

    private RedisUtils redisUtils;

    private static final String SEPERATOR = ",";

    public static enum LINK_RECUDER_COUNTERS {

        TARGET_RECORDS_EXAMINED,

    @Override

    protected void setup(Context context) throws IOException, InterruptedException {

        try {

            redisUtils = new RedisUtils(context.getConfiguration());

        } catch (Exception e) {

            log.error("Error connecting to Redis " + e.toString());

            throw new RuntimeException(e);

        }

    }

    @Override

                //store target records to have less I/O ops per record

                //test also version where we get each record again in each loop

                //TODO

                //String targetRecord = redisUtils.getValue(targetId);}

                String targetRecord = targetRecords.get(targetId);

                context.getCounter(LINK_RECUDER_COUNTERS.RECORD_COMPARISONS).increment(1);

                    try {

                                        + SEPERATOR + targetRecord.replaceAll(targetId, "")));

                        context.getCounter(LINK_RECUDER_COUNTERS.MATCHING_PAIRS).increment(1);

                    } catch (Exception ex) {

package eu.dnetlib.data.mapreduce.hbase.lodExport.linkage;

import eu.dnetlib.data.mapreduce.hbase.lodExport.utils.caching.RedisUtils;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.log4j.Logger;

    private RedisUtils redisUtils;

    private int optimalBlockSize;

    private static final char SEPERATOR = ',';

        OPTIMAL_BLOCK_SIZE,

        ERROR_COUNTERS,

        DISCARDED_BLOCKS,

    protected void setup(Context context) throws IOException, InterruptedException {

        try {

            redisUtils = new RedisUtils(context.getConfiguration());

            log.info("optimalBlockSize " + optimalBlockSize);

        } catch (Exception e) {

            log.error("ERROR CONNECTING TO REDIS ", e);

            int recordsNumber = countRecords(result);

            //purge blocks that contain only source or target entities

            //how many comparisons we have purged

            if (recordsNumber >= optimalBlockSize) {

                //TODO count source / target records and get actual comparison number

            } else {

                context.write(keyIn, result);

            }

        } catch (Exception e) {

            log.error("key causing error is " + keyIn);

            log.error("ERROR ", e);

            throw new IOException(e.toString(), e);

        }

    }

stopwordsReducers=1

lodBlocksInput=/tmp/lod_dump/

lodBlocksOutput=/tmp/lod_dump/stopwords

lodPrefix=http://lod.openaire.eu/data/result/

lodStopwords=a,able,about,above,abst,accordance,according,accordingly,across,act,actually,added,adj,affected,affecting,affects,after,afterwards,again,against,ah,all,almost,alone,along,already,also,although,always,am,among,amongst,an,and,announce,another,any,anybody,anyhow,anymore,anyone,anything,anyway,anyways,anywhere,apparently,approximately,are,aren,arent,arise,around,as,aside,ask,asking,at,auth,available,away,awfully,b,back,be,became,because,become,becomes,becoming,been,before,beforehand,begin,beginning,beginnings,begins,behind,being,believe,below,beside,besides,between,beyond,biol,both,brief,briefly,but,by,c,ca,came,can,cannot,cant,cause,causes,certain,certainly,co,com,come,comes,contain,containing,contains,could,couldnt,d,date,did,didnt,different,do,does,doesnt,doing,done,dont,down,downwards,due,during,e,each,ed,edu,effect,eg,eight,eighty,either,else,elsewhere,end,ending,enough,especially,et,et-al,etc,even,ever,every,everybody,everyone,everything,everywhere,ex,except,f,far,few,ff,fifth,first,five,fix,followed,following,follows,for,former,formerly,forth,found,four,from,further,furthermore,g,gave,get,gets,getting,give,given,gives,giving,go,goes,gone,got,gotten,h,had,happens,hardly,has,hasnt,have,havent,having,he,hed,hence,her,here,hereafter,hereby,herein,heres,hereupon,hers,herself,hes,hi,hid,him,himself,his,hither,home,how,howbeit,however,hundred,i,id,ie,if,ill,im,immediate,immediately,importance,important,in,inc,indeed,index,information,instead,into,invention,inward,is,isnt,it,itd,itll,its,itself,ive,j,just,k,keep,	keeps,kept,kg,km,know,known,knows,l,largely,last,lately,later,latter,latterly,least,less,lest,let,lets,like,liked,likely,line,little,ll,look,looking,looks,ltd,m,made,mainly,make,makes,many,may,maybe,me,mean,means,meantime,meanwhile,merely,mg,might,million,miss,ml,more,moreover,most,mostly,mr,mrs,much,mug,must,my,myself,n,na,name,namely,nay,nd,near,nearly,necessarily,necessary,need,needs,neither,never,nevertheless,new,next,nine,ninety,no,nobody,non,none,nonetheless,noone,nor,normally,nos,not,noted,nothing,now,nowhere,o,obtain,obtained,obviously,of,off,often,oh,ok,okay,old,omitted,on,once,one,ones,only,onto,or,ord,other,others,otherwise,ought,our,ours,ourselves,out,outside,over,overall,owing,own,p,page,pages,part,particular,particularly,past,per,perhaps,placed,please,plus,poorly,possible,possibly,potentially,pp,predominantly,present,previously,primarily,probably,promptly,proud,provides,put,q,que,quickly,quite,qv,r,ran,rather,rd,re,readily,really,recent,recently,ref,refs,regarding,regardless,regards,related,relatively,research,respectively,resulted,resulting,results,right,run,s,said,same,saw,say,saying,says,sec,section,see,seeing,seem,seemed,seeming,seems,seen,self,selves,sent,seven,several,shall,she,shed,shell,shes,should,shouldnt,show,showed,shown,showns,shows,significant,significantly,similar,similarly,since,six,slightly,so,some,somebody,somehow,someone,somethan,something,sometime,sometimes,somewhat,somewhere,soon,sorry,specifically,specified,specify,specifying,still,stop,strongly,sub,substantially,successfully,such,sufficiently,suggest,sup,sure,t,take,taken,taking,tell,tends,th,than,thank,thanks,thanx,that,thatll,thats,thatve,the,their,theirs,them,themselves,then,thence,there,thereafter,thereby,thered,therefore,therein,therell,thereof,therere,theres,thereto,thereupon,thereve,these,they,theyd,theyll,theyre,theyve,think,this,those,thou,though,thoughh,thousand,throug,through,throughout,thru,thus,til,tip,to,together,too,took,toward,towards,tried,tries,truly,try,trying,ts,twice,two,u,un,under,unfortunately,unless,unlike,unlikely,until,unto,up,upon,ups,us,use,used,useful,usefully,usefulness,uses,using,usually,v,value,various,ve,very,via,viz,vol,vols,vs,w,want,wants,was,wasnt,way,we,wed,welcome,well,went,were,werent,weve,what,whatever,whatll,whats,when,whence,whenever,where,whereafter,whereas,whereby,wherein,wheres,whereupon,wherever,whether,which,while,whim,whither,who,whod,whoever,whole,wholl,whom,whomever,whos,whose,why,widely,willing,wish,with,within,without,wont,words,world,would,wouldnt,www,x,y,yes,yet,you,youd,youll,your,youre,yours,yourself,yourselves,youve,z,zero

lod_configXML=<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE LIMES SYSTEM "limes.dtd"> <LIMES> <PREFIX> <NAMESPACE>http://www.w3.org/1999/02/22-rdf-syntax-ns#</NAMESPACE> <LABEL>rdf</LABEL> </PREFIX> <PREFIX> <NAMESPACE>http://www.w3.org/2000/01/rdf-schema#</NAMESPACE> <LABEL>rdfs</LABEL> </PREFIX> <SOURCE> <ID>source1</ID> <ENDPOINT>/user/kanakakis/groundTruth/sourceNT</ENDPOINT> <VAR>?x</VAR> <PAGESIZE>100</PAGESIZE> <RESTRICTION>?x rdf:type http://www.eurocris.org/ontologies/cerif/1.3#ResultEntity</RESTRICTION> <PROPERTY>http://lod.openaire.eu/vocab/year RENAME Year</PROPERTY> <PROPERTY>http://www.w3.org/1999/02/22-rdf-syntax-ns# RENAME type</PROPERTY> <PROPERTY>http://purl.org/dc/terms/identifier RENAME id</PROPERTY> <PROPERTY>http://www.eurocris.org/ontologies/cerif/1.3#name AS lowercase->regexreplace("[^A-Za-z0-9]"," ") RENAME publicationName</PROPERTY> </SOURCE> <TARGET> <ID>source2</ID> <ENDPOINT>/user/kanakakis/groundTruth/targetNT</ENDPOINT> <VAR>?y</VAR> <PAGESIZE>100</PAGESIZE> <RESTRICTION>?y rdf:type http://swrc.ontoware.org/ontology#Article</RESTRICTION> <PROPERTY>http://www.w3.org/1999/02/22-rdf-syntax-ns# RENAME type</PROPERTY> <PROPERTY>http://purl.org/dc/terms/issued RENAME Year</PROPERTY> <PROPERTY>http://purl.org/dc/terms/identifier RENAME id</PROPERTY> <PROPERTY>http://www.w3.org/2000/01/rdf-schema#label AS lowercase->regexreplace("[^A-Za-z0-9]"," ") RENAME articleName</PROPERTY> </TARGET> <METRIC>AND(jaro(x.publicationName,y.articleName)|0.8,jaro(x.Year,y.Year)|1.0)</METRIC> <!-- <METRIC>jaro(x.publicatioName,y.articleName)|0.7</METRIC> --> <ACCEPTANCE> <THRESHOLD>0.8</THRESHOLD> <FILE>/user/kanakakis/groundTruth/accepted_links_0.8_no_purge</FILE> <RELATION>owl:sameAs</RELATION> </ACCEPTANCE> <REVIEW> <THRESHOLD>0.8</THRESHOLD> <FILE>/user/kanakakis/groundTruth/verified_links_0.8</FILE> <RELATION>owl:sameAs</RELATION> </REVIEW> <EXECUTION>Default</EXECUTION> <OUTPUT>TTL</OUTPUT> </LIMES>

        </configuration>

    </global>

    <action name="preProcessing">

        <map-reduce>

            <configuration>

                    <name>mapred.input.dir.formats</name>

                    <value>

                    </value>

                </property>

                </property>

                <!-- ## Workflow node parameters -->

                <property>

                    <name>mapred.reduce.tasks</name>

                </property>

                <!-- ## Workflow node parameters -->

                <property>

                    <name>mapred.reduce.tasks</name>