D-Net

package eu.dnetlib.data.mapreduce.hbase.lodExport.utils;

import org.apache.log4j.Logger;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.HashMap;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

public class Blocking {

    private static Logger log = Logger.getLogger(Blocking.class);

    public static List<String> tokenBlocking(String[] record, Map<String, Integer> stopwordsMap) {

        List<String> blockingKeys = new ArrayList<>();

        Map<String, Integer> blockingKeysMap = new HashMap<>();

        String value = null;

        try {

            for (int i = 1; i < record.length; i++) {

                value = record[i].substring(record[i].indexOf("\t") + 1).replaceAll("_", " ");

                String[] tokens = value.toLowerCase().split("[\\W_]");

                for (int j = 0; j < tokens.length; j++) {

                    String currentToken = tokens[j];

                    if (currentToken.length() > 0 && !stopwordsMap.containsKey(currentToken)) {

                        //        blockingKeys.add(currentToken);

                        if (blockingKeysMap.containsKey(currentToken)) {

                            blockingKeysMap.put(currentToken, blockingKeysMap.get(currentToken) + 1);

                        } else {

                            blockingKeysMap.put(currentToken, 1);

                        }

                    }

                }

                StringBuilder tokenBuilder = new StringBuilder();

                Iterator<Map.Entry<String, Integer>> mapIterator = blockingKeysMap.entrySet().iterator();

                //throw away frequent words - tokens that occur more than once within a record field

                while (mapIterator.hasNext()) {

                    Map.Entry<String, Integer> entry = mapIterator.next();

           /* if (entry.getValue() > 1) {

                log.debug("Removed token : " + entry.getKey() + " with " + entry.getValue() + "occurences");

                blockingKeysMap.remove(entry.getKey());

            }

           */

                    if (entry.getValue() == 1) {

                        tokenBuilder.append(entry).append(" ");

                    }

                }

                blockingKeys.add(tokenBuilder.toString());

            }

        } catch (Exception e) {

            log.error(e);

        }

        return blockingKeys;

    }

}