|
1
|
|
|
2
|
package eu.dnetlib.data.mapreduce.hbase.lodExport.utils.compators;
|
|
3
|
|
|
4
|
import java.util.HashMap;
|
|
5
|
import java.util.Map;
|
|
6
|
|
|
7
|
|
|
8
|
public class RecordComparator {
|
|
9
|
private static final String LINE_SEPERATOR = "\t.\t";
|
|
10
|
private static final String FIELD_DELIM = "\t";
|
|
11
|
private static Map<String, String> sourceRecordMappings = new HashMap<>();
|
|
12
|
|
|
13
|
public static double computeSimilarity(DistanceAlgorithms algorithm, String source, String target) {
|
|
14
|
switch (algorithm) {
|
|
15
|
case LEVENSHTEIN:
|
|
16
|
return computeLevenshteinSimilarity(source, target);
|
|
17
|
case HAMMING:
|
|
18
|
default:
|
|
19
|
return computeDistanceSimilarity(source, target);
|
|
20
|
}
|
|
21
|
|
|
22
|
}
|
|
23
|
|
|
24
|
private static double computeDistanceSimilarity(String source, String target) {
|
|
25
|
Map<String, String> sourceRecordMap = getRecordsFiledMap(source);
|
|
26
|
Map<String, String> targetRecordMap = getRecordsFiledMap(target);
|
|
27
|
double totalFields = getTotalFields(sourceRecordMap, targetRecordMap);
|
|
28
|
double recordSimilarity = 0.0D;
|
|
29
|
int maxFieldLength = 1;
|
|
30
|
|
|
31
|
for (Map.Entry<String, String> sourceField : sourceRecordMap.entrySet()) {
|
|
32
|
String correspondingTargetField = sourceRecordMappings.get(sourceField.getKey());
|
|
33
|
String targetFieldValue = targetRecordMap.get(correspondingTargetField);
|
|
34
|
if (targetFieldValue != null) {
|
|
35
|
if (sourceField.getValue().length() > maxFieldLength) {
|
|
36
|
maxFieldLength = sourceField.getValue().length();
|
|
37
|
}
|
|
38
|
if (targetFieldValue.length() > maxFieldLength) {
|
|
39
|
maxFieldLength = targetFieldValue.length();
|
|
40
|
}
|
|
41
|
|
|
42
|
double fieldsSimilarity = DistanceCalculator.getSimpleDistance(sourceField.getValue(), targetFieldValue);
|
|
43
|
recordSimilarity += fieldsSimilarity;
|
|
44
|
}
|
|
45
|
}
|
|
46
|
|
|
47
|
return recordSimilarity / totalFields;
|
|
48
|
}
|
|
49
|
|
|
50
|
private static double getTotalFields(Map<String, String> sourceRecordMap, Map<String, String> targetRecordMap) {
|
|
51
|
return (double) (sourceRecordMap.size() > targetRecordMap.size() ? sourceRecordMap.size() : targetRecordMap.size());
|
|
52
|
}
|
|
53
|
|
|
54
|
private static double computeLevenshteinSimilarity(String source, String target) {
|
|
55
|
Map<String, String> sourceRecordMap = getRecordsFiledMap(source);
|
|
56
|
Map<String, String> targetRecordMap = getRecordsFiledMap(target);
|
|
57
|
double recordSimilarity = 0.0D;
|
|
58
|
int maxFieldLength = 1;
|
|
59
|
double totalFields = getTotalFields(sourceRecordMap, targetRecordMap);
|
|
60
|
|
|
61
|
for (Map.Entry<String, String> sourceField : sourceRecordMap.entrySet()) {
|
|
62
|
String correspondingTargetField = sourceRecordMappings.get(sourceField.getKey());
|
|
63
|
String targetFieldValue = targetRecordMap.get(correspondingTargetField);
|
|
64
|
if (targetFieldValue != null) {
|
|
65
|
if (sourceField.getValue().length() > maxFieldLength) {
|
|
66
|
maxFieldLength = sourceField.getValue().length();
|
|
67
|
}
|
|
68
|
if (targetFieldValue.length() > maxFieldLength) {
|
|
69
|
maxFieldLength = targetFieldValue.length();
|
|
70
|
}
|
|
71
|
|
|
72
|
double levenshteinDistance = DistanceCalculator.getLevenshteinDistance(sourceField.getValue(), targetFieldValue);
|
|
73
|
double longerStringSize = sourceField.getValue().length() > targetFieldValue.length() ? sourceField.getValue().length() : targetFieldValue.length();
|
|
74
|
double fieldsSimilarity = (longerStringSize - levenshteinDistance) / longerStringSize;
|
|
75
|
recordSimilarity += fieldsSimilarity;
|
|
76
|
}
|
|
77
|
}
|
|
78
|
|
|
79
|
return recordSimilarity / totalFields;
|
|
80
|
}
|
|
81
|
|
|
82
|
|
|
83
|
private static Map<String, String> getRecordsFiledMap(String source) {
|
|
84
|
String sourceRecord = source.substring(source.indexOf(FIELD_DELIM) + 1, source.length()).trim();
|
|
85
|
String[] sourceTriples = sourceRecord.split(LINE_SEPERATOR);
|
|
86
|
Map<String, String> sourceFieldsMap = new HashMap<>();
|
|
87
|
|
|
88
|
|
|
89
|
for (int i = 0; i < sourceTriples.length - 1; i++) {
|
|
90
|
String sourceTriple = sourceTriples[i];
|
|
91
|
String[] split = sourceTriple.split("\t");
|
|
92
|
if (split.length > 2) {
|
|
93
|
sourceFieldsMap.put(split[1], split[2]);
|
|
94
|
} else {
|
|
95
|
System.out.println("ERROR: Retrieved triples missing fields " + sourceTriple);
|
|
96
|
}
|
|
97
|
}
|
|
98
|
|
|
99
|
return sourceFieldsMap;
|
|
100
|
}
|
|
101
|
|
|
102
|
static {
|
|
103
|
sourceRecordMappings.put("<http://www.eurocris.org/ontologies/cerif/1.3#name>", "<http://www.w3.org/2000/01/rdf-schema#label>");
|
|
104
|
sourceRecordMappings.put("<http://lod.openaire.eu/vocab/year>", "<http://purl.org/dc/terms/issued>");
|
|
105
|
sourceRecordMappings.put("<http://purl.org/dc/terms/identifier>", "<http://purl.org/dc/terms/identifier>");
|
|
106
|
}
|
|
107
|
|
|
108
|
public static void main(String[] args) {
|
|
109
|
|
|
110
|
String target = "target_conf/clef/LarsonNJ08\tconf/clef/LarsonNJ08\t<http://www.w3.org/2000/01/rdf-schema#label>\tOverview of VideoCLEF 2008: Automatic Generation of Topic-Based Feeds for Dual Language Audio-Visual Content.\t.\t,";
|
|
111
|
String source = "source_<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://purl.org/dc/terms/identifier>\t\"od_______119::60f21cae791a925a78d0844ad00cea5a\"\t.\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://purl.org/dc/terms/identifier>\t\"oai:doras.dcu.ie:16187\"\t.\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://www.eurocris.org/ontologies/cerif/1.3#name>\t\"Overview of VideoCLEF 2008: Automatic generation of topic based feeds for dual language audio visual content\"\t.\t,";
|
|
112
|
/*
|
|
113
|
System.out.println("Hamming :" + RecordComparator.computeSimilarity(DistanceAlgorithms.HAMMING, source, target));
|
|
114
|
System.out.println("Lev :" + RecordComparator.computeSimilarity(DistanceAlgorithms.LEVENSHTEIN, source, target));
|
|
115
|
*/
|
|
116
|
|
|
117
|
target = "target_conf/clef/LarsonNJ08\tconf/clef/LarsonNJ08\t<http://www.w3.org/2000/01/rdf-schema#label>\tOverview of VideoCLEF 2008: Automatic Generation of Topic-Based Feeds for Dual Language Audio-Visual Content.\t.\t,";
|
|
118
|
source = "source_<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://purl.org/dc/terms/identifier>\t\"od_______119::60f21cae791a925a78d0844ad00cea5a\"\t.\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://purl.org/dc/terms/identifier>\t\"oai:doras.dcu.ie:16187\"\t.\t<http://lod.openaire.eu/data/result/od_______119::60f21cae791a925a78d0844ad00cea5a>\t<http://www.eurocris.org/ontologies/cerif/1.3#name>\t\"Overview of VideoCLEF 2008: Automatic generation of topic based feeds for visual content\"\t.\t,";
|
|
119
|
|
|
120
|
System.out.println("Hamming :" + RecordComparator.computeSimilarity(DistanceAlgorithms.HAMMING, source, target));
|
|
121
|
System.out.println("Lev :" + RecordComparator.computeSimilarity(DistanceAlgorithms.LEVENSHTEIN, source, target));
|
|
122
|
|
|
123
|
|
|
124
|
}
|
|
125
|
|
|
126
|
}
|
|
127
|
|