Revision 48939
Added by Tsampikos Livisianos over 6 years ago
| modules/dnet-openaire-lodimport/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/lodImport/utils/MapLanguages.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.mapreduce.hbase.lodImport.utils; |
|
| 2 |
import java.io.BufferedReader; |
|
| 3 |
import java.io.IOException; |
|
| 4 |
import java.io.InputStream; |
|
| 5 |
import java.io.InputStreamReader; |
|
| 6 |
|
|
| 7 |
/** |
|
| 8 |
* Method: getLangURI |
|
| 9 |
* |
|
| 10 |
* @author Giorgos Alexiou |
|
| 11 |
* |
|
| 12 |
*/ |
|
| 13 |
|
|
| 14 |
public class MapLanguages {
|
|
| 15 |
|
|
| 16 |
/** |
|
| 17 |
* This Method takes ISO iso639_3 codes for languages as input and returns the respective URI from lexvo.org as output. |
|
| 18 |
* For example iso639_3 for Greek is "ell" and the respective URI from lexvo is <http://lexvo.org/id/iso639-3/ell> |
|
| 19 |
* |
|
| 20 |
* @param iso639_3_Code |
|
| 21 |
* @return |
|
| 22 |
* @throws IOException |
|
| 23 |
*/ |
|
| 24 |
|
|
| 25 |
public MapLanguages(){
|
|
| 26 |
|
|
| 27 |
} |
|
| 28 |
|
|
| 29 |
// @SuppressWarnings("resource")
|
|
| 30 |
public String getLangURI(String iso639_3_Code) throws IOException{
|
|
| 31 |
|
|
| 32 |
// File inputFile = new File(MapCountries.class.getClassLoader().getResource("rdfData/lexvo-iso639-3.tsv").getFile());
|
|
| 33 |
// InputStream is = new FileInputStream(path); |
|
| 34 |
//InputStream is = this.getClass().getResourceAsStream("lodImport/lexvo-iso639-3.tsv");
|
|
| 35 |
InputStream is = ClassLoader.getSystemResourceAsStream("eu/dnetlib/data/mapreduce/hbase/lodImport/lexvo-iso639-3.tsv");
|
|
| 36 |
|
|
| 37 |
// File inputFile = new File(path); |
|
| 38 |
BufferedReader reader = new BufferedReader(new InputStreamReader(is)); |
|
| 39 |
String currentLine; |
|
| 40 |
while((currentLine = reader.readLine()) != null) {
|
|
| 41 |
String[] line = currentLine.split("\t");
|
|
| 42 |
// System.out.println(line[0].trim()+"--------------"+line[1].trim()); |
|
| 43 |
|
|
| 44 |
if(line[0].toString().trim().equals(iso639_3_Code)){
|
|
| 45 |
return line[1].toString().trim(); |
|
| 46 |
} |
|
| 47 |
|
|
| 48 |
} |
|
| 49 |
reader.close(); |
|
| 50 |
return ""; |
|
| 51 |
} |
|
| 52 |
|
|
| 53 |
|
|
| 54 |
|
|
| 55 |
// public static void main(String[] args) throws IOException{
|
|
| 56 |
// |
|
| 57 |
// System.out.println(MapLanguages.getLangURI("ell"));
|
|
| 58 |
// |
|
| 59 |
// } |
|
| 60 |
|
|
| 61 |
|
|
| 62 |
} |
|
| 63 |
|
|
| 0 | 64 | |
| modules/dnet-openaire-lodimport/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/lodImport/utils/DB.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.mapreduce.hbase.lodImport.utils; |
|
| 2 |
|
|
| 3 |
import java.beans.PropertyVetoException; |
|
| 4 |
import java.io.IOException; |
|
| 5 |
import java.sql.SQLException; |
|
| 6 |
|
|
| 7 |
import org.apache.log4j.Logger; |
|
| 8 |
|
|
| 9 |
import com.jolbox.bonecp.BoneCP; |
|
| 10 |
import com.jolbox.bonecp.BoneCPConfig; |
|
| 11 |
import com.jolbox.bonecp.BoneCPDataSource; |
|
| 12 |
|
|
| 13 |
public class DB {
|
|
| 14 |
|
|
| 15 |
private Logger log = Logger.getLogger(DB.class); |
|
| 16 |
|
|
| 17 |
private BoneCPDataSource ds; |
|
| 18 |
private BoneCP connectionPool; |
|
| 19 |
|
|
| 20 |
public DB(){
|
|
| 21 |
|
|
| 22 |
} |
|
| 23 |
|
|
| 24 |
// public DB(String conLine, String username, String password, String minCpart, String maxCpart, String part) throws IOException, SQLException, PropertyVetoException {
|
|
| 25 |
// try {
|
|
| 26 |
// // load the database driver (make sure this is in your classpath!) |
|
| 27 |
// Class.forName("virtuoso.jdbc4.Driver");
|
|
| 28 |
// } catch (Exception e) {
|
|
| 29 |
// log.error(e.getMessage(), e); |
|
| 30 |
// return; |
|
| 31 |
// } |
|
| 32 |
// |
|
| 33 |
// try {
|
|
| 34 |
// |
|
| 35 |
//// Will have to read Iterable via an external File (oozie Configuration maybe) |
|
| 36 |
// |
|
| 37 |
// // setup the connection pool using BoneCP Configuration |
|
| 38 |
// BoneCPConfig config = new BoneCPConfig(); |
|
| 39 |
// // jdbc url specific to the virtuoso, eg jdbc:mysql://127.0.0.1/yourdb |
|
| 40 |
// config.setJdbcUrl(conLine); |
|
| 41 |
// config.setUsername(username); |
|
| 42 |
// config.setPassword(password); |
|
| 43 |
// config.setMinConnectionsPerPartition(Integer.parseInt(minCpart)); |
|
| 44 |
// config.setMaxConnectionsPerPartition(Integer.parseInt(maxCpart)); |
|
| 45 |
// config.setPartitionCount(Integer.parseInt(part)); |
|
| 46 |
// // setup the connection pool |
|
| 47 |
// connectionPool = new BoneCP(config); |
|
| 48 |
// } catch (Exception e) {
|
|
| 49 |
// log.error(e.getMessage(), e); |
|
| 50 |
// return; |
|
| 51 |
// } |
|
| 52 |
// |
|
| 53 |
// } |
|
| 54 |
|
|
| 55 |
public BoneCPDataSource getDatasource(String conLine, String username, String password, String minCpart, String maxCpart, String part) throws IOException, SQLException, PropertyVetoException {
|
|
| 56 |
try {
|
|
| 57 |
// // load the database driver (make sure this is in your classpath!) |
|
| 58 |
Class.forName("virtuoso.jdbc4.Driver");
|
|
| 59 |
} catch (Exception e) {
|
|
| 60 |
log.error(e.getMessage(), e); |
|
| 61 |
} |
|
| 62 |
|
|
| 63 |
BoneCPConfig config = new BoneCPConfig(); |
|
| 64 |
config.setJdbcUrl(conLine); |
|
| 65 |
config.setUsername(username); |
|
| 66 |
config.setPassword(password); |
|
| 67 |
config.setMinConnectionsPerPartition(Integer.parseInt(minCpart)); |
|
| 68 |
config.setMaxConnectionsPerPartition(Integer.parseInt(maxCpart)); |
|
| 69 |
config.setPartitionCount(Integer.parseInt(part)); |
|
| 70 |
config.setPoolStrategy("CACHED");
|
|
| 71 |
|
|
| 72 |
ds = new BoneCPDataSource(config); |
|
| 73 |
return ds; |
|
| 74 |
|
|
| 75 |
} |
|
| 76 |
|
|
| 77 |
|
|
| 78 |
public BoneCP getConnectionPool(String conLine, String username, String password, String minCpart, String maxCpart, String part) throws IOException, SQLException, PropertyVetoException {
|
|
| 79 |
try {
|
|
| 80 |
// // load the database driver (make sure this is in your classpath!) |
|
| 81 |
Class.forName("virtuoso.jdbc4.Driver");
|
|
| 82 |
} catch (Exception e) {
|
|
| 83 |
log.error(e.getMessage(), e); |
|
| 84 |
} |
|
| 85 |
|
|
| 86 |
BoneCPConfig config = new BoneCPConfig(); |
|
| 87 |
config.setJdbcUrl(conLine); |
|
| 88 |
config.setUsername(username); |
|
| 89 |
config.setPassword(password); |
|
| 90 |
config.setMinConnectionsPerPartition(Integer.parseInt(minCpart)); |
|
| 91 |
config.setMaxConnectionsPerPartition(Integer.parseInt(maxCpart)); |
|
| 92 |
config.setPartitionCount(Integer.parseInt(part)); |
|
| 93 |
connectionPool = new BoneCP(config); |
|
| 94 |
|
|
| 95 |
return connectionPool; |
|
| 96 |
|
|
| 97 |
} |
|
| 98 |
|
|
| 99 |
public void closeConnectionPool() throws SQLException {
|
|
| 100 |
connectionPool.shutdown(); |
|
| 101 |
} |
|
| 102 |
|
|
| 103 |
public void closeDatasource() throws SQLException {
|
|
| 104 |
ds.close(); |
|
| 105 |
} |
|
| 106 |
|
|
| 107 |
} |
|
| 0 | 108 | |
| modules/dnet-openaire-lodimport/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/lodImport/utils/RDFizer.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.mapreduce.hbase.lodImport.utils; |
|
| 2 |
|
|
| 3 |
import java.net.URI; |
|
| 4 |
import java.sql.Connection; |
|
| 5 |
import java.sql.ResultSet; |
|
| 6 |
import java.sql.Statement; |
|
| 7 |
import java.util.List; |
|
| 8 |
|
|
| 9 |
import org.apache.hadoop.conf.Configuration; |
|
| 10 |
import org.apache.log4j.Logger; |
|
| 11 |
import org.json.JSONArray; |
|
| 12 |
import org.json.JSONObject; |
|
| 13 |
|
|
| 14 |
import com.jolbox.bonecp.BoneCPDataSource; |
|
| 15 |
|
|
| 16 |
public class RDFizer {
|
|
| 17 |
|
|
| 18 |
private static Logger log = Logger.getLogger(RDFizer.class); |
|
| 19 |
|
|
| 20 |
public RDFizer() {
|
|
| 21 |
} |
|
| 22 |
|
|
| 23 |
// public static String RDFizeEntityRow(List<String> row,VirtGraph graph, JSONObject mappings, Configuration conf, MapCountries mapCountries,MapLanguages mapLanguages){
|
|
| 24 |
public static String[] RDFizeEntityRow(List<String> row,BoneCPDataSource ds, JSONObject mappings, Configuration conf, MapCountries mapCountries,MapLanguages mapLanguages, String defaultGraph){
|
|
| 25 |
|
|
| 26 |
// VirtModel model = new VirtModel(graph); |
|
| 27 |
|
|
| 28 |
// String graph = conf.get("lod.defaultGraph");
|
|
| 29 |
|
|
| 30 |
String[] buildQuery = new String[2]; |
|
| 31 |
buildQuery[0]=""; |
|
| 32 |
buildQuery[1]=""; |
|
| 33 |
|
|
| 34 |
// String insertQuery =""; |
|
| 35 |
// String deleteQuery=""; |
|
| 36 |
|
|
| 37 |
String baseURI = conf.get("lod.baseURI");
|
|
| 38 |
String id = row.get(1).toString(); |
|
| 39 |
String type = row.get(0).toString(); |
|
| 40 |
String resourceURI = baseURI+type+"/"+id; |
|
| 41 |
|
|
| 42 |
// #################################################################################### |
|
| 43 |
// THIS IS FOR INCREMENTAL UPDATE ON-HOLD FOR THE BATCH |
|
| 44 |
// boolean ask = false; |
|
| 45 |
// try{
|
|
| 46 |
// ask = resourceExists(ds.getConnection(), resourceURI, defaultGraph); |
|
| 47 |
// }catch(Exception e){
|
|
| 48 |
// log.error("Could not ASK "+e.toString(),e);
|
|
| 49 |
// } |
|
| 50 |
|
|
| 51 |
|
|
| 52 |
|
|
| 53 |
// buildQuery[1]+= "<"+resourceURI+"> ?p ?o. "; |
|
| 54 |
|
|
| 55 |
// if(ask){
|
|
| 56 |
// try{
|
|
| 57 |
// deleteResource(ds.getConnection(), resourceURI, defaultGraph); |
|
| 58 |
// }catch(Exception e){
|
|
| 59 |
// log.error("Could not DELETE "+resourceURI+" "+e.toString(),e);
|
|
| 60 |
// } |
|
| 61 |
//#################################################################################### |
|
| 62 |
|
|
| 63 |
JSONArray typeMappings = new JSONArray(); |
|
| 64 |
JSONObject propertyObject = new JSONObject(); |
|
| 65 |
try{
|
|
| 66 |
typeMappings = mappings.getJSONArray(type); |
|
| 67 |
propertyObject = typeMappings.getJSONObject(0); |
|
| 68 |
}catch(Exception e){
|
|
| 69 |
log.error("ROW "+row.toString()+" "+e.toString(),e);
|
|
| 70 |
} |
|
| 71 |
|
|
| 72 |
for(int i=0; i<row.size(); i++){
|
|
| 73 |
String index = new Integer(i).toString(); |
|
| 74 |
String propertyString; |
|
| 75 |
|
|
| 76 |
|
|
| 77 |
if(type.equals("project") && i>24) continue;
|
|
| 78 |
if(type.equals("organization") && i>11) continue;
|
|
| 79 |
|
|
| 80 |
try{
|
|
| 81 |
propertyString = propertyObject.getString(index); |
|
| 82 |
}catch(Exception e){
|
|
| 83 |
// log.error("Could not get the property for type "+type+" and ID"+row.get(1).toString()+" : "+e.toString(),e);
|
|
| 84 |
continue; |
|
| 85 |
} |
|
| 86 |
|
|
| 87 |
if(i==0){
|
|
| 88 |
String resourceType = propertyObject.getString(propertyString); |
|
| 89 |
buildQuery[0]+="<"+resourceURI+"> <"+propertyString+"> <"+resourceType+">"; |
|
| 90 |
continue; |
|
| 91 |
} |
|
| 92 |
|
|
| 93 |
String value = row.get(i).trim(); |
|
| 94 |
value = value.replace("\\", "");
|
|
| 95 |
value = value.replace("\""," ");
|
|
| 96 |
|
|
| 97 |
if(value.trim().equals("null") || value==null || value.trim().equals("")) continue;
|
|
| 98 |
|
|
| 99 |
if(i==16 && type.equals("result") && !value.equals("und") || i==11 && type.equals("person") && !value.equals("und") || i==9 && type.equals("organization") && !value.equals("und")){
|
|
| 100 |
try{
|
|
| 101 |
// log.info("Country code is "+value);
|
|
| 102 |
String countryURI = mapCountries.getCountryURI(value); |
|
| 103 |
URI uri = new URI(countryURI); |
|
| 104 |
if(countryURI.equals("")){
|
|
| 105 |
buildQuery[0]+="; <"+propertyString+"> \""+value+"\""; |
|
| 106 |
continue; |
|
| 107 |
} |
|
| 108 |
buildQuery[0]+="; <"+propertyString+"> <"+uri+">"; |
|
| 109 |
}catch(Exception e){
|
|
| 110 |
buildQuery[0]+="; <"+propertyString+"> \""+value+"\""; |
|
| 111 |
continue; |
|
| 112 |
} |
|
| 113 |
continue; |
|
| 114 |
} |
|
| 115 |
|
|
| 116 |
if(type.equals("result") && i==9 && !value.equals("und")){
|
|
| 117 |
try{
|
|
| 118 |
String langURI = mapLanguages.getLangURI(value); |
|
| 119 |
URI uri = new URI(langURI); |
|
| 120 |
if(langURI.equals("")){
|
|
| 121 |
buildQuery[0]+="; <"+propertyString+"> \""+value+"\""; |
|
| 122 |
continue; |
|
| 123 |
} |
|
| 124 |
buildQuery[0]+="; <"+propertyString+"> <"+uri+">"; |
|
| 125 |
}catch(Exception e){
|
|
| 126 |
buildQuery[0]+="; <"+propertyString+"> \""+value+"\""; |
|
| 127 |
continue; |
|
| 128 |
} |
|
| 129 |
continue; |
|
| 130 |
} |
|
| 131 |
|
|
| 132 |
if(type.equals("datasource") && i==21 && !value.equals("und") && !value.isEmpty()){
|
|
| 133 |
if(value.contains(conf.get("lod.seperator"))){
|
|
| 134 |
String[] splittedValue = value.split(conf.get("lod.seperator"));
|
|
| 135 |
for(String v:splittedValue){
|
|
| 136 |
try{
|
|
| 137 |
String langURI = mapLanguages.getLangURI(v); |
|
| 138 |
URI uri = new URI(langURI); |
|
| 139 |
if(langURI.equals("")){
|
|
| 140 |
buildQuery[0]+="; <"+propertyString+"> \""+v+"\""; |
|
| 141 |
continue; |
|
| 142 |
} |
|
| 143 |
buildQuery[0]+="; <"+propertyString+"> <"+uri+">"; |
|
| 144 |
}catch(Exception e){
|
|
| 145 |
buildQuery[0]+="; <"+propertyString+"> \""+v+"\""; |
|
| 146 |
continue; |
|
| 147 |
} |
|
| 148 |
} |
|
| 149 |
continue; |
|
| 150 |
} |
|
| 151 |
} |
|
| 152 |
|
|
| 153 |
if(value.contains(conf.get("lod.seperator"))){
|
|
| 154 |
String[] splittedValue = value.split(conf.get("lod.seperator"));
|
|
| 155 |
for(String v:splittedValue){
|
|
| 156 |
v= v.replace(conf.get("lod.seperator"), "").trim();
|
|
| 157 |
buildQuery[0]+="; <"+propertyString+"> \""+v+"\""; |
|
| 158 |
} |
|
| 159 |
}else{
|
|
| 160 |
if(value.startsWith("http://")){
|
|
| 161 |
try{
|
|
| 162 |
if(value.contains(" ")){
|
|
| 163 |
buildQuery[0]+="; <"+propertyString+"> \""+value+"\""; |
|
| 164 |
}else{
|
|
| 165 |
value = value.replaceAll("\\s","-");
|
|
| 166 |
URI uri = new URI(value); |
|
| 167 |
buildQuery[0]+="; <"+propertyString+"> <"+uri+">"; |
|
| 168 |
} |
|
| 169 |
}catch(Exception e){
|
|
| 170 |
buildQuery[0]+="; <"+propertyString+"> \""+value+"\""; |
|
| 171 |
continue; |
|
| 172 |
} |
|
| 173 |
} |
|
| 174 |
else buildQuery[0]+="; <"+propertyString+"> \""+value+"\""; |
|
| 175 |
} |
|
| 176 |
|
|
| 177 |
} |
|
| 178 |
|
|
| 179 |
buildQuery[0]+=". "; |
|
| 180 |
|
|
| 181 |
return buildQuery; |
|
| 182 |
|
|
| 183 |
} |
|
| 184 |
|
|
| 185 |
public static String[] RDFizeRelationRow(List<String> row, JSONObject mappings, Configuration conf){
|
|
| 186 |
|
|
| 187 |
// String insertQuery = ""; |
|
| 188 |
String[] buildQuery = new String[1]; |
|
| 189 |
buildQuery[0]=""; |
|
| 190 |
String baseURI = conf.get("lod.baseURI");
|
|
| 191 |
JSONObject typeMappings = mappings.getJSONArray(row.get(0)).getJSONObject(0); |
|
| 192 |
String sourceType = typeMappings.getString("sourceType");
|
|
| 193 |
String sourceId = typeMappings.getString("sourceId");
|
|
| 194 |
String targetType = typeMappings.getString("targetType");
|
|
| 195 |
String targetId = typeMappings.getString("targetId");
|
|
| 196 |
String property = typeMappings.getString("property");
|
|
| 197 |
String sourceURI = baseURI+row.get(Integer.parseInt(sourceType))+"/"+row.get(Integer.parseInt(sourceId)); |
|
| 198 |
String targetURI = baseURI+row.get(Integer.parseInt(targetType))+"/"+row.get(Integer.parseInt(targetId)); |
|
| 199 |
buildQuery[0]="<"+sourceURI+"> <"+property+"> <"+targetURI+">. "; |
|
| 200 |
return buildQuery; |
|
| 201 |
} |
|
| 202 |
|
|
| 203 |
public static boolean resourceExists(Connection conn, String resourceURI, String graph){
|
|
| 204 |
boolean exists = false; |
|
| 205 |
String askQuery = "SPARQL SELECT ?o FROM <"+graph+"> WHERE {<"+resourceURI+"> ?p ?o}";
|
|
| 206 |
Statement stmt; |
|
| 207 |
try {
|
|
| 208 |
stmt = conn.createStatement(); |
|
| 209 |
ResultSet rs=stmt.executeQuery(askQuery); |
|
| 210 |
if(rs.next()) exists=true; |
|
| 211 |
rs.close(); |
|
| 212 |
stmt.close(); |
|
| 213 |
conn.close(); |
|
| 214 |
} catch (Exception e) {
|
|
| 215 |
log.error("Virtuoso ask Query failed. Query was: "+ askQuery +"\n" + e.toString(), e);
|
|
| 216 |
} |
|
| 217 |
return exists; |
|
| 218 |
} |
|
| 219 |
|
|
| 220 |
public static void deleteResource(Connection conn, String resourceURI, String graph){
|
|
| 221 |
String deleteQueryString ="SPARQL DELETE FROM <"+graph+"> {<"+resourceURI+"> ?p ?o} WHERE {<"+resourceURI+"> ?p ?o}" ;
|
|
| 222 |
Statement stmt; |
|
| 223 |
try {
|
|
| 224 |
stmt = conn.createStatement(); |
|
| 225 |
stmt.execute(deleteQueryString); |
|
| 226 |
conn.commit(); |
|
| 227 |
stmt.close(); |
|
| 228 |
conn.close(); |
|
| 229 |
} catch (Exception e) {
|
|
| 230 |
log.error("Virtuoso ask Query failed. "+ deleteQueryString +"\n" + e.toString(), e);
|
|
| 231 |
} |
|
| 232 |
} |
|
| 233 |
|
|
| 234 |
public static void clearGraph(String graph, Connection conn){
|
|
| 235 |
String clearQuery="SPARQL DEFINE sql:log-enable 3 CLEAR GRAPH <"+graph+">"; |
|
| 236 |
Statement stmt; |
|
| 237 |
try {
|
|
| 238 |
stmt = conn.createStatement(); |
|
| 239 |
stmt.execute(clearQuery); |
|
| 240 |
conn.commit(); |
|
| 241 |
stmt.close(); |
|
| 242 |
} catch (Exception e) {
|
|
| 243 |
log.error("Virtuoso FAILED TO CLEAR graph "+ clearQuery +"\n" + e.toString(), e);
|
|
| 244 |
} |
|
| 245 |
} |
|
| 246 |
|
|
| 247 |
public static void setCheckpoint(Connection conn, int checkpointValue){
|
|
| 248 |
Statement stmt; |
|
| 249 |
try {
|
|
| 250 |
|
|
| 251 |
stmt = conn.createStatement(); |
|
| 252 |
stmt.execute("checkpoint_interval("+checkpointValue+")");
|
|
| 253 |
if(checkpointValue==120){
|
|
| 254 |
log.info("CheckPoint Started");
|
|
| 255 |
stmt.execute("checkpoint");
|
|
| 256 |
// stmt.execute("log_enable(3,1)");
|
|
| 257 |
conn.commit(); |
|
| 258 |
log.info("CheckPoint Finished");
|
|
| 259 |
} |
|
| 260 |
|
|
| 261 |
stmt.close(); |
|
| 262 |
} catch (Exception e) {
|
|
| 263 |
log.error("Virtuoso set checkpoint failed. Checkpoint was "+ checkpointValue +"\n" + e.toString(), e);
|
|
| 264 |
} |
|
| 265 |
} |
|
| 266 |
|
|
| 267 |
public static void autoIndexing(Connection conn, boolean state){
|
|
| 268 |
Statement stmt; |
|
| 269 |
String query=""; |
|
| 270 |
if(state) query = "DB.DBA.RDF_OBJ_FT_RULE_ADD (null, null, 'All')"; |
|
| 271 |
else query = "DB.DBA.VT_BATCH_UPDATE ('DB.DBA.RDF_OBJ', 'ON', NULL)";
|
|
| 272 |
try {
|
|
| 273 |
stmt = conn.createStatement(); |
|
| 274 |
stmt.execute(query); |
|
| 275 |
log.info("autoIndex "+query);
|
|
| 276 |
stmt.close(); |
|
| 277 |
} catch (Exception e) {
|
|
| 278 |
log.error("Virtuoso autoIndexing failed. State was "+ query +"\n" + e.toString(), e);
|
|
| 279 |
} |
|
| 280 |
|
|
| 281 |
} |
|
| 282 |
} |
|
| 0 | 283 | |
| modules/dnet-openaire-lodimport/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/lodImport/utils/MapCountries.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.mapreduce.hbase.lodImport.utils; |
|
| 2 |
import java.io.InputStream; |
|
| 3 |
|
|
| 4 |
import org.apache.log4j.Logger; |
|
| 5 |
|
|
| 6 |
import com.hp.hpl.jena.query.QueryExecution; |
|
| 7 |
import com.hp.hpl.jena.query.QueryExecutionFactory; |
|
| 8 |
import com.hp.hpl.jena.query.QuerySolution; |
|
| 9 |
import com.hp.hpl.jena.query.ResultSet; |
|
| 10 |
import com.hp.hpl.jena.rdf.model.Model; |
|
| 11 |
import com.hp.hpl.jena.rdf.model.ModelFactory; |
|
| 12 |
|
|
| 13 |
/** |
|
| 14 |
* Method: getCoutryURI |
|
| 15 |
* |
|
| 16 |
* @author Giorgos Alexiou |
|
| 17 |
* |
|
| 18 |
*/ |
|
| 19 |
|
|
| 20 |
public class MapCountries {
|
|
| 21 |
|
|
| 22 |
/** |
|
| 23 |
* This Method takes ISO ISO_3166_1 codes for Countries as input and returns the respective URI from dbpedia.org as output. |
|
| 24 |
* For example iso639_3 for Germany is "DE" and the respective URI from dbpedia.org is <http://dbpedia.org/resource/Germany> |
|
| 25 |
* |
|
| 26 |
* @param iso_3166_1_Code |
|
| 27 |
* @return country URI <String> |
|
| 28 |
*/ |
|
| 29 |
|
|
| 30 |
public MapCountries(){
|
|
| 31 |
|
|
| 32 |
} |
|
| 33 |
|
|
| 34 |
private Logger log = Logger.getLogger(this.getClass()); |
|
| 35 |
public String getCountryURI(String iso_3166_1_Code){
|
|
| 36 |
String countryURI=""; |
|
| 37 |
try{
|
|
| 38 |
iso_3166_1_Code = iso_3166_1_Code.trim(); |
|
| 39 |
|
|
| 40 |
if(iso_3166_1_Code.equals("UK") || iso_3166_1_Code.equals("GB")) return "http://dbpedia.org/resource/United_Kingdom";
|
|
| 41 |
|
|
| 42 |
Model dbpediaCountries = ModelFactory.createDefaultModel(); |
|
| 43 |
// URL file = MapCountries.class.getClassLoader().getResource("rdfData/dbpedia_Countries.rdf");
|
|
| 44 |
// InputStream is = new FileInputStream(path); |
|
| 45 |
// InputStream is = this.getClass().getResourceAsStream("lodImport/dbpedia_Countries.rdf");
|
|
| 46 |
InputStream is = ClassLoader.getSystemResourceAsStream("eu/dnetlib/data/mapreduce/hbase/lodImport/dbpedia_Countries.rdf");
|
|
| 47 |
dbpediaCountries.read(is, "RDF/XML"); |
|
| 48 |
String isoURI = "http://dbpedia.org/resource/ISO_3166-1:"+iso_3166_1_Code; |
|
| 49 |
String query = "PREFIX dbpedia-owl:<http://dbpedia.org/ontology/> select distinct ?s ?r where {<"+isoURI+"> dbpedia-owl:wikiPageRedirects ?s}";
|
|
| 50 |
// System.out.println(query); |
|
| 51 |
QueryExecution queryExecution = QueryExecutionFactory.create(query,dbpediaCountries); |
|
| 52 |
ResultSet resultSet = queryExecution.execSelect(); |
|
| 53 |
|
|
| 54 |
while (resultSet.hasNext()) {
|
|
| 55 |
QuerySolution qs = resultSet.next(); |
|
| 56 |
String uriString = qs.getResource("s").toString().trim();
|
|
| 57 |
countryURI=uriString; |
|
| 58 |
} |
|
| 59 |
}catch(Exception e){
|
|
| 60 |
|
|
| 61 |
log.error("MapCountries error"+e.toString(),e);
|
|
| 62 |
} |
|
| 63 |
return countryURI; |
|
| 64 |
|
|
| 65 |
} |
|
| 66 |
|
|
| 67 |
} |
|
| 0 | 68 | |
| modules/dnet-openaire-lodimport/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/lodImport/LodImportReducer.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.mapreduce.hbase.lodImport; |
|
| 2 |
|
|
| 3 |
import java.io.IOException; |
|
| 4 |
import java.sql.Connection; |
|
| 5 |
import java.sql.PreparedStatement; |
|
| 6 |
import java.util.ArrayList; |
|
| 7 |
import java.util.Arrays; |
|
| 8 |
import java.util.Iterator; |
|
| 9 |
import java.util.List; |
|
| 10 |
|
|
| 11 |
import org.apache.hadoop.conf.Configuration; |
|
| 12 |
import org.apache.hadoop.io.NullWritable; |
|
| 13 |
import org.apache.hadoop.io.Text; |
|
| 14 |
import org.apache.hadoop.mapreduce.Reducer; |
|
| 15 |
import org.apache.log4j.Logger; |
|
| 16 |
import org.json.JSONObject; |
|
| 17 |
|
|
| 18 |
import com.jolbox.bonecp.BoneCPDataSource; |
|
| 19 |
|
|
| 20 |
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.DB; |
|
| 21 |
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.MapCountries; |
|
| 22 |
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.MapLanguages; |
|
| 23 |
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.RDFizer; |
|
| 24 |
|
|
| 25 |
|
|
| 26 |
public class LodImportReducer extends Reducer<Text, Text, NullWritable, NullWritable> {
|
|
| 27 |
|
|
| 28 |
private Logger log = Logger.getLogger(LodImportReducer.class); |
|
| 29 |
private BoneCPDataSource ds; |
|
| 30 |
private String fileName = ""; |
|
| 31 |
private Configuration hadoopConf; |
|
| 32 |
// private VirtGraph graph; |
|
| 33 |
private int counter; |
|
| 34 |
private String[] buildQuery = new String[2]; |
|
| 35 |
private String insertQuery = ""; |
|
| 36 |
private String deleteQuery = ""; |
|
| 37 |
private String defaultGraph = ""; |
|
| 38 |
private int entitiesPerQuery; |
|
| 39 |
private MapCountries mapCountries; |
|
| 40 |
private MapLanguages mapLanguages; |
|
| 41 |
private Connection conn; |
|
| 42 |
private Connection dConn; |
|
| 43 |
private JSONObject entitiesMappings; |
|
| 44 |
private JSONObject relationsMappings; |
|
| 45 |
|
|
| 46 |
|
|
| 47 |
public static enum REDUCES_COUNTER {
|
|
| 48 |
RESULT, |
|
| 49 |
PROJECT, |
|
| 50 |
DATASOURCE, |
|
| 51 |
PERSON, |
|
| 52 |
ORGANIZATION, |
|
| 53 |
TOTAL_ENTITIES, |
|
| 54 |
TOTAL_RELATIONS |
|
| 55 |
} |
|
| 56 |
|
|
| 57 |
|
|
| 58 |
@Override |
|
| 59 |
protected void setup(Context context) throws IOException, InterruptedException {
|
|
| 60 |
this.hadoopConf = context.getConfiguration(); |
|
| 61 |
this.fileName = hadoopConf.get("lod.inputFile");
|
|
| 62 |
if (!fileName.contains("relations")) defaultGraph = hadoopConf.get("lod.defaultGraph");
|
|
| 63 |
else defaultGraph = hadoopConf.get("lod.relationsGraph");
|
|
| 64 |
this.counter = 0; |
|
| 65 |
mapCountries = new MapCountries(); |
|
| 66 |
mapLanguages = new MapLanguages(); |
|
| 67 |
|
|
| 68 |
buildQuery[0] = ""; |
|
| 69 |
buildQuery[1] = ""; |
|
| 70 |
|
|
| 71 |
entitiesMappings = new JSONObject(hadoopConf.get("lod.jsonEntities"));
|
|
| 72 |
relationsMappings = new JSONObject(hadoopConf.get("lod.jsonRels"));
|
|
| 73 |
|
|
| 74 |
if (!fileName.contains("relations")) entitiesPerQuery = Integer.parseInt(hadoopConf.get("lod.entitiesPerQuery"));
|
|
| 75 |
else entitiesPerQuery = Integer.parseInt(hadoopConf.get("lod.relationsPerQuery"));
|
|
| 76 |
|
|
| 77 |
try {
|
|
| 78 |
DB db = new DB(); |
|
| 79 |
ds = db.getDatasource(hadoopConf.get("lod.conLine"), hadoopConf.get("lod.username"), hadoopConf.get("lod.password"), hadoopConf.get("lod.minCpart"), hadoopConf.get("lod.maxCpart"), hadoopConf.get("lod.part"));
|
|
| 80 |
conn = ds.getConnection(); |
|
| 81 |
conn.setAutoCommit(false); |
|
| 82 |
} catch (Exception e) {
|
|
| 83 |
log.error(e.toString(), e); |
|
| 84 |
} |
|
| 85 |
|
|
| 86 |
} |
|
| 87 |
|
|
| 88 |
@Override |
|
| 89 |
protected void reduce(final Text key, final Iterable<Text> values, final Context context) throws IOException, InterruptedException {
|
|
| 90 |
|
|
| 91 |
if (entitiesMappings == null || relationsMappings == null) {
|
|
| 92 |
throw new InterruptedException("Could not read json mappings!");
|
|
| 93 |
} |
|
| 94 |
Iterator<Text> it = values.iterator(); |
|
| 95 |
|
|
| 96 |
while (it.hasNext()) {
|
|
| 97 |
String data = it.next().toString(); |
|
| 98 |
String[] split = data.split(hadoopConf.get("lod.delim"));
|
|
| 99 |
List<String> row = new ArrayList<String>(Arrays.asList(split)); |
|
| 100 |
|
|
| 101 |
if (!fileName.contains("relations")) {
|
|
| 102 |
buildQuery = RDFizer.RDFizeEntityRow(row, ds, entitiesMappings, hadoopConf, mapCountries, mapLanguages, defaultGraph); |
|
| 103 |
insertQuery += buildQuery[0]; |
|
| 104 |
deleteQuery += buildQuery[1]; |
|
| 105 |
} else {
|
|
| 106 |
buildQuery = RDFizer.RDFizeRelationRow(row, relationsMappings, hadoopConf); |
|
| 107 |
insertQuery += buildQuery[0]; |
|
| 108 |
} |
|
| 109 |
|
|
| 110 |
} |
|
| 111 |
|
|
| 112 |
counter++; |
|
| 113 |
if (counter > entitiesPerQuery ) {
|
|
| 114 |
try {
|
|
| 115 |
PreparedStatement ps = conn.prepareStatement("DB.DBA.TTLP_MT (?, ?, ?, ?, ?, ?,?)");
|
|
| 116 |
ps.setString(1, insertQuery); |
|
| 117 |
ps.setString(2, ""); |
|
| 118 |
ps.setString(3, defaultGraph); |
|
| 119 |
final int NQUAD_LEVEL = 64; |
|
| 120 |
ps.setInt(4, NQUAD_LEVEL); |
|
| 121 |
ps.setInt(5, 0); |
|
| 122 |
ps.setInt(6, 15); |
|
| 123 |
ps.setInt(7, 0); |
|
| 124 |
ps.execute(); |
|
| 125 |
|
|
| 126 |
conn.commit(); |
|
| 127 |
|
|
| 128 |
ps.close(); |
|
| 129 |
|
|
| 130 |
buildQuery = new String[2]; |
|
| 131 |
buildQuery[0] = ""; |
|
| 132 |
buildQuery[1] = ""; |
|
| 133 |
insertQuery = ""; |
|
| 134 |
|
|
| 135 |
if (!fileName.contains("relations")) {
|
|
| 136 |
//s// TODO: keep counters for all entity types |
|
| 137 |
context.getCounter(REDUCES_COUNTER.TOTAL_ENTITIES).increment(counter); |
|
| 138 |
} else {
|
|
| 139 |
context.getCounter(REDUCES_COUNTER.TOTAL_RELATIONS).increment(counter); |
|
| 140 |
} |
|
| 141 |
|
|
| 142 |
counter = 0; |
|
| 143 |
}catch(Exception e){
|
|
| 144 |
log.error("Virtuoso write failed at query: " + insertQuery + " \n and with error " + e.toString(), e);
|
|
| 145 |
try {
|
|
| 146 |
if (conn != null) {
|
|
| 147 |
conn.rollback(); |
|
| 148 |
conn.close(); |
|
| 149 |
conn = ds.getConnection(); |
|
| 150 |
conn.setAutoCommit(false); |
|
| 151 |
} else {
|
|
| 152 |
conn.close(); |
|
| 153 |
conn = ds.getConnection(); |
|
| 154 |
conn.setAutoCommit(false); |
|
| 155 |
} |
|
| 156 |
|
|
| 157 |
PreparedStatement ps = conn.prepareStatement("DB.DBA.TTLP_MT (?, ?, ?, ?, ?, ?,?)");
|
|
| 158 |
ps.setString(1, insertQuery); |
|
| 159 |
ps.setString(2, ""); |
|
| 160 |
ps.setString(3, defaultGraph); |
|
| 161 |
final int NQUAD_LEVEL = 64; |
|
| 162 |
ps.setInt(4, NQUAD_LEVEL); |
|
| 163 |
ps.setInt(5, 0); |
|
| 164 |
ps.setInt(6, 15); |
|
| 165 |
ps.setInt(7, 0); |
|
| 166 |
ps.execute(); |
|
| 167 |
|
|
| 168 |
conn.commit(); |
|
| 169 |
|
|
| 170 |
ps.close(); |
|
| 171 |
|
|
| 172 |
buildQuery = new String[2]; |
|
| 173 |
buildQuery[0] = ""; |
|
| 174 |
buildQuery[1] = ""; |
|
| 175 |
insertQuery = ""; |
|
| 176 |
|
|
| 177 |
if (!fileName.contains("relations")) {
|
|
| 178 |
//s// TODO: keep counters for all entity types |
|
| 179 |
context.getCounter(REDUCES_COUNTER.TOTAL_ENTITIES).increment(counter); |
|
| 180 |
} else {
|
|
| 181 |
context.getCounter(REDUCES_COUNTER.TOTAL_RELATIONS).increment(counter); |
|
| 182 |
} |
|
| 183 |
|
|
| 184 |
counter = 0; |
|
| 185 |
} catch (Exception e1) {
|
|
| 186 |
log.error("Getting another Connection fail:" + e1.toString(), e1);
|
|
| 187 |
} |
|
| 188 |
} |
|
| 189 |
|
|
| 190 |
} |
|
| 191 |
|
|
| 192 |
} |
|
| 193 |
|
|
| 194 |
|
|
| 195 |
|
|
| 196 |
@Override |
|
| 197 |
protected void cleanup(Context context) throws IOException, InterruptedException {
|
|
| 198 |
try {
|
|
| 199 |
log.info("########### \n Cleanup Started. Commiting last Triples...\n##########");
|
|
| 200 |
PreparedStatement ps = conn.prepareStatement("DB.DBA.TTLP_MT (?, ?, ?, ?, ?, ?,?)");
|
|
| 201 |
ps.setString(1, insertQuery); |
|
| 202 |
ps.setString(2, ""); |
|
| 203 |
ps.setString(3, defaultGraph); |
|
| 204 |
final int NQUAD_LEVEL = 64; |
|
| 205 |
ps.setInt(4, NQUAD_LEVEL); |
|
| 206 |
ps.setInt(5, 0); |
|
| 207 |
ps.setInt(6, 15); |
|
| 208 |
ps.setInt(7, 0); |
|
| 209 |
ps.execute(); |
|
| 210 |
|
|
| 211 |
conn.commit(); |
|
| 212 |
|
|
| 213 |
ps.close(); |
|
| 214 |
if (!fileName.contains("relations")) {
|
|
| 215 |
//s// TODO: keep counters for all entity types |
|
| 216 |
context.getCounter(REDUCES_COUNTER.TOTAL_ENTITIES).increment(counter); |
|
| 217 |
} else {
|
|
| 218 |
context.getCounter(REDUCES_COUNTER.TOTAL_RELATIONS).increment(counter); |
|
| 219 |
} |
|
| 220 |
|
|
| 221 |
} catch (Exception e) {
|
|
| 222 |
log.error("Virtuoso write failed at query: " + insertQuery + " \n and with error " + e.toString(), e);
|
|
| 223 |
try {
|
|
| 224 |
if (conn != null) {
|
|
| 225 |
conn.rollback(); |
|
| 226 |
conn.close(); |
|
| 227 |
conn = ds.getConnection(); |
|
| 228 |
conn.setAutoCommit(false); |
|
| 229 |
PreparedStatement ps = conn.prepareStatement("DB.DBA.TTLP_MT (?, ?, ?, ?, ?, ?,?)");
|
|
| 230 |
ps.setString(1, insertQuery); |
|
| 231 |
ps.setString(2, ""); |
|
| 232 |
ps.setString(3, defaultGraph); |
|
| 233 |
final int NQUAD_LEVEL = 64; |
|
| 234 |
ps.setInt(4, NQUAD_LEVEL); |
|
| 235 |
ps.setInt(5, 0); |
|
| 236 |
ps.setInt(6, 15); |
|
| 237 |
ps.setInt(7, 0); |
|
| 238 |
ps.execute(); |
|
| 239 |
|
|
| 240 |
conn.commit(); |
|
| 241 |
|
|
| 242 |
ps.close(); |
|
| 243 |
if (!fileName.contains("relations")) {
|
|
| 244 |
//s// TODO: keep counters for all entity types |
|
| 245 |
context.getCounter(REDUCES_COUNTER.TOTAL_ENTITIES).increment(counter); |
|
| 246 |
} else {
|
|
| 247 |
context.getCounter(REDUCES_COUNTER.TOTAL_RELATIONS).increment(counter); |
|
| 248 |
} |
|
| 249 |
} else {
|
|
| 250 |
conn.close(); |
|
| 251 |
conn = ds.getConnection(); |
|
| 252 |
conn.setAutoCommit(false); |
|
| 253 |
PreparedStatement ps = conn.prepareStatement("DB.DBA.TTLP_MT (?, ?, ?, ?, ?, ?,?)");
|
|
| 254 |
ps.setString(1, insertQuery); |
|
| 255 |
ps.setString(2, ""); |
|
| 256 |
ps.setString(3, defaultGraph); |
|
| 257 |
final int NQUAD_LEVEL = 64; |
|
| 258 |
ps.setInt(4, NQUAD_LEVEL); |
|
| 259 |
ps.setInt(5, 0); |
|
| 260 |
ps.setInt(6, 15); |
|
| 261 |
ps.setInt(7, 0); |
|
| 262 |
ps.execute(); |
|
| 263 |
|
|
| 264 |
conn.commit(); |
|
| 265 |
|
|
| 266 |
ps.close(); |
|
| 267 |
if (!fileName.contains("relations")) {
|
|
| 268 |
//s// TODO: keep counters for all entity types |
|
| 269 |
context.getCounter(REDUCES_COUNTER.TOTAL_ENTITIES).increment(counter); |
|
| 270 |
} else {
|
|
| 271 |
context.getCounter(REDUCES_COUNTER.TOTAL_RELATIONS).increment(counter); |
|
| 272 |
} |
|
| 273 |
} |
|
| 274 |
|
|
| 275 |
} catch (Exception e1) {
|
|
| 276 |
log.error("Getting another Connection fail:" + e1.toString(), e1);
|
|
| 277 |
} |
|
| 278 |
} finally {
|
|
| 279 |
log.info("Cleaning up reducer...\nClosing Graph and Datasource");
|
|
| 280 |
try {
|
|
| 281 |
conn.commit(); |
|
| 282 |
conn.close(); |
|
| 283 |
} catch (Exception e) {
|
|
| 284 |
log.error("Could not Close connection \n" + e.toString(), e);
|
|
| 285 |
} |
|
| 286 |
|
|
| 287 |
log.info("Graph and Datasource Closed...");
|
|
| 288 |
} |
|
| 289 |
} |
|
| 290 |
|
|
| 291 |
} |
|
| 0 | 292 | |
| modules/dnet-openaire-lodimport/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/lodImport/LodImportMapper.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.data.mapreduce.hbase.lodImport; |
|
| 2 |
|
|
| 3 |
|
|
| 4 |
import java.io.IOException; |
|
| 5 |
import java.sql.Connection; |
|
| 6 |
import java.sql.SQLException; |
|
| 7 |
import java.sql.Statement; |
|
| 8 |
|
|
| 9 |
import org.apache.hadoop.conf.Configuration; |
|
| 10 |
import org.apache.hadoop.io.LongWritable; |
|
| 11 |
import org.apache.hadoop.io.Text; |
|
| 12 |
import org.apache.hadoop.mapreduce.Mapper; |
|
| 13 |
import org.apache.log4j.Logger; |
|
| 14 |
import org.joda.time.DateTime; |
|
| 15 |
import org.joda.time.format.DateTimeFormat; |
|
| 16 |
import org.joda.time.format.DateTimeFormatter; |
|
| 17 |
|
|
| 18 |
import com.jolbox.bonecp.BoneCPDataSource; |
|
| 19 |
|
|
| 20 |
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.DB; |
|
| 21 |
|
|
| 22 |
|
|
| 23 |
public class LodImportMapper extends Mapper<LongWritable, Text, Text, Text> {
|
|
| 24 |
private Logger log = Logger.getLogger(this.getClass()); |
|
| 25 |
|
|
| 26 |
private String fileName = ""; |
|
| 27 |
private String lastExecutionDate = ""; |
|
| 28 |
|
|
| 29 |
|
|
| 30 |
private Configuration hadoopConf; |
|
| 31 |
private BoneCPDataSource ds; |
|
| 32 |
private Connection conn; |
|
| 33 |
String baseURI; |
|
| 34 |
private String defaultGraph = ""; |
|
| 35 |
private String deleteQuery = ""; |
|
| 36 |
private int counter; |
|
| 37 |
private int entitiesPerQuery; |
|
| 38 |
|
|
| 39 |
public static enum ENTITIES_COUNTER {
|
|
| 40 |
STORED, DISCARDED, |
|
| 41 |
ENTITIES, RELATIONS, NULL_T_DATES, DELETED |
|
| 42 |
} |
|
| 43 |
|
|
| 44 |
@Override |
|
| 45 |
protected void setup(Mapper.Context context) throws IOException, InterruptedException {
|
|
| 46 |
|
|
| 47 |
this.fileName = context.getConfiguration().get("lod.inputFile");
|
|
| 48 |
|
|
| 49 |
this.hadoopConf = context.getConfiguration(); |
|
| 50 |
|
|
| 51 |
try {
|
|
| 52 |
DB db = new DB(); |
|
| 53 |
ds = db.getDatasource(hadoopConf.get("lod.conLine"), hadoopConf.get("lod.username"), hadoopConf.get("lod.password"), "1", "1", "1");
|
|
| 54 |
|
|
| 55 |
conn = ds.getConnection(); |
|
| 56 |
conn.setAutoCommit(false); |
|
| 57 |
|
|
| 58 |
Statement stmt1; |
|
| 59 |
stmt1 = conn.createStatement(); |
|
| 60 |
stmt1.execute("__dbf_set ('enable_qp', 1)");
|
|
| 61 |
stmt1.close(); |
|
| 62 |
conn.close(); |
|
| 63 |
|
|
| 64 |
} catch (Exception e) {
|
|
| 65 |
log.error(e.toString(), e); |
|
| 66 |
|
|
| 67 |
} |
|
| 68 |
|
|
| 69 |
} |
|
| 70 |
|
|
| 71 |
@Override |
|
| 72 |
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
|
|
| 73 |
//TODO remove enclosing delim from csv -not needed |
|
| 74 |
|
|
| 75 |
String split[] = value.toString().split(context.getConfiguration().get("lod.delim"));
|
|
| 76 |
if (split.length > 1) {
|
|
| 77 |
Text outputKey = null; |
|
| 78 |
Text outputValue = null; |
|
| 79 |
try {
|
|
| 80 |
outputKey = new Text(split[1]); |
|
| 81 |
outputValue = new Text(value.toString().replace("\\", " "));
|
|
| 82 |
} catch (Exception e) {
|
|
| 83 |
log.error(split.toString(), e); |
|
| 84 |
} |
|
| 85 |
|
|
| 86 |
if (fileName.contains("relations")) {
|
|
| 87 |
outputKey = new Text(key.toString()); |
|
| 88 |
writeOut(context, outputKey, outputValue); |
|
| 89 |
context.getCounter(ENTITIES_COUNTER.RELATIONS).increment(1); |
|
| 90 |
} else {
|
|
| 91 |
context.getCounter(ENTITIES_COUNTER.ENTITIES).increment(1); |
|
| 92 |
writeOut(context, outputKey, outputValue); |
|
| 93 |
} |
|
| 94 |
} |
|
| 95 |
} |
|
| 96 |
|
|
| 97 |
@Override |
|
| 98 |
protected void cleanup(Context context) throws IOException, InterruptedException {
|
|
| 99 |
super.cleanup(context); |
|
| 100 |
if(conn!=null){
|
|
| 101 |
try {
|
|
| 102 |
conn.close(); |
|
| 103 |
} catch (SQLException e) {
|
|
| 104 |
log.error(e.toString(), e); |
|
| 105 |
} |
|
| 106 |
} |
|
| 107 |
} |
|
| 108 |
|
|
| 109 |
private void writeOut(Context context, Text outputKey, Text outputValue) throws IOException, InterruptedException {
|
|
| 110 |
context.write(outputKey, outputValue); |
|
| 111 |
} |
|
| 112 |
|
|
| 113 |
} |
|
| 0 | 114 | |
| modules/dnet-openaire-lodimport/trunk/src/main/resources/eu/dnetlib/data/mapreduce/hbase/lodExport/log4j.properties | ||
|---|---|---|
| 1 |
# Root logger option |
|
| 2 |
log4j.rootLogger=INFO, file, stdout |
|
| 3 |
|
|
| 4 |
# Direct log messages to a log file |
|
| 5 |
log4j.appender.file=org.apache.log4j.RollingFileAppender |
|
| 6 |
log4j.appender.file.File=/tmp/statsExportLog |
|
| 7 |
log4j.appender.file.MaxFileSize=1MB |
|
| 8 |
log4j.appender.file.MaxBackupIndex=1 |
|
| 9 |
log4j.appender.file.layout=org.apache.log4j.PatternLayout |
|
| 10 |
log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
|
|
| 11 |
|
|
| 12 |
# Direct log messages to stdout |
|
| 13 |
log4j.appender.stdout=org.apache.log4j.ConsoleAppender |
|
| 14 |
log4j.appender.stdout.Target=System.out |
|
| 15 |
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout |
|
| 16 |
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
|
|
| 17 |
|
|
| 18 |
#log4j.rootLogger = WARN, R |
|
| 19 |
|
|
| 20 |
#log4j.logger.gr.uoa = DEBUG |
|
| 21 |
#log4j.logger.org.springframework = DEBUG, S |
|
| 22 |
#log4j.additivity.org.springframework = false |
|
| 23 |
# |
|
| 24 |
#log4j.logger.com.opensymphony.xwork2.ognl.OgnlValueStack = FATAL |
|
| 25 |
#log4j.logger.com.opensymphony.xwork2.ObjectFactory = FATAL |
|
| 26 |
# |
|
| 27 |
#log4j.appender.R=org.apache.log4j.RollingFileAppender |
|
| 28 |
#log4j.appender.R.File=/tmp/log/validator.log |
|
| 29 |
#log4j.appender.R.MaxFileSize=10MB |
|
| 30 |
#log4j.appender.R.MaxBackupIndex=10 |
|
| 31 |
#log4j.appender.R.layout=org.apache.log4j.PatternLayout |
|
| 32 |
#log4j.appender.R.layout.ConversionPattern= %d %p %t [%c] - %m%n |
|
| 33 |
# |
|
| 34 |
#log4j.appender.S=org.apache.log4j.RollingFileAppender |
|
| 35 |
#log4j.appender.S.File=/tmp/log/spring-validator.log |
|
| 36 |
#log4j.appender.S.MaxFileSize=10MB |
|
| 37 |
#log4j.appender.S.MaxBackupIndex=10 |
|
| 38 |
#log4j.appender.S.layout=org.apache.log4j.PatternLayout |
|
| 39 |
#log4j.appender.S.layout.ConversionPattern= %d %p %t [%c] - %m%n |
|
| 0 | 40 | |
| modules/dnet-openaire-lodimport/trunk/src/main/resources/eu/dnetlib/data/mapreduce/hbase/lodImport/lexvo-iso639-3.tsv | ||
|---|---|---|
| 1 |
aaa http://lexvo.org/id/iso639-3/aaa |
|
| 2 |
aab http://lexvo.org/id/iso639-3/aab |
|
| 3 |
aac http://lexvo.org/id/iso639-3/aac |
|
| 4 |
aad http://lexvo.org/id/iso639-3/aad |
|
| 5 |
aae http://lexvo.org/id/iso639-3/aae |
|
| 6 |
aaf http://lexvo.org/id/iso639-3/aaf |
|
| 7 |
aag http://lexvo.org/id/iso639-3/aag |
|
| 8 |
aah http://lexvo.org/id/iso639-3/aah |
|
| 9 |
aai http://lexvo.org/id/iso639-3/aai |
|
| 10 |
aak http://lexvo.org/id/iso639-3/aak |
|
| 11 |
aal http://lexvo.org/id/iso639-3/aal |
|
| 12 |
aam http://lexvo.org/id/iso639-3/aam |
|
| 13 |
aan http://lexvo.org/id/iso639-3/aan |
|
| 14 |
aao http://lexvo.org/id/iso639-3/aao |
|
| 15 |
aap http://lexvo.org/id/iso639-3/aap |
|
| 16 |
aaq http://lexvo.org/id/iso639-3/aaq |
|
| 17 |
aar http://lexvo.org/id/iso639-3/aar |
|
| 18 |
aas http://lexvo.org/id/iso639-3/aas |
|
| 19 |
aat http://lexvo.org/id/iso639-3/aat |
|
| 20 |
aau http://lexvo.org/id/iso639-3/aau |
|
| 21 |
aaw http://lexvo.org/id/iso639-3/aaw |
|
| 22 |
aax http://lexvo.org/id/iso639-3/aax |
|
| 23 |
aaz http://lexvo.org/id/iso639-3/aaz |
|
| 24 |
aba http://lexvo.org/id/iso639-3/aba |
|
| 25 |
abb http://lexvo.org/id/iso639-3/abb |
|
| 26 |
abc http://lexvo.org/id/iso639-3/abc |
|
| 27 |
abd http://lexvo.org/id/iso639-3/abd |
|
| 28 |
abe http://lexvo.org/id/iso639-3/abe |
|
| 29 |
abf http://lexvo.org/id/iso639-3/abf |
|
| 30 |
abg http://lexvo.org/id/iso639-3/abg |
|
| 31 |
abh http://lexvo.org/id/iso639-3/abh |
|
| 32 |
abi http://lexvo.org/id/iso639-3/abi |
|
| 33 |
abj http://lexvo.org/id/iso639-3/abj |
|
| 34 |
abk http://lexvo.org/id/iso639-3/abk |
|
| 35 |
abl http://lexvo.org/id/iso639-3/abl |
|
| 36 |
abm http://lexvo.org/id/iso639-3/abm |
|
| 37 |
abn http://lexvo.org/id/iso639-3/abn |
|
| 38 |
abo http://lexvo.org/id/iso639-3/abo |
|
| 39 |
abp http://lexvo.org/id/iso639-3/abp |
|
| 40 |
abq http://lexvo.org/id/iso639-3/abq |
|
| 41 |
abr http://lexvo.org/id/iso639-3/abr |
|
| 42 |
abs http://lexvo.org/id/iso639-3/abs |
|
| 43 |
abt http://lexvo.org/id/iso639-3/abt |
|
| 44 |
abu http://lexvo.org/id/iso639-3/abu |
|
| 45 |
abv http://lexvo.org/id/iso639-3/abv |
|
| 46 |
abw http://lexvo.org/id/iso639-3/abw |
|
| 47 |
abx http://lexvo.org/id/iso639-3/abx |
|
| 48 |
aby http://lexvo.org/id/iso639-3/aby |
|
| 49 |
abz http://lexvo.org/id/iso639-3/abz |
|
| 50 |
aca http://lexvo.org/id/iso639-3/aca |
|
| 51 |
acb http://lexvo.org/id/iso639-3/acb |
|
| 52 |
acd http://lexvo.org/id/iso639-3/acd |
|
| 53 |
ace http://lexvo.org/id/iso639-3/ace |
|
| 54 |
acf http://lexvo.org/id/iso639-3/acf |
|
| 55 |
ach http://lexvo.org/id/iso639-3/ach |
|
| 56 |
aci http://lexvo.org/id/iso639-3/aci |
|
| 57 |
ack http://lexvo.org/id/iso639-3/ack |
|
| 58 |
acl http://lexvo.org/id/iso639-3/acl |
|
| 59 |
acm http://lexvo.org/id/iso639-3/acm |
|
| 60 |
acn http://lexvo.org/id/iso639-3/acn |
|
| 61 |
acp http://lexvo.org/id/iso639-3/acp |
|
| 62 |
acq http://lexvo.org/id/iso639-3/acq |
|
| 63 |
acr http://lexvo.org/id/iso639-3/acr |
|
| 64 |
acs http://lexvo.org/id/iso639-3/acs |
|
| 65 |
act http://lexvo.org/id/iso639-3/act |
|
| 66 |
acu http://lexvo.org/id/iso639-3/acu |
|
| 67 |
acv http://lexvo.org/id/iso639-3/acv |
|
| 68 |
acw http://lexvo.org/id/iso639-3/acw |
|
| 69 |
acx http://lexvo.org/id/iso639-3/acx |
|
| 70 |
acy http://lexvo.org/id/iso639-3/acy |
|
| 71 |
acz http://lexvo.org/id/iso639-3/acz |
|
| 72 |
ada http://lexvo.org/id/iso639-3/ada |
|
| 73 |
adb http://lexvo.org/id/iso639-3/adb |
|
| 74 |
add http://lexvo.org/id/iso639-3/add |
|
| 75 |
ade http://lexvo.org/id/iso639-3/ade |
|
| 76 |
adf http://lexvo.org/id/iso639-3/adf |
|
| 77 |
adg http://lexvo.org/id/iso639-3/adg |
|
| 78 |
adh http://lexvo.org/id/iso639-3/adh |
|
| 79 |
adi http://lexvo.org/id/iso639-3/adi |
|
| 80 |
adj http://lexvo.org/id/iso639-3/adj |
|
| 81 |
adl http://lexvo.org/id/iso639-3/adl |
|
| 82 |
adn http://lexvo.org/id/iso639-3/adn |
|
| 83 |
ado http://lexvo.org/id/iso639-3/ado |
|
| 84 |
adp http://lexvo.org/id/iso639-3/adp |
|
| 85 |
adq http://lexvo.org/id/iso639-3/adq |
|
| 86 |
adr http://lexvo.org/id/iso639-3/adr |
|
| 87 |
ads http://lexvo.org/id/iso639-3/ads |
|
| 88 |
adt http://lexvo.org/id/iso639-3/adt |
|
| 89 |
adu http://lexvo.org/id/iso639-3/adu |
|
| 90 |
adw http://lexvo.org/id/iso639-3/adw |
|
| 91 |
adx http://lexvo.org/id/iso639-3/adx |
|
| 92 |
ady http://lexvo.org/id/iso639-3/ady |
|
| 93 |
adz http://lexvo.org/id/iso639-3/adz |
|
| 94 |
aea http://lexvo.org/id/iso639-3/aea |
|
| 95 |
aeb http://lexvo.org/id/iso639-3/aeb |
|
| 96 |
aec http://lexvo.org/id/iso639-3/aec |
|
| 97 |
aed http://lexvo.org/id/iso639-3/aed |
|
| 98 |
aee http://lexvo.org/id/iso639-3/aee |
|
| 99 |
aek http://lexvo.org/id/iso639-3/aek |
|
| 100 |
ael http://lexvo.org/id/iso639-3/ael |
|
| 101 |
aem http://lexvo.org/id/iso639-3/aem |
|
| 102 |
aen http://lexvo.org/id/iso639-3/aen |
|
| 103 |
aeq http://lexvo.org/id/iso639-3/aeq |
|
| 104 |
aer http://lexvo.org/id/iso639-3/aer |
|
| 105 |
aes http://lexvo.org/id/iso639-3/aes |
|
| 106 |
aeu http://lexvo.org/id/iso639-3/aeu |
|
| 107 |
aew http://lexvo.org/id/iso639-3/aew |
|
| 108 |
aey http://lexvo.org/id/iso639-3/aey |
|
| 109 |
aez http://lexvo.org/id/iso639-3/aez |
|
| 110 |
afb http://lexvo.org/id/iso639-3/afb |
|
| 111 |
afd http://lexvo.org/id/iso639-3/afd |
|
| 112 |
afe http://lexvo.org/id/iso639-3/afe |
|
| 113 |
afg http://lexvo.org/id/iso639-3/afg |
|
| 114 |
afh http://lexvo.org/id/iso639-3/afh |
|
| 115 |
afi http://lexvo.org/id/iso639-3/afi |
|
| 116 |
afk http://lexvo.org/id/iso639-3/afk |
|
| 117 |
afn http://lexvo.org/id/iso639-3/afn |
|
| 118 |
afo http://lexvo.org/id/iso639-3/afo |
|
| 119 |
afp http://lexvo.org/id/iso639-3/afp |
|
| 120 |
afr http://lexvo.org/id/iso639-3/afr |
|
| 121 |
afs http://lexvo.org/id/iso639-3/afs |
|
| 122 |
aft http://lexvo.org/id/iso639-3/aft |
|
| 123 |
afu http://lexvo.org/id/iso639-3/afu |
|
| 124 |
afz http://lexvo.org/id/iso639-3/afz |
|
| 125 |
aga http://lexvo.org/id/iso639-3/aga |
|
| 126 |
agb http://lexvo.org/id/iso639-3/agb |
|
| 127 |
agc http://lexvo.org/id/iso639-3/agc |
|
| 128 |
agd http://lexvo.org/id/iso639-3/agd |
|
| 129 |
age http://lexvo.org/id/iso639-3/age |
|
| 130 |
agf http://lexvo.org/id/iso639-3/agf |
|
| 131 |
agg http://lexvo.org/id/iso639-3/agg |
|
| 132 |
agh http://lexvo.org/id/iso639-3/agh |
|
| 133 |
agi http://lexvo.org/id/iso639-3/agi |
|
| 134 |
agj http://lexvo.org/id/iso639-3/agj |
|
| 135 |
agk http://lexvo.org/id/iso639-3/agk |
|
| 136 |
agl http://lexvo.org/id/iso639-3/agl |
|
| 137 |
agm http://lexvo.org/id/iso639-3/agm |
|
| 138 |
agn http://lexvo.org/id/iso639-3/agn |
|
| 139 |
ago http://lexvo.org/id/iso639-3/ago |
|
| 140 |
agq http://lexvo.org/id/iso639-3/agq |
|
| 141 |
agr http://lexvo.org/id/iso639-3/agr |
|
| 142 |
ags http://lexvo.org/id/iso639-3/ags |
|
| 143 |
agt http://lexvo.org/id/iso639-3/agt |
|
| 144 |
agu http://lexvo.org/id/iso639-3/agu |
|
| 145 |
agv http://lexvo.org/id/iso639-3/agv |
|
| 146 |
agw http://lexvo.org/id/iso639-3/agw |
|
| 147 |
agx http://lexvo.org/id/iso639-3/agx |
|
| 148 |
agy http://lexvo.org/id/iso639-3/agy |
|
| 149 |
agz http://lexvo.org/id/iso639-3/agz |
|
| 150 |
aha http://lexvo.org/id/iso639-3/aha |
|
| 151 |
ahb http://lexvo.org/id/iso639-3/ahb |
|
| 152 |
ahg http://lexvo.org/id/iso639-3/ahg |
|
| 153 |
ahh http://lexvo.org/id/iso639-3/ahh |
|
| 154 |
ahi http://lexvo.org/id/iso639-3/ahi |
|
| 155 |
ahk http://lexvo.org/id/iso639-3/ahk |
|
| 156 |
ahl http://lexvo.org/id/iso639-3/ahl |
|
| 157 |
ahm http://lexvo.org/id/iso639-3/ahm |
|
| 158 |
ahn http://lexvo.org/id/iso639-3/ahn |
|
| 159 |
aho http://lexvo.org/id/iso639-3/aho |
|
| 160 |
ahp http://lexvo.org/id/iso639-3/ahp |
|
| 161 |
ahr http://lexvo.org/id/iso639-3/ahr |
|
| 162 |
ahs http://lexvo.org/id/iso639-3/ahs |
|
| 163 |
aht http://lexvo.org/id/iso639-3/aht |
|
| 164 |
aia http://lexvo.org/id/iso639-3/aia |
|
| 165 |
aib http://lexvo.org/id/iso639-3/aib |
|
| 166 |
aic http://lexvo.org/id/iso639-3/aic |
|
| 167 |
aid http://lexvo.org/id/iso639-3/aid |
|
| 168 |
aie http://lexvo.org/id/iso639-3/aie |
|
| 169 |
aif http://lexvo.org/id/iso639-3/aif |
|
| 170 |
aig http://lexvo.org/id/iso639-3/aig |
|
| 171 |
aih http://lexvo.org/id/iso639-3/aih |
|
| 172 |
aii http://lexvo.org/id/iso639-3/aii |
|
| 173 |
aij http://lexvo.org/id/iso639-3/aij |
|
| 174 |
aik http://lexvo.org/id/iso639-3/aik |
|
| 175 |
ail http://lexvo.org/id/iso639-3/ail |
|
| 176 |
aim http://lexvo.org/id/iso639-3/aim |
|
| 177 |
ain http://lexvo.org/id/iso639-3/ain |
|
| 178 |
aio http://lexvo.org/id/iso639-3/aio |
|
| 179 |
aip http://lexvo.org/id/iso639-3/aip |
|
| 180 |
aiq http://lexvo.org/id/iso639-3/aiq |
|
| 181 |
air http://lexvo.org/id/iso639-3/air |
|
| 182 |
ais http://lexvo.org/id/iso639-3/ais |
|
| 183 |
ait http://lexvo.org/id/iso639-3/ait |
|
| 184 |
aiw http://lexvo.org/id/iso639-3/aiw |
|
| 185 |
aix http://lexvo.org/id/iso639-3/aix |
|
| 186 |
aiy http://lexvo.org/id/iso639-3/aiy |
|
| 187 |
aja http://lexvo.org/id/iso639-3/aja |
|
| 188 |
ajg http://lexvo.org/id/iso639-3/ajg |
|
| 189 |
aji http://lexvo.org/id/iso639-3/aji |
|
| 190 |
ajn http://lexvo.org/id/iso639-3/ajn |
|
| 191 |
ajp http://lexvo.org/id/iso639-3/ajp |
|
| 192 |
ajt http://lexvo.org/id/iso639-3/ajt |
|
| 193 |
aju http://lexvo.org/id/iso639-3/aju |
|
| 194 |
ajw http://lexvo.org/id/iso639-3/ajw |
|
| 195 |
ajz http://lexvo.org/id/iso639-3/ajz |
|
| 196 |
aka http://lexvo.org/id/iso639-3/aka |
|
| 197 |
akb http://lexvo.org/id/iso639-3/akb |
|
| 198 |
akc http://lexvo.org/id/iso639-3/akc |
|
| 199 |
akd http://lexvo.org/id/iso639-3/akd |
|
| 200 |
ake http://lexvo.org/id/iso639-3/ake |
|
| 201 |
akf http://lexvo.org/id/iso639-3/akf |
|
| 202 |
akg http://lexvo.org/id/iso639-3/akg |
|
| 203 |
akh http://lexvo.org/id/iso639-3/akh |
|
| 204 |
aki http://lexvo.org/id/iso639-3/aki |
|
| 205 |
akj http://lexvo.org/id/iso639-3/akj |
|
| 206 |
akk http://lexvo.org/id/iso639-3/akk |
|
| 207 |
akl http://lexvo.org/id/iso639-3/akl |
|
| 208 |
akm http://lexvo.org/id/iso639-3/akm |
|
| 209 |
ako http://lexvo.org/id/iso639-3/ako |
|
| 210 |
akp http://lexvo.org/id/iso639-3/akp |
|
| 211 |
akq http://lexvo.org/id/iso639-3/akq |
|
| 212 |
akr http://lexvo.org/id/iso639-3/akr |
|
| 213 |
aks http://lexvo.org/id/iso639-3/aks |
|
| 214 |
akt http://lexvo.org/id/iso639-3/akt |
|
| 215 |
aku http://lexvo.org/id/iso639-3/aku |
|
| 216 |
akv http://lexvo.org/id/iso639-3/akv |
|
| 217 |
akw http://lexvo.org/id/iso639-3/akw |
|
| 218 |
akx http://lexvo.org/id/iso639-3/akx |
|
| 219 |
aky http://lexvo.org/id/iso639-3/aky |
|
| 220 |
akz http://lexvo.org/id/iso639-3/akz |
|
| 221 |
ala http://lexvo.org/id/iso639-3/ala |
|
| 222 |
alc http://lexvo.org/id/iso639-3/alc |
|
| 223 |
ald http://lexvo.org/id/iso639-3/ald |
|
| 224 |
ale http://lexvo.org/id/iso639-3/ale |
|
| 225 |
alf http://lexvo.org/id/iso639-3/alf |
|
| 226 |
alh http://lexvo.org/id/iso639-3/alh |
|
| 227 |
ali http://lexvo.org/id/iso639-3/ali |
|
| 228 |
alj http://lexvo.org/id/iso639-3/alj |
|
| 229 |
alk http://lexvo.org/id/iso639-3/alk |
|
| 230 |
all http://lexvo.org/id/iso639-3/all |
|
| 231 |
alm http://lexvo.org/id/iso639-3/alm |
|
| 232 |
aln http://lexvo.org/id/iso639-3/aln |
|
| 233 |
alo http://lexvo.org/id/iso639-3/alo |
|
| 234 |
alp http://lexvo.org/id/iso639-3/alp |
|
| 235 |
alq http://lexvo.org/id/iso639-3/alq |
|
| 236 |
alr http://lexvo.org/id/iso639-3/alr |
|
| 237 |
als http://lexvo.org/id/iso639-3/als |
|
| 238 |
alt http://lexvo.org/id/iso639-3/alt |
|
| 239 |
alu http://lexvo.org/id/iso639-3/alu |
|
| 240 |
alw http://lexvo.org/id/iso639-3/alw |
|
| 241 |
alx http://lexvo.org/id/iso639-3/alx |
|
| 242 |
aly http://lexvo.org/id/iso639-3/aly |
|
| 243 |
alz http://lexvo.org/id/iso639-3/alz |
|
| 244 |
ama http://lexvo.org/id/iso639-3/ama |
|
| 245 |
amb http://lexvo.org/id/iso639-3/amb |
|
| 246 |
amc http://lexvo.org/id/iso639-3/amc |
|
| 247 |
ame http://lexvo.org/id/iso639-3/ame |
|
| 248 |
amf http://lexvo.org/id/iso639-3/amf |
|
| 249 |
amg http://lexvo.org/id/iso639-3/amg |
|
| 250 |
amh http://lexvo.org/id/iso639-3/amh |
|
| 251 |
ami http://lexvo.org/id/iso639-3/ami |
|
| 252 |
amj http://lexvo.org/id/iso639-3/amj |
|
| 253 |
amk http://lexvo.org/id/iso639-3/amk |
|
| 254 |
aml http://lexvo.org/id/iso639-3/aml |
|
| 255 |
amm http://lexvo.org/id/iso639-3/amm |
|
| 256 |
amn http://lexvo.org/id/iso639-3/amn |
|
| 257 |
amo http://lexvo.org/id/iso639-3/amo |
|
| 258 |
amp http://lexvo.org/id/iso639-3/amp |
|
| 259 |
amq http://lexvo.org/id/iso639-3/amq |
|
| 260 |
amr http://lexvo.org/id/iso639-3/amr |
|
| 261 |
ams http://lexvo.org/id/iso639-3/ams |
|
| 262 |
amt http://lexvo.org/id/iso639-3/amt |
|
| 263 |
amu http://lexvo.org/id/iso639-3/amu |
|
| 264 |
amv http://lexvo.org/id/iso639-3/amv |
|
| 265 |
amw http://lexvo.org/id/iso639-3/amw |
|
| 266 |
amx http://lexvo.org/id/iso639-3/amx |
|
| 267 |
amy http://lexvo.org/id/iso639-3/amy |
|
| 268 |
amz http://lexvo.org/id/iso639-3/amz |
|
| 269 |
ana http://lexvo.org/id/iso639-3/ana |
|
| 270 |
anb http://lexvo.org/id/iso639-3/anb |
|
| 271 |
anc http://lexvo.org/id/iso639-3/anc |
|
| 272 |
and http://lexvo.org/id/iso639-3/and |
|
| 273 |
ane http://lexvo.org/id/iso639-3/ane |
|
| 274 |
anf http://lexvo.org/id/iso639-3/anf |
|
| 275 |
ang http://lexvo.org/id/iso639-3/ang |
|
| 276 |
anh http://lexvo.org/id/iso639-3/anh |
|
| 277 |
ani http://lexvo.org/id/iso639-3/ani |
|
| 278 |
anj http://lexvo.org/id/iso639-3/anj |
|
| 279 |
ank http://lexvo.org/id/iso639-3/ank |
|
| 280 |
anl http://lexvo.org/id/iso639-3/anl |
|
| 281 |
anm http://lexvo.org/id/iso639-3/anm |
|
| 282 |
ann http://lexvo.org/id/iso639-3/ann |
|
| 283 |
ano http://lexvo.org/id/iso639-3/ano |
|
| 284 |
anp http://lexvo.org/id/iso639-3/anp |
|
| 285 |
anq http://lexvo.org/id/iso639-3/anq |
|
| 286 |
anr http://lexvo.org/id/iso639-3/anr |
|
| 287 |
ans http://lexvo.org/id/iso639-3/ans |
|
| 288 |
ant http://lexvo.org/id/iso639-3/ant |
|
| 289 |
anu http://lexvo.org/id/iso639-3/anu |
|
| 290 |
anv http://lexvo.org/id/iso639-3/anv |
|
| 291 |
anw http://lexvo.org/id/iso639-3/anw |
|
| 292 |
anx http://lexvo.org/id/iso639-3/anx |
|
| 293 |
any http://lexvo.org/id/iso639-3/any |
|
| 294 |
anz http://lexvo.org/id/iso639-3/anz |
|
| 295 |
aoa http://lexvo.org/id/iso639-3/aoa |
|
| 296 |
aob http://lexvo.org/id/iso639-3/aob |
|
| 297 |
aoc http://lexvo.org/id/iso639-3/aoc |
|
| 298 |
aod http://lexvo.org/id/iso639-3/aod |
|
| 299 |
aoe http://lexvo.org/id/iso639-3/aoe |
|
| 300 |
aof http://lexvo.org/id/iso639-3/aof |
|
| 301 |
aog http://lexvo.org/id/iso639-3/aog |
|
| 302 |
aoh http://lexvo.org/id/iso639-3/aoh |
|
| 303 |
aoi http://lexvo.org/id/iso639-3/aoi |
|
| 304 |
aoj http://lexvo.org/id/iso639-3/aoj |
|
| 305 |
aok http://lexvo.org/id/iso639-3/aok |
|
| 306 |
aol http://lexvo.org/id/iso639-3/aol |
|
| 307 |
aom http://lexvo.org/id/iso639-3/aom |
|
| 308 |
aon http://lexvo.org/id/iso639-3/aon |
|
| 309 |
aor http://lexvo.org/id/iso639-3/aor |
|
| 310 |
aos http://lexvo.org/id/iso639-3/aos |
|
| 311 |
aot http://lexvo.org/id/iso639-3/aot |
|
| 312 |
aou http://lexvo.org/id/iso639-3/aou |
|
| 313 |
aox http://lexvo.org/id/iso639-3/aox |
|
| 314 |
aoz http://lexvo.org/id/iso639-3/aoz |
|
| 315 |
apb http://lexvo.org/id/iso639-3/apb |
|
| 316 |
apc http://lexvo.org/id/iso639-3/apc |
|
| 317 |
apd http://lexvo.org/id/iso639-3/apd |
|
| 318 |
ape http://lexvo.org/id/iso639-3/ape |
|
| 319 |
apf http://lexvo.org/id/iso639-3/apf |
|
| 320 |
apg http://lexvo.org/id/iso639-3/apg |
|
| 321 |
aph http://lexvo.org/id/iso639-3/aph |
|
| 322 |
api http://lexvo.org/id/iso639-3/api |
|
| 323 |
apj http://lexvo.org/id/iso639-3/apj |
|
| 324 |
apk http://lexvo.org/id/iso639-3/apk |
|
| 325 |
apl http://lexvo.org/id/iso639-3/apl |
|
| 326 |
apm http://lexvo.org/id/iso639-3/apm |
|
| 327 |
apn http://lexvo.org/id/iso639-3/apn |
|
| 328 |
apo http://lexvo.org/id/iso639-3/apo |
|
| 329 |
app http://lexvo.org/id/iso639-3/app |
|
| 330 |
apq http://lexvo.org/id/iso639-3/apq |
|
| 331 |
apr http://lexvo.org/id/iso639-3/apr |
|
| 332 |
aps http://lexvo.org/id/iso639-3/aps |
|
| 333 |
apt http://lexvo.org/id/iso639-3/apt |
|
| 334 |
apu http://lexvo.org/id/iso639-3/apu |
|
| 335 |
apv http://lexvo.org/id/iso639-3/apv |
|
| 336 |
apw http://lexvo.org/id/iso639-3/apw |
|
| 337 |
apx http://lexvo.org/id/iso639-3/apx |
|
| 338 |
apy http://lexvo.org/id/iso639-3/apy |
|
| 339 |
apz http://lexvo.org/id/iso639-3/apz |
|
| 340 |
aqc http://lexvo.org/id/iso639-3/aqc |
|
| 341 |
aqd http://lexvo.org/id/iso639-3/aqd |
|
| 342 |
aqg http://lexvo.org/id/iso639-3/aqg |
|
| 343 |
aqm http://lexvo.org/id/iso639-3/aqm |
|
| 344 |
aqn http://lexvo.org/id/iso639-3/aqn |
|
| 345 |
aqp http://lexvo.org/id/iso639-3/aqp |
|
| 346 |
aqr http://lexvo.org/id/iso639-3/aqr |
|
| 347 |
aqz http://lexvo.org/id/iso639-3/aqz |
|
| 348 |
ara http://lexvo.org/id/iso639-3/ara |
|
| 349 |
arb http://lexvo.org/id/iso639-3/arb |
|
| 350 |
arc http://lexvo.org/id/iso639-3/arc |
|
| 351 |
ard http://lexvo.org/id/iso639-3/ard |
|
| 352 |
are http://lexvo.org/id/iso639-3/are |
|
| 353 |
arg http://lexvo.org/id/iso639-3/arg |
|
| 354 |
arh http://lexvo.org/id/iso639-3/arh |
|
| 355 |
ari http://lexvo.org/id/iso639-3/ari |
|
| 356 |
arj http://lexvo.org/id/iso639-3/arj |
|
| 357 |
ark http://lexvo.org/id/iso639-3/ark |
|
| 358 |
arl http://lexvo.org/id/iso639-3/arl |
|
| 359 |
arn http://lexvo.org/id/iso639-3/arn |
|
| 360 |
aro http://lexvo.org/id/iso639-3/aro |
|
| 361 |
arp http://lexvo.org/id/iso639-3/arp |
|
| 362 |
arq http://lexvo.org/id/iso639-3/arq |
|
| 363 |
arr http://lexvo.org/id/iso639-3/arr |
|
| 364 |
ars http://lexvo.org/id/iso639-3/ars |
|
| 365 |
aru http://lexvo.org/id/iso639-3/aru |
|
| 366 |
arv http://lexvo.org/id/iso639-3/arv |
|
| 367 |
arw http://lexvo.org/id/iso639-3/arw |
|
| 368 |
arx http://lexvo.org/id/iso639-3/arx |
|
| 369 |
ary http://lexvo.org/id/iso639-3/ary |
|
| 370 |
arz http://lexvo.org/id/iso639-3/arz |
|
| 371 |
asa http://lexvo.org/id/iso639-3/asa |
|
| 372 |
asb http://lexvo.org/id/iso639-3/asb |
|
| 373 |
asc http://lexvo.org/id/iso639-3/asc |
|
| 374 |
asd http://lexvo.org/id/iso639-3/asd |
|
| 375 |
ase http://lexvo.org/id/iso639-3/ase |
|
| 376 |
asf http://lexvo.org/id/iso639-3/asf |
|
| 377 |
asg http://lexvo.org/id/iso639-3/asg |
|
| 378 |
ash http://lexvo.org/id/iso639-3/ash |
|
| 379 |
asi http://lexvo.org/id/iso639-3/asi |
|
| 380 |
asj http://lexvo.org/id/iso639-3/asj |
|
| 381 |
ask http://lexvo.org/id/iso639-3/ask |
|
| 382 |
asl http://lexvo.org/id/iso639-3/asl |
|
| 383 |
asm http://lexvo.org/id/iso639-3/asm |
|
| 384 |
asn http://lexvo.org/id/iso639-3/asn |
|
| 385 |
aso http://lexvo.org/id/iso639-3/aso |
|
| 386 |
asp http://lexvo.org/id/iso639-3/asp |
|
| 387 |
asq http://lexvo.org/id/iso639-3/asq |
|
| 388 |
asr http://lexvo.org/id/iso639-3/asr |
|
| 389 |
ass http://lexvo.org/id/iso639-3/ass |
|
| 390 |
ast http://lexvo.org/id/iso639-3/ast |
|
| 391 |
asu http://lexvo.org/id/iso639-3/asu |
|
| 392 |
asv http://lexvo.org/id/iso639-3/asv |
|
| 393 |
asw http://lexvo.org/id/iso639-3/asw |
|
| 394 |
asx http://lexvo.org/id/iso639-3/asx |
|
| 395 |
asy http://lexvo.org/id/iso639-3/asy |
|
| 396 |
asz http://lexvo.org/id/iso639-3/asz |
|
| 397 |
ata http://lexvo.org/id/iso639-3/ata |
|
| 398 |
atb http://lexvo.org/id/iso639-3/atb |
|
| 399 |
atc http://lexvo.org/id/iso639-3/atc |
|
| 400 |
atd http://lexvo.org/id/iso639-3/atd |
|
| 401 |
ate http://lexvo.org/id/iso639-3/ate |
|
| 402 |
atg http://lexvo.org/id/iso639-3/atg |
|
| 403 |
ati http://lexvo.org/id/iso639-3/ati |
|
| 404 |
atj http://lexvo.org/id/iso639-3/atj |
|
| 405 |
atk http://lexvo.org/id/iso639-3/atk |
|
| 406 |
atl http://lexvo.org/id/iso639-3/atl |
|
| 407 |
atm http://lexvo.org/id/iso639-3/atm |
|
| 408 |
atn http://lexvo.org/id/iso639-3/atn |
|
| 409 |
ato http://lexvo.org/id/iso639-3/ato |
|
| 410 |
atp http://lexvo.org/id/iso639-3/atp |
|
| 411 |
atq http://lexvo.org/id/iso639-3/atq |
|
| 412 |
atr http://lexvo.org/id/iso639-3/atr |
|
| 413 |
ats http://lexvo.org/id/iso639-3/ats |
|
| 414 |
att http://lexvo.org/id/iso639-3/att |
|
| 415 |
atu http://lexvo.org/id/iso639-3/atu |
|
| 416 |
atv http://lexvo.org/id/iso639-3/atv |
|
| 417 |
atw http://lexvo.org/id/iso639-3/atw |
|
| 418 |
atx http://lexvo.org/id/iso639-3/atx |
|
| 419 |
aty http://lexvo.org/id/iso639-3/aty |
|
| 420 |
atz http://lexvo.org/id/iso639-3/atz |
|
| 421 |
aua http://lexvo.org/id/iso639-3/aua |
|
| 422 |
aub http://lexvo.org/id/iso639-3/aub |
|
| 423 |
auc http://lexvo.org/id/iso639-3/auc |
|
| 424 |
aud http://lexvo.org/id/iso639-3/aud |
|
| 425 |
aue http://lexvo.org/id/iso639-3/aue |
|
| 426 |
aug http://lexvo.org/id/iso639-3/aug |
|
| 427 |
auh http://lexvo.org/id/iso639-3/auh |
|
| 428 |
aui http://lexvo.org/id/iso639-3/aui |
|
| 429 |
auj http://lexvo.org/id/iso639-3/auj |
|
| 430 |
auk http://lexvo.org/id/iso639-3/auk |
|
| 431 |
aul http://lexvo.org/id/iso639-3/aul |
|
| 432 |
aum http://lexvo.org/id/iso639-3/aum |
|
| 433 |
aun http://lexvo.org/id/iso639-3/aun |
|
| 434 |
auo http://lexvo.org/id/iso639-3/auo |
|
| 435 |
aup http://lexvo.org/id/iso639-3/aup |
|
| 436 |
auq http://lexvo.org/id/iso639-3/auq |
|
| 437 |
aur http://lexvo.org/id/iso639-3/aur |
|
| 438 |
aut http://lexvo.org/id/iso639-3/aut |
|
| 439 |
auu http://lexvo.org/id/iso639-3/auu |
|
Also available in: Unified diff
dnet45