Project

General

Profile

« Previous | Next » 

Revision 40728

Added by Giorgos Alexiou over 8 years ago

epitelous

View differences:

LodImportReducer.java
1 1 
package eu.dnetlib.data.mapreduce.hbase.lodImport;
2 2 

  		




  3
  
  
    
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.DB;


  		




  4
  
  
    
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.RDFizer;


  		




  
  3
  
    
import java.io.IOException;


  		




  
  4
  
    
import java.util.Iterator;


  		




  
  5
  
    
import java.util.List;


  		




  
  6
  
    

  		




  
  7
  
    
import javax.sql.DataSource;


  		




  
  8
  
    

  		




  
  9
  
    
import org.apache.hadoop.conf.Configuration;


  		




  5
  10
  
    
import org.apache.hadoop.io.Text;


  		




  6
  11
  
    
import org.apache.hadoop.mapreduce.Reducer;


  		




  7
  12
  
    
import org.apache.log4j.Logger;


  		




  8
  13
  
    
import org.json.JSONObject;


  		




  9
  14
  
    

  		




  10
  
  
    
import javax.sql.DataSource;


  		




  11
  
  
    
import java.io.IOException;


  		




  12
  
  
    
import java.util.Iterator;


  		




  13
  
  
    
import java.util.List;


  		




  
  15
  
    
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.DB;


  		




  
  16
  
    
import eu.dnetlib.data.mapreduce.hbase.lodImport.utils.RDFizer;


  		




  14
  17
  
    

  		




  15
  18
  
    

  		




  16
  19
  
    
public class LodImportReducer extends Reducer<Text, List<Text>, Text, Text> {


  		




  ......




  18
  21
  
    
    private Logger log = Logger.getLogger(LodImportReducer.class);


  		




  19
  22
  
    
    private DataSource ds;


  		




  20
  23
  
    
    private String fileName = "";


  		




  
  24
  
    
    private Configuration hadoopConf; 


  		




  21
  25
  
    

  		




  22
  26
  
    
    @Override


  		




  23
  27
  
    
    protected void setup(Context context) throws IOException, InterruptedException {


  		




  24
  28
  
    
        this.fileName = context.getConfiguration().get("lod.inputFile");


  		




  
  29
  
    
        this.hadoopConf = context.getConfiguration();


  		




  25
  30
  
    
        try {


  		




  26
  31
  
    
            ds = DB.getDatasource(context.getConfiguration().get("lod.conLine"), context.getConfiguration().get("lod.username"), context.getConfiguration().get("lod.password"), context.getConfiguration().get("lod.minCpart"), context.getConfiguration().get("lod.maxCpart"), context.getConfiguration().get("lod.part"));


  		




  27
  32
  
    
        } catch (Exception e) {


  		




  28
  33
  
    
            log.error("", e);


  		




  29
  
  
    

  		




  
  34
  
    
            	


  		




  30
  35
  
    
        }


  		




  31
  36
  
    

  		




  32
  37
  
    
    }


  		




  ......




  36
  41
  
    
        JSONObject entitiesMappings = new JSONObject(context.getConfiguration().get("lod.jsonEntities"));


  		




  37
  42
  
    
        JSONObject relationsMappings = new JSONObject(context.getConfiguration().get("lod.jsonRels"));


  		




  38
  43
  
    

  		




  39
  
  
    
        List<Text> row = null;


  		




  40
  
  
    
        if (fileName.startsWith("entities")) RDFizer.RDFizeEntityRow(row, ds, entitiesMappings);


  		




  41
  
  
    
        else RDFizer.RDFizeRelationRow(row, ds, relationsMappings);


  		




  42
  
  
    

  		




  43
  44
  
    
//		Connection conn = ds.getConnection();


  		




  44
  45
  
    

  		




  45
  46
  
    
        Iterator<List<Text>> it = values.iterator();


  		




  46
  47
  
    

  		




  47
  48
  
    
        while (it.hasNext()) {


  		




  48
  
  
    
            List<Text> value = it.next();


  		




  
  49
  
    
            List<Text> row = it.next();


  		




  49
  50
  
    

  		




  50
  
  
    
            //TODO do stuff


  		




  51
  
  
    

  		




  52
  51
  
    
            System.out.println(" Values : ");


  		




  53
  52
  
    
            log.info(" Values : ");


  		




  54
  
  
    
            for (Text v : value) {


  		




  55
  
  
    
                System.out.println(v + " ");


  		




  56
  
  
    
                log.info(v + " ");


  		




  57
  
  
    
            }


  		




  
  53
  
    
            if (fileName.startsWith("entities")) RDFizer.RDFizeEntityRow(row, ds, entitiesMappings,hadoopConf);


  		




  
  54
  
    
            else RDFizer.RDFizeRelationRow(row, ds, relationsMappings,hadoopConf);


  		




  
  55
  
    
            


  		




  
  56
  
    
//            for (Text v : value) {


  		




  
  57
  
    
//                System.out.println(v + " ");


  		




  
  58
  
    
//                log.info(v + " ");


  		




  
  59
  
    
//            }


  		




  58
  60
  
    

  		




  59
  61
  
    
        }


  		




  60
  62
  
    

  		












Also available in:  Unified diff