Project

General

Profile

« Previous | Next » 

Revision 30986

Added by Marek Horst over 9 years ago

#757 fixing pmid and doi matching, fixing sourceDocumentId and destinationDocumentId generation

View differences:

ResolvedCitationsImporter.java
20 20 

  		




  21
  21
  
    
/**


  		




  22
  22
  
    
 * @author Mateusz Fedoryszak (m.fedoryszak@icm.edu.pl)


  		




  
  23
  
    
 * @author mhorst


  		




  23
  24
  
    
 */


  		




  24
  25
  
    
public class ResolvedCitationsImporter {


  		




  25
  26
  
    
    public static void main(String[] args) throws IOException {


  		




  26
  27
  
    
        String inPath = args[0];


  		




  27
  28
  
    
        String dedupPath = args[1];


  		




  28
  
  
    
        String idsPath = args[2];


  		




  29
  
  
    
        String outPath = args[3];


  		




  
  29
  
    
        String pmidToOaidPath = args[2];


  		




  
  30
  
    
        String doiToOaidPath = args[3];


  		




  
  31
  
    
        String outPath = args[4];


  		




  30
  32
  
    

  		




  31
  33
  
    
        Properties properties = new Properties();


  		




  32
  34
  
    
        AppProps.setApplicationJarClass(properties, ResolvedCitationsImporter.class);


  		




  ......




  37
  39
  
    

  		




  38
  40
  
    
        Tap docTap = new Hfs(new AvroScheme(DocumentText.getClassSchema()), inPath);


  		




  39
  41
  
    
        Tap dedupTap = new Hfs(new AvroScheme(DeduplicationMapping.getClassSchema()), dedupPath);


  		




  40
  
  
    
        Tap idsTap = new Hfs(new AvroScheme(DocumentId.getClassSchema()), idsPath);


  		




  
  42
  
    
        Tap pmidToOaidTap = new Hfs(new AvroScheme(DeduplicationMapping.getClassSchema()), pmidToOaidPath);


  		




  
  43
  
    
        Tap doiToOaidTap = new Hfs(new AvroScheme(DeduplicationMapping.getClassSchema()), doiToOaidPath);


  		




  41
  44
  
    

  		




  42
  45
  
    
        Pipe docPipe = new Pipe("doc");


  		




  43
  46
  
    
        Pipe dedupMapPipe = new Pipe("dedup");


  		




  44
  
  
    
        Pipe existentDocIdsPipe = new Pipe("existent_ids");


  		




  
  47
  
    
        Pipe pmidToOaidPipe = new Pipe("pmid_to_oaid");


  		




  
  48
  
    
        Pipe doiToOaidPipe = new Pipe("doi_to_oaid");


  		




  45
  49
  
    

  		




  46
  50
  
    
        Tap outTap = new Hfs(new PackedAvroScheme<Citation>(Citation.getClassSchema()), outPath);


  		




  47
  51
  
    

  		




  48
  
  
    
        SubAssembly main = new ResolvedCitationsSubAssembly(docPipe, dedupMapPipe, existentDocIdsPipe);


  		




  
  52
  
    
        SubAssembly main = new ResolvedCitationsSubAssembly(docPipe, dedupMapPipe, 


  		




  
  53
  
    
        		pmidToOaidPipe, doiToOaidPipe);


  		




  49
  54
  
    

  		




  50
  55
  
    
        FlowDef flowDef = FlowDef.flowDef()


  		




  51
  56
  
    
                .addSource(docPipe, docTap)


  		




  52
  57
  
    
                .addSource(dedupMapPipe, dedupTap)


  		




  53
  
  
    
                .addSource(existentDocIdsPipe, idsTap)


  		




  
  58
  
    
                .addSource(pmidToOaidPipe, pmidToOaidTap)


  		




  
  59
  
    
                .addSource(doiToOaidPipe, doiToOaidTap)


  		




  54
  60
  
    
                .addTailSink(main.getTails()[0], outTap);


  		




  55
  61
  
    

  		




  56
  62
  
    
        Flow flow = flowConnector.connect(flowDef);


  		












Also available in:  Unified diff