Project

General

Profile

« Previous | Next » 

Revision 29030

Added by Alessia Bardi about 10 years ago

Always new records to test how faster we go

View differences:

modules/dnet-mapreduce-jobs/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/oai/OaiFeedMapper.java
32 32 
import eu.dnetlib.data.mapreduce.hbase.oai.utils.PublisherField;
33 33 
import eu.dnetlib.data.mapreduce.hbase.oai.utils.RecordFieldsExtractor;
34 34 
import eu.dnetlib.data.mapreduce.hbase.oai.utils.SetInfo;
35 
import eu.dnetlib.miscutils.functional.xml.IndentXmlString;
35 36 

  		




  36
  37
  
    
public class OaiFeedMapper extends Mapper<Text, Text, NullWritable, NullWritable> {


  		




  37
  38
  
    

  		




  ......




  78
  79
  
    
		layout = formatIntLayout[2];


  		




  79
  80
  
    

  		




  80
  81
  
    
		String oaiConfigurationProfile = context.getConfiguration().get("oaiConfiguration");


  		




  81
  
  
    
		System.out.println("oaiConfiguration:\n" + oaiConfigurationProfile);


  		




  
  82
  
    
		System.out.println("oaiConfiguration:\n" + IndentXmlString.apply(oaiConfigurationProfile));


  		




  82
  83
  
    
		oaiConfigurationReader = new OAIConfigurationStringReader(oaiConfigurationProfile);


  		




  83
  84
  
    
		oaiConfiguration = oaiConfigurationReader.getOaiConfiguration();


  		




  84
  85
  
    

  		




  ......




  195
  196
  
    
	}


  		




  196
  197
  
    

  		




  197
  198
  
    
	private boolean isNewRecord(final String oaiIdentifier) {


  		




  198
  
  
    
		return this.collection.findOne(new BasicDBObject(OAIConfigurationReader.ID_FIELD, oaiIdentifier)) == null;


  		




  
  199
  
    
		// return this.collection.findOne(new BasicDBObject(OAIConfigurationReader.ID_FIELD, oaiIdentifier)) == null;


  		




  
  200
  
    
		return true;


  		




  199
  201
  
    
	}


  		




  200
  202
  
    

  		




  201
  203
  
    
	/**


  		












Also available in:  Unified diff