Project

General

Profile

« Previous | Next » 

Revision 29657

Added by Claudio Atzori over 10 years ago

defined limit to the maximum number of counters

View differences:

modules/dnet-mapreduce-jobs/trunk/src/main/java/eu/dnetlib/data/mapreduce/hbase/dedup/DedupPersonReducer.java
33 33 

  		




  34
  34
  
    
	private static final int MAX_Q_SIZE = 3000;


  		




  35
  35
  
    

  		




  
  36
  
    
	private static final int MAX_COUNTERS = 20;


  		




  
  37
  
    

  		




  36
  38
  
    
	private DedupConfig dedupConf;


  		




  37
  39
  
    

  		




  38
  40
  
    
	@Override


  		




  ......




  45
  47
  
    
		try {


  		




  46
  48
  
    
			Queue<OafDecoder> q = prepare(key, values, context);


  		




  47
  49
  
    
			if (q.size() > 1) {


  		




  48
  
  
    
				context.getCounter(dedupConf.getEntityType() + " count", lpad(q.size())).increment(1);


  		




  
  50
  
    

  		




  
  51
  
    
				if (q.size() < MAX_COUNTERS) {


  		




  
  52
  
    
					context.getCounter(dedupConf.getEntityType() + " count", lpad(q.size())).increment(1);


  		




  
  53
  
    
				} else {


  		




  
  54
  
    
					context.getCounter(dedupConf.getEntityType() + " count", "> " + MAX_COUNTERS).increment(1);


  		




  
  55
  
    
				}


  		




  49
  56
  
    
				String min = findMin(Iterables.transform(q, OafUtils.idDecoder()));


  		




  50
  57
  
    
				if (min == null) {


  		




  51
  58
  
    
					context.getCounter(dedupConf.getEntityType(), "unable to find min").increment(1);


  		












Also available in:  Unified diff