Project

General

Profile

« Previous | Next » 

Revision 48701

Added by Sandro La Bruzzo almost 7 years ago

Upgraded incremental harvesting and other stuff

View differences:

FindDateRangeForIncrementalHarvestingJobNode.java
1 1 
package eu.dnetlib.msro.workflows.nodes.collect;
2 2 

  		




  3
  3
  
    
import java.text.SimpleDateFormat;


  		




  4
  
  
    
import java.util.Date;


  		




  5
  
  
    
import java.util.HashMap;


  		




  6
  
  
    
import java.util.Map;


  		




  
  4
  
    
import java.time.Instant;


  		




  
  5
  
    
import java.time.LocalDateTime;


  		




  
  6
  
    
import java.time.ZoneId;


  		




  
  7
  
    
import java.time.ZonedDateTime;


  		




  
  8
  
    
import java.time.format.DateTimeFormatter;


  		




  
  9
  
    
import java.util.*;


  		




  7
  10
  
    

  		




  8
  11
  
    
import javax.annotation.Resource;


  		




  9
  12
  
    

  		




  
  13
  
    
import eu.dnetlib.enabling.tools.DnetStreamSupport;


  		




  
  14
  
    
import eu.dnetlib.miscutils.datetime.DateUtils;


  		




  10
  15
  
    
import org.apache.commons.logging.Log;


  		




  11
  16
  
    
import org.apache.commons.logging.LogFactory;


  		




  12
  17
  
    

  		




  ......




  49
  54
  
    

  		




  50
  55
  
    
	private String calculateFromDate() {


  		




  51
  56
  
    
		final long d = findLastSuccessStartDate();


  		




  52
  
  
    
		return (d > 0) ? (new SimpleDateFormat("yyyy-MM-dd")).format(new Date(d - ONE_DAY)) : null;


  		




  53
  
  
    
	}


  		




  54
  57
  
    

  		




  
  58
  
    

  		




  
  59
  
    
        LocalDateTime zdt = LocalDateTime.ofInstant(Instant.ofEpochMilli(d), ZoneId.of("Etc/UTC"));


  		




  
  60
  
    

  		




  
  61
  
    
        return DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").format(zdt);


  		




  
  62
  
    

  		




  
  63
  
    
//		return (d > 0) ? (new SimpleDateFormat("yyyy-MM-dd")).format(new Date(d - ONE_DAY)) : null;


  		




  
  64
  
    
    }


  		




  
  65
  
    

  		




  55
  66
  
    
	private long findLastSuccessStartDate() {


  		




  56
  67
  
    
		final long res = -1;


  		




  57
  68
  
    

  		




  58
  69
  
    
		// TODO find the right condition to obtain the list of the previous executions


  		




  59
  70
  
    

  		




  60
  
  
    
		final Map<String, Object> query = new HashMap<String, Object>();


  		




  61
  
  
    
		query.put("", "");


  		




  
  71
  
    
        //dnetLogger.


  		




  62
  72
  
    

  		




  63
  
  
    
		/*


  		




  64
  
  
    
		 * final Iterator<Map<String, String>> iter = this.dnetLogger.find(query); while (iter.hasNext()) { final Map<String, String> map =


  		




  65
  
  
    
		 * iter.next(); if ("true".equalsIgnoreCase(map.get(WorkflowsConstants.SYSTEM_COMPLETED_SUCCESSFULLY))) { final long curr =


  		




  66
  
  
    
		 * NumberUtils.toLong(map.get(WorkflowsConstants.SYSTEM_START_DATE), -1); if (curr > res) { res = curr; } } }


  		




  67
  
  
    
		 */


  		




  68
  
  
    
		return res;


  		




  69
  
  
    
	}


  		




  
  73
  
    
        final Map<String, Object> query = new HashMap<>();


  		




  70
  74
  
    

  		




  
  75
  
    
        query.put("system:profileTemplateId", process.getProfileId());


  		




  
  76
  
    
        query.put("system:parentProfileId", process.getParentProfileId());


  		




  
  77
  
    
        query.put("system:processStatus", "SUCCESS");


  		




  
  78
  
    

  		




  
  79
  
    
        Iterator<Map<String, String>> mapIterator = dnetLogger.find(query);


  		




  
  80
  
    
        Optional<Long> maxDate = DnetStreamSupport.generateStreamFromIterator(mapIterator).map(it -> it.get("system:startDate")).map(Long::parseLong).max(Long::compare);


  		




  
  81
  
    

  		




  
  82
  
    
        if (maxDate.isPresent())


  		




  
  83
  
    
            return maxDate.get();


  		




  
  84
  
    
        else


  		




  
  85
  
    
            return -1;


  		




  
  86
  
    
    }


  		




  
  87
  
    

  		




  71
  88
  
    
	public String getFromDateParam() {


  		




  72
  89
  
    
		return this.fromDateParam;


  		




  73
  90
  
    
	}


  		












Also available in:  Unified diff