Project

General

Profile

« Previous | Next » 

Revision 52823

Added by Claudio Atzori almost 6 years ago

adjusted ids export procedure

View differences:

modules/dnet-mapreduce-jobs/branches/beta/src/main/java/eu/dnetlib/data/mapreduce/hbase/dataexport/ExportResultIdentifiersMapper.java
4 4 
import java.util.ArrayList;
5 5 
import java.util.Collections;
6 6 
import java.util.List;
7 
import java.util.Map;
7 8 
import java.util.stream.Collectors;
8 9 

  		




  9
  10
  
    
import com.google.common.collect.Iterables;


  		




  10
  11
  
    
import com.google.common.collect.Lists;


  		




  11
  12
  
    
import eu.dnetlib.data.mapreduce.util.DedupUtils;


  		




  12
  13
  
    
import eu.dnetlib.data.mapreduce.util.OafDecoder;


  		




  
  14
  
    
import eu.dnetlib.data.proto.FieldTypeProtos.StructuredProperty;


  		




  13
  15
  
    
import eu.dnetlib.data.proto.OafProtos.OafEntity;


  		




  14
  16
  
    
import org.apache.commons.logging.Log;


  		




  15
  17
  
    
import org.apache.commons.logging.LogFactory;


  		




  ......




  60
  62
  
    

  		




  61
  63
  
    
			id.setDeleted(d.getOaf().getDataInfo().getDeletedbyinference());


  		




  62
  64
  
    
			id.setId(d.getEntityId());


  		




  63
  
  
    
			id.setPids(d.getOaf().getEntity().getPidList().stream().map(p -> p.getValue()).collect(Collectors.toList()));


  		




  
  65
  
    
			id.setPids(d.getOaf().getEntity().getPidList().stream()


  		




  
  66
  
    
					.collect(Collectors.groupingBy(


  		




  
  67
  
    
							p -> p.getQualifier().getClassid()


  		




  
  68
  
    
					)).entrySet().stream()


  		




  
  69
  
    
					.collect(Collectors.toMap(


  		




  
  70
  
    
							Map.Entry::getKey,


  		




  
  71
  
    
							e -> e.getValue().stream()


  		




  
  72
  
    
									.map(StructuredProperty::getValue)


  		




  
  73
  
    
									.collect(Collectors.toList()))));


  		




  64
  74
  
    

  		




  
  75
  
    

  		




  65
  76
  
    
			final List<OafEntity> childrenList = d.getEntity().getChildrenList();


  		




  66
  77
  
    
			if (childrenList != null && !childrenList.isEmpty()) {


  		




  67
  78
  
    
				final ArrayList<String> mergedIds = Lists.newArrayList(Iterables.transform(childrenList, oafEntity -> oafEntity.getId()));


  		












  

    
      modules/dnet-mapreduce-jobs/branches/beta/src/main/java/eu/dnetlib/data/mapreduce/hbase/dataexport/OpenaireEntityId.java
    
  





  1
  1
  
    
package eu.dnetlib.data.mapreduce.hbase.dataexport;


  		




  2
  2
  
    

  		




  3
  3
  
    
import java.util.List;


  		




  
  4
  
    
import java.util.Map;


  		




  4
  5
  
    

  		




  5
  6
  
    
import com.google.gson.Gson;


  		




  6
  7
  
    

  		




  ......




  11
  12
  
    

  		




  12
  13
  
    
	private String id;


  		




  13
  14
  
    

  		




  14
  
  
    
	private List<String> pids;


  		




  
  15
  
    
	private Map<String, List<String>> pids;


  		




  15
  16
  
    

  		




  16
  17
  
    
	private List<String> mergedIds;


  		




  17
  18
  
    

  		




  ......




  34
  35
  
    
		this.id = id;


  		




  35
  36
  
    
	}


  		




  36
  37
  
    

  		




  37
  
  
    
	public List<String> getPids() {


  		




  
  38
  
    
	public Map<String, List<String>> getPids() {


  		




  38
  39
  
    
		return pids;


  		




  39
  40
  
    
	}


  		




  40
  41
  
    

  		




  41
  
  
    
	public void setPids(final List<String> pids) {


  		




  
  42
  
    
	public void setPids(final Map<String, List<String>> pids) {


  		




  42
  43
  
    
		this.pids = pids;


  		




  43
  44
  
    
	}


  		




  44
  45
  
    

  		












Also available in:  Unified diff