Project

General

Profile

« Previous | Next » 

Revision 60988

Added by Claudio Atzori almost 3 years ago

using openaire datasource id instead of the repository profile identifier in the hadoop-based aggregation workflows

View differences:

modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/PrepareEnvCollectHadoopJobNode.java
107 107 
    @Override
108 108 
    protected String execute(final NodeToken token) throws Exception {
109 109 

  		




  110
  
  
    
        final String dsId = token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ID);


  		




  
  110
  
    
        final String dsId = token.getEnv().getAttribute("parentDatasourceId");


  		




  111
  111
  
    
        log.info("dsId: " + dsId);


  		




  112
  112
  
    
        final String apiId = token.getEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE);


  		




  113
  113
  
    
        log.info("apiId: " + apiId);


  		












  

    
      modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/dhp/CreateMDStoreHadoopJobNode.java
    
  





  4
  4
  
    
import java.util.Map;


  		




  5
  5
  
    

  		




  6
  6
  
    
import com.google.gson.Gson;


  		




  
  7
  
    
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;


  		




  
  8
  
    
import eu.dnetlib.enabling.locators.UniqueServiceLocator;


  		




  
  9
  
    
import eu.dnetlib.msro.rmi.MSROException;


  		




  
  10
  
    
import org.apache.commons.lang3.StringUtils;


  		




  7
  11
  
    
import org.apache.commons.logging.Log;


  		




  8
  12
  
    
import org.apache.commons.logging.LogFactory;


  		




  9
  13
  
    
import org.springframework.beans.factory.annotation.Required;


  		




  ......




  17
  21
  
    
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;


  		




  18
  22
  
    
import eu.dnetlib.msro.workflows.util.WorkflowsConstants;


  		




  19
  23
  
    

  		




  
  24
  
    
import javax.annotation.Resource;


  		




  
  25
  
    

  		




  20
  26
  
    
public class CreateMDStoreHadoopJobNode extends SimpleJobNode {


  		




  21
  27
  
    

  		




  22
  28
  
    
    private static final Log log = LogFactory.getLog(CreateMDStoreHadoopJobNode.class);


  		




  ......




  30
  36
  
    
    /* Spring managed params */


  		




  31
  37
  
    
    private String mdStoreManagerUrl;


  		




  32
  38
  
    

  		




  
  39
  
    
    @Resource


  		




  
  40
  
    
    private UniqueServiceLocator serviceLocator;


  		




  
  41
  
    

  		




  33
  42
  
    
    @Override


  		




  34
  43
  
    
    protected String execute(final NodeToken token) throws Exception {


  		




  35
  44
  
    
        final String url = getMdStoreManagerUrl() + "/new/{format}/{layout}/{interpretation}";


  		




  ......




  39
  48
  
    
        params.put("layout", getLayout());


  		




  40
  49
  
    
        params.put("interpretation", getInterpretation());


  		




  41
  50
  
    

  		




  
  51
  
    
        final String repositoryId = token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID);


  		




  
  52
  
    
        final String xquery = String.format(


  		




  
  53
  
    
                "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType') \n" +


  		




  
  54
  
    
                "where .//RESOURCE_IDENTIFIER[./@value='%s']\n" +


  		




  
  55
  
    
                "return $x//EXTRA_FIELDS/FIELD[./key/text() = 'OpenAireDataSourceId']/value/text()",


  		




  
  56
  
    
                repositoryId);


  		




  
  57
  
    

  		




  
  58
  
    
        final String openAireDataSourceId = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);


  		




  
  59
  
    

  		




  
  60
  
    
        if (StringUtils.isBlank(openAireDataSourceId)) {


  		




  
  61
  
    
            throw new MSROException("unable to find openAireDataSourceId in repository profile " + repositoryId);


  		




  
  62
  
    
        }


  		




  
  63
  
    

  		




  
  64
  
    
        log.info(String.format("got OpenAIRE datasource id '%s' for repository profile '%s'", openAireDataSourceId, repositoryId));


  		




  
  65
  
    

  		




  42
  66
  
    
        final URI uri = UriComponentsBuilder.fromUriString(url)


  		




  43
  67
  
    
                .queryParam("dsName", token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_NAME))


  		




  44
  
  
    
                .queryParam("dsId", token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_ORIGINALID))


  		




  
  68
  
    
                .queryParam("dsId", openAireDataSourceId)


  		




  45
  69
  
    
                .queryParam("apiId", token.getFullEnv().getAttribute(WorkflowsConstants.DATAPROVIDER_INTERFACE))


  		




  46
  70
  
    
                .buildAndExpand(params)


  		




  47
  71
  
    
                .toUri();


  		












  

    
      modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_PubsRepository_Hadoop.xml
    
  





  10
  10
  
    
        <WORKFLOW_NAME>Aggregate Metadata DublinCore format [Hadoop]</WORKFLOW_NAME>


  		




  11
  11
  
    
        <WORKFLOW_INFO>


  		




  12
  12
  
    
            <FIELD name="Action">Aggregate Metadata</FIELD>


  		




  13
  
  
    
            <FIELD name="Consequence IS">Support</FIELD>


  		




  
  13
  
    
            <FIELD name="Consequence IS">Ingestion</FIELD>


  		




  14
  14
  
    
            <FIELD name="Datasource class">PubsRepository</FIELD>


  		




  15
  15
  
    
            <FIELD name="Content">publications</FIELD>


  		




  16
  16
  
    
        </WORKFLOW_INFO>


  		












  

    
      modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/repo-hi/Aggregate_Metadata_from_DataRepository_Hadoop.xml
    
  





  10
  10
  
    
        <WORKFLOW_NAME>Aggregate Metadata Datacite format [Hadoop]</WORKFLOW_NAME>


  		




  11
  11
  
    
        <WORKFLOW_INFO>


  		




  12
  12
  
    
            <FIELD name="Action">Aggregate Metadata</FIELD>


  		




  13
  
  
    
            <FIELD name="Consequence IS">Support</FIELD>


  		




  
  13
  
    
            <FIELD name="Consequence IS">Ingestion</FIELD>


  		




  14
  14
  
    
            <FIELD name="Datasource class">PubsRepository</FIELD>


  		




  15
  15
  
    
            <FIELD name="Content">publications</FIELD>


  		




  16
  16
  
    
        </WORKFLOW_INFO>


  		












Also available in:  Unified diff