Project

General

Profile

« Previous | Next » 

Revision 51177

Added by Claudio Atzori over 6 years ago

adapted the broker event generation workflow to directly feed ElasticSearch

View differences:

calculateEnrichmentEvents.xml
20 20 
					<PARAM required="true" type="string" name="table" managedBy="user"/>
21 21 
				</PARAMETERS>
22 22 
				<ARCS>
23 
					<ARC to="cleanupHDFS"/>
23 
					<ARC to="enrichment"/>
24 24 
				</ARCS>
25 25 
			</NODE>
26 26 

  		




  27
  
  
    
			<NODE name="setHdfsFile" type="SetHdfsFile" isStart="true">


  		




  28
  
  
    
				<DESCRIPTION/>


  		




  29
  
  
    
				<PARAMETERS>


  		




  30
  
  
    
					<PARAM name="hdfsPath" type="string" managedBy="user" required="true">/tmp/brokerEvents.tmp.seq</PARAM>


  		




  31
  
  
    
					<PARAM name="hdfsPathParam" type="string" managedBy="system" required="true">brokerEventsPath</PARAM>


  		




  32
  
  
    
				</PARAMETERS>


  		




  33
  
  
    
				<ARCS>


  		




  34
  
  
    
					<ARC to="cleanupHDFS"/>


  		




  35
  
  
    
				</ARCS>


  		




  36
  
  
    
			</NODE>


  		




  37
  
  
    

  		




  38
  27
  
    
			<NODE name="setDedupConfigs" type="SetDedupConfiguration" isStart="true">


  		




  39
  28
  
    
				<DESCRIPTION>Set Dedup conf</DESCRIPTION>


  		




  40
  29
  
    
				<PARAMETERS>


  		




  ......




  42
  31
  
    
					<PARAM required="true" type="string" name="dedupConfigSequenceParam" managedBy="system">dedup.conf.queue</PARAM>


  		




  43
  32
  
    
				</PARAMETERS>


  		




  44
  33
  
    
				<ARCS>


  		




  45
  
  
    
					<ARC to="cleanupHDFS"/>


  		




  46
  
  
    
				</ARCS>


  		




  47
  
  
    
			</NODE>


  		




  48
  
  
    

  		




  49
  
  
    
			<NODE name="cleanupHDFS" type="DeleteHdfsPathJob" isJoin="true">


  		




  50
  
  
    
				<DESCRIPTION>hdfs cleanup (xml)</DESCRIPTION>


  		




  51
  
  
    
				<PARAMETERS>


  		




  52
  
  
    
					<PARAM required="true" type="string" name="cluster" managedBy="system">DM</PARAM>


  		




  53
  
  
    
					<PARAM required="true" type="string" name="envParams" managedBy="system">


  		




  54
  
  
    
						{


  		




  55
  
  
    
						'path' : 'brokerEventsPath'


  		




  56
  
  
    
						}


  		




  57
  
  
    
					</PARAM>


  		




  58
  
  
    
				</PARAMETERS>


  		




  59
  
  
    
				<ARCS>


  		




  60
  34
  
    
					<ARC to="enrichment"/>


  		




  61
  35
  
    
				</ARCS>


  		




  62
  36
  
    
			</NODE>


  		




  63
  37
  
    

  		




  64
  38
  
    

  		




  65
  
  
    
			<NODE name="enrichment" type="DedupConfigurationAwareJob">


  		




  
  39
  
    
			<NODE name="enrichment" type="DedupConfigurationAwareJob" isJoin="true">


  		




  66
  40
  
    
				<DESCRIPTION>Run M/R Job</DESCRIPTION>


  		




  67
  41
  
    
				<PARAMETERS>


  		




  68
  42
  
    
					<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>


  		




  ......




  70
  44
  
    
					<PARAM required="true" type="string" name="dedupConfigSequenceParam" managedBy="system">dedup.conf.queue</PARAM>


  		




  71
  45
  
    
					<PARAM managedBy="system" name="envParams" required="false" type="string">


  		




  72
  46
  
    
						{


  		




  73
  
  
    
						'mapred.output.dir' : 'brokerEventsPath',


  		




  74
  47
  
    
						'hbase.mapred.inputtable' : 'tableName',


  		




  75
  48
  
    
						'hbase.mapreduce.inputtable' : 'tableName'


  		




  76
  49
  
    
						}


  		












Also available in:  Unified diff