Project

General

Profile

« Previous | Next » 

Revision 51177

adapted the broker event generation workflow to directly feed ElasticSearch

View differences:

modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentProjectEvents.xml
24 24
				</ARCS>
25 25
			</NODE>
26 26

  
27
			<NODE name="setHdfsFile" type="SetHdfsFile" isStart="true">
28
				<DESCRIPTION/>
29
				<PARAMETERS>
30
					<PARAM name="hdfsPath" type="string" managedBy="user" required="true">/tmp/brokerEvents.tmp.seq</PARAM>
31
					<PARAM name="hdfsPathParam" type="string" managedBy="system" required="true">brokerEventsPath</PARAM>
32
				</PARAMETERS>
33
				<ARCS>
34
					<ARC to="cleanupHDFS"/>
35
				</ARCS>
36
			</NODE>
37

  
38 27
			<NODE name="setDedupConfigs" type="SetDedupConfiguration" isStart="true">
39 28
				<DESCRIPTION>Set Dedup conf</DESCRIPTION>
40 29
				<PARAMETERS>
......
70 59
					<PARAM required="true" type="string" name="dedupConfigSequenceParam" managedBy="system">dedup.conf.queue</PARAM>
71 60
					<PARAM managedBy="system" name="envParams" required="false" type="string">
72 61
						{
73
						'mapred.output.dir' : 'brokerEventsPath',
74 62
						'hbase.mapred.inputtable' : 'tableName',
75 63
						'hbase.mapreduce.inputtable' : 'tableName'
76 64
						}
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/broker/calculateEnrichmentEvents.xml
20 20
					<PARAM required="true" type="string" name="table" managedBy="user"/>
21 21
				</PARAMETERS>
22 22
				<ARCS>
23
					<ARC to="cleanupHDFS"/>
23
					<ARC to="enrichment"/>
24 24
				</ARCS>
25 25
			</NODE>
26 26

  
27
			<NODE name="setHdfsFile" type="SetHdfsFile" isStart="true">
28
				<DESCRIPTION/>
29
				<PARAMETERS>
30
					<PARAM name="hdfsPath" type="string" managedBy="user" required="true">/tmp/brokerEvents.tmp.seq</PARAM>
31
					<PARAM name="hdfsPathParam" type="string" managedBy="system" required="true">brokerEventsPath</PARAM>
32
				</PARAMETERS>
33
				<ARCS>
34
					<ARC to="cleanupHDFS"/>
35
				</ARCS>
36
			</NODE>
37

  
38 27
			<NODE name="setDedupConfigs" type="SetDedupConfiguration" isStart="true">
39 28
				<DESCRIPTION>Set Dedup conf</DESCRIPTION>
40 29
				<PARAMETERS>
......
42 31
					<PARAM required="true" type="string" name="dedupConfigSequenceParam" managedBy="system">dedup.conf.queue</PARAM>
43 32
				</PARAMETERS>
44 33
				<ARCS>
45
					<ARC to="cleanupHDFS"/>
46
				</ARCS>
47
			</NODE>
48

  
49
			<NODE name="cleanupHDFS" type="DeleteHdfsPathJob" isJoin="true">
50
				<DESCRIPTION>hdfs cleanup (xml)</DESCRIPTION>
51
				<PARAMETERS>
52
					<PARAM required="true" type="string" name="cluster" managedBy="system">DM</PARAM>
53
					<PARAM required="true" type="string" name="envParams" managedBy="system">
54
						{
55
						'path' : 'brokerEventsPath'
56
						}
57
					</PARAM>
58
				</PARAMETERS>
59
				<ARCS>
60 34
					<ARC to="enrichment"/>
61 35
				</ARCS>
62 36
			</NODE>
63 37

  
64 38

  
65
			<NODE name="enrichment" type="DedupConfigurationAwareJob">
39
			<NODE name="enrichment" type="DedupConfigurationAwareJob" isJoin="true">
66 40
				<DESCRIPTION>Run M/R Job</DESCRIPTION>
67 41
				<PARAMETERS>
68 42
					<PARAM managedBy="system" name="cluster" required="true" type="string">DM</PARAM>
......
70 44
					<PARAM required="true" type="string" name="dedupConfigSequenceParam" managedBy="system">dedup.conf.queue</PARAM>
71 45
					<PARAM managedBy="system" name="envParams" required="false" type="string">
72 46
						{
73
						'mapred.output.dir' : 'brokerEventsPath',
74 47
						'hbase.mapred.inputtable' : 'tableName',
75 48
						'hbase.mapreduce.inputtable' : 'tableName'
76 49
						}

Also available in: Unified diff