Revision 34908
Added by Marek Horst over 9 years ago
modules/icm-iis-transformers/trunk/src/main/resources/eu/dnetlib/iis/transformers/common/union4/oozie_app/lib/scripts/union.pig | ||
---|---|---|
1 |
define avro_load_input_a |
|
2 |
org.apache.pig.piggybank.storage.avro.AvroStorage( |
|
3 |
'input_schema_class', '$schema'); |
|
4 |
|
|
5 |
define avro_load_input_b |
|
6 |
org.apache.pig.piggybank.storage.avro.AvroStorage( |
|
7 |
'input_schema_class', '$schema'); |
|
8 |
|
|
9 |
define avro_load_input_c |
|
10 |
org.apache.pig.piggybank.storage.avro.AvroStorage( |
|
11 |
'input_schema_class', '$schema'); |
|
12 |
|
|
13 |
define avro_load_input_d |
|
14 |
org.apache.pig.piggybank.storage.avro.AvroStorage( |
|
15 |
'input_schema_class', '$schema'); |
|
16 |
|
|
17 |
define avro_store_output |
|
18 |
org.apache.pig.piggybank.storage.avro.AvroStorage( |
|
19 |
'index', '0', |
|
20 |
'output_schema_class', '$schema'); |
|
21 |
|
|
22 |
input_a = load '$input_a' using avro_load_input_a; |
|
23 |
input_b = load '$input_b' using avro_load_input_b; |
|
24 |
input_c = load '$input_c' using avro_load_input_c; |
|
25 |
input_d = load '$input_d' using avro_load_input_d; |
|
26 |
|
|
27 |
output_final = union input_a, input_b, input_c, input_d; |
|
28 |
|
|
29 |
store output_final into '$output' using avro_store_output; |
modules/icm-iis-transformers/trunk/src/main/resources/eu/dnetlib/iis/transformers/common/union4/oozie_app/workflow.xml | ||
---|---|---|
1 |
<workflow-app xmlns="uri:oozie:workflow:0.4" name="transformers_common_union4"> |
|
2 |
|
|
3 |
<parameters> |
|
4 |
<property> |
|
5 |
<name>input_a</name> |
|
6 |
</property> |
|
7 |
<property> |
|
8 |
<name>input_b</name> |
|
9 |
</property> |
|
10 |
<property> |
|
11 |
<name>input_c</name> |
|
12 |
</property> |
|
13 |
<property> |
|
14 |
<name>input_d</name> |
|
15 |
</property> |
|
16 |
<property> |
|
17 |
<name>output</name> |
|
18 |
</property> |
|
19 |
<property> |
|
20 |
<name>schema</name> |
|
21 |
</property> |
|
22 |
</parameters> |
|
23 |
|
|
24 |
<start to="transformer"/> |
|
25 |
<action name="transformer"> |
|
26 |
<pig> |
|
27 |
<job-tracker>${jobTracker}</job-tracker> |
|
28 |
<name-node>${nameNode}</name-node> |
|
29 |
<!-- The data generated by this node is deleted in this section --> |
|
30 |
<prepare> |
|
31 |
<delete path="${nameNode}${output}" /> |
|
32 |
</prepare> |
|
33 |
<configuration> |
|
34 |
<property> |
|
35 |
<name>mapred.job.queue.name</name> |
|
36 |
<value>${queueName}</value> |
|
37 |
</property> |
|
38 |
</configuration> |
|
39 |
<!-- Path to PIG script the workflow executes. --> |
|
40 |
<script>lib/scripts/union.pig</script> |
|
41 |
<param>input_a=${input_a}</param> |
|
42 |
<param>input_b=${input_b}</param> |
|
43 |
<param>input_c=${input_c}</param> |
|
44 |
<param>input_d=${input_d}</param> |
|
45 |
<param>output=${output}</param> |
|
46 |
<param>schema=${schema}</param> |
|
47 |
</pig> |
|
48 |
<ok to="end"/> |
|
49 |
<error to="fail"/> |
|
50 |
</action> |
|
51 |
<kill name="fail"> |
|
52 |
<message>Unfortunately, the workflow failed -- error message: |
|
53 |
[${wf:errorMessage(wf:lastErrorNode())}]</message> |
|
54 |
</kill> |
|
55 |
<end name="end"/> |
|
56 |
</workflow-app> |
|
0 | 57 |
Also available in: Unified diff
#1147 introducing union4 pig script