Project

General

Profile

« Previous | Next » 

Revision 34908

Added by Marek Horst over 9 years ago

#1147 introducing union4 pig script

View differences:

modules/icm-iis-transformers/trunk/src/main/resources/eu/dnetlib/iis/transformers/common/union4/oozie_app/lib/scripts/union.pig
1
define avro_load_input_a
2
org.apache.pig.piggybank.storage.avro.AvroStorage(
3
'input_schema_class', '$schema');
4

  
5
define avro_load_input_b
6
org.apache.pig.piggybank.storage.avro.AvroStorage(
7
'input_schema_class', '$schema');
8

  
9
define avro_load_input_c
10
org.apache.pig.piggybank.storage.avro.AvroStorage(
11
'input_schema_class', '$schema');
12

  
13
define avro_load_input_d
14
org.apache.pig.piggybank.storage.avro.AvroStorage(
15
'input_schema_class', '$schema');
16

  
17
define avro_store_output
18
org.apache.pig.piggybank.storage.avro.AvroStorage(
19
'index', '0',
20
'output_schema_class', '$schema');
21

  
22
input_a = load '$input_a' using avro_load_input_a;
23
input_b = load '$input_b' using avro_load_input_b;
24
input_c = load '$input_c' using avro_load_input_c;
25
input_d = load '$input_d' using avro_load_input_d;
26

  
27
output_final = union input_a, input_b, input_c, input_d; 
28

  
29
store output_final into '$output' using avro_store_output;
modules/icm-iis-transformers/trunk/src/main/resources/eu/dnetlib/iis/transformers/common/union4/oozie_app/workflow.xml
1
<workflow-app xmlns="uri:oozie:workflow:0.4" name="transformers_common_union4">
2
	
3
	<parameters>
4
		<property>
5
			<name>input_a</name>
6
		</property>
7
		<property>
8
			<name>input_b</name>
9
		</property>
10
		<property>
11
			<name>input_c</name>
12
		</property>
13
		<property>
14
			<name>input_d</name>
15
		</property>
16
		<property>
17
			<name>output</name>
18
		</property>
19
		<property>
20
			<name>schema</name>
21
		</property>
22
	</parameters>
23
    
24
    <start to="transformer"/>
25
    <action name="transformer">
26
        <pig>
27
            <job-tracker>${jobTracker}</job-tracker>
28
            <name-node>${nameNode}</name-node>
29
			<!-- The data generated by this node is deleted in this section -->
30
			<prepare>
31
				<delete path="${nameNode}${output}" />
32
			</prepare>
33
            <configuration>
34
                <property>
35
                    <name>mapred.job.queue.name</name>
36
                    <value>${queueName}</value>
37
                </property>
38
            </configuration>
39
            <!-- Path to PIG script the workflow executes. -->
40
            <script>lib/scripts/union.pig</script>
41
            <param>input_a=${input_a}</param>
42
            <param>input_b=${input_b}</param>
43
            <param>input_c=${input_c}</param>
44
            <param>input_d=${input_d}</param>
45
            <param>output=${output}</param>
46
            <param>schema=${schema}</param>
47
        </pig>
48
        <ok to="end"/>
49
        <error to="fail"/>
50
    </action>
51
    <kill name="fail">
52
		<message>Unfortunately, the workflow failed -- error message:
53
			[${wf:errorMessage(wf:lastErrorNode())}]</message>
54
    </kill>
55
    <end name="end"/>
56
</workflow-app>
0 57

  

Also available in: Unified diff