|
1
|
package eu.dnetlib.data.hadoop.oozie;
|
|
2
|
|
|
3
|
import java.io.IOException;
|
|
4
|
import java.util.*;
|
|
5
|
|
|
6
|
import com.google.common.collect.Maps;
|
|
7
|
import com.google.common.collect.Sets;
|
|
8
|
import org.apache.commons.io.IOUtils;
|
|
9
|
import org.apache.commons.lang.StringUtils;
|
|
10
|
import org.apache.commons.logging.Log;
|
|
11
|
import org.apache.commons.logging.LogFactory;
|
|
12
|
import org.apache.oozie.client.OozieClient;
|
|
13
|
import org.apache.oozie.client.OozieClientException;
|
|
14
|
import org.apache.oozie.client.WorkflowAction;
|
|
15
|
import org.apache.oozie.client.WorkflowJob;
|
|
16
|
import org.apache.oozie.client.WorkflowJob.Status;
|
|
17
|
|
|
18
|
import eu.dnetlib.data.hadoop.action.JobCompletion;
|
|
19
|
import eu.dnetlib.data.hadoop.action.JobMonitor;
|
|
20
|
import eu.dnetlib.data.hadoop.rmi.HadoopServiceException;
|
|
21
|
|
|
22
|
public class OozieJobMonitor extends JobMonitor {
|
|
23
|
|
|
24
|
private static final Log log = LogFactory.getLog(JobMonitor.class); // NOPMD by marko on 11/24/08 5:02 PM
|
|
25
|
|
|
26
|
private final OozieClient oozieClient;
|
|
27
|
|
|
28
|
private final String jobId;
|
|
29
|
|
|
30
|
public static final String ACTION_TYPE_SUBWORKFLOW = "sub-workflow";
|
|
31
|
|
|
32
|
private Set<String> workflowActions = Sets.newHashSet();
|
|
33
|
|
|
34
|
public OozieJobMonitor(final OozieClient oozieClient, String jobId, final JobCompletion callback) {
|
|
35
|
super(callback);
|
|
36
|
this.oozieClient = oozieClient;
|
|
37
|
this.jobId = jobId;
|
|
38
|
}
|
|
39
|
|
|
40
|
public OozieJobMonitor(final OozieClient oozieClient, String jobId, final JobCompletion callback, final Set<String> workflowActions) {
|
|
41
|
super(callback);
|
|
42
|
this.oozieClient = oozieClient;
|
|
43
|
this.jobId = jobId;
|
|
44
|
this.workflowActions = workflowActions;
|
|
45
|
}
|
|
46
|
|
|
47
|
@Override
|
|
48
|
public void run() {
|
|
49
|
try {
|
|
50
|
log.info("waiting for oozie job completion: " + getHadoopId());
|
|
51
|
|
|
52
|
Status status = doGetStatus();
|
|
53
|
while (status.equals(Status.RUNNING)) {
|
|
54
|
Thread.sleep(monitorSleepTimeSecs * 1000);
|
|
55
|
|
|
56
|
Status currentStatus = doGetStatus();
|
|
57
|
if (!status.equals(currentStatus)) {
|
|
58
|
status = currentStatus;
|
|
59
|
lastActivity = new Date(System.currentTimeMillis());
|
|
60
|
}
|
|
61
|
}
|
|
62
|
|
|
63
|
log.debug("job " + jobId + " finihsed with status: " + status);
|
|
64
|
if (status == Status.SUCCEEDED) {
|
|
65
|
// TODO set some content to return to the blackboard msg.
|
|
66
|
|
|
67
|
final Properties report = getReport(getOozieClient(), getHadoopId(), workflowActions);
|
|
68
|
if (report != null) {
|
|
69
|
final Map<String, String> map = Maps.newHashMap();
|
|
70
|
report.forEach((k, v) -> map.put(k.toString(), v.toString()));
|
|
71
|
getCallback().done(map);
|
|
72
|
} else {
|
|
73
|
getCallback().done(new HashMap<>());
|
|
74
|
}
|
|
75
|
} else {
|
|
76
|
// TODO retrieve some failure information from the oozie client.
|
|
77
|
String msg = "hadoop job: " + getHadoopId() + " failed with status: " + getStatus() + ", oozie log:\n "
|
|
78
|
+ getOozieClient().getJobLog(getHadoopId()) + "\n\n";
|
|
79
|
getCallback().failed(msg, new HadoopServiceException(msg));
|
|
80
|
}
|
|
81
|
} catch (Throwable e) {
|
|
82
|
getCallback().failed(getHadoopId(), e);
|
|
83
|
}
|
|
84
|
}
|
|
85
|
|
|
86
|
/**
|
|
87
|
* Provides report entries when found for given oozie job identifier. Returns null when report not found.
|
|
88
|
*/
|
|
89
|
private static Properties getReport(final OozieClient oozieClient, final String oozieJobId, final Set<String> workflowActions) throws OozieClientException, IOException {
|
|
90
|
WorkflowJob mainIisWfJob = oozieClient.getJobInfo(oozieJobId);
|
|
91
|
for (WorkflowAction currentAction : mainIisWfJob.getActions()) {
|
|
92
|
if (workflowActions.contains(currentAction.getName())) {
|
|
93
|
if (ACTION_TYPE_SUBWORKFLOW.equals(currentAction.getType())) {
|
|
94
|
Properties subworkflowProperties = getReport(oozieClient, currentAction.getExternalId(), workflowActions);
|
|
95
|
if (subworkflowProperties != null) {
|
|
96
|
return subworkflowProperties;
|
|
97
|
}
|
|
98
|
} else if (StringUtils.isNotBlank(currentAction.getData())) {
|
|
99
|
Properties properties = new Properties();
|
|
100
|
properties.load(IOUtils.toInputStream(currentAction.getData()));
|
|
101
|
return properties;
|
|
102
|
}
|
|
103
|
}
|
|
104
|
}
|
|
105
|
|
|
106
|
return null;
|
|
107
|
}
|
|
108
|
|
|
109
|
@Override
|
|
110
|
public String getHadoopId() {
|
|
111
|
return jobId;
|
|
112
|
}
|
|
113
|
|
|
114
|
public OozieClient getOozieClient() {
|
|
115
|
return oozieClient;
|
|
116
|
}
|
|
117
|
|
|
118
|
@Override
|
|
119
|
public String getStatus() {
|
|
120
|
try {
|
|
121
|
return doGetStatus().toString();
|
|
122
|
} catch (OozieClientException e) {
|
|
123
|
log.error("error accessing job status", e);
|
|
124
|
return "UNKNOWN";
|
|
125
|
}
|
|
126
|
}
|
|
127
|
|
|
128
|
private Status doGetStatus() throws OozieClientException {
|
|
129
|
return getOozieClient().getJobInfo(getHadoopId()).getStatus();
|
|
130
|
}
|
|
131
|
|
|
132
|
@Override
|
|
133
|
public Date getLastActivity() {
|
|
134
|
return lastActivity;
|
|
135
|
}
|
|
136
|
|
|
137
|
@Override
|
|
138
|
public Date getStartTime() throws HadoopServiceException {
|
|
139
|
try {
|
|
140
|
return getOozieClient().getJobInfo(getHadoopId()).getStartTime();
|
|
141
|
} catch (OozieClientException e) {
|
|
142
|
throw new HadoopServiceException("unable to read job start time", e);
|
|
143
|
}
|
|
144
|
}
|
|
145
|
|
|
146
|
@Override
|
|
147
|
public String getTrackerUrl() {
|
|
148
|
return getOozieClient().getOozieUrl();
|
|
149
|
}
|
|
150
|
|
|
151
|
@Override
|
|
152
|
public void kill() {
|
|
153
|
try {
|
|
154
|
getOozieClient().kill(getHadoopId());
|
|
155
|
} catch (OozieClientException e) {
|
|
156
|
log.error("unable to kill job: " + getHadoopId(), e);
|
|
157
|
}
|
|
158
|
}
|
|
159
|
|
|
160
|
}
|