Revision 46794
Added by Michele Artini over 7 years ago
| modules/dnet-springboot-apps/trunk/dnet-msro-application/src/main/java/eu/dnetlib/msro/workflows/util/WorkflowDispatcher.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.msro.workflows.util; |
| 2 | 2 |
|
| 3 |
import java.io.IOException; |
|
| 3 | 4 |
import java.io.StringReader; |
| 4 | 5 |
import java.util.HashMap; |
| 5 | 6 |
import java.util.Map; |
| 6 | 7 |
|
| 8 |
import org.antlr.stringtemplate.StringTemplate; |
|
| 9 |
import org.apache.commons.io.IOUtils; |
|
| 7 | 10 |
import org.apache.commons.lang3.StringUtils; |
| 8 | 11 |
import org.apache.commons.lang3.math.NumberUtils; |
| 9 | 12 |
import org.apache.commons.logging.Log; |
| ... | ... | |
| 14 | 17 |
import org.springframework.beans.factory.annotation.Autowired; |
| 15 | 18 |
import org.springframework.stereotype.Component; |
| 16 | 19 |
|
| 20 |
import com.fasterxml.jackson.core.type.TypeReference; |
|
| 21 |
import com.fasterxml.jackson.databind.ObjectMapper; |
|
| 22 |
|
|
| 17 | 23 |
import eu.dnetlib.clients.is.InformationServiceClient; |
| 18 | 24 |
import eu.dnetlib.clients.locators.ServiceClientFactory; |
| 19 | 25 |
import eu.dnetlib.clients.msro.MsroWorkerClient; |
| 20 | 26 |
import eu.dnetlib.clients.msro.Workflow; |
| 21 | 27 |
import eu.dnetlib.clients.msro.WorkflowTemplate; |
| 28 |
import eu.dnetlib.exceptions.DnetGenericRuntimeException; |
|
| 29 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
|
| 22 | 30 |
import eu.dnetlib.msro.exceptions.MSROException; |
| 23 | 31 |
import eu.dnetlib.msro.workflows.WorkflowInstance; |
| 24 | 32 |
import eu.dnetlib.services.async.AsyncClientCallback; |
| ... | ... | |
| 30 | 38 |
private static final Log log = LogFactory.getLog(WorkflowDispatcher.class); |
| 31 | 39 |
|
| 32 | 40 |
@Autowired |
| 33 |
private InformationServiceClient isLookup;
|
|
| 41 |
private InformationServiceClient isClient;
|
|
| 34 | 42 |
|
| 35 | 43 |
@Autowired |
| 36 | 44 |
private GraphLoader graphLoader; |
| ... | ... | |
| 71 | 79 |
} |
| 72 | 80 |
|
| 73 | 81 |
private WorkflowInstance newWorkflowInstance(final String profileId) throws Exception {
|
| 74 |
final String profile = isLookup.getProfile(profileId);
|
|
| 82 |
final String profile = isClient.getProfile(profileId);
|
|
| 75 | 83 |
final Document doc = new SAXReader().read(new StringReader(profile)); |
| 76 | 84 |
|
| 77 | 85 |
final boolean isReady = doc.valueOf("//CONFIGURATION/@status").equals(WorkflowsConstants.WorkflowStatus.EXECUTABLE.toString());
|
| ... | ... | |
| 116 | 124 |
final String localBaseUrl) throws MSROException {
|
| 117 | 125 |
|
| 118 | 126 |
try {
|
| 119 |
final String profile = isLookup.getProfile(profileId);
|
|
| 127 |
final String profile = isClient.getProfile(profileId);
|
|
| 120 | 128 |
final Document doc = new SAXReader().read(new StringReader(profile)); |
| 121 | 129 |
|
| 122 | 130 |
final Map<String, String> globalParams = new HashMap<String, String>(); |
| ... | ... | |
| 160 | 168 |
|
| 161 | 169 |
worker.execute(wf, localBaseUrl, new AsyncClientCallback() {
|
| 162 | 170 |
|
| 171 |
private ObjectMapper mapper = new ObjectMapper(); |
|
| 172 |
|
|
| 163 | 173 |
@Override |
| 164 | 174 |
public void onGoing(final AsyncResponse res) {
|
| 165 | 175 |
log.info("ONGOING - " + res.getResponseJson());
|
| ... | ... | |
| 168 | 178 |
@Override |
| 169 | 179 |
public void onFailed(final AsyncResponse res) {
|
| 170 | 180 |
log.info("FAILED - " + res.getResponseJson());
|
| 171 |
|
|
| 181 |
try {
|
|
| 182 |
saveResponse(wf, false, mapper.readValue(res.getResponseJson(), new TypeReference<HashMap<String, Object>>() {}));
|
|
| 183 |
} catch (final IOException e) {
|
|
| 184 |
log.error("Error parsing async response: " + res.getResponseJson(), e);
|
|
| 185 |
throw new DnetGenericRuntimeException("Error final parsing async response: " + res.getResponseJson(), e);
|
|
| 186 |
} |
|
| 172 | 187 |
} |
| 173 | 188 |
|
| 174 | 189 |
@Override |
| 175 | 190 |
public void onDone(final AsyncResponse res) {
|
| 176 | 191 |
log.info("DONE - " + res.getResponseJson());
|
| 192 |
try {
|
|
| 193 |
saveResponse(wf, true, mapper.readValue(res.getResponseJson(), new TypeReference<HashMap<String, Object>>() {}));
|
|
| 194 |
} catch (final IOException e) {
|
|
| 195 |
log.error("Error parsing async response: " + res.getResponseJson(), e);
|
|
| 196 |
throw new DnetGenericRuntimeException("Error final parsing async response: " + res.getResponseJson(), e);
|
|
| 197 |
} |
|
| 177 | 198 |
} |
| 178 | 199 |
}); |
| 179 | 200 |
|
| ... | ... | |
| 181 | 202 |
|
| 182 | 203 |
} |
| 183 | 204 |
|
| 205 |
private void saveResponse(final WorkflowInstance wf, final boolean success, final Map<String, String> res) {
|
|
| 206 |
final String profileId = wf.getProfileId(); |
|
| 207 |
|
|
| 208 |
if (!wf.isTemplate() && StringUtils.isNotBlank(profileId)) {
|
|
| 209 |
try {
|
|
| 210 |
final String template = IOUtils.toString(getClass().getResourceAsStream("/templates/workflow_status.xml.st"));
|
|
| 211 |
final StringTemplate st = new StringTemplate(template); |
|
| 212 |
st.setAttribute("procId", res.getOrDefault(WorkflowsConstants.LOG_WF_PROCESS_ID, ""));
|
|
| 213 |
st.setAttribute("date", res.getOrDefault(WorkflowsConstants.LOG_WF_PROCESS_END_DATE, DateUtils.now_ISO8601()));
|
|
| 214 |
st.setAttribute("params", res);
|
|
| 215 |
if (!success) {
|
|
| 216 |
st.setAttribute("error", res.getOrDefault(WorkflowsConstants.LOG_SYSTEM_ERROR, "Unknown error"));
|
|
| 217 |
} |
|
| 218 |
isClient.updateProfileNode(profileId, "//STATUS", st.toString()); |
|
| 219 |
} catch (final Exception e) {
|
|
| 220 |
log.error("Error saving wf response", e);
|
|
| 221 |
throw new DnetGenericRuntimeException("Error saving wf response", e);
|
|
| 222 |
} |
|
| 223 |
} |
|
| 224 |
|
|
| 225 |
/* TODO : generare la risposta da passare al manager server */ |
|
| 226 |
/* |
|
| 227 |
* dnetLogger.newLogMessage() .addDetails(process.getOutputParams()) .addDetail(WorkflowsConstants.LOG_WF_NAME, process.getName()) |
|
| 228 |
* .addDetail(WorkflowsConstants.LOG_WF_FAMILY, process.getFamily()) .addDetail(WorkflowsConstants.LOG_WF_PRIORITY, "" + |
|
| 229 |
* process.getPriority()) .addDetail(WorkflowsConstants.LOG_WF_PROCESS_ID, process.getId()) |
|
| 230 |
* .addDetail(WorkflowsConstants.LOG_WF_PROCESS_STATUS, process.getStatus().toString()) |
|
| 231 |
* .addDetail(WorkflowsConstants.LOG_WF_PROCESS_START_DATE, Long.toString(process.getStartDate())) |
|
| 232 |
* .addDetail(WorkflowsConstants.LOG_WF_PROCESS_END_DATE, Long.toString(process.getEndDate())) |
|
| 233 |
* .addDetail(WorkflowsConstants.LOG_WF_PROFILE_ID, process.isTemplate() ? null : process.getProfileId()) |
|
| 234 |
* .addDetail(WorkflowsConstants.LOG_WF_PROFILE_TEMPLATE_ID, process.isTemplate() ? process.getProfileId() : null) |
|
| 235 |
* .addDetail(WorkflowsConstants.LOG_WF_PARENT, process.getParentProfileId()) .addDetail(WorkflowsConstants.LOG_DATASOURCE_ID, |
|
| 236 |
* process.getDsId()) .addDetail(WorkflowsConstants.LOG_DATASOURCE_NAME, process.getDsName()) |
|
| 237 |
* .addDetail(WorkflowsConstants.LOG_DATASOURCE_INTERFACE, process.getDsInterface()) .addDetail(WorkflowsConstants.LOG_SYSTEM_ERROR, |
|
| 238 |
* process.getError()) .addDetail(WorkflowsConstants.LOG_SYSTEM_ERROR_STACKTRACE, process.getErrorStacktrace()) .flush(); |
|
| 239 |
* |
|
| 240 |
* emailDispatcher.sendMails(process); |
|
| 241 |
*/} |
|
| 242 |
|
|
| 184 | 243 |
} |
| modules/dnet-springboot-apps/trunk/dnet-msro-application/src/main/resources/templates/workflow_status.xml.st | ||
|---|---|---|
| 1 |
<STATUS> |
|
| 2 |
<LAST_EXECUTION_ID>$procId$</LAST_EXECUTION_ID> |
|
| 3 |
<LAST_EXECUTION_DATE>$date$</LAST_EXECUTION_DATE> |
|
| 4 |
$if(error)$ |
|
| 5 |
<LAST_EXECUTION_STATUS>FAILURE</LAST_EXECUTION_STATUS> |
|
| 6 |
<LAST_EXECUTION_ERROR>$error$</LAST_EXECUTION_ERROR> |
|
| 7 |
$else$ |
|
| 8 |
<LAST_EXECUTION_STATUS>SUCCESS</LAST_EXECUTION_STATUS> |
|
| 9 |
<LAST_EXECUTION_ERROR /> |
|
| 10 |
$endif$ |
|
| 11 |
$params.keys:{k|<LAST_EXECUTION_OUTPUT name='$k$'>$params.(k)$</LAST_EXECUTION_OUTPUT>}$
|
|
| 12 |
</STATUS> |
|
| modules/dnet-springboot-apps/trunk/dnet-administration-uis/src/main/resources/static/html/wf/wf-form-update.html | ||
|---|---|---|
| 58 | 58 |
<label class="col-sm-3 control-label">Email notification(s)</label> |
| 59 | 59 |
|
| 60 | 60 |
<div class="col-sm-9" ng-if="!emailMessages || emailMessages.length == 0"> |
| 61 |
<p class="form-control-static text-danger">No message template found. Please <a href="isManager.do#/register">register</a> a new message template.</p>
|
|
| 61 |
<p class="form-control-static text-danger">No message template found. Please <a href="is#/register">register</a> a new message template.</p> |
|
| 62 | 62 |
</div> |
| 63 | 63 |
|
| 64 | 64 |
<div class="col-sm-9" ng-if="emailMessages && emailMessages.length > 0"> |
| modules/dnet-springboot-apps/trunk/dnet-administration-uis/src/main/resources/static/html/wf/wf-sub-workflows.html | ||
|---|---|---|
| 31 | 31 |
<button type="button" class="btn btn-sm btn-default" ng-click="showGraph(swf.id)"> |
| 32 | 32 |
<span class="glyphicon glyphicon-picture"></span> |
| 33 | 33 |
</button> |
| 34 |
<a class="btn btn-sm btn-default" href="isManager.do#/profile/{{swf.id}}">
|
|
| 34 |
<a class="btn btn-sm btn-default" href="is#/profile/{{swf.id}}">
|
|
| 35 | 35 |
<span class="glyphicon glyphicon-link"></span> |
| 36 | 36 |
</a> |
| 37 | 37 |
<button class="btn btn-sm" ng-class="{'btn-default': swf.disabled != 'true', 'btn-warning': swf.disabled == 'true'}" ng-click="executeWf(swf.id, swf.name, swf.parent, swf.isTemplate=='true', swf.disabled == 'true')">
|
| modules/dnet-springboot-apps/trunk/dnet-administration-uis/src/main/resources/static/js/wfs/wf_form_fields.js | ||
|---|---|---|
| 6 | 6 |
], |
| 7 | 7 |
MDSTORE_ID : [ |
| 8 | 8 |
{ label : 'show records', url : '../inspector/mdstore.do?id=%s', paramRequired : true },
|
| 9 |
{ label : 'show profile', url : 'isManager.do#/profile/%s', paramRequired : true }
|
|
| 9 |
{ label : 'show profile', url : 'is#/profile/%s', paramRequired : true }
|
|
| 10 | 10 |
], |
| 11 | 11 |
OBJECTSTORE_ID : [ |
| 12 | 12 |
{ label : 'show objects', url : '../inspector/objectstore.do?id=%s', paramRequired : true },
|
| 13 |
{ label : 'show profile', url : 'isManager.do#/profile/%s', paramRequired : true }
|
|
| 13 |
{ label : 'show profile', url : 'is#/profile/%s', paramRequired : true }
|
|
| 14 | 14 |
], |
| 15 | 15 |
TRANSFORMATION_RULE_ID : [ |
| 16 |
{ label : 'register profile', url : 'isManager.do#/register', paramRequired : false },
|
|
| 17 |
{ label : 'show profile', url : 'isManager.do#/profile/%s', paramRequired : true }
|
|
| 16 |
{ label : 'register profile', url : 'is#/register', paramRequired : false },
|
|
| 17 |
{ label : 'show profile', url : 'is#/profile/%s', paramRequired : true }
|
|
| 18 | 18 |
], |
| 19 | 19 |
CLEANER_RULE_ID : [ |
| 20 |
{ label : 'register profile', url : 'isManager.do#/register', paramRequired : false },
|
|
| 21 |
{ label : 'show profile', url : 'isManager.do#/profile/%s', paramRequired : true }
|
|
| 20 |
{ label : 'register profile', url : 'is#/register', paramRequired : false },
|
|
| 21 |
{ label : 'show profile', url : 'is#/profile/%s', paramRequired : true }
|
|
| 22 | 22 |
] |
| 23 | 23 |
}); |
| 24 | 24 |
|
| modules/dnet-springboot-apps/trunk/dnet-administration-uis/src/main/resources/isStartupResources/profiles/workflows/workflow_test_02.xml | ||
|---|---|---|
| 7 | 7 |
<DATE_OF_CREATION value="2006-05-04T18:13:51.0Z"/> |
| 8 | 8 |
</HEADER> |
| 9 | 9 |
<BODY> |
| 10 |
<WORKFLOW_NAME menuSection="Example workflows">Clean database</WORKFLOW_NAME>
|
|
| 10 |
<WORKFLOW_NAME menuSection="Example workflows">Hello workflow</WORKFLOW_NAME>
|
|
| 11 | 11 |
<WORKFLOW_DESCRIPTION>This is a test workflow</WORKFLOW_DESCRIPTION> |
| 12 | 12 |
<WORKFLOW_INFO/> |
| 13 |
<WORKFLOW_FAMILY>CLEAN</WORKFLOW_FAMILY>
|
|
| 13 |
<WORKFLOW_FAMILY>TEST</WORKFLOW_FAMILY>
|
|
| 14 | 14 |
<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY> |
| 15 | 15 |
|
| 16 | 16 |
<CONFIGURATION status="EXECUTABLE" start="AUTO"> |
| 17 | 17 |
<PARAMETERS/> |
| 18 | 18 |
<WORKFLOW> |
| 19 |
<NODE isStart="true" name="TODO">
|
|
| 19 |
<NODE isStart="true" name="Hello">
|
|
| 20 | 20 |
<DESCRIPTION/> |
| 21 | 21 |
<PARAMETERS/> |
| 22 | 22 |
<ARCS> |
| modules/dnet-springboot-apps/trunk/dnet-administration-uis/src/main/resources/xslt/wf.xslt | ||
|---|---|---|
| 83 | 83 |
<hr/> |
| 84 | 84 |
</xsl:for-each> |
| 85 | 85 |
|
| 86 |
<wf-form-row-static name="Workflow" value="{$name}" value-description="{//WORKFLOW_DESCRIPTION}" url="isManager.do#/profile/{$id}"/>
|
|
| 86 |
<wf-form-row-static name="Workflow" value="{$name}" value-description="{//WORKFLOW_DESCRIPTION}" url="is#/profile/{$id}"/>
|
|
| 87 | 87 |
<wf-form-row-static name="Family" value="{//WORKFLOW_FAMILY}"/>
|
| 88 | 88 |
|
| 89 | 89 |
<xsl:choose> |
| modules/dnet-springboot-apps/trunk/dnet-administration-uis/src/main/resources/xslt/datasource_api.xslt | ||
|---|---|---|
| 240 | 240 |
<div ng-repeat="e in availableRepohiWfs" class="panel panel-default"> |
| 241 | 241 |
<div class="panel-heading"> |
| 242 | 242 |
<a ng-click="newRepoWorkflow('{$profileId}', '{$ifaceId}', e.id)" data-dismiss="modal" href="javascript:void(0)">{{e.name}}</a>
|
| 243 |
<a href="isManager.do#/profile/{{{{e.id}}}}" class="pull-right">[link]</a>
|
|
| 243 |
<a href="is#/profile/{{{{e.id}}}}" class="pull-right">[link]</a>
|
|
| 244 | 244 |
<br /><span class="text-muted small">{{e.description}}</span>
|
| 245 | 245 |
</div> |
| 246 | 246 |
<table class="table table-striped table-condensed small"> |
| modules/dnet-springboot-apps/trunk/dnet-simple-aggregation-worker/src/main/java/eu/dnetlib/msro/workflows/procs/ProcessEngine.java | ||
|---|---|---|
| 7 | 7 |
import java.util.Map; |
| 8 | 8 |
import java.util.stream.Collectors; |
| 9 | 9 |
|
| 10 |
import org.antlr.stringtemplate.StringTemplate; |
|
| 11 |
import org.apache.commons.io.IOUtils; |
|
| 12 | 10 |
import org.apache.commons.lang3.StringEscapeUtils; |
| 13 | 11 |
import org.apache.commons.lang3.StringUtils; |
| 14 | 12 |
import org.apache.commons.logging.Log; |
| ... | ... | |
| 18 | 16 |
|
| 19 | 17 |
import com.google.common.base.Throwables; |
| 20 | 18 |
|
| 21 |
import eu.dnetlib.clients.is.InformationServiceClient; |
|
| 22 | 19 |
import eu.dnetlib.miscutils.datetime.DateUtils; |
| 23 | 20 |
import eu.dnetlib.msro.workflows.GraphNode; |
| 24 | 21 |
import eu.dnetlib.msro.workflows.ProcessStatus; |
| ... | ... | |
| 40 | 37 |
@Autowired |
| 41 | 38 |
private NodeHelper nodeHelper; |
| 42 | 39 |
|
| 43 |
@Autowired |
|
| 44 |
private InformationServiceClient isClient; |
|
| 45 |
|
|
| 46 | 40 |
public void startProcess(final WorkflowProcess process) {
|
| 47 | 41 |
log.info(process.getGraph()); |
| 48 | 42 |
|
| ... | ... | |
| 179 | 173 |
|
| 180 | 174 |
process.getCallback().notify(asyncResponse); |
| 181 | 175 |
|
| 182 |
final String profileId = process.getProfileId(); |
|
| 183 |
if (!process.isTemplate() && StringUtils.isNotBlank(profileId)) {
|
|
| 184 |
try {
|
|
| 185 |
final String template = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/msro/workflows/templates/workflow_status.xml.st"));
|
|
| 186 |
final StringTemplate st = new StringTemplate(template); |
|
| 187 |
st.setAttribute("procId", StringEscapeUtils.escapeXml11(process.getId()));
|
|
| 188 |
st.setAttribute("date", StringEscapeUtils.escapeXml11(DateUtils.calculate_ISO8601(now)));
|
|
| 189 |
st.setAttribute("params", process.getOutputParams());
|
|
| 190 |
if (process.getStatus() == ProcessStatus.FAILURE) {
|
|
| 191 |
st.setAttribute("error", StringEscapeUtils.escapeXml11(process.getError()));
|
|
| 192 |
} |
|
| 193 |
isClient.updateProfileNode(profileId, "//STATUS", st.toString()); |
|
| 194 |
} catch (final Exception e) {
|
|
| 195 |
log.error("Error updating workflow profile: " + profileId, e);
|
|
| 196 |
process.setStatus(ProcessStatus.FAILURE); |
|
| 197 |
process.setError("Error updating workflow profile: " + profileId);
|
|
| 198 |
process.setErrorStacktrace(Throwables.getStackTraceAsString(e)); |
|
| 199 |
} |
|
| 200 |
} |
|
| 201 |
|
|
| 202 |
/* TODO : generare la risposta da passare al manager server */ |
|
| 203 |
/* |
|
| 204 |
* dnetLogger.newLogMessage() .addDetails(process.getOutputParams()) .addDetail(WorkflowsConstants.LOG_WF_NAME, process.getName()) |
|
| 205 |
* .addDetail(WorkflowsConstants.LOG_WF_FAMILY, process.getFamily()) .addDetail(WorkflowsConstants.LOG_WF_PRIORITY, "" + |
|
| 206 |
* process.getPriority()) .addDetail(WorkflowsConstants.LOG_WF_PROCESS_ID, process.getId()) |
|
| 207 |
* .addDetail(WorkflowsConstants.LOG_WF_PROCESS_STATUS, process.getStatus().toString()) |
|
| 208 |
* .addDetail(WorkflowsConstants.LOG_WF_PROCESS_START_DATE, Long.toString(process.getStartDate())) |
|
| 209 |
* .addDetail(WorkflowsConstants.LOG_WF_PROCESS_END_DATE, Long.toString(process.getEndDate())) |
|
| 210 |
* .addDetail(WorkflowsConstants.LOG_WF_PROFILE_ID, process.isTemplate() ? null : process.getProfileId()) |
|
| 211 |
* .addDetail(WorkflowsConstants.LOG_WF_PROFILE_TEMPLATE_ID, process.isTemplate() ? process.getProfileId() : null) |
|
| 212 |
* .addDetail(WorkflowsConstants.LOG_WF_PARENT, process.getParentProfileId()) .addDetail(WorkflowsConstants.LOG_DATASOURCE_ID, |
|
| 213 |
* process.getDsId()) .addDetail(WorkflowsConstants.LOG_DATASOURCE_NAME, process.getDsName()) |
|
| 214 |
* .addDetail(WorkflowsConstants.LOG_DATASOURCE_INTERFACE, process.getDsInterface()) .addDetail(WorkflowsConstants.LOG_SYSTEM_ERROR, |
|
| 215 |
* process.getError()) .addDetail(WorkflowsConstants.LOG_SYSTEM_ERROR_STACKTRACE, process.getErrorStacktrace()) .flush(); |
|
| 216 |
* |
|
| 217 |
* emailDispatcher.sendMails(process); |
|
| 218 |
*/ |
|
| 219 | 176 |
} |
| 220 | 177 |
|
| 221 | 178 |
private Map<String, String> filterOutputParams(final Map<String, Object> map) {
|
| modules/dnet-springboot-apps/trunk/dnet-common-utils/src/main/java/eu/dnetlib/services/BaseService.java | ||
|---|---|---|
| 149 | 149 |
log.info("* Message : " + res.getResponseJson());
|
| 150 | 150 |
log.info("***************************************************");
|
| 151 | 151 |
|
| 152 |
return (new RestTemplate()).postForObject(url, res, ResponseAckStatus.class);
|
|
| 152 |
return (new RestTemplate()).postForObject(url, res, ResponseAck.class).getStatus();
|
|
| 153 | 153 |
} |
| 154 | 154 |
|
| 155 | 155 |
protected String baseDir() {
|
Also available in: Unified diff
wf execution