Revision 51038
Added by Sandro La Bruzzo over 6 years ago
| modules/dnet-openaireplus-workflows/branches/solr7/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/IncrementalTransformationJobNode.java | ||
|---|---|---|
| 41 | 41 |
final String currentWfProfileId = findCurrentWfProfileId(nodeToken); |
| 42 | 42 |
final Long lastSuccessEndDate = findLastSuccessEndDate(currentWfProfileId); |
| 43 | 43 |
if (lastSuccessEndDate<0) {
|
| 44 |
log.info("Last success date <0 the transformation will be forced to REFRESH");
|
|
| 44 |
nodeToken.getFullEnv().setAttribute("TransformationTypeInfo", "Last success date <0, the transformation will be forced to REFRESH");
|
|
| 45 | 45 |
nodeToken.getFullEnv().setAttribute("transformationType", "REFRESH");
|
| 46 | 46 |
return Arc.DEFAULT_ARC; |
| 47 | 47 |
} |
| ... | ... | |
| 50 | 50 |
log.info("last update date of the trId is "+lastUpdateDate);
|
| 51 | 51 |
|
| 52 | 52 |
if (lastUpdateDate> lastSuccessEndDate) {
|
| 53 |
log.info("Transformation Rule has benn updated, hence the transformation process will be forced to REFRESH");
|
|
| 53 |
nodeToken.getFullEnv().setAttribute("TransformationTypeInfo", "Transformation Rule has benn updated, hence the transformation process will be forced to REFRESH");
|
|
| 54 | 54 |
nodeToken.getFullEnv().setAttribute("transformationType", "REFRESH");
|
| 55 | 55 |
return Arc.DEFAULT_ARC; |
| 56 | 56 |
} |
| 57 |
nodeToken.getFullEnv().setAttribute("TransformationTypeInfo", "Transformation type setted to INCREMENTAL with dateFROMFIlter "+lastSuccessEndDate);
|
|
| 57 | 58 |
nodeToken.getFullEnv().setAttribute("transformationType", "INCREMENTAL");
|
| 58 |
nodeToken.getFullEnv().setAttribute("DateFromFilter", lastSuccessEndDate);
|
|
| 59 |
log.info("Transformation type setted to INCREMENTAL");
|
|
| 59 |
nodeToken.getFullEnv().setAttribute("DateFromFilter", lastSuccessEndDate);
|
|
| 60 | 60 |
return Arc.DEFAULT_ARC; |
| 61 | 61 |
} |
| 62 | 62 |
nodeToken.getFullEnv().setAttribute("transformationType", "REFRESH");
|
| 63 |
log.info("Transformation type setted to REFRESH");
|
|
| 63 |
nodeToken.getFullEnv().setAttribute("TransformationTypeInfo", "Transformation type setted to REFRESH");
|
|
| 64 | 64 |
return Arc.DEFAULT_ARC; |
| 65 | 65 |
} |
| 66 | 66 |
|
| modules/dnet-openaireplus-workflows/branches/solr7/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggrDatarepoOaiSets/transform.wf.st | ||
|---|---|---|
| 6 | 6 |
<PARAM required="true" type="string" name="api" managedBy="system">$params.("dataprovider:interface")$</PARAM>
|
| 7 | 7 |
</PARAMETERS> |
| 8 | 8 |
<ARCS> |
| 9 |
<ARC to="fetchOriginals"/>
|
|
| 9 |
<ARC to="selectIncrementalTransformation"/>
|
|
| 10 | 10 |
<ARC to="obtainParams" /> |
| 11 | 11 |
</ARCS> |
| 12 | 12 |
</NODE> |
| 13 | 13 |
|
| 14 |
<NODE name="selectIncrementalTransformation" type="IncrementalTransformation"> |
|
| 15 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
|
| 16 |
<PARAMETERS> |
|
| 17 |
<PARAM required="true" type="string" name="transformationType" managedBy="user" function="validValues(['REFRESH','INCREMENTAL'])"></PARAM> |
|
| 18 |
</PARAMETERS> |
|
| 19 |
<ARCS> |
|
| 20 |
<ARC to="fetchOriginals"/> |
|
| 21 |
</ARCS> |
|
| 22 |
</NODE> |
|
| 23 |
|
|
| 14 | 24 |
<NODE name="fetchOriginals" type="FetchMDStoreRecords"> |
| 15 | 25 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
| 16 | 26 |
<PARAMETERS> |
| modules/dnet-openaireplus-workflows/branches/solr7/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/dataRepository/transform.wf.st | ||
|---|---|---|
| 6 | 6 |
<PARAM required="true" type="string" name="api" managedBy="system">$params.("dataprovider:interface")$</PARAM>
|
| 7 | 7 |
</PARAMETERS> |
| 8 | 8 |
<ARCS> |
| 9 |
<ARC to="fetchOriginals"/>
|
|
| 9 |
<ARC to="selectIncrementalTransformation"/>
|
|
| 10 | 10 |
<ARC to="obtainParams" /> |
| 11 | 11 |
</ARCS> |
| 12 | 12 |
</NODE> |
| 13 | 13 |
|
| 14 |
<NODE name="selectIncrementalTransformation" type="IncrementalTransformation"> |
|
| 15 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
|
| 16 |
<PARAMETERS> |
|
| 17 |
<PARAM required="true" type="string" name="transformationType" managedBy="user" function="validValues(['REFRESH','INCREMENTAL'])"></PARAM> |
|
| 18 |
</PARAMETERS> |
|
| 19 |
<ARCS> |
|
| 20 |
<ARC to="fetchOriginals"/> |
|
| 21 |
</ARCS> |
|
| 22 |
</NODE> |
|
| 23 |
|
|
| 14 | 24 |
<NODE name="fetchOriginals" type="FetchMDStoreRecords"> |
| 15 | 25 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
| 16 | 26 |
<PARAMETERS> |
| modules/dnet-openaireplus-workflows/branches/solr7/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorPubsRepository/transform.wf.st | ||
|---|---|---|
| 6 | 6 |
<PARAM required="true" type="string" name="api" managedBy="system">$params.("dataprovider:interface")$</PARAM>
|
| 7 | 7 |
</PARAMETERS> |
| 8 | 8 |
<ARCS> |
| 9 |
<ARC to="fetchOriginals"/>
|
|
| 9 |
<ARC to="selectIncrementalTransformation"/>
|
|
| 10 | 10 |
<ARC to="obtainParams" /> |
| 11 | 11 |
</ARCS> |
| 12 | 12 |
</NODE> |
| 13 | 13 |
|
| 14 |
<NODE name="selectIncrementalTransformation" type="IncrementalTransformation"> |
|
| 15 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
|
| 16 |
<PARAMETERS> |
|
| 17 |
<PARAM required="true" type="string" name="transformationType" managedBy="user" function="validValues(['REFRESH','INCREMENTAL'])"></PARAM> |
|
| 18 |
</PARAMETERS> |
|
| 19 |
<ARCS> |
|
| 20 |
<ARC to="fetchOriginals"/> |
|
| 21 |
</ARCS> |
|
| 22 |
</NODE> |
|
| 23 |
|
|
| 14 | 24 |
<NODE name="fetchOriginals" type="FetchMDStoreRecords"> |
| 15 | 25 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
| 16 | 26 |
<PARAMETERS> |
| modules/dnet-openaireplus-workflows/branches/solr7/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/aggregatorDataRepository/transform.wf.st | ||
|---|---|---|
| 6 | 6 |
<PARAM required="true" type="string" name="api" managedBy="system">$params.("dataprovider:interface")$</PARAM>
|
| 7 | 7 |
</PARAMETERS> |
| 8 | 8 |
<ARCS> |
| 9 |
<ARC to="fetchOriginals"/>
|
|
| 9 |
<ARC to="selectIncrementalTransformation"/>
|
|
| 10 | 10 |
<ARC to="obtainParams" /> |
| 11 | 11 |
</ARCS> |
| 12 | 12 |
</NODE> |
| 13 | 13 |
|
| 14 |
<NODE name="selectIncrementalTransformation" type="IncrementalTransformation"> |
|
| 15 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
|
| 16 |
<PARAMETERS> |
|
| 17 |
<PARAM required="true" type="string" name="transformationType" managedBy="user" function="validValues(['REFRESH','INCREMENTAL'])"></PARAM> |
|
| 18 |
</PARAMETERS> |
|
| 19 |
<ARCS> |
|
| 20 |
<ARC to="fetchOriginals"/> |
|
| 21 |
</ARCS> |
|
| 22 |
</NODE> |
|
| 23 |
|
|
| 14 | 24 |
<NODE name="fetchOriginals" type="FetchMDStoreRecords"> |
| 15 | 25 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
| 16 | 26 |
<PARAMETERS> |
| modules/dnet-openaireplus-workflows/branches/solr7/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/repo-hi/datacite/transform.wf.st | ||
|---|---|---|
| 6 | 6 |
<PARAM required="true" type="string" name="api" managedBy="system">$params.("dataprovider:interface")$</PARAM>
|
| 7 | 7 |
</PARAMETERS> |
| 8 | 8 |
<ARCS> |
| 9 |
<ARC to="fetchOriginals"/>
|
|
| 9 |
<ARC to="selectIncrementalTransformation"/>
|
|
| 10 | 10 |
<ARC to="obtainParams" /> |
| 11 | 11 |
</ARCS> |
| 12 | 12 |
</NODE> |
| 13 | 13 |
|
| 14 |
<NODE name="selectIncrementalTransformation" type="IncrementalTransformation"> |
|
| 15 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
|
| 16 |
<PARAMETERS> |
|
| 17 |
<PARAM required="true" type="string" name="transformationType" managedBy="user" function="validValues(['REFRESH','INCREMENTAL'])"></PARAM> |
|
| 18 |
</PARAMETERS> |
|
| 19 |
<ARCS> |
|
| 20 |
<ARC to="fetchOriginals"/> |
|
| 21 |
</ARCS> |
|
| 22 |
</NODE> |
|
| 23 |
|
|
| 14 | 24 |
<NODE name="fetchOriginals" type="FetchMDStoreRecords"> |
| 15 | 25 |
<DESCRIPTION>Fetch records from MDStore</DESCRIPTION> |
| 16 | 26 |
<PARAMETERS> |
Also available in: Unified diff
Added node to every transform workflows added property that explain if the transformation is incremental