Revision 53542
Added by Michele Artini over 5 years ago
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidWorkflowsJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency; |
|
| 2 |
|
|
| 3 |
import java.util.HashSet; |
|
| 4 |
import java.util.Set; |
|
| 5 |
|
|
| 6 |
import org.apache.commons.logging.Log; |
|
| 7 |
import org.apache.commons.logging.LogFactory; |
|
| 8 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 9 |
|
|
| 10 |
import com.googlecode.sarasvati.Arc; |
|
| 11 |
import com.googlecode.sarasvati.NodeToken; |
|
| 12 |
|
|
| 13 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
| 14 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; |
|
| 15 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
| 16 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
|
| 17 |
import eu.dnetlib.msro.workflows.util.WorkflowsConstants; |
|
| 18 |
|
|
| 19 |
public class FindInvalidWorkflowsJobNode extends SimpleJobNode {
|
|
| 20 |
|
|
| 21 |
@Autowired |
|
| 22 |
private UniqueServiceLocator serviceLocator; |
|
| 23 |
|
|
| 24 |
private static final Log log = LogFactory.getLog(FindInvalidWorkflowsJobNode.class); |
|
| 25 |
|
|
| 26 |
private boolean deleteInvalidProfiles = false; |
|
| 27 |
|
|
| 28 |
@Override |
|
| 29 |
protected String execute(final NodeToken token) throws Exception {
|
|
| 30 |
final ISLookUpService lookup = serviceLocator.getService(ISLookUpService.class); |
|
| 31 |
final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class); |
|
| 32 |
|
|
| 33 |
final String q1 = |
|
| 34 |
"for $x in collection('/db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType')//WORKFLOW/@id|//CONFIGURATION/@destroyWorkflow return $x/string()";
|
|
| 35 |
|
|
| 36 |
final String q2 = "for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
|
|
| 37 |
|
|
| 38 |
final Set<String> valids = new HashSet<>(lookup.quickSearchProfile(q1)); |
|
| 39 |
|
|
| 40 |
int count = 0; |
|
| 41 |
int countInvalids = 0; |
|
| 42 |
int countDeleted = 0; |
|
| 43 |
for (final String wfId : lookup.quickSearchProfile(q2)) {
|
|
| 44 |
count++; |
|
| 45 |
if (!valids.contains(wfId)) {
|
|
| 46 |
log.warn("Invalid wf: " + wfId);
|
|
| 47 |
countInvalids++; |
|
| 48 |
if (deleteInvalidProfiles) {
|
|
| 49 |
registry.deleteProfile(wfId); |
|
| 50 |
log.info(" -- Deleted");
|
|
| 51 |
countDeleted++; |
|
| 52 |
} |
|
| 53 |
|
|
| 54 |
} |
|
| 55 |
} |
|
| 56 |
|
|
| 57 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", count); |
|
| 58 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalid", countInvalids); |
|
| 59 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "deleted", countDeleted); |
|
| 60 |
|
|
| 61 |
return Arc.DEFAULT_ARC; |
|
| 62 |
} |
|
| 63 |
|
|
| 64 |
public boolean isDeleteInvalidProfiles() {
|
|
| 65 |
return deleteInvalidProfiles; |
|
| 66 |
} |
|
| 67 |
|
|
| 68 |
public void setDeleteInvalidProfiles(final boolean deleteInvalidProfiles) {
|
|
| 69 |
this.deleteInvalidProfiles = deleteInvalidProfiles; |
|
| 70 |
} |
|
| 71 |
|
|
| 72 |
} |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidStoresJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency; |
|
| 2 |
|
|
| 3 |
import java.util.HashSet; |
|
| 4 |
import java.util.Set; |
|
| 5 |
|
|
| 6 |
import org.apache.commons.logging.Log; |
|
| 7 |
import org.apache.commons.logging.LogFactory; |
|
| 8 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 9 |
|
|
| 10 |
import com.googlecode.sarasvati.Arc; |
|
| 11 |
import com.googlecode.sarasvati.NodeToken; |
|
| 12 |
|
|
| 13 |
import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao; |
|
| 14 |
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao; |
|
| 15 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
| 16 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; |
|
| 17 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
| 18 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
|
| 19 |
import eu.dnetlib.msro.workflows.util.WorkflowsConstants; |
|
| 20 |
|
|
| 21 |
public class FindInvalidStoresJobNode extends SimpleJobNode {
|
|
| 22 |
|
|
| 23 |
@Autowired |
|
| 24 |
private UniqueServiceLocator serviceLocator; |
|
| 25 |
|
|
| 26 |
@Autowired |
|
| 27 |
private MDStoreDao mdstoreDao; |
|
| 28 |
|
|
| 29 |
@Autowired |
|
| 30 |
private ObjectStoreDao objDao; |
|
| 31 |
|
|
| 32 |
private static final Log log = LogFactory.getLog(FindInvalidStoresJobNode.class); |
|
| 33 |
|
|
| 34 |
private boolean deleteInvalidStores = false; |
|
| 35 |
|
|
| 36 |
@Override |
|
| 37 |
protected String execute(final NodeToken token) throws Exception {
|
|
| 38 |
final ISLookUpService lookup = serviceLocator.getService(ISLookUpService.class); |
|
| 39 |
final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class); |
|
| 40 |
|
|
| 41 |
final String q1 = "distinct-values("
|
|
| 42 |
+ "for $x in collection('/db/DRIVER/WorkflowDSResources/WorkflowDSResourceType')//PARAM[@category='MDSTORE_ID' or @category='OBJECTSTORE_ID'] "
|
|
| 43 |
+ "return $x/text())"; |
|
| 44 |
|
|
| 45 |
final String q2 = "for $x in collection('/db/DRIVER/MDStoreDSResources/MDStoreDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
|
|
| 46 |
final String q3 = "for $x in collection('/db/DRIVER/ObjectStoreDSResources/ObjectStoreDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
|
|
| 47 |
|
|
| 48 |
final Set<String> valids = new HashSet<>(lookup.quickSearchProfile(q1)); |
|
| 49 |
|
|
| 50 |
int countMdstores = 0; |
|
| 51 |
int countInvalidMdStores = 0; |
|
| 52 |
int countDeletedMdStores = 0; |
|
| 53 |
int countObjstores = 0; |
|
| 54 |
int countInvalidObjStores = 0; |
|
| 55 |
int countDeletedObjStores = 0; |
|
| 56 |
|
|
| 57 |
for (final String id : lookup.quickSearchProfile(q2)) {
|
|
| 58 |
countMdstores++; |
|
| 59 |
if (!valids.contains(id)) {
|
|
| 60 |
log.warn("Invalid mdstore: " + id);
|
|
| 61 |
countInvalidMdStores++; |
|
| 62 |
if (deleteInvalidStores) {
|
|
| 63 |
registry.deleteProfile(id); |
|
| 64 |
mdstoreDao.deleteMDStore(id); |
|
| 65 |
log.info(" -- Deleted");
|
|
| 66 |
countDeletedMdStores++; |
|
| 67 |
} |
|
| 68 |
|
|
| 69 |
} |
|
| 70 |
} |
|
| 71 |
|
|
| 72 |
for (final String id : lookup.quickSearchProfile(q3)) {
|
|
| 73 |
countObjstores++; |
|
| 74 |
if (!valids.contains(id)) {
|
|
| 75 |
log.warn("Invalid objstore: " + id);
|
|
| 76 |
countInvalidObjStores++; |
|
| 77 |
if (deleteInvalidStores) {
|
|
| 78 |
registry.deleteProfile(id); |
|
| 79 |
objDao.deleteObjectStore(id); |
|
| 80 |
log.info(" -- Deleted");
|
|
| 81 |
countDeletedObjStores++; |
|
| 82 |
} |
|
| 83 |
|
|
| 84 |
} |
|
| 85 |
} |
|
| 86 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "totalMdStores", countMdstores); |
|
| 87 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalidMdStores", countInvalidMdStores); |
|
| 88 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "deletedMdStores", countDeletedMdStores); |
|
| 89 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "totalObjStores", countObjstores); |
|
| 90 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalidObjStores", countInvalidObjStores); |
|
| 91 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "deletedObjStores", countDeletedObjStores); |
|
| 92 |
|
|
| 93 |
return Arc.DEFAULT_ARC; |
|
| 94 |
} |
|
| 95 |
|
|
| 96 |
public boolean isDeleteInvalidStores() {
|
|
| 97 |
return deleteInvalidStores; |
|
| 98 |
} |
|
| 99 |
|
|
| 100 |
public void setDeleteInvalidStores(final boolean deleteInvalidStores) {
|
|
| 101 |
this.deleteInvalidStores = deleteInvalidStores; |
|
| 102 |
} |
|
| 103 |
} |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/consistency/FindInvalidMetaWorkflowsJobNode.java | ||
|---|---|---|
| 1 |
package eu.dnetlib.msro.openaireplus.workflows.nodes.consistency; |
|
| 2 |
|
|
| 3 |
import java.util.HashSet; |
|
| 4 |
import java.util.Set; |
|
| 5 |
|
|
| 6 |
import org.apache.commons.lang.StringUtils; |
|
| 7 |
import org.apache.commons.logging.Log; |
|
| 8 |
import org.apache.commons.logging.LogFactory; |
|
| 9 |
import org.springframework.beans.factory.annotation.Autowired; |
|
| 10 |
|
|
| 11 |
import com.googlecode.sarasvati.Arc; |
|
| 12 |
import com.googlecode.sarasvati.NodeToken; |
|
| 13 |
|
|
| 14 |
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; |
|
| 15 |
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService; |
|
| 16 |
import eu.dnetlib.enabling.locators.UniqueServiceLocator; |
|
| 17 |
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode; |
|
| 18 |
import eu.dnetlib.msro.workflows.util.WorkflowsConstants; |
|
| 19 |
|
|
| 20 |
public class FindInvalidMetaWorkflowsJobNode extends SimpleJobNode {
|
|
| 21 |
|
|
| 22 |
@Autowired |
|
| 23 |
private UniqueServiceLocator serviceLocator; |
|
| 24 |
|
|
| 25 |
private static final Log log = LogFactory.getLog(FindInvalidMetaWorkflowsJobNode.class); |
|
| 26 |
|
|
| 27 |
private boolean deleteInvalidProfiles = false; |
|
| 28 |
|
|
| 29 |
@Override |
|
| 30 |
protected String execute(final NodeToken token) throws Exception {
|
|
| 31 |
|
|
| 32 |
final ISLookUpService lookup = serviceLocator.getService(ISLookUpService.class); |
|
| 33 |
final ISRegistryService registry = serviceLocator.getService(ISRegistryService.class); |
|
| 34 |
|
|
| 35 |
final String q1 = "for $x in collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')\n" +
|
|
| 36 |
"let $id := $x//RESOURCE_IDENTIFIER/@value\n" + |
|
| 37 |
"for $api in $x//INTERFACE/@id\n" + |
|
| 38 |
"return concat($id,'|',$api)"; |
|
| 39 |
|
|
| 40 |
final String q2 = "for $x in collection(' /db/DRIVER/MetaWorkflowDSResources/MetaWorkflowDSResourceType') \n" +
|
|
| 41 |
"where $x//DATAPROVIDER/@id and $x//DATAPROVIDER/@interface\n" + |
|
| 42 |
"return concat($x//RESOURCE_IDENTIFIER/@value,'@@@',$x//DATAPROVIDER/@id,'|',$x//DATAPROVIDER/@interface)"; |
|
| 43 |
|
|
| 44 |
final Set<String> valids = new HashSet<>(lookup.quickSearchProfile(q1)); |
|
| 45 |
|
|
| 46 |
int count = 0; |
|
| 47 |
int countInvalids = 0; |
|
| 48 |
int countDeleted = 0; |
|
| 49 |
for (final String s : lookup.quickSearchProfile(q2)) {
|
|
| 50 |
count++; |
|
| 51 |
final String metaWfId = StringUtils.substringBefore(s, "@@@"); |
|
| 52 |
final String dsapi = StringUtils.substringAfter(s, "@@@"); |
|
| 53 |
if (!valids.contains(dsapi)) {
|
|
| 54 |
log.warn("Invalid metaWf: " + metaWfId);
|
|
| 55 |
countInvalids++; |
|
| 56 |
if (deleteInvalidProfiles) {
|
|
| 57 |
registry.deleteProfile(metaWfId); |
|
| 58 |
log.info(" -- Deleted");
|
|
| 59 |
countDeleted++; |
|
| 60 |
} |
|
| 61 |
|
|
| 62 |
} |
|
| 63 |
} |
|
| 64 |
|
|
| 65 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "total", count); |
|
| 66 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "invalid", countInvalids); |
|
| 67 |
token.getEnv().setAttribute(WorkflowsConstants.MAIN_LOG_PREFIX + "deleted", countDeleted); |
|
| 68 |
|
|
| 69 |
return Arc.DEFAULT_ARC; |
|
| 70 |
} |
|
| 71 |
|
|
| 72 |
public boolean isDeleteInvalidProfiles() {
|
|
| 73 |
return deleteInvalidProfiles; |
|
| 74 |
} |
|
| 75 |
|
|
| 76 |
public void setDeleteInvalidProfiles(final boolean deleteInvalidProfiles) {
|
|
| 77 |
this.deleteInvalidProfiles = deleteInvalidProfiles; |
|
| 78 |
} |
|
| 79 |
} |
|
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/findInvalidRepoProfiles.xml | ||
|---|---|---|
| 7 | 7 |
<DATE_OF_CREATION value="2015-10-12T14:10:32+00:00"/> |
| 8 | 8 |
</HEADER> |
| 9 | 9 |
<BODY> |
| 10 |
<WORKFLOW_NAME>Find Invalid Repo Profiles</WORKFLOW_NAME>
|
|
| 10 |
<WORKFLOW_NAME>Find Invalid Profiles</WORKFLOW_NAME> |
|
| 11 | 11 |
<WORKFLOW_TYPE>Consistency</WORKFLOW_TYPE> |
| 12 | 12 |
<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY> |
| 13 | 13 |
<CONFIGURATION start="manual"> |
| 14 |
<NODE name="find" type="FindInvalidRepoProfiles" isStart="true"> |
|
| 14 |
<NODE name="findRepos" type="FindInvalidRepoProfiles" isStart="true">
|
|
| 15 | 15 |
<DESCRIPTION>Find Invalid Repo Profiles</DESCRIPTION> |
| 16 | 16 |
<PARAMETERS> |
| 17 | 17 |
<PARAM name="deleteInvalidProfiles" managedBy="user" required="true" type="boolean">false</PARAM> |
| 18 | 18 |
<PARAM name="registerMissingProfiles" managedBy="user" required="true" type="boolean">false</PARAM> |
| 19 | 19 |
</PARAMETERS> |
| 20 | 20 |
<ARCS> |
| 21 |
<ARC to="findMetaWfs"/> |
|
| 22 |
</ARCS> |
|
| 23 |
</NODE> |
|
| 24 |
|
|
| 25 |
<NODE name="findMetaWfs" type="FindInvalidMetaWorkflows"> |
|
| 26 |
<DESCRIPTION>Find Invalid Meta Wf Profiles</DESCRIPTION> |
|
| 27 |
<PARAMETERS> |
|
| 28 |
<PARAM name="deleteInvalidProfiles" managedBy="user" required="true" type="boolean">false</PARAM> |
|
| 29 |
</PARAMETERS> |
|
| 30 |
<ARCS> |
|
| 31 |
<ARC to="findWfs"/> |
|
| 32 |
</ARCS> |
|
| 33 |
</NODE> |
|
| 34 |
|
|
| 35 |
<NODE name="findWfs" type="FindInvalidWorkflows"> |
|
| 36 |
<DESCRIPTION>Find Invalid Wf Profiles</DESCRIPTION> |
|
| 37 |
<PARAMETERS> |
|
| 38 |
<PARAM name="deleteInvalidProfiles" managedBy="user" required="true" type="boolean">false</PARAM> |
|
| 39 |
</PARAMETERS> |
|
| 40 |
<ARCS> |
|
| 41 |
<ARC to="findStores"/> |
|
| 42 |
</ARCS> |
|
| 43 |
</NODE> |
|
| 44 |
|
|
| 45 |
<NODE name="findStores" type="FindInvalidStores"> |
|
| 46 |
<DESCRIPTION>Find Invalid [Md/Obj]Stores</DESCRIPTION> |
|
| 47 |
<PARAMETERS> |
|
| 48 |
<PARAM name="deleteInvalidStores" managedBy="user" required="true" type="boolean">false</PARAM> |
|
| 49 |
</PARAMETERS> |
|
| 50 |
<ARCS> |
|
| 21 | 51 |
<ARC to="success"/> |
| 22 | 52 |
</ARCS> |
| 23 | 53 |
</NODE> |
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/consistency/reposStoresConsistency.meta.xml | ||
|---|---|---|
| 12 | 12 |
<METAWORKFLOW_SECTION>InfoSpace Consistency</METAWORKFLOW_SECTION> |
| 13 | 13 |
<ADMIN_EMAIL>michele.artini@isti.cnr.it</ADMIN_EMAIL> |
| 14 | 14 |
<CONFIGURATION status="EXECUTABLE"> |
| 15 |
<WORKFLOW id="e5bb911b-3fe2-403a-a757-6c1f89ff27e8_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" name="Find Invalid Repo Profiles">
|
|
| 15 |
<WORKFLOW id="e5bb911b-3fe2-403a-a757-6c1f89ff27e8_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" name="Find Invalid Profiles"> |
|
| 16 | 16 |
<WORKFLOW id="3a5aea4a-cbeb-445e-b39a-52f40f42fa50_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" name="Refresh Mdstore Sizes"> |
| 17 | 17 |
<WORKFLOW id="671e4507-3b0e-4318-8065-ddb856182e9f_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl" name="Repos-Stores consistency"/> |
| 18 | 18 |
</WORKFLOW> |
| modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/applicationContext-msro-openaire-nodes.xml | ||
|---|---|---|
| 317 | 317 |
p:xquery="${dnet.openaire.bulktag.xquery}"
|
| 318 | 318 |
scope="prototype"/> |
| 319 | 319 |
|
| 320 |
<!-- Consistency nodes --> |
|
| 320 |
<!-- Consistency nodes -->
|
|
| 321 | 321 |
<bean id="wfNodeFixRepoMdstoreSizes" |
| 322 | 322 |
class="eu.dnetlib.msro.openaireplus.workflows.nodes.consistency.FixRepoMdstoreSizesJobNode" |
| 323 | 323 |
scope="prototype"/> |
| ... | ... | |
| 329 | 329 |
<bean id="wfNodeFindInvalidRepoProfiles" |
| 330 | 330 |
class="eu.dnetlib.msro.openaireplus.workflows.nodes.consistency.FindInvalidRepoProfilesJobNode" |
| 331 | 331 |
scope="prototype"/> |
| 332 |
|
|
| 333 |
<bean id="wfNodeFindInvalidMetaWorkflows" |
|
| 334 |
class="eu.dnetlib.msro.openaireplus.workflows.nodes.consistency.FindInvalidMetaWorkflowsJobNode" |
|
| 335 |
scope="prototype"/> |
|
| 332 | 336 |
|
| 337 |
<bean id="wfNodeFindInvalidWorkflows" |
|
| 338 |
class="eu.dnetlib.msro.openaireplus.workflows.nodes.consistency.FindInvalidWorkflowsJobNode" |
|
| 339 |
scope="prototype"/> |
|
| 340 |
|
|
| 341 |
<bean id="wfNodeFindInvalidStores" |
|
| 342 |
class="eu.dnetlib.msro.openaireplus.workflows.nodes.consistency.FindInvalidStoresJobNode" |
|
| 343 |
scope="prototype"/> |
|
| 344 |
|
|
| 333 | 345 |
</beans> |
Also available in: Unified diff
new consistency nodes