Project

General

Profile

« Previous | Next » 

Revision 45641

Added by Claudio Atzori about 7 years ago

added LOD export workflow

View differences:

modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/PrepareLodParamJobNode.java
1 
package eu.dnetlib.msro.openaireplus.workflows.nodes;
2 

  		




  
  3
  
    
import java.text.SimpleDateFormat;


  		




  
  4
  
    
import java.util.Date;


  		




  
  5
  
    

  		




  
  6
  
    
import com.googlecode.sarasvati.Arc;


  		




  
  7
  
    
import com.googlecode.sarasvati.NodeToken;


  		




  
  8
  
    
import com.mongodb.DBObject;


  		




  
  9
  
    
import com.mongodb.client.MongoCollection;


  		




  
  10
  
    
import com.mongodb.client.model.Filters;


  		




  
  11
  
    
import com.mongodb.client.model.Sorts;


  		




  
  12
  
    
import eu.dnetlib.common.logging.DnetLogger;


  		




  
  13
  
    
import eu.dnetlib.common.logging.dao.DnetLoggerMongoDao;


  		




  
  14
  
    
import eu.dnetlib.msro.workflows.nodes.SimpleJobNode;


  		




  
  15
  
    
import org.apache.commons.logging.Log;


  		




  
  16
  
    
import org.apache.commons.logging.LogFactory;


  		




  
  17
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  
  18
  
    
import org.springframework.beans.factory.annotation.Value;


  		




  
  19
  
    

  		




  
  20
  
    
public class PrepareLodParamJobNode extends SimpleJobNode {


  		




  
  21
  
    

  		




  
  22
  
    
	/**


  		




  
  23
  
    
	 * logger.


  		




  
  24
  
    
	 */


  		




  
  25
  
    
	private static final Log log = LogFactory.getLog(PrepareLodParamJobNode.class); // NOPMD by marko on 11/24/08 5:02 PM


  		




  
  26
  
    
	private static final String SYSTEM_PROFILE_ID = "system:profileId";


  		




  
  27
  
    
	private static final String LOG_DATE = "log:date";


  		




  
  28
  
    
	private static final String IS_COMPLETED_SUCCESSFULLY = "system:isCompletedSuccessfully";


  		




  
  29
  
    

  		




  
  30
  
    
	@Value("${dnet.openaire.lod.jsonrels}")


  		




  
  31
  
    
	private String lodJsonRels;


  		




  
  32
  
    

  		




  
  33
  
    
	@Value("${dnet.openaire.lod.wfId}")


  		




  
  34
  
    
	private String lodWfId;


  		




  
  35
  
    

  		




  
  36
  
    
	@Value("${dnet.openaire.lod.default.date}")


  		




  
  37
  
    
	private String defaultLodDate;


  		




  
  38
  
    

  		




  
  39
  
    
	@Autowired


  		




  
  40
  
    
	private DnetLogger dnetLogger;


  		




  
  41
  
    

  		




  
  42
  
    
	@Autowired


  		




  
  43
  
    
	private DnetLoggerMongoDao mongoDao;


  		




  
  44
  
    

  		




  
  45
  
    
	private final SimpleDateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");


  		




  
  46
  
    

  		




  
  47
  
    
	@Override


  		




  
  48
  
    
	protected String execute(final NodeToken token) throws Exception {


  		




  
  49
  
    

  		




  
  50
  
    
		log.info("prepare LOD update job");


  		




  
  51
  
    

  		




  
  52
  
    
		token.getEnv().setAttribute("lod_jsonRels", lodJsonRels);


  		




  
  53
  
    
		token.getEnv().setAttribute("lod_lastExecutionDate", getFormattedDate());


  		




  
  54
  
    

  		




  
  55
  
    
		return Arc.DEFAULT_ARC;


  		




  
  56
  
    
	}


  		




  
  57
  
    

  		




  
  58
  
    
	/**


  		




  
  59
  
    
	 * Queries the MSRO logger to find the last time the LOD update workflow was successfully executed.


  		




  
  60
  
    
	 * @return


  		




  
  61
  
    
	 *  the formatted date


  		




  
  62
  
    
	 */


  		




  
  63
  
    
	private String getFormattedDate() {


  		




  
  64
  
    
		final DBObject dbo = getLoggerDao()


  		




  
  65
  
    
				.find(Filters.and(Filters.eq(SYSTEM_PROFILE_ID, lodWfId), Filters.eq(IS_COMPLETED_SUCCESSFULLY, "true")))


  		




  
  66
  
    
				.sort(Sorts.descending(LOG_DATE))


  		




  
  67
  
    
				.limit(1)


  		




  
  68
  
    
				.first();


  		




  
  69
  
    

  		




  
  70
  
    
		if (dbo == null || !dbo.containsField(LOG_DATE)) {


  		




  
  71
  
    
			log.warn(String.format("Cannot find log entry for LOD update workflow, using default date: '%s'", defaultLodDate));


  		




  
  72
  
    
			return defaultLodDate;


  		




  
  73
  
    
		}


  		




  
  74
  
    

  		




  
  75
  
    
		final String lastExecutionDate = dateFormat.format(new Date((Long) dbo.get(LOG_DATE)));


  		




  
  76
  
    
		log.debug(String.format("Last LOD execution date: '%s'", lastExecutionDate));


  		




  
  77
  
    
		return lastExecutionDate;


  		




  
  78
  
    
	}


  		




  
  79
  
    

  		




  
  80
  
    
	private MongoCollection<DBObject> getLoggerDao() {


  		




  
  81
  
    
		return mongoDao.getDb().getCollection(dnetLogger.getName(), DBObject.class);


  		




  
  82
  
    
	}


  		




  
  83
  
    

  		




  
  84
  
    
}


  		












  

    
      modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/provision/lod.update.xml
    
  





  
  1
  
    
<?xml version="1.0" encoding="UTF-8"?>


  		




  
  2
  
    
<RESOURCE_PROFILE xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">


  		




  
  3
  
    
	<HEADER>


  		




  
  4
  
    
		<RESOURCE_IDENTIFIER value="42c42eeb-8cf4-4410-bdf5-a20db879d23b_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl"/>


  		




  
  5
  
    
		<RESOURCE_TYPE value="WorkflowDSResourceType"/>


  		




  
  6
  
    
		<RESOURCE_KIND value="WorkflowDSResources"/>


  		




  
  7
  
    
		<RESOURCE_URI value=""/>


  		




  
  8
  
    
		<DATE_OF_CREATION value="2014-12-12T12:13:51.0Z" />


  		




  
  9
  
    
	</HEADER>


  		




  
  10
  
    
	<BODY>


  		




  
  11
  
    
		<WORKFLOW_NAME>Update LOD</WORKFLOW_NAME>


  		




  
  12
  
    
		<WORKFLOW_TYPE>Provision</WORKFLOW_TYPE>


  		




  
  13
  
    
		<WORKFLOW_PRIORITY>30</WORKFLOW_PRIORITY>


  		




  
  14
  
    
		<CONFIGURATION start="manual">


  		




  
  15
  
    
			<NODE name="fetchEntityLinks" type="LoadEntityLinksJob" isStart="true">


  		




  
  16
  
    
				<DESCRIPTION />


  		




  
  17
  
    
				<PARAMETERS/>


  		




  
  18
  
    
				<ARCS>


  		




  
  19
  
    
					<ARC to="updateLod" />


  		




  
  20
  
    
				</ARCS>


  		




  
  21
  
    
			</NODE>


  		




  
  22
  
    
			<NODE name="prepareOozieParams" type="PrepareStatsParams" isStart="true">


  		




  
  23
  
    
				<DESCRIPTION />


  		




  
  24
  
    
				<PARAMETERS/>


  		




  
  25
  
    
				<ARCS>


  		




  
  26
  
    
					<ARC to="updateLod" />


  		




  
  27
  
    
				</ARCS>


  		




  
  28
  
    
			</NODE>


  		




  
  29
  
    
			<NODE name="prepareLodParams" type="PrepareLodParams" isStart="true">


  		




  
  30
  
    
				<DESCRIPTION />


  		




  
  31
  
    
				<PARAMETERS/>


  		




  
  32
  
    
				<ARCS>


  		




  
  33
  
    
					<ARC to="updateLod" />


  		




  
  34
  
    
				</ARCS>


  		




  
  35
  
    
			</NODE>


  		




  
  36
  
    
			<NODE name="updateLod" type="SubmitHadoopJob" isJoin="true">


  		




  
  37
  
    
				<DESCRIPTION>Update stats DB</DESCRIPTION>


  		




  
  38
  
    
				<PARAMETERS>


  		




  
  39
  
    
					<PARAM required="true" type="string" name="cluster" managedBy="system">DM</PARAM>


  		




  
  40
  
    
					<PARAM required="true" type="string" name="hadoopJob" managedBy="system">lodExportJob</PARAM>


  		




  
  41
  
    
					<PARAM required="true" type="string" name="sysParams" managedBy="system">


  		




  
  42
  
    
						{


  		




  
  43
  
    
						'lod_hbase_table' : 'hbase.mapred.datatable'


  		




  
  44
  
    
						}


  		




  
  45
  
    
					</PARAM>


  		




  
  46
  
    
					<PARAM required="true" type="string" name="envParams" managedBy="system">


  		




  
  47
  
    
						{


  		




  
  48
  
    
						'nameNode' : 'nameNode',


  		




  
  49
  
    
						'jobTracker' : 'jobTracker',


  		




  
  50
  
    
						'isLookupEndpoint' : 'isLookupEndpoint',


  		




  
  51
  
    
						'lod_indexConf' : 'index.entity.links'.


  		




  
  52
  
    
						'lod_jsonRels' : 'lod_jsonRels',


  		




  
  53
  
    
						'lod_lastExecutionDate' : , 'lod_lastExecutionDate'


  		




  
  54
  
    
						}


  		




  
  55
  
    
					</PARAM>


  		




  
  56
  
    
				</PARAMETERS>


  		




  
  57
  
    
				<ARCS>


  		




  
  58
  
    
					<ARC to="success" />


  		




  
  59
  
    
				</ARCS>


  		




  
  60
  
    
			</NODE>


  		




  
  61
  
    
		</CONFIGURATION>


  		




  
  62
  
    
		<STATUS />


  		




  
  63
  
    
	</BODY>


  		




  
  64
  
    
</RESOURCE_PROFILE>


  		




  
  65
  
    

  		




  
  66
  
    

  		












  

    
      modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/applicationContext-msro-openaireplus.properties
    
  





  30
  30
  
    
dnet.actionmanager.garbage.sets.xupdate=for $x in /RESOURCE_PROFILE[.//RESOURCE_TYPE/@value = 'ActionManagerSetDSResourceType'] let $latest:=$x//LATEST return update replace $x//RAW_SETS with <RAW_SETS>{$latest}</RAW_SETS>


  		




  31
  31
  
    

  		




  32
  32
  
    
oaf.schema.location = http://www.openaire.eu/schema/0.2/oaf-0.2.xsd


  		




  
  33
  
    

  		




  
  34
  
    
dnet.openaire.lod.jsonrels = { "result": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://www.eurocris.org/ontologies/cerif/1.3#name", "6": "http://purl.org/dc/terms/dateAccepted", "7": "http://purl.org/dc/terms/publisher", "8": "http://purl.org/dc/terms/identifier", "9": "http://purl.org/dc/terms/language", "10": "http://purl.org/dc/terms/date", "11": "http://lod.openaire.eu/vocab/resultSubject", "12": "http://lod.openaire.eu/vocab/externalReference", "13": "http://purl.org/dc/terms/source", "14": "http://purl.org/dc/terms/format", "15": "http://lod.openaire.eu/vocab/context", "16": "http://dbpedia.org/ontology/country", "17": "http://purl.org/dc/terms/accessRights", "18": "http://purl.org/dc/terms/description", "19": "http://lsdis.cs.uga.edu/projects/semdis/opus#journal_name", "20": "http://lod.openaire.eu/vocab/dataSourceType", "21": "http://lod.openaire.eu/vocab/device", "22": "http://lod.openaire.eu/vocab/size", "23": "http://lod.openaire.eu/vocab/version", "24": "http://lod.openaire.eu/vocab/lastMetadataUpdate", "25": "http://lod.openaire.eu/vocab/metadataVersion", "26": "http://lod.openaire.eu/vocab/year", "27": "http://lod.openaire.eu/vocab/resultType", "28": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://www.eurocris.org/ontologies/cerif/1.3#ResultEntity" }], "person": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://xmlns.com/foaf/0.1/firstName", "6": "http://xmlns.com/foaf/spec/lastName", "7": "http://xmlns.com/foaf/0.1/name", "8": "http://schema.org/faxNumber", "9": "http://xmlns.com/foaf/0.1/mbox", "10": "http://xmlns.com/foaf/0.1/phone", "11": "http://schema.org/nationality", "12": "http://purl.org/dc/terms/identifier", "13": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http: //xmlns.com/foaf/0.1/Person" }], "datasource": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://lod.openaire.eu/vocab/datasourceType", "6": "http://lod.openaire.eu/vocab/openAIRECompatibility", "7": "http://dbpedia.org/ontology/officialName", "8": "http://lod.openaire.eu/vocab/englishName", "9": "http://schema.org/url", "10": "http://xmlns.com/foaf/0.1/logo", "11": "http://xmlns.com/foaf/0.1/mbox", "12": "http://purl.org/vocab/vann/preferredNamespacePrefix", "13": "http://www.w3.org/2003/01/geo/wgs84_pos#lat", "14": "http://www.w3.org/2003/01/geo/wgs84_pos#long", "15": "http://lod.openaire.eu/vocab/dateOfValidity", "16": "http://purl.org/dc/terms/description", "17": "http://lod.openaire.eu/vocab/subjectList", "18": "http://lod.openaire.eu/numberOfItems", "19": "http://purl.org/dc/terms/date", "20": "http://lod.openaire.eu/vocab/policies", "21": "http://lod.openaire.eu/vocab/languages", "22": "http://lod.openaire.eu/vocab/contentType", "23": "http://lod.openaire.eu/vocab/accessInfoPackage", "24": "http://lod.openaire.eu/vocab/releaseStartDate", "25": "http://lod.openaire.eu/vocab/releaseEndDate", "26": "http://lod.openaire.eu/vocab/missionStatementUrl", "27": "http://www.europeana.eu/schemas/edm/dataProvider", "28": "http://lod.openaire.eu/vocab/serviceProvider", "29": "http://lod.openaire.eu/vocab/databaseAccessType", "30": "http://lod.openaire.eu/vocab/dataUploadType", "31": "http://lod.openaire.eu/vocab/dataUploadRestrictions", "32": "http://lod.openaire.eu/vocab/versioning", "33": "http://lod.openaire.eu/vocab/citationGuidelineUrl", "34": "http://lod.openaire.eu/vocab/qualityManagementKind", "35": "http://lod.openaire.eu/vocab/pidSystems", "36": "http://lod.openaire.eu/vocab/certificates", "37": "http://purl.org/dc/terms/accessRights", "38": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http: //www.w3.org/ns/prov#Entity" }], "organization": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://www.w3.org/2004/02/skos/core#altLabel", "6": "http://www.w3.org/2004/02/skos/core#prefLabel", "7": "http://lod.openaire.eu/vocab/webSiteUrl", "8": "http://xmlns.com/foaf/0.1/logo", "9": "http://dbpedia.org/ontology/country", "10": "http://lod.openaire.eu/vocab/entityType", "11": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http: //xmlns.com/foaf/0.1/Organization" }], "project": [{ "0": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "1": "http://purl.org/dc/terms/identifier", "2": "http://lod.openaire.eu/vocab/dateOfTransformation", "3": "http://lod.openaire.eu/vocab/dateOfCollection", "4": "http://purl.org/dc/terms/identifier", "5": "http://lod.openaire.eu/vocab/projectCode", "6": "http://schema.org/url", "7": "http://www.eurocris.org/ontologies/cerif/1.3#acronym", "8": "http://www.eurocris.org/ontologies/cerif/1.3#name", "9": "http://www.eurocris.org/ontologies/cerif/1.3#startDate", "10": "http://www.eurocris.org/ontologies/cerif/1.3#endDate", "11": "http://purl.org/cerif/frapo/hasCallIdentifier", "12": "http://www.eurocris.org/ontologies/cerif/1.3#keyword", "13": "http://www.w3.org/2006/time#hasDurationDescription", "14": "http://lod.openaire.eu/vocab/ec_SC39", "15": "http://lod.openaire.eu/vocab/contractType", "16": "http://lod.openaire.eu/vocab/oaMandatePublications", "17": "http://lod.openaire.eu/vocab/projectSubjects", "18": "http://od.openaire.eu/vocab/ec_article29-3", "19": "http://lod.openaire.eu/vocab/funder", "20": "http://lod.openaire.eu/vocab/fundingLevel0", "21": "http://lod.openaire.eu/vocab/fundingLevel1", "22": "http://lod.openaire.eu/vocab/fundingLevel2", "23": "http://lod.openaire.eu/vocab/fundingLevel3", "24": "http://lod.openaire.eu/vocab/trust", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "http://www.eurocris.org/ontologies/cerif/1.3#Project" }] }


  		




  
  35
  
    
dnet.openaire.lod.wfId = 42c42eeb-8cf4-4410-bdf5-a20db879d23b_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl


  		




  
  36
  
    
dnet.openaire.lod.default.date = 01/01/1970


  		












  

    
      modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/workflows/applicationContext-msro-openaire-nodes.xml
    
  





  130
  130
  
    
          class="eu.dnetlib.msro.openaireplus.workflows.nodes.PrepareStatsParamJobNode"


  		




  131
  131
  
    
          scope="prototype"/>


  		




  132
  132
  
    

  		




  
  133
  
    
    <bean id="wfNodePrepareLodParams"


  		




  
  134
  
    
          class="eu.dnetlib.msro.openaireplus.workflows.nodes.PrepareLodParamJobNode"


  		




  
  135
  
    
          scope="prototype"/>


  		




  
  136
  
    

  		




  133
  137
  
    
    <bean id="wfNodeFinalizeIndexFeeding"


  		




  134
  138
  
    
          class="eu.dnetlib.msro.openaireplus.workflows.nodes.index.FinalizeIndexJobNode"


  		




  135
  139
  
    
          scope="prototype"/>


  		












Also available in:  Unified diff