Project

General

Profile

« Previous | Next » 

Revision 48603

Updated parameters of the OAI feed wf branch.

View differences:

modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/test/profiles/openaireplus/workflows/dm/dm.provision.xml
130 130
				</PARAMETERS>
131 131
				<ARCS>
132 132
					<ARC to="updateIndex"/>
133
					<ARC to="setOAIFormat"/>
133
					<ARC to="prepareOAI"/>
134 134
					<ARC to="prepareStats"/>
135 135
				</ARCS>
136 136
			</NODE>
......
206 206
				</ARCS>
207 207
			</NODE>
208 208

  
209

  
209 210
			<!-- OAI STORE UPDATE -->
210
			<NODE name="setOAIFormat" type="SetFormatInfo">
211
				<DESCRIPTION>set format, layout, interpretation for OAI publisher</DESCRIPTION>
212
				<PARAMETERS>
213
					<PARAM name="format" type="string" managedBy="user" required="true">oaf</PARAM>
214
					<PARAM name="layout" type="string" managedBy="user" required="true">index</PARAM>
215
					<PARAM name="interpretation" type="string" managedBy="user" required="true">openaire</PARAM>
216
					<PARAM name="formatParam" type="string" managedBy="system" required="true">oai_format</PARAM>
217
					<PARAM name="layoutParam" type="string" managedBy="system" required="true">oai_layout</PARAM>
218
					<PARAM name="interpretationParam" type="string" managedBy="system" required="true">oai_interpretation</PARAM>
219
				</PARAMETERS>
220
				<ARCS>
221
					<ARC to="prepareOAI"/>
222
				</ARCS>
223
			</NODE>
224 211
			<NODE name="prepareOAI" type="PrepareOaiJob">
225
				<DESCRIPTION>Prepare params for OAI store feeding</DESCRIPTION>
212
				<DESCRIPTION>Prepare target OAI store</DESCRIPTION>
226 213
				<PARAMETERS>
227
					<PARAM name="oaiStoreCollectionParam" type="string" managedBy="system" required="true">oaiStoreCollection</PARAM>
228
					<PARAM name="oaiDBName" type="string" managedBy="user" required="true">oaistore</PARAM>
229
					<PARAM name="oaiDBNameParam" type="string" managedBy="system" required="true">oai_dbName</PARAM>
230
					<PARAM name="formatParam" type="string" managedBy="system" required="true">oai_format</PARAM>
231
					<PARAM name="layoutParam" type="string" managedBy="system" required="true">oai_layout</PARAM>
232
					<PARAM name="interpretationParam" type="string" managedBy="system" required="true">oai_interpretation</PARAM>
214
					<PARAM name="oaiDbName" type="string" managedBy="user" required="true">oaistore</PARAM>
215
					<PARAM name="oaiFormat" type="string" managedBy="user" required="true">oaf</PARAM>
216
					<PARAM name="oaiLayout" type="string" managedBy="user" required="true">index</PARAM>
217
					<PARAM name="oaiInterpretation" type="string" managedBy="user" required="true">openaire</PARAM>
218
					<PARAM name="oaiSource" type="string" managedBy="user" required="false"></PARAM>
233 219
					<PARAM name="skipDuplicates" type="boolean" managedBy="user" required="true">true</PARAM>
234 220
					<PARAM name="duplicateXPath" type="string" managedBy="user" required="true">//*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']</PARAM>
235 221
				</PARAMETERS>
236 222
				<ARCS>
237
					<ARC to="OAIDropStore"/>
223
					<ARC to="OAIDropStore" />
238 224
				</ARCS>
239 225
			</NODE>
240 226
			<NODE name="OAIDropStore" type="OAIDropStore">
241 227
				<DESCRIPTION>Drop the store: upserts are too expensive!</DESCRIPTION>
242
				<PARAMETERS>
243
					<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM>
244
					<PARAM name="formatParam" type="string" managedBy="system" required="true">oai_format</PARAM>
245
					<PARAM name="layoutParam" type="string" managedBy="system" required="true">oai_layout</PARAM>
246
					<PARAM name="interpretationParam" type="string" managedBy="system" required="true">oai_interpretation</PARAM>
247
				</PARAMETERS>
228
				<PARAMETERS/>
248 229
				<ARCS>
249 230
					<ARC to="OAICreateStore"/>
250 231
				</ARCS>
251 232
			</NODE>
252 233
			<NODE name="OAICreateStore" type="OAICreateStore">
253 234
				<DESCRIPTION/>
254
				<PARAMETERS>
255
					<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM>
256
					<PARAM name="formatParam" type="string" managedBy="system" required="true">oai_format</PARAM>
257
					<PARAM name="layoutParam" type="string" managedBy="system" required="true">oai_layout</PARAM>
258
					<PARAM name="interpretationParam" type="string" managedBy="system" required="true">oai_interpretation</PARAM>
259
				</PARAMETERS>
235
				<PARAMETERS/>
260 236
				<ARCS>
261 237
					<ARC to="feedOAI"/>
262 238
				</ARCS>
......
269 245
					<PARAM required="true" type="string" name="envParams" managedBy="system">
270 246
						{
271 247
						'mapred.input.dir' : 'hdfsRecordsPath',
272
						'services.publisher.oai.collection' : 'oaiStoreCollection',
273
						'oaiConfiguration' : 'oaiConfiguration',
274
						'oai.feed.date' : 'oai.feed.date',
248
						'services.publisher.oai.collection' : 'oai_collection',
249
						'oaiConfiguration' : 'oai_configuration',
250
						'oai.feed.date' : 'oai_actionDate',
275 251
						'services.publisher.oai.db' : 'oai_dbName',
276
						'services.publisher.oai.skipDuplicates':'services.publisher.oai.skipDuplicates',
277
						'services.publisher.oai.duplicateXPath':'services.publisher.oai.duplicateXPath'
252
						'services.publisher.oai.skipDuplicates':'oai_skipDuplicates',
253
						'services.publisher.oai.duplicateXPath':'oai_duplicateXPath'
278 254
						}
279 255
					</PARAM>
280 256
					<PARAM required="true" type="string" name="sysParams" managedBy="system">
......
292 268
			<NODE name="CompoundIndexes" type="OAICreateIndex">
293 269
				<DESCRIPTION>Create composite indexes for the OAI store</DESCRIPTION>
294 270
				<PARAMETERS>
295
					<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM>
296
					<PARAM name="oaiStoreCollectionParam" type="string" managedBy="system" required="true">oaiStoreCollection</PARAM>
297
					<PARAM required="true" type="string" name="fieldNames" managedBy="user">set,datestamp;license,oaftype;set,oaftype;oaftype,funder;resulttypeid,oaftype</PARAM>
271
					<PARAM required="true" type="string" name="fieldNames" managedBy="user">set,resulttypeid;set,datestamp;license,oaftype;set,oaftype;oaftype,funder;resulttypeid,oaftype;oaftype,funding;resulttypeid,funder;resulttypeid,funding</PARAM>
298 272
				</PARAMETERS>
299 273
				<ARCS>
300 274
					<ARC to="ConfigIndexes"/>
......
302 276
			</NODE>
303 277
			<NODE name="ConfigIndexes" type="OAIEnsureIndexes">
304 278
				<DESCRIPTION>Ensure an index exists on fields as specified in the configuration profile</DESCRIPTION>
305
				<PARAMETERS>
306
					<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM>
307
					<PARAM name="oaiStoreCollectionParam" type="string" managedBy="system" required="true">oaiStoreCollection</PARAM>
308
				</PARAMETERS>
279
				<PARAMETERS/>
309 280
				<ARCS>
310 281
					<ARC to="SetsCount"/>
311 282
					<ARC to="ConfigSets"/>
......
314 285
			<NODE name="SetsCount" type="OAISetsCountUpdate">
315 286
				<DESCRIPTION>Count records in each OAI set, for each exported metadata format</DESCRIPTION>
316 287
				<PARAMETERS>
317
					<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM>
318
					<PARAM name="oaiStoreCollectionParam" type="string" managedBy="system" required="true">oaiStoreCollection</PARAM>
319 288
					<PARAM name="configuredOnly" type="boolean" managedBy="user" required="true">false</PARAM>
320 289
				</PARAMETERS>
321 290
				<ARCS>
322 291
					<ARC to="waitAll"/>
323 292
				</ARCS>
324 293
			</NODE>
325
			<NODE name="ConfigSets" type="RefreshSetsFromConfig">
294
			<NODE name="ConfigSets" type="OAIRefreshConfiguration">
326 295
				<DESCRIPTION>Reads the current OAI configuration and upsert OAI sets accordingly (counts are updated as well)</DESCRIPTION>
327 296
				<PARAMETERS>
328
					<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM>
329 297
				</PARAMETERS>
330 298
				<ARCS>
331 299
					<ARC to="waitAll"/>
332 300
				</ARCS>
333 301
			</NODE>
334

  
335 302
			<!-- WAIT FOR ALL THE WF BRANCHES TO COMPLETE -->
336 303
			<NODE name="waitAll" isJoin="true">
337 304
				<DESCRIPTION>wait for all the branches to complete</DESCRIPTION>

Also available in: Unified diff