Revision 48603
Added by Alessia Bardi almost 6 years ago
| dm.provision.xml | ||
|---|---|---|
| 130 | 130 |
</PARAMETERS> |
| 131 | 131 |
<ARCS> |
| 132 | 132 |
<ARC to="updateIndex"/> |
| 133 |
<ARC to="setOAIFormat"/>
|
|
| 133 |
<ARC to="prepareOAI"/>
|
|
| 134 | 134 |
<ARC to="prepareStats"/> |
| 135 | 135 |
</ARCS> |
| 136 | 136 |
</NODE> |
| ... | ... | |
| 206 | 206 |
</ARCS> |
| 207 | 207 |
</NODE> |
| 208 | 208 |
|
| 209 |
|
|
| 209 | 210 |
<!-- OAI STORE UPDATE --> |
| 210 |
<NODE name="setOAIFormat" type="SetFormatInfo"> |
|
| 211 |
<DESCRIPTION>set format, layout, interpretation for OAI publisher</DESCRIPTION> |
|
| 212 |
<PARAMETERS> |
|
| 213 |
<PARAM name="format" type="string" managedBy="user" required="true">oaf</PARAM> |
|
| 214 |
<PARAM name="layout" type="string" managedBy="user" required="true">index</PARAM> |
|
| 215 |
<PARAM name="interpretation" type="string" managedBy="user" required="true">openaire</PARAM> |
|
| 216 |
<PARAM name="formatParam" type="string" managedBy="system" required="true">oai_format</PARAM> |
|
| 217 |
<PARAM name="layoutParam" type="string" managedBy="system" required="true">oai_layout</PARAM> |
|
| 218 |
<PARAM name="interpretationParam" type="string" managedBy="system" required="true">oai_interpretation</PARAM> |
|
| 219 |
</PARAMETERS> |
|
| 220 |
<ARCS> |
|
| 221 |
<ARC to="prepareOAI"/> |
|
| 222 |
</ARCS> |
|
| 223 |
</NODE> |
|
| 224 | 211 |
<NODE name="prepareOAI" type="PrepareOaiJob"> |
| 225 |
<DESCRIPTION>Prepare params for OAI store feeding</DESCRIPTION>
|
|
| 212 |
<DESCRIPTION>Prepare target OAI store</DESCRIPTION>
|
|
| 226 | 213 |
<PARAMETERS> |
| 227 |
<PARAM name="oaiStoreCollectionParam" type="string" managedBy="system" required="true">oaiStoreCollection</PARAM> |
|
| 228 |
<PARAM name="oaiDBName" type="string" managedBy="user" required="true">oaistore</PARAM> |
|
| 229 |
<PARAM name="oaiDBNameParam" type="string" managedBy="system" required="true">oai_dbName</PARAM> |
|
| 230 |
<PARAM name="formatParam" type="string" managedBy="system" required="true">oai_format</PARAM> |
|
| 231 |
<PARAM name="layoutParam" type="string" managedBy="system" required="true">oai_layout</PARAM> |
|
| 232 |
<PARAM name="interpretationParam" type="string" managedBy="system" required="true">oai_interpretation</PARAM> |
|
| 214 |
<PARAM name="oaiDbName" type="string" managedBy="user" required="true">oaistore</PARAM> |
|
| 215 |
<PARAM name="oaiFormat" type="string" managedBy="user" required="true">oaf</PARAM> |
|
| 216 |
<PARAM name="oaiLayout" type="string" managedBy="user" required="true">index</PARAM> |
|
| 217 |
<PARAM name="oaiInterpretation" type="string" managedBy="user" required="true">openaire</PARAM> |
|
| 218 |
<PARAM name="oaiSource" type="string" managedBy="user" required="false"></PARAM> |
|
| 233 | 219 |
<PARAM name="skipDuplicates" type="boolean" managedBy="user" required="true">true</PARAM> |
| 234 | 220 |
<PARAM name="duplicateXPath" type="string" managedBy="user" required="true">//*[local-name()='entity']//*[local-name()='datainfo']/*[local-name()='deletedbyinference'][./text() = 'true']</PARAM> |
| 235 | 221 |
</PARAMETERS> |
| 236 | 222 |
<ARCS> |
| 237 |
<ARC to="OAIDropStore"/> |
|
| 223 |
<ARC to="OAIDropStore" />
|
|
| 238 | 224 |
</ARCS> |
| 239 | 225 |
</NODE> |
| 240 | 226 |
<NODE name="OAIDropStore" type="OAIDropStore"> |
| 241 | 227 |
<DESCRIPTION>Drop the store: upserts are too expensive!</DESCRIPTION> |
| 242 |
<PARAMETERS> |
|
| 243 |
<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM> |
|
| 244 |
<PARAM name="formatParam" type="string" managedBy="system" required="true">oai_format</PARAM> |
|
| 245 |
<PARAM name="layoutParam" type="string" managedBy="system" required="true">oai_layout</PARAM> |
|
| 246 |
<PARAM name="interpretationParam" type="string" managedBy="system" required="true">oai_interpretation</PARAM> |
|
| 247 |
</PARAMETERS> |
|
| 228 |
<PARAMETERS/> |
|
| 248 | 229 |
<ARCS> |
| 249 | 230 |
<ARC to="OAICreateStore"/> |
| 250 | 231 |
</ARCS> |
| 251 | 232 |
</NODE> |
| 252 | 233 |
<NODE name="OAICreateStore" type="OAICreateStore"> |
| 253 | 234 |
<DESCRIPTION/> |
| 254 |
<PARAMETERS> |
|
| 255 |
<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM> |
|
| 256 |
<PARAM name="formatParam" type="string" managedBy="system" required="true">oai_format</PARAM> |
|
| 257 |
<PARAM name="layoutParam" type="string" managedBy="system" required="true">oai_layout</PARAM> |
|
| 258 |
<PARAM name="interpretationParam" type="string" managedBy="system" required="true">oai_interpretation</PARAM> |
|
| 259 |
</PARAMETERS> |
|
| 235 |
<PARAMETERS/> |
|
| 260 | 236 |
<ARCS> |
| 261 | 237 |
<ARC to="feedOAI"/> |
| 262 | 238 |
</ARCS> |
| ... | ... | |
| 269 | 245 |
<PARAM required="true" type="string" name="envParams" managedBy="system"> |
| 270 | 246 |
{
|
| 271 | 247 |
'mapred.input.dir' : 'hdfsRecordsPath', |
| 272 |
'services.publisher.oai.collection' : 'oaiStoreCollection',
|
|
| 273 |
'oaiConfiguration' : 'oaiConfiguration',
|
|
| 274 |
'oai.feed.date' : 'oai.feed.date',
|
|
| 248 |
'services.publisher.oai.collection' : 'oai_collection',
|
|
| 249 |
'oaiConfiguration' : 'oai_configuration',
|
|
| 250 |
'oai.feed.date' : 'oai_actionDate',
|
|
| 275 | 251 |
'services.publisher.oai.db' : 'oai_dbName', |
| 276 |
'services.publisher.oai.skipDuplicates':'services.publisher.oai.skipDuplicates',
|
|
| 277 |
'services.publisher.oai.duplicateXPath':'services.publisher.oai.duplicateXPath'
|
|
| 252 |
'services.publisher.oai.skipDuplicates':'oai_skipDuplicates',
|
|
| 253 |
'services.publisher.oai.duplicateXPath':'oai_duplicateXPath'
|
|
| 278 | 254 |
} |
| 279 | 255 |
</PARAM> |
| 280 | 256 |
<PARAM required="true" type="string" name="sysParams" managedBy="system"> |
| ... | ... | |
| 292 | 268 |
<NODE name="CompoundIndexes" type="OAICreateIndex"> |
| 293 | 269 |
<DESCRIPTION>Create composite indexes for the OAI store</DESCRIPTION> |
| 294 | 270 |
<PARAMETERS> |
| 295 |
<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM> |
|
| 296 |
<PARAM name="oaiStoreCollectionParam" type="string" managedBy="system" required="true">oaiStoreCollection</PARAM> |
|
| 297 |
<PARAM required="true" type="string" name="fieldNames" managedBy="user">set,datestamp;license,oaftype;set,oaftype;oaftype,funder;resulttypeid,oaftype</PARAM> |
|
| 271 |
<PARAM required="true" type="string" name="fieldNames" managedBy="user">set,resulttypeid;set,datestamp;license,oaftype;set,oaftype;oaftype,funder;resulttypeid,oaftype;oaftype,funding;resulttypeid,funder;resulttypeid,funding</PARAM> |
|
| 298 | 272 |
</PARAMETERS> |
| 299 | 273 |
<ARCS> |
| 300 | 274 |
<ARC to="ConfigIndexes"/> |
| ... | ... | |
| 302 | 276 |
</NODE> |
| 303 | 277 |
<NODE name="ConfigIndexes" type="OAIEnsureIndexes"> |
| 304 | 278 |
<DESCRIPTION>Ensure an index exists on fields as specified in the configuration profile</DESCRIPTION> |
| 305 |
<PARAMETERS> |
|
| 306 |
<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM> |
|
| 307 |
<PARAM name="oaiStoreCollectionParam" type="string" managedBy="system" required="true">oaiStoreCollection</PARAM> |
|
| 308 |
</PARAMETERS> |
|
| 279 |
<PARAMETERS/> |
|
| 309 | 280 |
<ARCS> |
| 310 | 281 |
<ARC to="SetsCount"/> |
| 311 | 282 |
<ARC to="ConfigSets"/> |
| ... | ... | |
| 314 | 285 |
<NODE name="SetsCount" type="OAISetsCountUpdate"> |
| 315 | 286 |
<DESCRIPTION>Count records in each OAI set, for each exported metadata format</DESCRIPTION> |
| 316 | 287 |
<PARAMETERS> |
| 317 |
<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM> |
|
| 318 |
<PARAM name="oaiStoreCollectionParam" type="string" managedBy="system" required="true">oaiStoreCollection</PARAM> |
|
| 319 | 288 |
<PARAM name="configuredOnly" type="boolean" managedBy="user" required="true">false</PARAM> |
| 320 | 289 |
</PARAMETERS> |
| 321 | 290 |
<ARCS> |
| 322 | 291 |
<ARC to="waitAll"/> |
| 323 | 292 |
</ARCS> |
| 324 | 293 |
</NODE> |
| 325 |
<NODE name="ConfigSets" type="RefreshSetsFromConfig">
|
|
| 294 |
<NODE name="ConfigSets" type="OAIRefreshConfiguration">
|
|
| 326 | 295 |
<DESCRIPTION>Reads the current OAI configuration and upsert OAI sets accordingly (counts are updated as well)</DESCRIPTION> |
| 327 | 296 |
<PARAMETERS> |
| 328 |
<PARAM name="dbNameParam" required="true" type="string" managedBy="system">oai_dbName</PARAM> |
|
| 329 | 297 |
</PARAMETERS> |
| 330 | 298 |
<ARCS> |
| 331 | 299 |
<ARC to="waitAll"/> |
| 332 | 300 |
</ARCS> |
| 333 | 301 |
</NODE> |
| 334 |
|
|
| 335 | 302 |
<!-- WAIT FOR ALL THE WF BRANCHES TO COMPLETE --> |
| 336 | 303 |
<NODE name="waitAll" isJoin="true"> |
| 337 | 304 |
<DESCRIPTION>wait for all the branches to complete</DESCRIPTION> |
Also available in: Unified diff
Updated parameters of the OAI feed wf branch.