Project

General

Profile

« Previous | Next » 

Revision 34211

Added by Marek Horst over 9 years ago

#1070 introducing support for multiple context identifiers, replacing import_project_concepts_context_id IIS input parameter with import_project_concepts_context_ids_csv

View differences:

modules/icm-iis-import/trunk/src/main/java/eu/dnetlib/iis/importer/concept/ISLookupServiceBasedConceptImporter.java
39 39
 */
40 40
public class ISLookupServiceBasedConceptImporter implements Process {
41 41

  
42
	public static final String PARAM_IMPORT_CONTEXT_ID = "import.context.id";
42
	public static final String PARAM_IMPORT_CONTEXT_IDS_CSV = "import.context.ids.csv";
43 43
	
44 44
	public static final String PARAM_IMPORT_RESULTSET_PAGESIZE = "import.resultset.pagesize";
45 45
	
......
73 73
			throw new InvalidParameterException("unknown ISLookup service location, "
74 74
					+ "required parameter '" + WorkflowRuntimeParameters.IMPORT_ISLOOKUP_SERVICE_LOCATION + "' is missing!");
75 75
		}
76
		String contextId;
77
		if (parameters.containsKey(PARAM_IMPORT_CONTEXT_ID)) {
78
			contextId = parameters.get(PARAM_IMPORT_CONTEXT_ID);
76
		String contextIdsCSV;
77
		if (parameters.containsKey(PARAM_IMPORT_CONTEXT_IDS_CSV)) {
78
			contextIdsCSV = parameters.get(PARAM_IMPORT_CONTEXT_IDS_CSV);
79 79
		} else {
80 80
			throw new InvalidParameterException("unknown context identifier");
81 81
		}
......
101 101
					new FileSystemPath(fs, portBindings.getOutput().get(PORT_OUT_CONCEPTS)), 
102 102
					Concept.SCHEMA$);
103 103
			
104
			W3CEndpointReference results = isLookupService.searchProfile("//BODY/CONFIGURATION/context"
105
					+ "[@id=\""+contextId+"\"]");
104
			String[] contextIds = StringUtils.split(contextIdsCSV, ',');
105
			if (contextIds.length==0) {
106
				throw new RuntimeException("got 0 context ids, "
107
						+ "unable to tokenize context identifiers: " + contextIdsCSV);
108
			}
109
			StringBuilder query = new StringBuilder("//BODY/CONFIGURATION/context[");
110
			int tokensCount = 0;
111
			for (String contextId : contextIds) {
112
				if (tokensCount>0) {
113
					query.append(" or ");		
114
				}
115
				query.append("@id=\"" + contextId + "\"");	
116
				tokensCount++;
117
			}
118
			query.append(']');
119
			
120
			W3CEndpointReference results = isLookupService.searchProfile(
121
					query.toString());
106 122
			ResultSetClientFactory rsFactory = new ResultSetClientFactory();
107 123
			if (rsClientReadTimeout!=null) {
108 124
				rsFactory.setTimeout(rsClientReadTimeout);	
......
123 139
							new ConceptXmlHandler(
124 140
									new DataFileRecordReceiver<Concept>(conceptWriter)));
125 141
				} else {
126
					log.error("got empty context when looking for for context id: " + 
127
							contextId + ", service location: " + isLookupServiceLocation);
142
					log.error("got empty context when looking for for context ids: " + 
143
							contextIdsCSV + ", service location: " + isLookupServiceLocation);
128 144
				}
129 145
			}
130 146
			if (count==0) {
131
				log.warn("got " + count + " profiles when looking for context id: " + 
132
							contextId + ", service location: " + isLookupServiceLocation);
147
				log.warn("got 0 profiles when looking for context ids: " + 
148
						contextIdsCSV + ", service location: " + isLookupServiceLocation);
133 149
			}
134 150
		} catch (ISLookUpDocumentNotFoundException e) {
135
			log.error("unable to find profile for context id: " + 
136
					contextId + ", service location: " + isLookupServiceLocation, e);
151
			log.error("unable to find profile for context ids: " + 
152
					contextIdsCSV + ", service location: " + isLookupServiceLocation, e);
137 153
		} finally {
138 154
			if (conceptWriter!=null) {
139 155
				conceptWriter.close();	
modules/icm-iis-import/trunk/src/main/resources/eu/dnetlib/iis/importer/concept/job.properties
1 1
#islookup_service_location=http://openaire-services.vls.icm.edu.pl:8280/is/services/isLookUp
2 2
islookup_service_location=http://beta.services.openaire.eu:8280/is/services/isLookUp
3
context_id=fet-fp7
3
context_ids_csv=fet-fp7
4 4
output=${workingDir}/out
modules/icm-iis-import/trunk/src/main/resources/eu/dnetlib/iis/importer/concept/oozie_app/workflow.xml
10 10
			<description>concept avro datastore output</description>
11 11
		</property>
12 12
		<property>
13
			<name>context_id</name>
14
			<description>context identifier</description>
13
			<name>context_ids_csv</name>
14
			<description>comma separated context identifiers</description>
15 15
		</property>
16 16
	</parameters>
17 17

  
......
35 35
			<arg>eu.dnetlib.iis.importer.concept.ISLookupServiceBasedConceptImporter</arg>
36 36
			<arg>-SworkingDir=${workingDir}</arg>
37 37
			<arg>-Pimport.islookup.service.location=${islookup_service_location}</arg>
38
			<arg>-Pimport.context.id=${context_id}</arg>
38
			<arg>-Pimport.context.ids.csv=${context_ids_csv}</arg>
39 39
			<arg>-Oconcepts=${output}</arg>
40 40
		</java>
41 41
		<ok to="end" />

Also available in: Unified diff