Project

General

Profile

« Previous | Next » 

Revision 52010

moved directIndex API in a new module

View differences:

modules/dnet-openaireplus-workflows/trunk/src/test/java/eu/dnetlib/msro/openaireplus/api/objects/ResultEntryTest.java
1
package eu.dnetlib.msro.openaireplus.api.objects;
2

  
3
import java.io.InputStreamReader;
4
import java.io.StringReader;
5
import java.util.ArrayList;
6
import java.util.Arrays;
7
import java.util.Properties;
8

  
9
import com.google.gson.Gson;
10
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
11
import eu.dnetlib.msro.rmi.MSROException;
12
import org.apache.commons.lang3.StringEscapeUtils;
13
import org.apache.velocity.app.VelocityEngine;
14
import org.dom4j.Document;
15
import org.dom4j.io.OutputFormat;
16
import org.dom4j.io.SAXReader;
17
import org.dom4j.io.XMLWriter;
18
import org.junit.Before;
19
import org.junit.Test;
20
import org.junit.runner.RunWith;
21
import org.mockito.Mock;
22
import org.mockito.invocation.InvocationOnMock;
23
import org.mockito.runners.MockitoJUnitRunner;
24
import org.mockito.stubbing.Answer;
25

  
26
import static org.mockito.Matchers.anyString;
27
import static org.mockito.Mockito.when;
28

  
29
/**
30
 * Created by michele on 14/12/15.
31
 */
32
@RunWith(MockitoJUnitRunner.class)
33
public class ResultEntryTest {
34

  
35
	private VelocityEngine ve;
36

  
37
	@Mock
38
	private ISLookUpService lookUpService;
39

  
40
	@Before
41
	public void setUp() throws Exception {
42

  
43
		final Properties props = new Properties();
44
		props.setProperty("resource.loader", "class");
45
		props.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader");
46
		props.setProperty("runtime.log.logsystem.class", "org.apache.velocity.runtime.log.Log4JLogChute");
47
		ve = new VelocityEngine();
48
		ve.init(props);
49

  
50
		when(lookUpService.quickSearchProfile(anyString())).thenAnswer(new Answer<Object>() {
51

  
52
			@Override
53
			public Object answer(final InvocationOnMock invocation) throws Throwable {
54
				final String query = invocation.getArguments()[0].toString();
55
				if (query.contains("dnet:result_typologies")) {
56
					return Arrays.asList("publication @@@ publication", "dataset @@@ dataset", "software @@@ software");
57
				} else if (query.contains("dnet:access_modes")) {
58
					return Arrays.asList("OPEN @@@ Open Access");
59
				} else if (query.contains("dnet:publication_resource")) {
60
					return Arrays.asList("0001 @@@ Journal Article");
61
				} else if (query.contains("dnet:pid_types")) {
62
					return Arrays.asList("oai @@@ Open Archive Initiative", "doi @@@ Digital object identifier");
63
				} else if (query.contains("dnet:languages")) {
64
					return Arrays.asList("ita @@@ Italian");
65
				} else if (query.contains("ContextDSResourceType")) {
66
					return Arrays.asList(
67
							"egi::classification::natsc::math::stats @@@ Statistics and Probability",
68
							"egi::classification::natsc::math::pure @@@ Pure Mathematics",
69
							"egi::classification::natsc::math @@@ Mathematics",
70
							"egi::classification::natsc @@@ Natural Sciences",
71
							"egi::classification @@@ EGI classification scheme",
72
							"egi @@@ EGI", "mes @@@ MES");
73
				} else {
74
					return new ArrayList<String>();
75
				}
76
			}
77
		});
78

  
79
		when(lookUpService.getResourceProfileByQuery(anyString())).thenAnswer(new Answer<Object>() {
80

  
81
			@Override
82
			public Object answer(final InvocationOnMock invocation) throws Throwable {
83
				return "REPO NAME @@@ repo________";
84
			}
85
		});
86
	}
87

  
88
	@Test
89
	public void testAsIndexRecord() throws Exception {
90
		final ResultEntry pub = new ResultEntry();
91
		pub.setOriginalId("ORIG_ID_1234");
92
		pub.setTitle("TEST TITLE <test>");
93
		pub.getAuthors().add("Michele Artini");
94
		pub.getAuthors().add("Claudio Atzori");
95
		pub.getAuthors().add("Alessia Bardi");
96
		pub.setPublisher("Test publisher");
97
		pub.setDescription("DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION ");
98
		pub.setLanguage("ita");
99
		pub.setLicenseCode("OPEN");
100
		pub.setResourceType("0001");
101
		pub.setUrl("http://test.it/a=1&b=2");
102
		pub.getPids().add(new PidEntry("doi", "10.000/xyz-123"));
103
		pub.getPids().add(new PidEntry("oai", "oai:1234"));
104
		pub.setCollectedFromId("test________::zenodo");
105
		pub.setHostedById("test________::UNKNOWN");
106
		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus");
107
		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble");
108
		pub.getContexts().add("egi::classification::natsc::math::pure");
109
		pub.getContexts().add("egi::classification::natsc::math::stats");
110
		final String xml = pub.asOafRecord(ve, lookUpService, "http://oaf/oaf.xsd");
111

  
112
		final Document doc = (new SAXReader()).read(new StringReader(xml));
113

  
114
		final OutputFormat format = OutputFormat.createPrettyPrint();
115

  
116
		final XMLWriter writer = new XMLWriter(System.out, format);
117

  
118
		writer.write(doc);
119

  
120
		/* writer.close(); */
121

  
122
	}
123

  
124
	@Test
125
	public void testAsIndexRecord_json() throws Exception {
126
		testAsIndexRecord_json("test_record.json");
127

  
128
	}
129

  
130
	@Test
131
	public void testAsIndexRecord_json_with_greek_chars() throws Exception {
132

  
133
		testAsIndexRecord_json("test_record_with_greek_chars.json");
134

  
135
	}
136

  
137
	@Test
138
	public void testAsIndexRecord_openaireId() throws Exception {
139

  
140
		testAsIndexRecord_json("test_record_openaireId.json");
141

  
142
	}
143

  
144
	@Test(expected = MSROException.class)
145
	public void testAsIndexRecord_wrongOpenaireId() throws Exception {
146

  
147
		testAsIndexRecord_json("test_record_wrong_openaireId.json");
148

  
149
	}
150

  
151
	@Test
152
	public void testAsIndexRecord_json_zenodo() throws Exception {
153

  
154
		testAsIndexRecord_json("test_zenodo.json");
155

  
156
	}
157

  
158
	@Test
159
	public void testAsIndexRecord_ZenodoSw() throws Exception {
160
		testAsIndexRecord_json("test_zenodo_software.json");
161

  
162
	}
163

  
164
	private void testAsIndexRecord_json(final String filename) throws Exception {
165
		final ResultEntry pub =
166
				(new Gson()).fromJson(new InputStreamReader(getClass().getResourceAsStream(filename)), ResultEntry.class);
167

  
168
		final String xml = pub.asOafRecord(ve, lookUpService, "http://oaf/oaf.xsd");
169
		System.out.println(xml);
170

  
171
		final Document doc = (new SAXReader()).read(new StringReader(xml));
172

  
173
		final OutputFormat format = OutputFormat.createPrettyPrint();
174

  
175
		final XMLWriter writer = new XMLWriter(System.out, format);
176

  
177
		writer.write(doc);
178

  
179
		/* writer.close(); */
180
	}
181

  
182
	@Test
183
	public void testEscapeUnicode(){
184
		String unicodeTxt = "i.e. closed curves of the form $t\ud835\udfc4 [0,2\u03c0] \u21a6 (\\cos t)u + (\\sin t)v$ for suitable orthogonal vectors $u$";
185
		System.out.println(StringEscapeUtils.escapeXml11(unicodeTxt));
186
	}
187

  
188

  
189
}
modules/dnet-openaireplus-workflows/trunk/src/test/java/eu/dnetlib/msro/openaireplus/api/OpenAIRESubmitterUtilsTest.java
1
package eu.dnetlib.msro.openaireplus.api;
2

  
3
import java.util.Map;
4
import java.util.Map.Entry;
5

  
6
import org.junit.Test;
7

  
8
/**
9
 * Created by Alessia Bardi on 26/05/2017.
10
 *
11
 * @author Alessia Bardi
12
 */
13
public class OpenAIRESubmitterUtilsTest {
14

  
15
	private OpenAIRESubmitterUtils utils = new OpenAIRESubmitterUtils();
16

  
17
	final String fullProject = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble";
18
	final String minimalProject = "info:eu-repo/grantAgreement/EC/FP7/244909///WorkAble";
19
	final String onlyId = "info:eu-repo/grantAgreement/EC/FP7/244909/";
20
	final String onlyTitle = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work";
21

  
22
	@Test
23
	public void testCalculateProjectInfoFull() {
24
		final Map<String, String> project = utils.calculateProjectInfo(fullProject);
25
		print(project);
26
	}
27

  
28
	@Test
29
	public void testCalculateProjectInfoOnlyId() {
30
		final Map<String, String> project = utils.calculateProjectInfo(onlyId);
31
		print(project);
32
	}
33

  
34
	@Test
35
	public void testCalculateProjectInfoMinimalAcro() {
36
		final Map<String, String> project = utils.calculateProjectInfo(minimalProject);
37
		print(project);
38
	}
39

  
40
	@Test
41
	public void testCalculateProjectInfoOnlyTitle() {
42
		final Map<String, String> project = utils.calculateProjectInfo(onlyTitle);
43
		print(project);
44
	}
45

  
46
	private void print(final Map<String, String> map) {
47
		for (final Entry e : map.entrySet()) {
48
			System.out.println(e.getKey() + " = " + e.getValue());
49
		}
50
	}
51
}
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/OpenaireResultSubmitter.java
1
package eu.dnetlib.msro.openaireplus.api;
2

  
3
import java.io.IOException;
4
import java.util.List;
5
import javax.annotation.Resource;
6

  
7
import com.google.common.collect.Lists;
8
import com.google.gson.Gson;
9
import eu.dnetlib.common.rmi.DNetRestDocumentation;
10
import eu.dnetlib.data.index.CloudIndexClient;
11
import eu.dnetlib.data.index.CloudIndexClientFactory;
12
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
13
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
14
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
15
import eu.dnetlib.msro.openaireplus.api.objects.ResultEntry;
16
import eu.dnetlib.msro.openaireplus.utils.OafToIndexRecordFactory;
17
import eu.dnetlib.msro.rmi.MSROException;
18
import org.apache.commons.io.IOUtils;
19
import org.apache.commons.lang.exception.ExceptionUtils;
20
import org.apache.commons.logging.Log;
21
import org.apache.commons.logging.LogFactory;
22
import org.apache.velocity.app.VelocityEngine;
23
import org.springframework.beans.factory.annotation.Value;
24
import org.springframework.core.io.ClassPathResource;
25
import org.springframework.http.HttpStatus;
26
import org.springframework.stereotype.Controller;
27
import org.springframework.web.bind.annotation.*;
28

  
29
/**
30
 * Created by michele on 11/11/15.
31
 */
32
@Controller
33
@DNetRestDocumentation
34
public class OpenaireResultSubmitter {
35

  
36
	private static final Log log = LogFactory.getLog(OpenaireResultSubmitter.class);
37

  
38
	public class IndexDsInfo {
39

  
40
		private final String indexBaseUrl;
41
		private final String indexDsId;
42
		private final String format;
43
		private final String coll;
44

  
45
		public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) {
46
			this.indexBaseUrl = indexBaseUrl;
47
			this.indexDsId = indexDsId;
48
			this.format = format;
49
			this.coll = coll;
50
		}
51

  
52
		public String getIndexBaseUrl() {
53
			return indexBaseUrl;
54
		}
55

  
56
		public String getIndexDsId() {
57
			return indexDsId;
58
		}
59

  
60
		public String getFormat() {
61
			return format;
62
		}
63

  
64
		public String getColl() {
65
			return coll;
66
		}
67

  
68
	}
69

  
70
	@Value(value = "oaf.schema.location")
71
	private String oafSchemaLocation;
72

  
73
	@Resource
74
	private UniqueServiceLocator serviceLocator;
75

  
76
	@Resource
77
	private OafToIndexRecordFactory oafToIndexRecordFactory;
78

  
79
	@Resource
80
	private RecentResultsQueue recentResultsQueue;
81

  
82
	@Resource(name = "openaireplusApisVelocityEngine")
83
	private VelocityEngine velocityEngine;
84

  
85
	@Value(value = "${openaireplus.msro.api.findSolrIndexUrl.xquery}")
86
	private ClassPathResource findSolrIndexUrl;
87

  
88
	@Value(value = "${openaireplus.msro.api.findIndexDsInfo.xquery}")
89
	private ClassPathResource findIndexDsInfo;
90

  
91
	@Deprecated
92
	@RequestMapping(value = { "/api/publications/feedJson", "/api/results/feedJson" }, method = RequestMethod.POST)
93
	public
94
	@ResponseBody
95
	String feedObjectJson(@RequestParam(value = "json", required = true) final String json,
96
			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws MSROException {
97
		final ResultEntry pub = new Gson().fromJson(json, ResultEntry.class);
98
		return feedObject(pub, commit);
99
	}
100

  
101
	@RequestMapping(value = { "/api/results/feedObject" }, method = RequestMethod.POST)
102
	public
103
	@ResponseBody
104
	String feedResult(@RequestBody final ResultEntry pub,
105
			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)
106
			throws MSROException {
107

  
108
		try {
109
			return feedObject(pub, commit);
110
		} catch (final Throwable e) {
111
			throw new MSROException("Error adding publication: " + e.getMessage(), e);
112
		}
113
	}
114

  
115
	@RequestMapping(value = { "/api/publications/feedObject" }, method = RequestMethod.POST)
116
	public
117
	@ResponseBody
118
	String feedObject(@RequestBody final ResultEntry pub,
119
			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit)
120
			throws MSROException {
121

  
122
		final List<IndexDsInfo> idxList;
123
		try {
124
			idxList = calculateCurrentIndexDsInfo();
125
			if (idxList == null || idxList.isEmpty()) {
126
				throw new MSROException("Cannot add result: " + pub.getAnyId() + " : No public Search Service found");
127
			}
128
			if (idxList.size() > 1) log.warn("Found more than 1 public search service");
129
			final String oafRecord = pub.asOafRecord(velocityEngine, serviceLocator.getService(ISLookUpService.class), oafSchemaLocation);
130

  
131
			for (IndexDsInfo idx : idxList) {
132
				CloudIndexClient idxClient = null;
133
				try {
134
					idxClient = CloudIndexClientFactory.newIndexClient(idx.getIndexBaseUrl(), idx.getColl(), false);
135
					idxClient.feed(oafRecord, idx.getIndexDsId(), oafToIndexRecordFactory.newTransformer(idx.getFormat()), commit);
136
				} catch (final Throwable e) {
137
					throw new MSROException("Error adding publication: " + e.getMessage(), e);
138
				} finally {
139
					if (idxClient != null) {
140
						idxClient.close();
141
					}
142
				}
143
			}
144
			recentResultsQueue.add(oafRecord);
145
			return pub.getOpenaireId();
146
		} catch (final Throwable e) {
147
			log.debug(pub.toString());
148
			throw new MSROException("Error adding publication: " + e.getMessage(), e);
149
		}
150
	}
151

  
152
	@RequestMapping(value = "/api/result/{openaireId}", method = RequestMethod.DELETE)
153
	public
154
	@ResponseBody
155
	boolean deleteResultWithOpenaireId(
156
			@PathVariable(value = "openaireId") final String openaireId,
157
			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws MSROException {
158

  
159
		return deleteResult(openaireId, commit);
160
	}
161

  
162
	@RequestMapping(value = "/api/results", method = RequestMethod.DELETE)
163
	public
164
	@ResponseBody
165
	boolean deleteResultWithOriginalId(
166
			@RequestParam(value = "originalId", required = true) final String originalId,
167
			@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,
168
			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws Exception {
169

  
170
		final String openaireId = ResultEntry.calculateOpenaireId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class));
171
		return deleteResult(openaireId, commit);
172
	}
173

  
174
	@Deprecated
175
	@RequestMapping(value = { "/api/publications/deleteObject", "/api/results/delete" }, method = RequestMethod.POST)
176
	public
177
	@ResponseBody
178
	boolean deleteResultPost(
179
			@RequestParam(value = "originalId", required = true) final String originalId,
180
			@RequestParam(value = "collectedFromId", required = true) final String collectedFromId,
181
			@RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws Exception {
182

  
183
		final String openaireId = ResultEntry.calculateOpenaireId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class));
184
		return deleteResult(openaireId, commit);
185
	}
186

  
187
	private boolean deleteResult(final String openaireId, final boolean commit) throws MSROException {
188

  
189
		final List<IndexDsInfo> idxList;
190
		try {
191
			idxList = calculateCurrentIndexDsInfo();
192

  
193
			if (idxList == null || idxList.isEmpty()) {
194
				throw new MSROException("Cannot delete result: " + openaireId + " : No public Search Service found");
195
			}
196
			if (idxList.size() > 1) log.warn("Found more than 1 public search service");
197

  
198
			//			final String objId = ResultEntry.calculateOpenaireId(originalId, collectedFromId, serviceLocator.getService(ISLookUpService.class));
199

  
200
			for (IndexDsInfo idx : idxList) {
201
				CloudIndexClient idxClient = null;
202
				try {
203
					idxClient = CloudIndexClientFactory.newIndexClient(idx.getIndexBaseUrl(), idx.getColl(), false);
204
					idxClient.remove(openaireId, commit);
205
					log.info("Deleted result with id: " + openaireId + " from: " + idx.getIndexBaseUrl());
206
				} catch (final Throwable e) {
207
					throw new MSROException("Error deleting publication: " + e.getMessage(), e);
208
				} finally {
209
					if (idxClient != null) {
210
						idxClient.close();
211
					}
212
				}
213
			}
214
			recentResultsQueue.remove(openaireId);
215
			return true;
216
		} catch (IOException | ISLookUpException e) {
217
			throw new MSROException("Error deleting publication: " + e.getMessage(), e);
218
		}
219
	}
220

  
221
	@ExceptionHandler(Exception.class)
222
	@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
223
	public
224
	@ResponseBody
225
	ErrorMessage handleException(final Exception e) {
226
		log.error("Error in direct index API", e);
227
		return new ErrorMessage(e);
228
	}
229

  
230
	private List<IndexDsInfo> calculateCurrentIndexDsInfo() throws IOException, ISLookUpException {
231
		final List<IndexDsInfo> list = Lists.newArrayList();
232

  
233
		final String queryUrl = IOUtils.toString(findSolrIndexUrl.getInputStream());
234
		final String queryDs = IOUtils.toString(findIndexDsInfo.getInputStream());
235

  
236
		final ISLookUpService lu = serviceLocator.getService(ISLookUpService.class);
237
		final String indexBaseUrl = lu.getResourceProfileByQuery(queryUrl);
238

  
239
		final List<String> idxDs = lu.quickSearchProfile(queryDs);
240
		for (String idx : idxDs) {
241
			final String[] arr = idx.split("@@@");
242
			list.add(new IndexDsInfo(indexBaseUrl, arr[0].trim(), arr[1].trim(), arr[2].trim()));
243
		}
244
		return list;
245
	}
246

  
247
	public class ErrorMessage {
248

  
249
		private final String message;
250
		private final String stacktrace;
251

  
252
		public ErrorMessage(final Exception e) {
253
			this(e.getMessage(), ExceptionUtils.getStackTrace(e));
254
		}
255

  
256
		public ErrorMessage(final String message, final String stacktrace) {
257
			this.message = message;
258
			this.stacktrace = stacktrace;
259
		}
260

  
261
		public String getMessage() {
262
			return this.message;
263
		}
264

  
265
		public String getStacktrace() {
266
			return this.stacktrace;
267
		}
268

  
269
	}
270

  
271
}
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/OpenAIRESubmitterUtils.java
1
package eu.dnetlib.msro.openaireplus.api;
2

  
3
import java.io.StringWriter;
4
import java.util.ArrayList;
5
import java.util.HashMap;
6
import java.util.List;
7
import java.util.Map;
8

  
9
import com.google.common.base.Function;
10
import com.google.common.collect.Lists;
11
import eu.dnetlib.miscutils.functional.hash.Hashing;
12
import org.apache.commons.lang.StringUtils;
13

  
14
/**
15
 * Created by michele on 15/01/16.
16
 */
17
public class OpenAIRESubmitterUtils {
18

  
19
	private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(OpenAIRESubmitterUtils.class);
20

  
21
	public Map<String, String> calculateProjectInfo(final String link) {
22
		final Map<String, String> info = new HashMap<String, String>();
23
		final String[] arr = link.split("/");
24
		// info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble
25

  
26
		if (arr.length > 4) {
27
			final String acronym = arr.length > 7 ? arr[7] : "";
28
			final String title = arr.length > 6 ? StringUtils.isNotBlank(arr[6]) ? arr[6] : acronym : "";
29
			final String jurisdiction = arr.length > 5 ? arr[5] : "";
30
			final String funderId = calculateFunderId(arr[2], arr[3]);
31
			info.put("id", calculateProjectId(arr[2], arr[3], arr[4]));
32
			info.put("funderShortName", arr[2]);
33
			info.put("fundingName", arr[3]);
34
			info.put("code", arr[4]);
35
			info.put("jurisdiction", jurisdiction);
36
			info.put("title", title);
37
			info.put("acronym", acronym);
38
			info.put("funderId", funderId);
39
			info.put("funderName", calculateFunderName(arr[2]));
40
			info.put("fundingId", funderId + "::" + arr[3]);
41
		}
42
		return info;
43
	}
44

  
45
	protected String calculateFunderPrefix(final String funderShortName, final String funding){
46
		switch(funderShortName.toLowerCase()){
47
		case "conicyt":
48
			return "conicytf____::";
49
		case "ec":
50
			if (funding.equalsIgnoreCase("fp7")) {
51
				return "corda_______::";
52
			} else {
53
				return "corda__h2020::";
54
			}
55
		case "hrzz":
56
		case "mzos":
57
			return "irb_hr______::";
58
		case "tara":
59
			return "taraexp_____::";
60
		case "tubitak":
61
			return "tubitakf____::";
62
		default:
63
			String prefix = funderShortName.toLowerCase();
64
			//ensure we have 12 chars
65
			while(prefix.length() < 12) prefix += "_";
66
			return prefix +"::";
67
		}
68
	}
69

  
70
	protected String calculateProjectId(final String funderShortName, final String funding, final String code) {
71
		final String suffix = Hashing.md5(code);
72
		final String funderPrefix = calculateFunderPrefix(funderShortName, funding);
73
		return funderPrefix + suffix;
74
	}
75

  
76
	protected String calculateFunderId(final String funderShortName, final String funding) {
77
		switch (funderShortName.toLowerCase()) {
78
		case "ec":
79
			return "ec__________::EC";
80
		default:
81
			String prefix = calculateFunderPrefix(funderShortName, funding);
82
			return prefix + funderShortName.toUpperCase();
83
		}
84
	}
85

  
86

  
87
	protected String calculateFunderName(final String funderShortName) {
88

  
89
		switch (funderShortName.toLowerCase()) {
90
		case "aff":
91
			return "Suomen Akatemia";
92
		case "arc":
93
			return "Australian Research Council (ARC)";
94
		case "conicyt":
95
			return "Comisión Nacional de Investigación Científica y Tecnológica";
96
		case "ec":
97
			return "European Commission";
98
		case "fct":
99
			return "Fundação para a Ciência e a Tecnologia, I.P.";
100
		case "fwf":
101
			return "Austrian Science Fund (FWF)";
102
		case "hrzz":
103
			return "Croatian Science Foundation (CSF)";
104
		case "mestd":
105
			return "Ministry of Education, Science and Technological Development of Republic of Serbia";
106
		case "mzos":
107
			return "Ministry of Science, Education and Sports of the Republic of Croatia (MSES)";
108
		case "nhmrc":
109
			return "National Health and Medical Research Council (NHMRC)";
110
		case "nih":
111
			return "National Institutes of Health";
112
		case "nsf":
113
			return "National Science Foundation";
114
		case "nwo":
115
			return "Netherlands Organisation for Scientific Research (NWO)";
116
		case "rcuk":
117
			return "Research Council UK";
118
		case "sfi":
119
			return "Science Foundation Ireland";
120
		case "sgov":
121
			return "Gobierno de España";
122
		case "snsf":
123
			return "Swiss National Science Foundation";
124
		case "tara":
125
			return "Tara Expeditions Foundation";
126
		case "tubitak":
127
			return "Türkiye Bilimsel ve Teknolojik Araştırma Kurumu";
128
		case "wt":
129
			return "Wellcome Trust";
130
		default:
131
			log.error("Funder short name '"+funderShortName+"' not managed");
132
			return "";
133
		}
134
	}
135

  
136
	public List<ContextInfo> processContexts(final List<String> list) {
137
		return Lists.newArrayList(Lists.transform(list, new Function<String, ContextInfo>() {
138

  
139
			@Override
140
			public ContextInfo apply(final String s) {
141
				return createContextInfo(s.split("::"), 0);
142
			}
143

  
144
			private ContextInfo createContextInfo(final String[] arr, final int pos) {
145
				final StringWriter id = new StringWriter();
146
				id.write(arr[0]);
147
				for (int i = 0; i < pos; i++) {
148
					id.write("::");
149
					id.write(arr[i + 1]);
150
				}
151
				final String elem = (pos == 0) ? "context" : (pos == 1) ? "category" : "concept";
152
				final ContextInfo info = new ContextInfo(elem, id.toString());
153
				if ((pos + 1) < arr.length) {
154
					info.getChildren().add(createContextInfo(arr, pos + 1));
155
				}
156
				return info;
157
			}
158
		}));
159

  
160
	}
161

  
162
	public class ContextInfo {
163

  
164
		private String elem;
165
		private String id;
166
		private List<ContextInfo> children = new ArrayList<ContextInfo>();
167

  
168
		public ContextInfo(final String elem,
169
				final String id) {
170
			this.elem = elem;
171
			this.id = id;
172
		}
173

  
174
		public String getElem() {
175
			return elem;
176
		}
177

  
178
		public void setElem(final String elem) {
179
			this.elem = elem;
180
		}
181

  
182
		public String getId() {
183
			return id;
184
		}
185

  
186
		public void setId(final String id) {
187
			this.id = id;
188
		}
189

  
190
		public List<ContextInfo> getChildren() {
191
			return children;
192
		}
193

  
194
		public void setChildren(final List<ContextInfo> children) {
195
			this.children = children;
196
		}
197

  
198
		public boolean isRoot() {
199
			return elem.equals("context");
200
		}
201
	}
202
}
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/RecentResultsQueue.java
1
package eu.dnetlib.msro.openaireplus.api;
2

  
3
import java.io.StringReader;
4
import java.util.Iterator;
5
import java.util.List;
6

  
7
import com.mongodb.*;
8
import eu.dnetlib.miscutils.datetime.DateUtils;
9
import org.apache.commons.logging.Log;
10
import org.apache.commons.logging.LogFactory;
11
import org.dom4j.io.SAXReader;
12
import org.springframework.beans.factory.annotation.Required;
13

  
14
/**
15
 * Created by michele on 11/11/15.
16
 */
17
public class RecentResultsQueue implements Iterable<String> {
18

  
19
	private static final Log log = LogFactory.getLog(RecentResultsQueue.class);
20

  
21
	private DB db;
22
	private String collection;
23

  
24
	public void init() {
25
		if (!db.collectionExists(collection)) {
26
			log.info(String.format("creating collection %s", collection));
27
			db.createCollection(collection, new BasicDBObject());
28
		}
29
	}
30

  
31
	@Override
32
	public Iterator<String> iterator() {
33

  
34
		final DBCursor cursor = db.getCollection(collection).find();
35

  
36
		return new Iterator<String>() {
37

  
38
			@Override
39
			public boolean hasNext() {
40
				return cursor.hasNext();
41
			}
42

  
43
			@Override
44
			public String next() {
45
				final DBObject obj = cursor.next();
46
				return ((obj != null) && obj.containsField("record")) ? obj.get("record").toString() : "";
47
			}
48

  
49
			@Override
50
			public void remove() {
51
				throw new RuntimeException("NOT IMPLEMENTED");
52
			}
53
		};
54
	}
55

  
56
	synchronized public void add(final String oaf) throws Exception {
57
		final String id = (new SAXReader()).read(new StringReader(oaf)).valueOf("//*[local-name() = 'objIdentifier']");
58

  
59
		log.info("Saving record " + id + " in db: " + db.getName() + ", coll: " + collection);
60

  
61
		final DBCollection coll = db.getCollection(collection);
62
		final DBObject obj = BasicDBObjectBuilder.start()
63
				.append("id", id)
64
				.append("record", oaf)
65
				.append("date", DateUtils.now())
66
				.get();
67
		coll.update(new BasicDBObject("id", id), obj, true, false);
68
	}
69

  
70
	public void remove(final List<String> list) {
71
		final DBCollection coll = db.getCollection(collection);
72
		for (final String id : list) {
73
			coll.remove(new BasicDBObject("id", id));
74
		}
75
	}
76

  
77
	public void remove(final String... ids) {
78
		final DBCollection coll = db.getCollection(collection);
79
		for (final String id : ids) {
80
			coll.remove(new BasicDBObject("id", id));
81
		}
82
	}
83

  
84
	public DB getDb() {
85
		return db;
86
	}
87

  
88
	@Required
89
	public void setDb(final DB db) {
90
		this.db = db;
91
	}
92

  
93
	public String getCollection() {
94
		return collection;
95
	}
96

  
97
	@Required
98
	public void setCollection(final String collection) {
99
		this.collection = collection;
100
	}
101
}
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/objects/DatasourceEntry.java
1
package eu.dnetlib.msro.openaireplus.api.objects;
2

  
3
import eu.dnetlib.miscutils.functional.hash.Hashing;
4
import org.apache.commons.lang.StringUtils;
5

  
6
/**
7
 * Created by michele on 02/12/15.
8
 */
9
public class DatasourceEntry {
10

  
11
	private String id;
12
	private String name;
13
	private String prefix;
14

  
15
	public DatasourceEntry() {
16
	}
17

  
18
	public DatasourceEntry(final String id, final String name, final String prefix) {
19
		this.id = id;
20
		this.name = name;
21
		this.prefix = prefix;
22
	}
23

  
24
	public String getId() {
25
		return id;
26
	}
27

  
28
	public void setId(final String id) {
29
		this.id = id;
30
	}
31

  
32
	public String getName() {
33
		return name;
34
	}
35

  
36
	public void setName(final String name) {
37
		this.name = name;
38
	}
39

  
40
	public String getPrefix() {
41
		return prefix;
42
	}
43

  
44
	public void setPrefix(final String prefix) {
45
		this.prefix = prefix;
46
	}
47

  
48
	public String calculateOpenaireId() {
49
		if (StringUtils.isNotBlank(id)) {
50
			final String[] arr = id.split("::");
51
			if (arr.length == 2) {
52
				return String.format("%s::%s", arr[0], Hashing.md5(arr[1]));
53
			}
54
		}
55
		return "";
56
	}
57
}
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/objects/PidEntry.java
1
package eu.dnetlib.msro.openaireplus.api.objects;
2

  
3
import io.swagger.annotations.ApiModelProperty;
4

  
5
/**
6
 * Created by michele on 02/12/15.
7
 */
8
public class PidEntry {
9

  
10
	private String type;
11
	private String value;
12

  
13
	public PidEntry() {
14
	}
15

  
16
	public PidEntry(final String type, final String value) {
17
		this.type = type;
18
		this.value = value;
19
	}
20

  
21
	@ApiModelProperty(required = true, value = "E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types")
22
	public String getType() {
23
		return type;
24
	}
25

  
26
	public void setType(final String type) {
27
		this.type = type;
28
	}
29

  
30
	@ApiModelProperty(required = true)
31
	public String getValue() {
32
		return value;
33
	}
34

  
35
	public void setValue(final String value) {
36
		this.value = value;
37
	}
38
}
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/api/objects/ResultEntry.java
1
package eu.dnetlib.msro.openaireplus.api.objects;
2

  
3
import java.text.SimpleDateFormat;
4
import java.util.*;
5
import java.util.concurrent.TimeUnit;
6

  
7
import com.google.gson.Gson;
8
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
9
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
10
import eu.dnetlib.miscutils.datetime.DateUtils;
11
import eu.dnetlib.miscutils.functional.hash.Hashing;
12
import eu.dnetlib.miscutils.functional.string.EscapeXml;
13
import eu.dnetlib.msro.openaireplus.api.OpenAIRESubmitterUtils;
14
import eu.dnetlib.msro.rmi.MSROException;
15
import io.swagger.annotations.ApiModelProperty;
16
import org.apache.commons.lang.StringUtils;
17
import org.apache.commons.logging.Log;
18
import org.apache.commons.logging.LogFactory;
19
import org.apache.velocity.app.VelocityEngine;
20
import org.springframework.ui.velocity.VelocityEngineUtils;
21

  
22
/**
23
 * Created by michele on 02/12/15.
24
 */
25
public class ResultEntry {
26

  
27
	private String openaireId;
28
	private String originalId;
29
	private String title;
30
	private List<String> authors = new ArrayList<String>();
31
	private String publisher;
32
	private String description;
33
	private String language;
34
	private List<PidEntry> pids = new ArrayList<PidEntry>();
35
	private String licenseCode;
36
	private String embargoEndDate;
37
	private String type = "publication";
38
	private String resourceType;
39
	private String url;
40
	private String collectedFromId;
41
	private String hostedById;
42

  
43
	// String according to the EGI context profile, example: egi::classification::natsc::math
44
	private List<String> contexts = new ArrayList<String>();
45

  
46
	// String according to openaire guidelines:
47
	// info:eu-repo/grantAgreement/Funder/FundingProgram/ProjectID/[Jurisdiction]/[ProjectName]/[ProjectAcronym]
48
	private List<String> linksToProjects = new ArrayList<String>();
49

  
50
	private static long last_cache_update = 0;
51
	private static final Map<String, Map<String, String>> cached_vocabularies = new HashMap<String, Map<String, String>>();
52
	private static final Map<String, DatasourceEntry> cached_datasources = new HashMap<String, DatasourceEntry>();
53
	private static final Map<String, String> cached_contexts = new HashMap<String, String>();
54

  
55
	private static final Log log = LogFactory.getLog(ResultEntry.class);
56

  
57
	public ResultEntry() {
58
	}
59

  
60
	public String getOpenaireId() {
61
		return openaireId;
62
	}
63

  
64
	public void setOpenaireId(String openaireId) {
65
		this.openaireId = openaireId;
66
	}
67

  
68
	public String getOriginalId() {
69
		return originalId;
70
	}
71

  
72
	public void setOriginalId(final String originalId) {
73
		this.originalId = originalId;
74
	}
75

  
76
	@ApiModelProperty(required = true)
77
	public String getTitle() {
78
		return title;
79
	}
80

  
81
	public void setTitle(final String title) {
82
		this.title = title;
83
	}
84

  
85
	public List<String> getAuthors() {
86
		return authors;
87
	}
88

  
89
	public void setAuthors(final List<String> authors) {
90
		this.authors = authors;
91
	}
92

  
93
	public String getPublisher() {
94
		return publisher;
95
	}
96

  
97
	public void setPublisher(final String publisher) {
98
		this.publisher = publisher;
99
	}
100

  
101
	public String getDescription() {
102
		return description;
103
	}
104

  
105
	public void setDescription(final String description) {
106
		this.description = description;
107
	}
108

  
109
	@ApiModelProperty(value = "ISO Alpha-3 code. E.g. 'eng', 'ita'")
110
	public String getLanguage() {
111
		return language;
112
	}
113

  
114
	public void setLanguage(final String language) {
115
		this.language = language;
116
	}
117

  
118
	public List<PidEntry> getPids() {
119
		return pids;
120
	}
121

  
122
	public void setPids(final List<PidEntry> pids) {
123
		this.pids = pids;
124
	}
125

  
126
	@ApiModelProperty(required = true, allowableValues = "OPEN, CLOSED, RESTRICTED, EMBARGO, UNKNOWN, OTHER")
127
	public String getLicenseCode() {
128
		return licenseCode;
129
	}
130

  
131
	public void setLicenseCode(final String licenseCode) {
132
		this.licenseCode = licenseCode;
133
	}
134

  
135
	@ApiModelProperty(required = true, value = "Use 001 for articles, 021 for datasets. See: http://api.openaire.eu/vocabularies/dnet:publication_resource.")
136
	public String getResourceType() {
137
		return resourceType;
138
	}
139

  
140
	public void setResourceType(final String resourceType) {
141
		this.resourceType = resourceType;
142
	}
143

  
144
	@ApiModelProperty(required = true)
145
	public String getUrl() {
146
		return url;
147
	}
148

  
149
	public void setUrl(final String url) {
150
		this.url = url;
151
	}
152

  
153
	@ApiModelProperty(required = true, value = "Use opendoar___::2659 for Zenodo Publications; re3data_____::r3d100010468 for Zenodo datasets; infrastruct::openaire for OpenAIRE portal.")
154
	public String getCollectedFromId() {
155
		return collectedFromId;
156
	}
157

  
158
	public void setCollectedFromId(final String collectedFromId) {
159
		this.collectedFromId = collectedFromId;
160
	}
161

  
162
	public String getHostedById() {
163
		return hostedById;
164
	}
165

  
166
	public void setHostedById(final String hostedById) {
167
		this.hostedById = hostedById;
168
	}
169

  
170
	@ApiModelProperty(value = "E.g. fet, egi::classification::natsc::math::pure, egi::projects::EMI")
171
	public List<String> getContexts() {
172
		return contexts;
173
	}
174

  
175
	public void setContexts(final List<String> contexts) {
176
		this.contexts = contexts;
177
	}
178

  
179
	@ApiModelProperty(value = "E.g. info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus")
180
	public List<String> getLinksToProjects() {
181
		return linksToProjects;
182
	}
183

  
184
	public void setLinksToProjects(final List<String> linksToProjects) {
185
		this.linksToProjects = linksToProjects;
186
	}
187

  
188
	@ApiModelProperty(allowableValues = "publication, dataset")
189
	public String getType() {
190
		return type;
191
	}
192

  
193
	public void setType(final String type) {
194
		this.type = type;
195
	}
196

  
197
	public String getEmbargoEndDate() {
198
		return embargoEndDate;
199
	}
200

  
201
	public void setEmbargoEndDate(final String embargoEndDate) {
202
		this.embargoEndDate = embargoEndDate;
203
	}
204

  
205
	public String asOafRecord(final VelocityEngine ve,
206
			final ISLookUpService lookupService,
207
			final String oafSchemaLocation) throws Exception {
208

  
209
		if (StringUtils.isBlank(getOriginalId()) && StringUtils.isBlank(getOpenaireId())) {
210
			throw new MSROException("One of the following fields is required: originalId or openaireId");
211
		}
212
		if (StringUtils.isBlank(getTitle())) { throw new MSROException("A required field is missing: title"); }
213
		if (StringUtils.isBlank(getUrl())) { throw new MSROException("A required field is missing: url"); }
214
		if (StringUtils.isBlank(getLicenseCode())) { throw new MSROException("A required field is missing: licenseCode"); }
215
		if (StringUtils.isBlank(getResourceType())) { throw new MSROException("A required field is missing: resourceType"); }
216
		if (StringUtils.isBlank(getCollectedFromId())) { throw new MSROException("A required field is missing: collectedFromId"); }
217
		if (StringUtils.isBlank(getType())) { throw new MSROException("A required field is missing: type"); }
218

  
219
		final DatasourceEntry collectedFromEntry = getDatasourceInfo(collectedFromId, lookupService);
220
		final DatasourceEntry hostedByEntry = getDatasourceInfo(hostedById, lookupService);
221

  
222
		if (StringUtils.isBlank(openaireId)) {
223
			setOpenaireId(calculateOpenaireId(originalId, collectedFromEntry));
224
		}
225

  
226
		if (!openaireId.matches("^\\w{12}::\\w{32}$")) {
227
			throw new MSROException("Invalid openaireId: " + openaireId + " - regex ^\\w{12}::\\w{32}$ not matched");
228
		}
229

  
230
		final Map<String, Object> model = new HashMap<String, Object>();
231
		model.put("esc", new EscapeXml());
232
		model.put("util", new OpenAIRESubmitterUtils());
233
		model.put("pub", this);
234
		model.put("objIdentifier", getOpenaireId());
235
		model.put("oafSchemaLocation", oafSchemaLocation);
236
		model.put("resultTypes", getVocabulary("dnet:result_typologies", lookupService));
237
		model.put("licenses", getVocabulary("dnet:access_modes", lookupService));
238
		model.put("resourceTypes", getVocabulary("dnet:publication_resource", lookupService));
239
		model.put("pidTypes", getVocabulary("dnet:pid_types", lookupService));
240
		model.put("languages", getVocabulary("dnet:languages", lookupService));
241
		model.put("contexts", getContexts(lookupService));
242
		model.put("dateOfCollection", (new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'")).format(new Date()));
243
		model.put("collectedFrom", collectedFromEntry);
244
		model.put("hostedBy", hostedByEntry);
245

  
246
		return VelocityEngineUtils.mergeTemplateIntoString(ve, "/eu/dnetlib/msro/openaireplus/api/indexRecord.xml.vm", "UTF-8", model);
247
	}
248

  
249
	private static String calculateOpenaireId(final String originalId, final DatasourceEntry collectedFromEntry) {
250
		return collectedFromEntry.getPrefix() + "::" + Hashing.md5(originalId);
251
	}
252

  
253
	public static String calculateOpenaireId(final String originalId, final String collectedFromId, final ISLookUpService lookupService)
254
			throws ISLookUpException {
255
		return calculateOpenaireId(originalId, getDatasourceInfo(collectedFromId, lookupService));
256
	}
257

  
258
	private synchronized static DatasourceEntry getDatasourceInfo(final String dsId, final ISLookUpService lookupService) throws ISLookUpException {
259
		if (StringUtils
260
				.isBlank(dsId)) { return new DatasourceEntry("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "unknown_____"); }
261

  
262
		if (!cached_datasources.containsKey(dsId)) {
263
			final String query =
264
					"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//CONFIGURATION[./DATASOURCE_ORIGINAL_ID='" + dsId
265
							+ "']/concat(./OFFICIAL_NAME, ' @@@ ', .//FIELD/value[../key='NamespacePrefix'])";
266
			final String s = lookupService.getResourceProfileByQuery(query);
267
			final String[] arr = s.split("@@@");
268

  
269
			final DatasourceEntry ds = new DatasourceEntry(dsId, arr[0].trim(), arr[1].trim());
270

  
271
			if (StringUtils.isBlank(ds.getName()) || StringUtils.isBlank(ds.getPrefix())) {
272
				log.error("Invalid datasource id: " + dsId);
273
				throw new ISLookUpException("Invalid datasource id: " + dsId);
274
			} else {
275
				cached_datasources.put(dsId, ds);
276
			}
277
		}
278

  
279
		return cached_datasources.get(dsId);
280

  
281
	}
282

  
283
	private synchronized static Map<String, String> getVocabulary(final String voc, final ISLookUpService lookupService) throws ISLookUpException {
284

  
285
		if (((DateUtils.now() - last_cache_update) < TimeUnit.MINUTES.toMillis(15)) && cached_vocabularies.containsKey(voc)) {
286
			return cached_vocabularies.get(voc);
287
		} else {
288
			final String query = "collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code='" + voc
289
					+ "']//TERM/concat(@code, ' @@@ ', @english_name)";
290

  
291
			final Map<String, String> map = new HashMap<String, String>();
292
			for (final String s : lookupService.quickSearchProfile(query)) {
293
				final String[] arr = s.split("@@@");
294
				map.put(arr[0].trim(), arr[1].trim());
295
			}
296

  
297
			cached_vocabularies.put(voc, map);
298

  
299
			last_cache_update = DateUtils.now();
300

  
301
			return map;
302
		}
303
	}
304

  
305
	private synchronized static Map<String, String> getContexts(final ISLookUpService lookupService) throws ISLookUpException {
306
		if (((DateUtils.now() - last_cache_update) > TimeUnit.MINUTES.toMillis(15)) || cached_contexts.isEmpty()) {
307
			final String query =
308
					"collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)";
309

  
310
			cached_contexts.clear();
311
			for (final String s : lookupService.quickSearchProfile(query)) {
312
				final String[] arr = s.split("@@@");
313
				cached_contexts.put(arr[0].trim(), arr[1].trim());
314
			}
315
			last_cache_update = DateUtils.now();
316
		}
317
		return cached_contexts;
318
	}
319

  
320
	@Override
321
	public String toString() {
322
		return new Gson().toJson(this);
323
	}
324

  
325
	public String getAnyId(){
326
		return StringUtils.isNotBlank(openaireId) ? openaireId : originalId;
327
	}
328
}
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/utils/OafToIndexRecordFactory.java
1
package eu.dnetlib.msro.openaireplus.utils;
2

  
3
import java.io.IOException;
4
import java.io.StringReader;
5
import java.io.StringWriter;
6
import javax.annotation.Resource;
7
import javax.xml.transform.Transformer;
8
import javax.xml.transform.TransformerException;
9
import javax.xml.transform.TransformerFactory;
10
import javax.xml.transform.stream.StreamResult;
11
import javax.xml.transform.stream.StreamSource;
12

  
13
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
14
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
15
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
16
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
17
import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
18
import org.apache.commons.io.IOUtils;
19
import org.springframework.beans.factory.annotation.Required;
20
import org.springframework.core.io.ClassPathResource;
21

  
22
/**
23
 * Created by michele on 15/12/15.
24
 */
25
public class OafToIndexRecordFactory {
26

  
27
	private ClassPathResource layoutToRecord;
28

  
29
	@Resource
30
	private UniqueServiceLocator serviceLocator;
31

  
32
	public ApplyXslt newTransformer(final String format) throws ISLookUpException, IOException, TransformerException {
33
		final TransformerFactory factory = TransformerFactory.newInstance();
34
		final Transformer layoutTransformer = factory.newTransformer(readLayoutToRecord());
35

  
36
		final StreamResult layoutToXsltXslt = new StreamResult(new StringWriter());
37

  
38
		layoutTransformer.setParameter("format", format);
39
		layoutTransformer.transform(new StreamSource(new StringReader(getLayoutSource(format))), layoutToXsltXslt);
40

  
41
		return new ApplyXslt(layoutToXsltXslt.getWriter().toString());
42
	}
43

  
44
	private StreamSource readLayoutToRecord() throws IOException {
45
		return new StreamSource(new StringReader(IOUtils.toString(layoutToRecord.getInputStream())));
46
	}
47

  
48
	private String getLayoutSource(final String format) throws ISLookUpDocumentNotFoundException, ISLookUpException {
49
		return serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(
50
				"collection('/db/DRIVER/MDFormatDSResources/MDFormatDSResourceType')[.//NAME='" + format + "']//LAYOUT[@name='index']");
51
	}
52

  
53
	public ClassPathResource getLayoutToRecord() {
54
		return layoutToRecord;
55
	}
56

  
57
	@Required
58
	public void setLayoutToRecord(final ClassPathResource layoutToRecord) {
59
		this.layoutToRecord = layoutToRecord;
60
	}
61

  
62
}
modules/dnet-openaireplus-workflows/trunk/src/main/java/eu/dnetlib/msro/openaireplus/workflows/nodes/index/FeedMissingClaimsJobNode.java
12 12
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
13 13
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
14 14
import eu.dnetlib.miscutils.functional.xml.ApplyXslt;
15
import eu.dnetlib.msro.openaireplus.api.RecentResultsQueue;
16
import eu.dnetlib.msro.openaireplus.utils.OafToIndexRecordFactory;
17 15
import eu.dnetlib.msro.rmi.MSROException;
18 16
import eu.dnetlib.msro.workflows.nodes.AsyncJobNode;
17
import eu.dnetlib.openaire.directindex.api.RecentResultsQueue;
18
import eu.dnetlib.openaire.directindex.utils.OafToIndexRecordFactory;
19

  
19 20
import org.apache.commons.io.IOUtils;
20 21
import org.apache.commons.logging.Log;
21 22
import org.apache.commons.logging.LogFactory;
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/api/applicationContext-api.properties
1
openaireplus.msro.api.mongo.host=localhost
2
openaireplus.msro.api.mongo.port=27017
3
openaireplus.msro.api.mongo.db=openaireplus_apis
4
openaireplus.msro.api.mongo.collection=recent_publications
5
openaireplus.msro.api.layoutToRecord.xslt=/eu/dnetlib/msro/openaireplus/workflows/index/openaireLayoutToRecordStylesheet.xsl
6
openaireplus.msro.api.findSolrIndexUrl.xquery=/eu/dnetlib/msro/openaireplus/api/findSolrIndexUrl.xquery
7
openaireplus.msro.api.findIndexDsInfo.xquery=/eu/dnetlib/msro/openaireplus/api/findIndexDsInfo.xquery
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/api/applicationContext-api.xml
1
<?xml version="1.0" encoding="UTF-8"?>
2
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3
       xmlns:p="http://www.springframework.org/schema/p"
4
       xmlns="http://www.springframework.org/schema/beans"
5
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
6

  
7
	<bean id="recentResultsQueue" class="eu.dnetlib.msro.openaireplus.api.RecentResultsQueue"
8
	      p:db-ref="msroApiMongoDB" p:collection="${openaireplus.msro.api.mongo.collection}"
9
	      init-method="init"/>
10

  
11

  
12
	<bean id="apiMongoServer" class="com.mongodb.Mongo">
13
		<constructor-arg index="0" type="com.mongodb.ServerAddress">
14
			<bean class="com.mongodb.ServerAddress">
15
				<constructor-arg index="0"
16
				                 value="${openaireplus.msro.api.mongo.host}"/>
17
				<constructor-arg index="1"
18
				                 value="${openaireplus.msro.api.mongo.port}"/>
19
			</bean>
20
		</constructor-arg>
21
	</bean>
22

  
23
	<bean id="apiOafToIndexRecordFactory" class="eu.dnetlib.msro.openaireplus.utils.OafToIndexRecordFactory"
24
	      p:layoutToRecord="${openaireplus.msro.api.layoutToRecord.xslt}"/>
25

  
26

  
27
	<bean id="msroApiMongoDB" factory-bean="apiMongoServer"
28
	      factory-method="getDB">
29
		<constructor-arg index="0" value="${openaireplus.msro.api.mongo.db}"/>
30
	</bean>
31

  
32
	<bean id="openaireplusApisVelocityEngine" class="org.springframework.ui.velocity.VelocityEngineFactoryBean">
33
		<property name="velocityProperties">
34
			<value>
35
				resource.loader=class
36
				class.resource.loader.class=org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader
37
				runtime.log.logsystem.class=org.apache.velocity.runtime.log.Log4JLogChute
38
			</value>
39
		</property>
40
	</bean>
41

  
42
</beans>
modules/dnet-openaireplus-workflows/trunk/src/main/resources/eu/dnetlib/msro/openaireplus/api/indexRecord.xml.vm
1
<?xml version="1.0" encoding="UTF-8"?>
2

  
3
<!--
4
#macro(context $ctx)
5
        <$!ctx.elem id="$!esc.evaluate($!ctx.id)" label="$!esc.evaluate($contexts.get($!ctx.id))" #if($!ctx.isRoot()) type="community"#end>
6
	#foreach($child in $!ctx.children)#context($child)#end
7
        </$!ctx.elem>
8
#end
9
-->
10

  
11
<record xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oaf="http://namespace.openaire.eu/oaf">
12
	<result>
13
		<header>
14
			<dri:objIdentifier>$!esc.evaluate($!objIdentifier)</dri:objIdentifier>
15
			<dri:dateOfCollection>$!esc.evaluate($!dateOfCollection)</dri:dateOfCollection>
16
			<dri:status>under curation</dri:status>
17
			<counters/>
18
		</header>
19
		<metadata>
20
			<oaf:entity schemaLocation="http://namespace.openaire.eu/oaf $!oafSchemaLocation">
21
				<oaf:result>
22
					<title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">
23
						$!esc.evaluate($!pub.title)
24
					</title>
25
					<dateofacceptance/>
26
					<resulttype classid="$!esc.evaluate($!pub.type)" classname="$!esc.evaluate($!resultTypes.get($!pub.type))" schemeid="dnet:result_typologies" schemename="dnet:result_typologies"/>
27
					<language classid="$!esc.evaluate($!pub.language)" classname="$!esc.evaluate($!languages.get($!pub.language))" schemeid="dnet:languages"
28
					          schemename="dnet:languages"/>
29
					<description>
30
						$!esc.evaluate($!pub.description)
31
					</description>
32
					<country classid="" classname="" schemeid="" schemename=""/>
33
					<subject classid="" classname="" schemeid="" schemename=""/>
34
					<relevantdate classid="" classname="" schemeid="" schemename=""/>
35
					<publisher>$!esc.evaluate($!pub.publisher)</publisher>
36
					<embargoenddate>$!esc.evaluate($!pub.embargoEndDate)</embargoenddate>
37
					<source/>
38
					<fulltext/>
39
					<format/>
40
					<storagedate/>
41
					<resourcetype classid="" classname="" schemeid="" schemename=""/>
42
					<device/>
43
					<size/>
44
					<version/>
45
					<lastmetadataupdate/>
46
					<metadataversionnumber/>
47
					<originalId>$!esc.evaluate($!pub.originalId)</originalId>
48
					<collectedfrom name="$!esc.evaluate($!collectedFrom.name)" id="$!esc.evaluate($!collectedFrom.calculateOpenaireId())"/>
49

  
50
					#foreach($pid in $!pub.pids)
51
						<pid classid="$!esc.evaluate($!pid.type)" classname="$!esc.evaluate($!pidTypes.get($!pid.type))" schemeid="dnet:pid_types"
52
						     schemename="dnet:pid_types">$!esc.evaluate($pid.value)</pid>
53
					#end
54

  
55
					<bestlicense classid="$!esc.evaluate($!pub.licenseCode)" classname="$!esc.evaluate($!licenses.get($!pub.licenseCode))"
56
					             schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
57

  
58
					#foreach($ctx in $util.processContexts($!pub.contexts))
59
						#context($ctx)
60
					#end
61

  
62
					<datainfo>
63
						<inferred>false</inferred>
64
						<deletedbyinference>false</deletedbyinference>
65
						<trust>0.9</trust>
66
						<inferenceprovenance/>
67
						<provenanceaction classid="user:insert" classname="user:insert" schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff