| 1 |
1 |
package eu.dnetlib.openaire.exporter.datasource.clients;
|
| 2 |
2 |
|
| 3 |
3 |
import java.io.IOException;
|
| 4 |
|
import java.util.Date;
|
| 5 |
|
import java.util.List;
|
|
4 |
import java.util.Calendar;
|
| 6 |
5 |
import java.util.Map;
|
| 7 |
6 |
import java.util.Queue;
|
| 8 |
7 |
import java.util.concurrent.*;
|
| ... | ... | |
| 28 |
27 |
import org.apache.solr.client.solrj.response.QueryResponse;
|
| 29 |
28 |
import org.apache.solr.common.SolrDocument;
|
| 30 |
29 |
import org.springframework.beans.factory.annotation.Autowired;
|
|
30 |
import org.springframework.cache.annotation.Cacheable;
|
| 31 |
31 |
import org.springframework.http.HttpStatus;
|
| 32 |
32 |
import org.springframework.stereotype.Component;
|
| 33 |
33 |
|
| ... | ... | |
| 147 |
147 |
// throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find document matching query: %s", queryTotal));
|
| 148 |
148 |
// }
|
| 149 |
149 |
return new IndexRecordsInfo()
|
| 150 |
|
.setDate(getDate(doc))
|
|
150 |
.setDate(getLastIndexingDate(indexClient))
|
| 151 |
151 |
.setTotal(rsp.getResults().getNumFound());
|
| 152 |
152 |
} catch (Throwable e) {
|
| 153 |
153 |
throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying information system", e);
|
| 154 |
154 |
}
|
| 155 |
155 |
}
|
| 156 |
156 |
|
| 157 |
|
private String getDate(final SolrDocument doc) throws ApiException {
|
| 158 |
|
final List<Date> dsversion = (List<Date>) doc.get(DSVERSION);
|
| 159 |
|
if (dsversion == null || dsversion.isEmpty()) {
|
| 160 |
|
throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find %s in matched solr document", DSVERSION));
|
| 161 |
|
}
|
| 162 |
|
final Date date = Iterables.getLast(dsversion);
|
|
157 |
@Cacheable("index-cache")
|
|
158 |
public String getLastIndexingDate(final CloudIndexClient indexClient) throws ApiException {
|
|
159 |
try {
|
|
160 |
final QueryResponse rsp = indexClient.query("oaftype:datasource", 1);
|
|
161 |
if (rsp.getResults().getNumFound() > 0) {
|
|
162 |
final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null);
|
| 163 |
163 |
|
| 164 |
|
return DateFormatUtils.format(date, DatasourceFunctions.DATE_FORMAT);
|
| 165 |
|
}
|
| 166 |
|
|
| 167 |
|
public String getLastIndexingDate(final IndexDsInfo info) throws ApiException {
|
| 168 |
|
try {
|
| 169 |
|
final QueryResponse rsp = getIndexClient(info).query("oaftype:datasource", 1);
|
| 170 |
|
final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null);
|
| 171 |
|
final String dsversion = doc.get("__dsversion").toString();
|
| 172 |
|
return StringUtils.substringBefore(dsversion, "T");
|
|
164 |
final String dsversion = doc.get(DSVERSION).toString();
|
|
165 |
return StringUtils.substringBefore(dsversion, "T");
|
|
166 |
} else {
|
|
167 |
final String defaultDate = getDefaultLastIndexingDate();
|
|
168 |
log.debug("unable to find documents, defaulting to " + defaultDate);
|
|
169 |
return defaultDate;
|
|
170 |
}
|
| 173 |
171 |
} catch (CloudIndexClientException e) {
|
| 174 |
|
throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info, e);
|
|
172 |
throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile", e);
|
| 175 |
173 |
}
|
| 176 |
174 |
}
|
| 177 |
175 |
|
|
176 |
private String getDefaultLastIndexingDate() {
|
|
177 |
final Calendar cal = Calendar.getInstance();
|
|
178 |
cal.add(Calendar.MONTH, -1);
|
|
179 |
return DateFormatUtils.format(cal.getTime(), DatasourceFunctions.DATE_FORMAT);
|
|
180 |
}
|
|
181 |
|
| 178 |
182 |
private synchronized CloudIndexClient getIndexClient(final IndexDsInfo info) throws CloudIndexClientException {
|
| 179 |
183 |
if (!indexClientMap.containsKey(info.getColl())) {
|
| 180 |
184 |
indexClientMap.put(info.getColl(), CloudIndexClientFactory.newIndexClient(info.getIndexBaseUrl(), info.getColl(), false));
|
Added by Claudio Atzori about 7 years ago
implemented new way to fetch the lastIndexingDate