Project

General

Profile

« Previous | Next » 

Revision 49819

Added by Claudio Atzori over 6 years ago

implemented new way to fetch the lastIndexingDate

View differences:

DatasourceIndexClient.java
1 1 
package eu.dnetlib.openaire.exporter.datasource.clients;
2 2 

  		




  3
  3
  
    
import java.io.IOException;


  		




  4
  
  
    
import java.util.Date;


  		




  5
  
  
    
import java.util.List;


  		




  
  4
  
    
import java.util.Calendar;


  		




  6
  5
  
    
import java.util.Map;


  		




  7
  6
  
    
import java.util.Queue;


  		




  8
  7
  
    
import java.util.concurrent.*;


  		




  ......




  28
  27
  
    
import org.apache.solr.client.solrj.response.QueryResponse;


  		




  29
  28
  
    
import org.apache.solr.common.SolrDocument;


  		




  30
  29
  
    
import org.springframework.beans.factory.annotation.Autowired;


  		




  
  30
  
    
import org.springframework.cache.annotation.Cacheable;


  		




  31
  31
  
    
import org.springframework.http.HttpStatus;


  		




  32
  32
  
    
import org.springframework.stereotype.Component;


  		




  33
  33
  
    

  		




  ......




  147
  147
  
    
			//				throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find document matching query: %s", queryTotal));


  		




  148
  148
  
    
			//			}


  		




  149
  149
  
    
			return new IndexRecordsInfo()


  		




  150
  
  
    
					.setDate(getDate(doc))


  		




  
  150
  
    
					.setDate(getLastIndexingDate(indexClient))


  		




  151
  151
  
    
					.setTotal(rsp.getResults().getNumFound());


  		




  152
  152
  
    
		} catch (Throwable e) {


  		




  153
  153
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying information system", e);


  		




  154
  154
  
    
		}


  		




  155
  155
  
    
	}


  		




  156
  156
  
    

  		




  157
  
  
    
	private String getDate(final SolrDocument doc) throws ApiException {


  		




  158
  
  
    
		final List<Date> dsversion = (List<Date>) doc.get(DSVERSION);


  		




  159
  
  
    
		if (dsversion == null || dsversion.isEmpty()) {


  		




  160
  
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), String.format("cannot find %s in matched solr document", DSVERSION));


  		




  161
  
  
    
		}


  		




  162
  
  
    
		final Date date = Iterables.getLast(dsversion);


  		




  
  157
  
    
	@Cacheable("index-cache")


  		




  
  158
  
    
	public String getLastIndexingDate(final CloudIndexClient indexClient) throws ApiException {


  		




  
  159
  
    
		try {


  		




  
  160
  
    
			final QueryResponse rsp = indexClient.query("oaftype:datasource", 1);


  		




  
  161
  
    
			if (rsp.getResults().getNumFound() > 0) {


  		




  
  162
  
    
				final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null);


  		




  163
  163
  
    

  		




  164
  
  
    
		return DateFormatUtils.format(date, DatasourceFunctions.DATE_FORMAT);


  		




  165
  
  
    
	}


  		




  166
  
  
    

  		




  167
  
  
    
	public String getLastIndexingDate(final IndexDsInfo info) throws ApiException {


  		




  168
  
  
    
		try {


  		




  169
  
  
    
			final QueryResponse rsp = getIndexClient(info).query("oaftype:datasource", 1);


  		




  170
  
  
    
			final SolrDocument doc = Iterables.getFirst(rsp.getResults(), null);


  		




  171
  
  
    
			final String dsversion = doc.get("__dsversion").toString();


  		




  172
  
  
    
			return StringUtils.substringBefore(dsversion, "T");


  		




  
  164
  
    
				final String dsversion = doc.get(DSVERSION).toString();


  		




  
  165
  
    
				return StringUtils.substringBefore(dsversion, "T");


  		




  
  166
  
    
			} else {


  		




  
  167
  
    
				final String defaultDate = getDefaultLastIndexingDate();


  		




  
  168
  
    
				log.debug("unable to find documents, defaulting to " + defaultDate);


  		




  
  169
  
    
				return defaultDate;


  		




  
  170
  
    
			}


  		




  173
  171
  
    
		} catch (CloudIndexClientException e) {


  		




  174
  
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile: " + info, e);


  		




  
  172
  
    
			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile", e);


  		




  175
  173
  
    
		}


  		




  176
  174
  
    
	}


  		




  177
  175
  
    

  		




  
  176
  
    
	private String getDefaultLastIndexingDate() {


  		




  
  177
  
    
		final Calendar cal = Calendar.getInstance();


  		




  
  178
  
    
		cal.add(Calendar.MONTH, -1);


  		




  
  179
  
    
		return DateFormatUtils.format(cal.getTime(), DatasourceFunctions.DATE_FORMAT);


  		




  
  180
  
    
	}


  		




  
  181
  
    

  		




  178
  182
  
    
	private synchronized CloudIndexClient getIndexClient(final IndexDsInfo info) throws CloudIndexClientException {


  		




  179
  183
  
    
		if (!indexClientMap.containsKey(info.getColl())) {


  		




  180
  184
  
    
			indexClientMap.put(info.getColl(), CloudIndexClientFactory.newIndexClient(info.getIndexBaseUrl(), info.getColl(), false));


  		












Also available in:  Unified diff