D-Net

package eu.dnetlib.openaire.exporter.datasource.clients;

import java.io.IOException;

import java.util.Calendar;

import java.util.Date;

import java.util.Map;

import java.util.Queue;

import java.util.concurrent.*;

import javax.annotation.PostConstruct;

import javax.annotation.PreDestroy;

import com.google.common.collect.Iterables;

import com.google.common.util.concurrent.*;

import eu.dnetlib.OpenaireExporterConfig;

import eu.dnetlib.miscutils.functional.hash.Hashing;

import eu.dnetlib.openaire.exporter.datasource.ApiException;

import eu.dnetlib.openaire.exporter.datasource.clients.utils.DatasourceFunctions;

import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexDsInfo;

import eu.dnetlib.openaire.exporter.datasource.clients.utils.IndexRecordsInfo;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.lang.time.DateFormatUtils;

import org.apache.commons.lang3.exception.ExceptionUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.apache.solr.client.solrj.SolrClient;

import org.apache.solr.client.solrj.SolrQuery;

import org.apache.solr.client.solrj.SolrServerException;

import org.apache.solr.client.solrj.impl.CloudSolrClient;

import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;

import org.apache.solr.client.solrj.response.QueryResponse;

import org.apache.solr.common.SolrDocument;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.cache.annotation.Cacheable;

import org.springframework.http.HttpStatus;

import org.springframework.stereotype.Component;

/**

 * Created by claudio on 20/10/2016.

 */

@Component

public class DatasourceIndexClient {

	private static final Log log = LogFactory.getLog(DatasourceIndexClient.class);

	public static final String SEPARATOR = "::";

	public static final String DSVERSION = "__dsversion";

	@Autowired

	private OpenaireExporterConfig config;

	private ListeningExecutorService executor;

	private static Map<String, SolrClient> indexClientMap = new ConcurrentHashMap<>();

	@PostConstruct

	public void init() {

		executor = MoreExecutors.listeningDecorator(

				new ScheduledThreadPoolExecutor(2,

						new ThreadFactoryBuilder().setNameFormat("datasource-index-client-%d").build()));

	}

	@PreDestroy

	public void tearDown() {

		indexClientMap.forEach((name, client) -> {

			try {

				client.close();

			} catch (IOException e) {

				log.warn(String.format("unable to gracefully shutdown client for index %s", name));

			}

		});

	}

	public IndexRecordsInfo getIndexInfo(final String dsId, final IndexDsInfo info, final Queue<Throwable> errors) throws ApiException {

		try {

			final String collectedFrom = StringUtils.substringBefore(dsId, SEPARATOR) + SEPARATOR + Hashing.md5(StringUtils.substringAfter(dsId, SEPARATOR));

			final SolrClient indexClient = getIndexClient(info);

			final CountDownLatch latch = new CountDownLatch(2);

			final IndexRecordsInfo indexRecordInfo = new IndexRecordsInfo();

			Futures.addCallback(

				executor.submit(() -> setTotal(collectedFrom, indexClient)),

						new FutureCallback<IndexRecordsInfo>() {

							@Override

							public void onSuccess(final IndexRecordsInfo info) {

								indexRecordInfo

										.setTotal(info.getTotal())

										.setDate(info.getDate());

								latch.countDown();

							}

							@Override

							public void onFailure(final Throwable e) {

								log.error(ExceptionUtils.getStackTrace(e));

								errors.offer(e);

								latch.countDown();

							}

						});

			Futures.addCallback(

					executor.submit(() -> setFunded(collectedFrom, indexClient)),

					new FutureCallback<Long>() {

						@Override

						public void onSuccess(final Long numFound) {

							indexRecordInfo.setFunded(numFound);

							latch.countDown();

						}

						@Override

						public void onFailure(final Throwable e) {

							log.error(ExceptionUtils.getStackTrace(e));

							errors.offer(e);

							latch.countDown();

						}

					});

			waitLatch(latch, errors, config.getRequestTimeout());

			return indexRecordInfo;

		} catch (final Throwable e) {

			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying publications from: " + dsId, e);

		}

	}

	private Long setFunded(

			final String collectedFrom,

			final SolrClient indexClient) throws ApiException {

		final SolrQuery query = new SolrQuery(

				String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\" AND relprojectid:*", collectedFrom))

				.setRows(0);

		log.debug(String.format("query: %s", query));

		try {

			return indexClient.query(query).getResults().getNumFound();

		} catch (Throwable e) {

			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying information system", e);

		}

	}

	private IndexRecordsInfo setTotal(

			final String collectedFrom,

			final SolrClient indexClient) throws ApiException {

		try {

			final SolrQuery query = new SolrQuery(

					String.format("oaftype:result AND deletedbyinference:false AND collectedfromdatasourceid:\"%s\"", collectedFrom))

					.setRows(1);

			log.debug(String.format("query: %s", query));

			final QueryResponse rsp = indexClient.query(query);

			final SolrDocument doc = Iterables.getFirst(rsp.getResults(), new SolrDocument());

			if (log.isDebugEnabled()) {

				log.debug(String.format("got document %s", doc.get("__indexrecordidentifier")));

			}

			return new IndexRecordsInfo()

					.setDate(getLastIndexingDate(indexClient))

					.setTotal(rsp.getResults().getNumFound());

		} catch (Throwable e) {

			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying information system", e);

		}

	}

	@Cacheable("index-cache")

	public String getLastIndexingDate(final SolrClient indexClient) throws ApiException {

		try {

			final QueryResponse rsp = indexClient.query(new SolrQuery("oaftype:datasource").setRows(1));

			if (rsp.getResults().getNumFound() > 0) {

				final SolrDocument doc = rsp.getResults().get(0);

				final Date dsversion = (Date) doc.getFirstValue(DSVERSION);

				log.debug("got __dsversion: " + dsversion);

				return DateFormatUtils.format(dsversion, DatasourceFunctions.DATE_FORMAT);

			} else {

				final String defaultDate = getDefaultLastIndexingDate();

				log.debug("unable to find documents, defaulting to " + defaultDate);

				return defaultDate;

			}

		} catch (SolrServerException | IOException e) {

			throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.value(), "Error querying index DS profile", e);

		}

	}

	private String getDefaultLastIndexingDate() {

		final Calendar cal = Calendar.getInstance();

		cal.add(Calendar.MONTH, -1);

		return DateFormatUtils.format(cal.getTime(), DatasourceFunctions.DATE_FORMAT);

	}

	private synchronized SolrClient getIndexClient(final IndexDsInfo info) {

		if (!indexClientMap.containsKey(info.getColl())) {

			final CloudSolrClient client = new Builder().withZkHost(info.getIndexBaseUrl()).build();

			client.setDefaultCollection(info.getColl());

			indexClientMap.put(info.getColl(), client);

		}

		return indexClientMap.get(info.getColl());

	}

	private void waitLatch(final CountDownLatch latch, final Queue<Throwable> errors, final int waitSeconds) {

		try {

			if (!latch.await(waitSeconds, TimeUnit.SECONDS)) {

				errors.offer(new TimeoutException("Waiting for requests to complete has timed out."));

			}

		} catch (final InterruptedException e) {

			errors.offer(e);

		}

	}

}