D-Net

package eu.dnetlib.functionality.modular.ui.dedup;

import java.util.List;

import java.util.Map;

import java.util.UUID;

import javax.annotation.Resource;

import javax.xml.ws.wsaddressing.W3CEndpointReference;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.RequestBody;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestParam;

import org.springframework.web.bind.annotation.ResponseBody;

import com.google.common.base.Function;

import com.google.common.base.Joiner;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.enabling.database.rmi.DatabaseService;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;

import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;

import eu.dnetlib.enabling.locators.UniqueServiceLocator;

import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;

import eu.dnetlib.functionality.index.client.IndexClient;

import eu.dnetlib.functionality.index.client.IndexClientException;

import eu.dnetlib.functionality.index.client.ResolvingIndexClientFactory;

import eu.dnetlib.functionality.index.client.response.LookupResponse;

import eu.dnetlib.functionality.modular.ui.AbstractAjaxController;

import eu.dnetlib.miscutils.datetime.DateUtils;

import eu.dnetlib.miscutils.functional.xml.ApplyXslt;

@Controller

public class DedupServiceInternalController extends AbstractAjaxController {

	private static final Log log = LogFactory.getLog(DedupServiceInternalController.class);

	@Resource

	private UniqueServiceLocator serviceLocator;

	@Autowired

	private ResultSetClientFactory resultSetClientFactory;

	@Value("${dnet.openaire.db.name}")

	private String dbName;

	/** The index client factory. */

	@Autowired

	private ResolvingIndexClientFactory indexClientFactory;

	@Value("${dnet.openaire.similaritygroups.index.format.xquery}")

	private String indexFormatXquery;

	@Value("${dnet.openaire.similaritygroups.indexrecord2html.xsl}")

	private org.springframework.core.io.Resource recordToHtmlXsl;

	@Value("${dnet.openaire.similaritygroups.indexrecord2htmldetails.xsl}")

	private org.springframework.core.io.Resource recordToHtmlDetailsXsl;

	private String currentIndex = "";

	private IndexClient indexClient = null;

	@ResponseBody

	@RequestMapping(value = "/ui/dedup/listSimRels.do")

	public List<SimilarityGroup> list(@RequestParam(value = "entitytype", required = true) final String entitytype,

			@RequestParam(value = "offset", required = true) final int offset,

			@RequestParam(value = "limit", required = true) final int limit) throws Exception {

		final StringBuilder query = new StringBuilder(

				"SELECT groupid as id, entitytype, date, array_agg(objidentifier) as group FROM similarity_groups sg " +

				"LEFT OUTER join similarities s ON (s.groupid = sg.id) ");

		if (!entitytype.equals("all")) {

			query.append("WHERE entitytype = '" + entitytype + "' ");

		}

		query.append("GROUP BY groupid, entitytype, date OFFSET %d LIMIT %d");

		final W3CEndpointReference epr = serviceLocator.getService(DatabaseService.class).searchSQL(dbName, String.format(query.toString(), offset, limit));

		final List<SimilarityGroup> res =

				Lists.newArrayList(Iterables.transform(resultSetClientFactory.getClient(epr), new SimilarityGroupFunction()));

		log.info(String.format("found %d similarity entries", res.size()));

		return res;

	}

	@ResponseBody

	@RequestMapping(value = "/ui/dedup/search.do")

	public HtmlSearchResults search(@RequestParam(value = "entitytype", required = true) final String type,

			@RequestParam(value = "query", required = true) final String userQuery,

			@RequestParam(value = "start", required = true) final int start,

			@RequestParam(value = "rows", required = true) final int rows) throws Exception {

		final String cqlQuery = "(>s=SOLR s.q.op=AND) and oaftype = " + type + " and deletedbyinference = false and " + userQuery;

		final LookupResponse rsp = getIndexClient().lookup(cqlQuery, null, start, (start + rows) - 1);

		final ApplyXslt recordToHtmlTable = new ApplyXslt(recordToHtmlXsl, getXslParams(type));

		return new HtmlSearchResults(rsp.getTotal(), recordToHtmlTable.evaluate("<records>" + Joiner.on("").join(rsp.getRecords()) + "</records>"));

	}

	@ResponseBody

	@RequestMapping(value = "/ui/dedup/searchById.do")

	public String searchById(@RequestParam(value = "entitytype", required = true) final String type,

			@RequestParam(value = "objidentifier", required = true) final String objidentifier) throws Exception {

		final String cqlQuery = "objidentifier exact \"" + objidentifier + "\"";

		final LookupResponse rsp = getIndexClient().lookup(cqlQuery, null, 0, 1);

		final ApplyXslt recordToHtmlDetail = new ApplyXslt(recordToHtmlDetailsXsl, getXslParams(type));

		return recordToHtmlDetail.evaluate(Iterables.getOnlyElement(rsp.getRecords()));

	}

	@ResponseBody

	@RequestMapping(value = "/ui/dedup/getExtGroup.do")

	public ExtendendSimilarityGroup getExtendedGroup(@RequestParam(value = "groupid", required = true) final String groupid) throws Exception {

		final SimilarityGroup group = getSimpleGroup(groupid);

		final String cqlQuery = Joiner.on(" or ").join(Iterables.transform(group.getGroup(), new Function<String, String>() {

			@Override

			public String apply(final String id) {

				return "objidentifier exact \"" + id + "\"";

			}

		}));

		final ApplyXslt recordToHtmlTable = new ApplyXslt(recordToHtmlXsl, getXslParams(group.getEntitytype().getType()));

		final LookupResponse indexRsp = getIndexClient().lookup(cqlQuery, null, 0, group.getGroup().size());

		log.debug(String.format("found %d index records among %d similar", indexRsp.getRecords().size(), group.getGroup().size()));

		return new ExtendendSimilarityGroup(group, recordToHtmlTable.evaluate("<records>" + Joiner.on("").join(indexRsp.getRecords()) + "</records>"));

	}

	@ResponseBody

	@RequestMapping(value = "/ui/dedup/getSimpleGroup.do")

	public SimilarityGroup getSimpleGroup(@RequestParam(value = "groupid", required = true) final String groupid) throws Exception {

		final String sqlQuery =

				String.format(

						"SELECT groupid as id, entitytype, date, array_agg(objidentifier) as group FROM similarity_groups sg " +

								"LEFT OUTER join similarities s ON (s.groupid = sg.id) " +

								"WHERE groupid = '%s' GROUP BY groupid, entitytype, date", groupid);

		final W3CEndpointReference epr = serviceLocator.getService(DatabaseService.class).searchSQL(dbName, sqlQuery);

		final SimilarityGroup group = Iterables.getOnlyElement(Iterables.transform(resultSetClientFactory.getClient(epr), new SimilarityGroupFunction()));

		log.debug(String.format("found %s similarity group, size %d", group.getId(), group.getGroup().size()));

		return group;

	}

	@ResponseBody

	@RequestMapping(value = "/ui/dedup/addSimRels.do")

	public boolean add(@RequestBody(required = true) final SimilarityGroup group) throws Exception {

		final DatabaseService dbService = serviceLocator.getService(DatabaseService.class);

		group.setId(UUID.randomUUID().toString());

		group.setDate(DateUtils.now_ISO8601());

		final StringBuilder sql =

				new StringBuilder(

						String.format("BEGIN; INSERT INTO similarity_groups(id, date, entitytype) VALUES('%s', '%s', '%s');", group.getId(), group.getDate(),

								group.getEntitytype().getType()));

		for (final String id : group.getGroup()) {

			if (dbService.contains(dbName, "similarities", "objidentifier", id)) throw new Exception("id already defined in a similarity group.");

			sql.append(String.format("INSERT INTO similarities(groupid, objidentifier) VALUES('%s', '%s'); ", group.getId(), id));

		}

		sql.append("COMMIT;");

		log.info("adding similarities: " + group.getGroup());

		return dbService.updateSQL(dbName, sql.toString());

	}

	@ResponseBody

	@RequestMapping(value = "/ui/dedup/updateSimRels.do")

	public boolean update(@RequestBody(required = true) final SimilarityGroup group) throws Exception {

		final DatabaseService dbService = serviceLocator.getService(DatabaseService.class);

		final StringBuilder sql =

				new StringBuilder(

						String.format("BEGIN; UPDATE similarity_groups SET date = '%s' WHERE id = now(); ", group.getId()));

		for (final String id : group.getGroup()) {

			if (!dbService.contains(dbName, "similarities", "objidentifier", id)) {

				sql.append(String.format("INSERT INTO similarities(groupid, objidentifier) VALUES('%s', '%s'); ", group.getId(), id));

			}

		}

		sql.append("COMMIT;");

		return dbService.updateSQL(dbName, sql.toString());

	}

	// helpers

	private IndexClient getIndexClient() throws IndexClientException, ISLookUpDocumentNotFoundException, ISLookUpException {

		final String format = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(indexFormatXquery);

		if (!currentIndex.equals(format)) {

			currentIndex = format;

			indexClient = indexClientFactory.getClient(currentIndex, "index", "openaire", "solr");

		}

		return indexClient;

	}

	private Map<String, String> getXslParams(final String type) {

		final Map<String, String> xslParam = Maps.newHashMap();

		xslParam.put("entitytype", type);

		return xslParam;

	}

}