D-Net

package eu.dnetlib.enabling.datasources;

import java.io.StringReader;

import java.util.Collections;

import java.util.Date;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import org.apache.commons.lang.StringEscapeUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.lang.math.NumberUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.dom4j.Document;

import org.dom4j.Node;

import org.dom4j.io.SAXReader;

import org.springframework.beans.factory.annotation.Required;

import com.google.common.base.Function;

import com.google.common.collect.Iterables;

import com.google.common.collect.Lists;

import com.google.common.collect.Maps;

import eu.dnetlib.enabling.datasources.rmi.BrowsableField;

import eu.dnetlib.enabling.datasources.rmi.BrowseTerm;

import eu.dnetlib.enabling.datasources.rmi.DatasourceConstants;

import eu.dnetlib.enabling.datasources.rmi.DatasourceDesc;

import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerService;

import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerServiceException;

import eu.dnetlib.enabling.datasources.rmi.IfaceDesc;

import eu.dnetlib.enabling.datasources.rmi.RepositoryMapEntry;

import eu.dnetlib.enabling.datasources.rmi.SearchInterfacesEntry;

import eu.dnetlib.enabling.datasources.rmi.SimpleDatasourceDesc;

import eu.dnetlib.enabling.tools.AbstractBaseService;

public class DatasourceManagerServiceImpl extends AbstractBaseService implements DatasourceManagerService {

	private DatasourceManagerClients datasourceManagerClients;

	private List<DbBrowsableField> browsableFields;

	private static final Log log = LogFactory.getLog(DatasourceManagerServiceImpl.class);

	@Override

	public boolean addDatasource(final DatasourceDesc ds) throws DatasourceManagerServiceException {

		if (StringUtils.isBlank(ds.getAggregator())) {

			ds.setAggregator("OPENAIRE");

		}

		final Map<String, Object> params = DatasourceFunctions.asMapOfSqlValues(ds);

		if ((ds.getOrganization() != null) && !ds.getOrganization().trim().isEmpty()) {

			params.put("hasOrganization", 1);

		}

		return datasourceManagerClients.updateSQL(ds.getId(), "addDatasource.sql.st", params, false, true);

	}

	@Override

	public boolean deleteDatasource(final String dsId) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));

		return datasourceManagerClients.updateSQL(dsId, "deleteDatasource.sql.st", params, true, true);

	}

	@Override

	public DatasourceDesc getDatasource(final String dsId) throws DatasourceManagerServiceException {

		final List<DatasourceDesc> list = datasourceManagerClients.getDatasourcesByCondition("ds.id = " + DatasourceFunctions.asSqlValue(dsId));

		if (list.size() != 1) { throw new DatasourceManagerServiceException("Datasource not found, id=" + dsId); }

		return list.get(0);

	}

	@Override

	public List<DatasourceDesc> listAllDatasources() throws DatasourceManagerServiceException {

		return datasourceManagerClients.getDatasourcesByCondition(null);

	}

	@Override

	public List<DatasourceDesc> listDatasourcesUsingFilter(final String compliance,

			final String contentDescription,

			final String iisProcessingWorkflow,

			final String collectedFrom) throws DatasourceManagerServiceException {

		String cond = "";

		if ((compliance != null) && !compliance.isEmpty()) {

			if (!cond.isEmpty()) {

				cond += " and ";

			}

			cond += "ag.compatibilityclass=" + DatasourceFunctions.asSqlValue(compliance);

		}

		if ((contentDescription != null) && !contentDescription.isEmpty()) {

			if (!cond.isEmpty()) {

				cond += " and ";

			}

			cond += "ag.contentdescriptionclass=" + DatasourceFunctions.asSqlValue(contentDescription);

		}

		if ((iisProcessingWorkflow != null) && !iisProcessingWorkflow.isEmpty()) {

			if (!cond.isEmpty()) {

				cond += " and ";

			}

			cond += "ag.accessinfopackage LIKE '%###" + DatasourceParams.iis_processing_workflow + "###" + iisProcessingWorkflow + "###%'";

		}

		if ((collectedFrom != null) && !collectedFrom.isEmpty()) {

			if (!cond.isEmpty()) {

				cond += " and ";

			}

			cond += "ds.collectedfrom=" + DatasourceFunctions.asSqlValue(collectedFrom);

		}

		return datasourceManagerClients.getDatasourcesByCondition(cond);

	}

	@Override

	public boolean updateActivationStatus(final String dsId, final String ifaceId, final boolean active) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("active", DatasourceFunctions.asSqlValue(active));

		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));

		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));

		return datasourceManagerClients.updateSQL(dsId, "updateActivationStatus.sql.st", params, false, true);

	}

	@Override

	public boolean updateLevelOfCompliance(final String dsId, final String ifaceId, final String level) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("level", DatasourceFunctions.asSqlValue(level));

		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));

		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));

		return datasourceManagerClients.updateSQL(dsId, "updateLevelOfCompliance.sql.st", params, false, true);

	}

	@Override

	public boolean updateBaseUrl(final String dsId, final String ifaceId, final String baseUrl) throws DatasourceManagerServiceException {

		return updateAccessParam(dsId, ifaceId, DatasourceParams.baseUrl.toString(), baseUrl, true);

	}

	@Override

	public boolean updateContentDescription(final String dsId, final String ifaceId, final String desc) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("desc", DatasourceFunctions.asSqlValue(desc));

		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));

		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));

		return datasourceManagerClients.updateSQL(dsId, "updateContentDescription.sql.st", params, false, true);

	}

	@Override

	public boolean setIisProcessingWorkflow(final String dsId, final String ifaceId, final String wf) throws DatasourceManagerServiceException {

		return updateExtraField(dsId, ifaceId, DatasourceParams.iis_processing_workflow.toString(), wf, false);

	}

	@Override

	public boolean addInterface(final String dsId, final IfaceDesc iface) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("datasource", DatasourceFunctions.asSqlValue(dsId));

		params.put("id", DatasourceFunctions.asSqlValue(iface.getId()));

		params.put("typology", DatasourceFunctions.asSqlValue(iface.getTypology()));

		params.put("protocol", DatasourceFunctions.asSqlValue(iface.getAccessProtocol()));

		params.put("baseUrl", DatasourceFunctions.asSqlValue(iface.getBaseUrl()));

		params.put("description", DatasourceFunctions.asSqlValue(iface.getContentDescription()));

		params.put("compliance", DatasourceFunctions.asSqlValue(iface.getCompliance()));

		final Map<String, String> accessParams = new HashMap<String, String>();

		if (iface.getAccessParams() != null) {

			for (final Entry<String, String> e : iface.getAccessParams().entrySet()) {

				accessParams.put(DatasourceFunctions.asSqlValue(e.getKey()), DatasourceFunctions.asSqlValue(e.getValue()));

			}

		}

		params.put("accessParams", accessParams);

		final Map<String, String> extraFields = new HashMap<String, String>();

		if (iface.getExtraFields() != null) {

			for (final Entry<String, String> e : iface.getExtraFields().entrySet()) {

				extraFields.put(DatasourceFunctions.asSqlValue(e.getKey()), DatasourceFunctions.asSqlValue(e.getValue()));

			}

		}

		params.put("extraFields", extraFields);

		return datasourceManagerClients.updateSQL(dsId, "insertInterface.sql.st", params, false, true);

	}

	@Override

	public boolean deleteInterface(final String dsId, final String ifcId) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("datasource", DatasourceFunctions.asSqlValue(dsId));

		params.put("id", DatasourceFunctions.asSqlValue(ifcId));

		return datasourceManagerClients.updateSQL(dsId, "deleteInterface.sql.st", params, false, true);

	}

	@Override

	public boolean updateExtraField(final String dsId, final String ifaceId, final String field, final String value, final boolean preserveOriginal)

			throws DatasourceManagerServiceException {

		return updateApiCollectionsRow(dsId, ifaceId, field, value, preserveOriginal, false);

	}

	@Override

	public boolean updateAccessParam(final String dsId, final String ifaceId, final String field, final String value, final boolean preserveOriginal)

			throws DatasourceManagerServiceException {

		return updateApiCollectionsRow(dsId, ifaceId, field, value, preserveOriginal, true);

	}

	private boolean updateApiCollectionsRow(final String dsId,

			final String ifaceId,

			final String field,

			final String value,

			final boolean preserveOriginal,

			final boolean accessParam) throws DatasourceManagerServiceException {

		final String openaireDsId = datasourceManagerClients.findDatasourceId(dsId);

		final Map<String, Object> params = Maps.newHashMap();

		params.put("dsId", DatasourceFunctions.asSqlValue(openaireDsId));

		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));

		params.put("field", DatasourceFunctions.asSqlValue(field));

		params.put("accessParam", accessParam);

		if (value != null) {

			params.put("value", DatasourceFunctions.asSqlValue(value));

		}

		if (datasourceManagerClients.isDefinedParam(ifaceId, field)) {

			params.put("update", 1);

		}

		if (preserveOriginal) {

			params.put("preserveOriginal", 1);

		}

		return datasourceManagerClients.updateSQL(openaireDsId, "updateApiCollectionsRow.sql.st", params, false, true);

	}

	@Override

	public boolean deleteAccessParamOrExtraField(final String dsId, final String ifaceId, final String field) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));

		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));

		params.put("field", DatasourceFunctions.asSqlValue(field));

		return datasourceManagerClients.updateSQL(dsId, "deleteApiCollectionsRow.sql.st", params, false, true);

	}

	@Override

	public boolean updateSQL(final String dsId, final String sql, final boolean delete) throws DatasourceManagerServiceException {

		return datasourceManagerClients.updateSQL(dsId, sql, delete, true);

	}

	@Override

	public Date findNextScheduledExecution(final String dsId, final String ifaceId)

			throws DatasourceManagerServiceException {

		return datasourceManagerClients.findNextScheduledExecution(dsId, ifaceId);

	}

	@Override

	public boolean bulkUpdateApiExtraFields(final String repoId, final String ifaceId, final Map<String, String> fields)

			throws DatasourceManagerServiceException {

		return performUpdate(repoId, ifaceId, fields, false);

	}

	@Override

	public boolean bulkUpdateApiAccessParams(final String repoId, final String ifaceId, final Map<String, String> params)

			throws DatasourceManagerServiceException {

		return performUpdate(repoId, ifaceId, params, true);

	}

	private boolean performUpdate(final String repoId, final String ifaceId, final Map<String, String> params, final boolean accessParam)

			throws DatasourceManagerServiceException {

		final String dsId = datasourceManagerClients.findDatasourceId(repoId);

		if (!accessParam) {

			deleteOldExtraFields(dsId, ifaceId);

		}

		for (final Map.Entry<String, String> e : params.entrySet()) {

			performSingleUpdate(dsId, ifaceId, e.getKey(), e.getValue(), accessParam);

		}

		datasourceManagerClients.regenerateProfile(dsId);

		return true;

	}

	private boolean deleteOldExtraFields(final String dsId, final String ifaceId) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));

		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));

		return datasourceManagerClients.updateSQL(dsId, "deleteOldExtraFields.sql.st", params, false, false);

	}

	private boolean performSingleUpdate(final String dsId, final String ifaceId, final String field, final String value, final boolean accessParam)

			throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));

		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));

		params.put("field", DatasourceFunctions.asSqlValue(field));

		params.put("accessParam", accessParam);

		if (value != null) {

			params.put("value", DatasourceFunctions.asSqlValue(value));

		}

		if (accessParam && datasourceManagerClients.isDefinedParam(ifaceId, field)) {

			params.put("update", 1);

			params.put("preserveOriginal", 1);

		}

		return datasourceManagerClients.updateSQL(dsId, "updateApiCollectionsRow.sql.st", params, false, false);

	}

	public DatasourceManagerClients getDatasourceManagerClients() {

		return datasourceManagerClients;

	}

	@Required

	public void setDatasourceManagerClients(final DatasourceManagerClients datasourceManagerClients) {

		this.datasourceManagerClients = datasourceManagerClients;

	}

	@Override

	public boolean overrideCompliance(final String repoId, final String ifaceId, final String compliance) throws DatasourceManagerServiceException {

		final String dsId = datasourceManagerClients.findDatasourceId(repoId);

		final Map<String, Object> params = Maps.newHashMap();

		params.put("dsId", DatasourceFunctions.asSqlValue(dsId));

		params.put("ifaceId", DatasourceFunctions.asSqlValue(ifaceId));

		params.put("field", DatasourceFunctions.asSqlValue(DatasourceConstants.OVERRIDING_COMPLIANCE_FIELD));

		if (StringUtils.isEmpty(compliance)) {

			params.put("delete", true);

			log.debug("Removing compliance");

		} else {

			params.put("value", DatasourceFunctions.asSqlValue(compliance));

			if (datasourceManagerClients.isDefinedParam(ifaceId, DatasourceConstants.OVERRIDING_COMPLIANCE_FIELD)) {

				params.put("update", true);

				log.debug("Updating compliance: " + compliance);

			} else {

				params.put("insert", true);

				log.debug("Adding compliance: " + compliance);

			}

		}

		return datasourceManagerClients.updateSQL(dsId, "overrideCompliance.sql.st", params, false, true);

	}

	@Override

	public List<BrowsableField> listBrowsableFields() throws DatasourceManagerServiceException {

		return Lists.transform(getBrowsableFields(), new Function<DbBrowsableField, BrowsableField>() {

			@Override

			public BrowsableField apply(final DbBrowsableField f) {

				return new BrowsableField(f.getId(), f.getLabel());

			}

		});

	}

	@Override

	public List<BrowseTerm> browseField(final String f) throws DatasourceManagerServiceException {

		final List<BrowseTerm> list = Lists.newArrayList();

		try {

			final DbBrowsableField field = findBrowseField(f);

			if (field != null) {

				final SAXReader reader = new SAXReader();

				for (final String s : datasourceManagerClients.searchSQL(field.getSql())) {

					final Document doc = reader.read(new StringReader(s));

					final String id = doc.valueOf("/ROW/FIELD[@name='id']");

					final String name = doc.valueOf("/ROW/FIELD[@name='name']");

					final int value = NumberUtils.toInt(doc.valueOf("/ROW/FIELD[@name='count']"), 0);

					if (StringUtils.isNotEmpty(id)) {

						list.add(new BrowseTerm(id, name, value));

					}

				}

			}

		} catch (final Exception e) {

			log.error("Error browsing field " + f, e);

		}

		return list;

	}

	@Override

	public List<SearchInterfacesEntry> searchInterface(final String field, final String value) throws DatasourceManagerServiceException {

		try {

			final Map<String, Object> params = Maps.newHashMap();

			params.put("value", value);

			if (!field.equalsIgnoreCase("__search__")) {

				params.put("field", field);

			}

			final DbBrowsableField f = findBrowseField(field);

			if ((f == null) || f.isText()) {

				params.put("delimeter", "'");

			}

			final SAXReader reader = new SAXReader();

			final Iterable<String> iter = datasourceManagerClients.searchSQL("searchRepoInterfaces.sql.st", params);

			return Lists.newArrayList(Iterables.transform(iter, new Function<String, SearchInterfacesEntry>() {

				@Override

				public SearchInterfacesEntry apply(final String s) {

					final SearchInterfacesEntry iface = new SearchInterfacesEntry();

					try {

						final Document doc = reader.read(new StringReader(s));

						final String country = doc.valueOf("/ROW/FIELD[@name='country']");

						iface.setRepoId(doc.valueOf("/ROW/FIELD[@name='repoid']"));

						iface.setRepoCountry(StringUtils.isEmpty(country) ? "-" : country.toUpperCase());

						iface.setRepoName(StringEscapeUtils.unescapeXml(doc.valueOf("/ROW/FIELD[@name='reponame']")));

						iface.setRepoPrefix(doc.valueOf("/ROW/FIELD[@name='repoprefix']"));

						iface.setId(doc.valueOf("/ROW/FIELD[@name='id']"));

						iface.setActive(Boolean.valueOf(doc.valueOf("/ROW/FIELD[@name='active']")));

						iface.setProtocol(doc.valueOf("/ROW/FIELD[@name='protocol']"));

						iface.setCompliance(doc.valueOf("/ROW/FIELD[@name='compliance']"));

						iface.setAggrDate(doc.valueOf("/ROW/FIELD[@name='aggrdate']"));

						iface.setAggrTotal(NumberUtils.toInt(doc.valueOf("/ROW/FIELD[@name='aggrtotal']"), 0));

					} catch (final Exception e) {

						log.error(e);

					}

					return iface;

				}

			}));

		} catch (final Exception e) {

			log.error("Error searching field " + field + " - value: " + value, e);

		}

		return Lists.newArrayList();

	}

	private DbBrowsableField findBrowseField(final String id) {

		for (final DbBrowsableField f : getBrowsableFields()) {

			if (f.getId().equals(id)) { return f; }

		}

		return null;

	}

	@Override

	public List<RepositoryMapEntry> getRepositoryMap() throws DatasourceManagerServiceException {

		final SAXReader reader = new SAXReader();

		try {

			final Iterable<String> iter = datasourceManagerClients.searchSQL("findReposMap.sql.st", null);

			return Lists.newArrayList(Iterables.transform(iter, s -> {

				final RepositoryMapEntry r = new RepositoryMapEntry();

				try {

					final Document doc = reader.read(new StringReader(s));

					r.setId(doc.valueOf("/ROW/FIELD[@name='id']"));

					r.setName(StringEscapeUtils.unescapeXml(doc.valueOf("/ROW/FIELD[@name='name']")));

					r.setLat(NumberUtils.toFloat(doc.valueOf("/ROW/FIELD[@name='lat']"), 0));

					r.setLng(NumberUtils.toFloat(doc.valueOf("/ROW/FIELD[@name='lng']"), 0));

				} catch (final Exception e) {

					log.error(e);

				}

				return r;

			}));

		} catch (final Exception e) {

			log.error("Error obtaing repo map entries", e);

		}

		return Lists.newArrayList();

	}

	@Override

	public List<SimpleDatasourceDesc> simpleListDatasourcesByType(final String type) throws DatasourceManagerServiceException {

		final Map<String, Object> params = Maps.newHashMap();

		params.put("type", type);

		final List<SimpleDatasourceDesc> list = Lists.newArrayList();

		try {

			final SAXReader reader = new SAXReader();

			for (final String s : datasourceManagerClients.searchSQL("simpleFindRepos.sql.st", params)) {

				final Document doc = reader.read(new StringReader(s));

				final SimpleDatasourceDesc r = new SimpleDatasourceDesc();

				r.setId(doc.valueOf("/ROW/FIELD[@name='id']"));

				r.setOrigId(doc.valueOf("/ROW/FIELD[@name='id']"));

				r.setName(StringEscapeUtils.unescapeXml(doc.valueOf("/ROW/FIELD[@name='name']")));

				for (final Object o : doc.selectNodes("/ROW/FIELD[@name='apis']/ITEM")) {

					r.getApis().add(((Node) o).getText());

				}

				r.setValid(true); // TODO

				r.setTypology(type);

				list.add(r);

			}

			Collections.sort(list);

		} catch (final Exception e) {

			log.error("Error listing repos", e);

		}

		return list;

	}

	public List<DbBrowsableField> getBrowsableFields() {

		return browsableFields;

	}

	@Required

	public void setBrowsableFields(final List<DbBrowsableField> browsableFields) {

		this.browsableFields = browsableFields;

	}

}

import eu.dnetlib.enabling.datasources.rmi.DatasourceConstants;

import eu.dnetlib.enabling.datasources.rmi.DatasourceDesc;

import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerServiceException;

import eu.dnetlib.enabling.datasources.rmi.IfaceDesc;

import eu.dnetlib.enabling.datasources.rmi.DatasourceDesc;

import eu.dnetlib.enabling.datasources.rmi.DatasourceManagerServiceException;

import eu.dnetlib.enabling.datasources.rmi.BrowsableField;

			<artifactId>dnet-datasource-manager-rmi</artifactId>