D-Net

package eu.dnetlib.enabling.datasources;

import java.io.StringWriter;

import java.util.*;

import java.util.stream.Collectors;

import com.google.common.collect.ImmutableMap;

import eu.dnetlib.enabling.datasources.DatasourceManagerClients.AfterSqlUpdate;

import eu.dnetlib.enabling.datasources.common.*;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.lang3.math.NumberUtils;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.springframework.beans.factory.annotation.Required;

import org.springframework.core.io.ClassPathResource;

import org.springframework.core.io.Resource;

import org.springframework.transaction.annotation.Transactional;

public class LocalOpenaireDatasourceManagerImpl implements LocalOpenaireDatasourceManager {

	private DatasourceManagerClients datasourceManagerClients;

	private List<DbBrowsableField> browsableFields;

	public static final String QUERY_BASEDIR = "/eu/dnetlib/enabling/datasources/queries/";

	private static final Resource searchDsByType = new ClassPathResource(QUERY_BASEDIR + "searchDsByType.sql");

	private static final Resource searchApis = new ClassPathResource(QUERY_BASEDIR + "searchApisNormal.sql");

	private static final Resource searchApisUsingField = new ClassPathResource(QUERY_BASEDIR + "searchApisUsingField.sql");

	private static final Resource addDs = new ClassPathResource(QUERY_BASEDIR + "addDatasource.sql");

	private static final Resource addOrg = new ClassPathResource(QUERY_BASEDIR + "addOrganization.sql");

	private static final Resource deleteDs = new ClassPathResource(QUERY_BASEDIR + "deleteDatasource.sql");

	private static final Resource setActive = new ClassPathResource(QUERY_BASEDIR + "setActive.sql");

	private static final Resource setManaged = new ClassPathResource(QUERY_BASEDIR + "setManaged.sql");

	private static final Resource setCompliance = new ClassPathResource(QUERY_BASEDIR + "setCompliance.sql");

	private static final Resource overrideCompliance = new ClassPathResource(QUERY_BASEDIR + "overrideCompliance.sql");

	private static final Resource resetCompliance = new ClassPathResource(QUERY_BASEDIR + "resetCompliance.sql");

	private static final Resource setLastCollectionInfo = new ClassPathResource(QUERY_BASEDIR + "setLastCollectionInfo.sql");

	private static final Resource setLastAggregationInfo = new ClassPathResource(QUERY_BASEDIR + "setLastAggregationInfo.sql");

	private static final Resource setLastDownloadInfo = new ClassPathResource(QUERY_BASEDIR + "setLastDownloadInfo.sql");

	private static final Resource setLastValidationJob = new ClassPathResource(QUERY_BASEDIR + "setLastValidationJob.sql");

	private static final Resource resetLastOperationsInfo = new ClassPathResource(QUERY_BASEDIR + "resetLastOperationsInfo.sql");

	private static final Resource insertApiParam = new ClassPathResource(QUERY_BASEDIR + "insertApiParam.sql");

	private static final Resource insertApi = new ClassPathResource(QUERY_BASEDIR + "insertApi.sql");

	private static final Log log = LogFactory.getLog(LocalOpenaireDatasourceManagerImpl.class);

	private static final String REPO_PROFILEID_SUFFIX = "_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=";

	@Override

	@Transactional(readOnly = true)

	public Set<String> listManagedDatasourceIds() throws DsmRuntimeException {

		try {

			return datasourceManagerClients.searchSQL("SELECT id FROM dsm_datasources WHERE managed = true", new HashMap<>())

					.stream()

					.map(m -> (String) m.get("id"))

					.collect(Collectors.toCollection(HashSet::new));

		} catch (final DsmException e) {

			throw new DsmRuntimeException(e);

		}

	}

	@Override

	@Transactional(readOnly = true)

	public List<SimpleDatasource> searchDatasourcesByType(final String type) throws DsmException {

		return datasourceManagerClients.searchSQL(searchDsByType, ImmutableMap.of("type", type))

				.stream()

				.map(DatasourceFunctions::mapToSimpleDs)

				.collect(Collectors.toList());

	}

	@Override

	@Transactional(readOnly = true)

	public List<? extends SearchApisEntry> searchApis(final String field, final Object value) throws DsmException {

		try {

			final StringWriter sql = new StringWriter();

			if (field.equalsIgnoreCase("__search__")) {

				sql.append(IOUtils.toString(searchApis.getInputStream()));

			} else {

				sql.append(IOUtils.toString(searchApisUsingField.getInputStream()));

				sql.append(field);

				sql.append("::text = ");

				sql.append(":value");

			}

			return datasourceManagerClients

					.searchSQL(sql.toString(), ImmutableMap.of("value", field.equalsIgnoreCase("__search__") ? "%" + value + "%" : value))

					.stream()

					.map(DatasourceFunctions::mapToSearchApisEntry)

					.collect(Collectors.toList());

		} catch (final Exception e) {

			log.error("Error searching field " + field + " - value: " + value, e);

		}

		return new ArrayList<>();

	}

	@Override

	@Transactional

	public void saveDs(final Datasource<Organization<?>, Identity> ds) throws DsmException {

		if (StringUtils.isBlank(ds.getAggregator())) {

			ds.setAggregator("OPENAIRE");

		}

		ds.setManaged(true);

		datasourceManagerClients.updateSQL(ds.getId(), addDs, AfterSqlUpdate.NONE, DatasourceFunctions.dsToMap(ds));

		if (ds.getOrganizations() != null) {

			for (final Organization<?> org : ds.getOrganizations()) {

				final Map<String, Object> orgParams = DatasourceFunctions.orgToMap(ds.getId(), org);

				datasourceManagerClients.updateSQL(ds.getId(), addOrg, AfterSqlUpdate.NONE, orgParams);

			}

		}

		datasourceManagerClients.regenerateProfile(ds.getId());

	}

	@Override

	@Transactional

	public void deleteDs(final String dsId) throws DsmException {

		datasourceManagerClients.updateSQL(fixDsId(dsId), deleteDs, AfterSqlUpdate.DELETE_DS_PROFILE, ImmutableMap.of("dsId", dsId));

	}

	@Override

	public Datasource<Organization<?>, Identity> getDs(final String dsId) throws DsmException {

		return datasourceManagerClients.getDatasourceById(fixDsId(dsId));

	}

	@Override

	public void regenerateProfiles() throws DsmException {

		datasourceManagerClients.regenerateProfiles();

	}

	@Override

	public List<Api<ApiParam>> getApis(final String dsId) throws DsmException {

		return datasourceManagerClients.getApis(fixDsId(dsId));

	}

	@Override

	public void setManaged(final String dsId, final boolean managed) throws DsmException {

		final String id = fixDsId(dsId);

		final Map<String, Object> params = new HashMap<>();

		params.put("managed", managed);

		params.put("dsId", id);

		datasourceManagerClients.updateSQL(id, setManaged, AfterSqlUpdate.UPDATE_DS_PROFILE, params);

	}

	@Override

	public boolean isManaged(final String dsId) throws DsmException {

		final String q = "SELECT * from dsm_datasources WHERE id = :dsId AND managed = true";

		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId))).isEmpty();

	}

	@Override

	@Transactional

	public void setActive(final String dsId, final String apiId, final boolean active) throws DsmException {

		final String id = fixDsId(dsId);

		final Map<String, Object> params = new HashMap<>();

		params.put("active", active);

		params.put("apiId", apiId);

		params.put("dsId", id);

		datasourceManagerClients.updateSQL(id, setActive, AfterSqlUpdate.NONE, params);

		if (!active) {

			datasourceManagerClients.updateSQL(dsId, resetLastOperationsInfo, AfterSqlUpdate.NONE, params);

		}

		setManaged(id, true); // It also update the IS profile

	}

	@Override

	@Transactional(readOnly = true)

	public boolean isActive(final String dsId, final String apiId) throws DsmException {

		final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active = true";

		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();

	}

	@Override

	@Transactional(readOnly = true)

	public boolean isRemovable(final String dsId, final String apiId) throws DsmException {

		final String q = "SELECT * from dsm_api WHERE id = :apiId AND datasource = :dsId AND active != true AND removable = true";

		return !datasourceManagerClients.searchSQL(q, ImmutableMap.of("dsId", fixDsId(dsId), "apiId", apiId)).isEmpty();

	}

	@Override

	@Transactional

	public void updateCompliance(final String dsId, final String apiId, final String level, final boolean override) throws DsmException {

		final String id = fixDsId(dsId);

		if (!override) {

			datasourceManagerClients.updateSQL(id, setCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));

		} else if (level != null) {

			datasourceManagerClients.updateSQL(id, overrideCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("level", level, "apiId", apiId, "dsId", id));

		} else {

			datasourceManagerClients.updateSQL(id, resetCompliance, AfterSqlUpdate.NONE, ImmutableMap.of("apiId", apiId, "dsId", id));

		}

		setManaged(id, true); // It also update the IS profile

	}

	@Override

	public void setLastCollectionInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)

			throws DsmException {

		setLastOperationInfo(setLastCollectionInfo, fixDsId(dsId), apiId, mdId, size, date);

	}

	@Override

	public void setLastAggregationInfo(final String dsId, final String apiId, final String mdId, final Integer size, final Date date)

			throws DsmException {

		setLastOperationInfo(setLastAggregationInfo, fixDsId(dsId), apiId, mdId, size, date);

	}

	@Override

	public void setLastDownloadInfo(final String dsId, final String apiId, final String objId, final Integer size, final Date date)

			throws DsmException {

		setLastOperationInfo(setLastDownloadInfo, fixDsId(dsId), apiId, objId, size, date);

	}

	@Override

	public void setLastValidationJob(String dsId, String apiId, String jobId) throws DsmException {

		final Map<String, Object> params = new HashMap<>();

		params.put("dsId", dsId);

		params.put("apiId", apiId);

		params.put("jobId", jobId);

		datasourceManagerClients.updateSQL(dsId, setLastValidationJob, AfterSqlUpdate.NONE, params);

		setManaged(dsId, true); // It also update the IS profile

	}

	@Transactional

	protected void setLastOperationInfo(final Resource sqlResource, final String dsId, final String apiId, final String mdId, final Integer size, final Date date)

			throws DsmException {

		final Map<String, Object> params = new HashMap<>();

		params.put("dsId", dsId);

		params.put("apiId", apiId);

		params.put("mdId", mdId);

		params.put("total", size);

		if (date != null) {

			params.put("date", new java.sql.Timestamp(date.getTime()));

		}

		datasourceManagerClients.updateSQL(dsId, sqlResource, AfterSqlUpdate.NONE, params);

		setManaged(dsId, true); // It also update the IS profile

	}

	@Override

	@Transactional

	public void updateApiDetails(final String dsId,

			final String apiId,

			final String metadataIdentifierPath,

			final String baseUrl,

			final Map<String, String> params)

			throws DsmException {

		final String id = fixDsId(dsId);

		// Delete old params

		datasourceManagerClients.updateSQL(id, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));

		// Insert new params

		for (final Map.Entry<String, String> e : params.entrySet()) {

			final Map<String, Object> sqlParams = ImmutableMap.of("param", e.getKey(), "value", e.getValue(), "api", apiId);

			datasourceManagerClients.updateSQL(id, insertApiParam, AfterSqlUpdate.NONE, sqlParams);

		}

		// Update the BaseURL

		datasourceManagerClients.updateSQL(id,

				"UPDATE dsm_api SET baseurl = :baseurl WHERE id = :api",

				AfterSqlUpdate.NONE,

				ImmutableMap.of("baseurl", baseUrl, "api", apiId));

		// Update the metadata_identifier_path

		datasourceManagerClients.updateSQL(id,

				"UPDATE dsm_api SET metadata_identifier_path = :path WHERE id = :api",

				AfterSqlUpdate.NONE,

				ImmutableMap.of("path", metadataIdentifierPath, "api", apiId));

		setManaged(id, true); // It also update the IS profile

	}

	@Override

	public List<? extends BrowsableField> listBrowsableFields() throws DsmException {

		return getBrowsableFields();

	}

	@Override

	public List<BrowseTerm> browseField(final String field) throws DsmException {

		final Optional<DbBrowsableField> bf = getBrowsableFields()

				.stream()

				.filter(f -> f.getId().equals(field))

				.findFirst();

		if (bf.isPresent()) {

			return datasourceManagerClients.searchSQL(bf.get().getSql(), new HashMap<>())

					.stream()

					.filter(m -> m.get("term") != null)

					.filter(m -> m.get("count") != null)

					.filter(m -> StringUtils.isNotBlank(m.get("term").toString()))

					.map(m -> new BrowseTermImpl(m.get("term").toString(), NumberUtils.toInt(m.get("count").toString(), 0)))

					.collect(Collectors.toList());

		} else {

			log.error("Not browsable field:" + field);

			throw new DsmException("Not browsable field:" + field);

		}

	}

	@Override

	@Transactional

	public void addApi(final Api<ApiParam> api) throws DsmException {

		datasourceManagerClients.updateSQL(api.getDatasource(), insertApi, AfterSqlUpdate.NONE, DatasourceFunctions.apiToMap(api));

		if (api.getApiParams() != null) {

			api.getApiParams().forEach(p -> {

				final ImmutableMap<String, Object> sqlParams = ImmutableMap.of("param", p.getParam(), "value", p.getValue(), "api", api.getId());

				try {

					datasourceManagerClients.updateSQL(api.getDatasource(), insertApiParam, AfterSqlUpdate.NONE, sqlParams);

				} catch (final DsmException e) {

					throw new RuntimeException(e);

				}

			});

		}

		setManaged(api.getDatasource(), true); // It also update the IS profile

	}

	@Override

	@Transactional

	public void deleteApi(final String dsId, final String apiId) throws DsmException {

		if (!isRemovable(dsId, apiId)) { throw new DsmException("The api " + apiId + " can't be deleted"); }

		datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_apiparams WHERE api = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));

		datasourceManagerClients.updateSQL(dsId, "DELETE FROM dsm_api WHERE id = :api", AfterSqlUpdate.NONE, ImmutableMap.of("api", apiId));

		setManaged(dsId, true); // It also update the IS profile

	}

	public DatasourceManagerClients getDatasourceManagerClients() {

		return datasourceManagerClients;

	}

	private String fixDsId(final String dsId) throws DsmException {

		return dsId.endsWith(REPO_PROFILEID_SUFFIX) ? datasourceManagerClients.findDatasourceId(dsId) : dsId;

	}

	@Required

	public void setDatasourceManagerClients(final DatasourceManagerClients datasourceManagerClients) {

		this.datasourceManagerClients = datasourceManagerClients;

	}

	public List<DbBrowsableField> getBrowsableFields() {

		return browsableFields;

	}

	@Required

	public void setBrowsableFields(final List<DbBrowsableField> browsableFields) {

		this.browsableFields = browsableFields;

	}

}