D-Net

package eu.dnetlib.repo.manager.service;

import com.fasterxml.jackson.databind.ObjectMapper;

import com.google.gson.JsonArray;

import com.google.gson.JsonElement;

import com.google.gson.JsonObject;

import eu.dnetlib.api.functionality.ValidatorServiceException;

import eu.dnetlib.domain.data.Repository;

import eu.dnetlib.domain.data.RepositoryInterface;

import eu.dnetlib.domain.enabling.Vocabulary;

import eu.dnetlib.domain.functionality.validator.JobForValidation;

import eu.dnetlib.repo.manager.domain.*;

import eu.dnetlib.repo.manager.domain.dto.Role;

import eu.dnetlib.repo.manager.exception.ResourceNotFoundException;

import eu.dnetlib.repo.manager.service.aai.registry.AaiRegistryService;

import eu.dnetlib.repo.manager.service.security.AuthoritiesUpdater;

import eu.dnetlib.repo.manager.service.security.AuthorizationService;

import eu.dnetlib.repo.manager.service.security.RoleMappingService;

import eu.dnetlib.repo.manager.utils.Converter;

import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader;

import org.apache.commons.codec.digest.DigestUtils;

import org.apache.log4j.Logger;

import org.json.JSONArray;

import org.json.JSONException;

import org.json.JSONObject;

import org.mitre.openid.connect.model.OIDCAuthenticationToken;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.context.annotation.Lazy;

import org.springframework.core.ParameterizedTypeReference;

import org.springframework.http.*;

import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;

import org.springframework.security.core.Authentication;

import org.springframework.security.core.context.SecurityContextHolder;

import org.springframework.stereotype.Service;

import org.springframework.web.client.HttpClientErrorException;

import org.springframework.web.client.RestTemplate;

import org.springframework.web.util.UriComponents;

import org.springframework.web.util.UriComponentsBuilder;

import javax.annotation.PostConstruct;

import java.io.IOException;

import java.sql.Timestamp;

import java.util.*;

import java.util.concurrent.ConcurrentHashMap;

import java.util.stream.Collectors;

@Service("repositoryService")

public class RepositoryServiceImpl implements RepositoryService {

    private static final Logger LOGGER = Logger.getLogger(RepositoryServiceImpl.class);

    private final AuthorizationService authorizationService;

    private final RoleMappingService roleMappingService;

    private final AaiRegistryService registryCalls;

    private final AuthoritiesUpdater authoritiesUpdater;

    private final RestTemplate restTemplate;

    private final VocabularyLoader vocabularyLoader;

    private final PiWikService piWikService;

    private final EmailUtils emailUtils;

    private final ValidatorService validatorService;

    @Value("${services.provide.clients.dsm}")

    private String baseAddress;

    @Value("${services.provide.adminEmail}")

    private String adminEmail;

    @Value("${services.provide.usageStatisticsDiagramsBaseURL}")

    private String usageStatisticsDiagramsBaseURL;

    @Value("${services.provide.usageStatisticsNumbersBaseURL}")

    private String usageStatisticsNumbersBaseURL;

    private static final Map<String, List<String>> dataSourceClass = new HashMap<>();

    private static final Map<String, String> invertedDataSourceClass = new HashMap<>();

    private final String[] vocabularyNames = {"dnet:countries", "dnet:datasource_typologies", "dnet:compatibilityLevel"};

    private final Map<String, Vocabulary> vocabularyMap = new ConcurrentHashMap<>();

    private final Map<String, String> countriesMap = new HashMap<>();

    private final Map<String, String> inverseCountriesMap = new HashMap<>();

    private HttpHeaders httpHeaders;

    @Autowired

    public RepositoryServiceImpl(AuthorizationService authorizationService,

                                 RoleMappingService roleMappingService,

                                 AaiRegistryService registryCalls,

                                 AuthoritiesUpdater authoritiesUpdater,

                                 VocabularyLoader vocabularyLoader,

                                 RestTemplate restTemplate,

                                 @Lazy EmailUtils emailUtils,

                                 @Lazy ValidatorService validatorService,

                                 @Lazy PiWikService piWikService) {

        this.authorizationService = authorizationService;

        this.roleMappingService = roleMappingService;

        this.registryCalls = registryCalls;

        this.authoritiesUpdater = authoritiesUpdater;

        this.vocabularyLoader = vocabularyLoader;

        this.piWikService = piWikService;

        this.emailUtils = emailUtils;

        this.validatorService = validatorService;

        this.restTemplate = restTemplate;

    }

    @PostConstruct

    private void init() {

        LOGGER.debug("Initialization method of repository api!");

        LOGGER.debug("Updated version!");

        for (String key : this.getVocabulary("dnet:datasource_typologies").getAsMap().keySet()) {

            if (key.contains("aggregator")) {

                dataSourceClass.putIfAbsent("aggregator", new ArrayList<>());

                dataSourceClass.get("aggregator").add(key);

            } else if (key.contains("crissystem")) {

                dataSourceClass.putIfAbsent("cris", new ArrayList<>());

                dataSourceClass.get("cris").add(key);

            } else if (key.contains("pubsrepository::journal")) { // do not change order -->

                dataSourceClass.putIfAbsent("journal", Collections.singletonList("pubsrepository::journal"));

            } else if (key.contains("pubsrepository")) { // do not change order <--

                dataSourceClass.putIfAbsent("opendoar", new ArrayList<>());

                dataSourceClass.get("opendoar").add(key);

            } else if (key.contains("datarepository")) {

                dataSourceClass.putIfAbsent("re3data", Collections.singletonList("datarepository::unknown"));

            }

        }

        for (Map.Entry<String, List<String>> entry : dataSourceClass.entrySet()) {

            entry.getValue().forEach(v -> invertedDataSourceClass.put(v, entry.getKey()));

        }

        httpHeaders = new HttpHeaders();

        httpHeaders.setContentType(MediaType.APPLICATION_JSON_UTF8);

        for (String vocName : vocabularyNames) {

            vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));

        }

        Country[] countries = getCountries();

        for (Country c : countries) {

            countriesMap.put(c.getName(), c.getCode());

            inverseCountriesMap.put(c.getCode(), c.getName());

        }

    }

    @Override

    public Country[] getCountries() {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/countries")

                .build().encode();

        return restTemplate.getForObject(uriComponents.toUri(), Country[].class);

    }

    // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used

    //  and the "requestFilter.setId(repoId)" should return only one result at a time, thus,

    //  another way for paging must be implemented.

    @Override

    public List<Repository> getRepositories(List<String> ids) throws JSONException {

        return getRepositories(ids, 0, 10);

    }

    // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used

    //  and the "requestFilter.setId(repoId)" should return only one result at a time, thus,

    //  another way for paging must be implemented.

    @Override

    public List<Repository> getRepositories(List<String> ids, int page, int size) throws JSONException {

        List<Repository> repos = new ArrayList<>();

        LOGGER.debug("Retrieving repositories with ids : " + String.join(", ", ids));

        UriComponents uriComponents = searchDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));

        RequestFilter requestFilter = new RequestFilter();

        for (String repoId : ids) {

            requestFilter.setId(repoId);

            String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

            repos.addAll(Converter.jsonToRepositoryList(new JSONObject(rs)));

        }

        for (Repository r : repos)

            r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));

        return repos;

    }

    // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used

    //  and the "requestFilter.setId(repoId)" should return only one result at a time, thus,

    //  another way for paging must be implemented.

    @Override

    public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids) throws Exception {

        return getRepositoriesSnippets(ids, 0, 10);

    }

    // FIXME: with the new roles of the users the "requestFilter.setRegisteredby(userEmail)" can no longer be used

    //  and the "requestFilter.setId(repoId)" should return only one result at a time, thus,

    //  another way for paging must be implemented.

    @Override

    public List<RepositorySnippet> getRepositoriesSnippets(List<String> ids, int page, int size) throws Exception {

        List<RepositorySnippet> resultSet = new ArrayList<>();

        ObjectMapper mapper = new ObjectMapper();

        // here page should be 0

        UriComponents uriComponents = searchSnipperDatasource(Integer.toString(Math.abs(page)), Integer.toString(Math.abs(size)));

        RequestFilter requestFilter = new RequestFilter();

        try {

            for (String repoId : ids) {

                requestFilter.setId(repoId);

                String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

                JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

                resultSet.addAll(mapper.readValue(String.valueOf(jsonArray),

                        mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));

            }

        } catch (Exception e) {

            LOGGER.debug("Exception on getRepositoriesSnippetOfUser", e);

            throw e;

        }

        LOGGER.debug("resultSet:" + resultSet);

        resultSet.parallelStream().forEach(repositorySnippet -> {

            repositorySnippet.setPiwikInfo(piWikService.getPiwikSiteForRepo(repositorySnippet.getId()));

        });

        return resultSet;

    }

    @Override

    public List<RepositorySnippet> getRepositoriesByCountry(String country,

                                                            String mode,

                                                            Boolean managed) throws JSONException, IOException {

        LOGGER.debug("Getting repositories by country!");

        int page = 0;

        int size = 100;

        List<RepositorySnippet> resultSet = new ArrayList<>();

        ObjectMapper mapper = new ObjectMapper();

        String filterKey = "UNKNOWN";

        if (mode.equalsIgnoreCase("opendoar"))

            filterKey = "openaire____::opendoar";

        else if (mode.equalsIgnoreCase("re3data"))

            filterKey = "openaire____::re3data";

        else if (mode.equalsIgnoreCase("cris"))

            filterKey = "eurocrisdris::dris";

        LOGGER.debug("Country code equals : " + country);

        LOGGER.debug("Filter mode equals : " + filterKey);

        UriComponents uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size));

        RequestFilter requestFilter = new RequestFilter();

        requestFilter.setCountry(country);

        requestFilter.setCollectedfrom(filterKey);

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

        JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

        while (jsonArray.length() > 0) {

            resultSet.addAll(mapper.readValue(String.valueOf(jsonArray),

                    mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));

            page += 1;

            uriComponents = searchSnipperDatasource(String.valueOf(page), String.valueOf(size));

            rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

            jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

        }

        return resultSet;

    }

    public List<RepositorySnippet> searchRegisteredRepositories(String country, String typology, String englishName,

                                                                String officialName, String requestSortBy, String order, int page, int pageSize) throws Exception {

        LOGGER.debug("Searching registered repositories");

        List<RepositorySnippet> resultSet = new ArrayList<>();

        ObjectMapper mapper = new ObjectMapper();

        UriComponents uriComponents = searchRegisteredDatasource(requestSortBy, order, Integer.toString(page), Integer.toString(pageSize));

        RequestFilter requestFilter = new RequestFilter();

        requestFilter.setCountry(country);

        requestFilter.setTypology(typology);

        requestFilter.setOfficialname(officialName);

        requestFilter.setEnglishname(englishName);

        try {

            String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

            JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

            resultSet.addAll(mapper.readValue(String.valueOf(jsonArray), mapper.getTypeFactory().constructCollectionType(List.class, RepositorySnippet.class)));

            return resultSet;

        } catch (Exception e) {

            LOGGER.error("Error searching registered datasources", e);

            throw e;

        }

    }

    @Override

    public int getTotalRegisteredRepositories() {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/countregistered")

                .queryParam("fromDate", "1900-01-01")

                .build().encode();

        return restTemplate.getForObject(uriComponents.toUri(), Integer.class);

    }

    private Repository updateRepositoryInfo(Repository r) throws JSONException {

        /*

         * from datasource class

         * we get the datasource type form the inverted map

         * */

        r.setDatasourceType(getRepositoryType(r.getDatasourceClass()));

        r.setInterfaces(this.getRepositoryInterface(r.getId()));

        r.setPiwikInfo(piWikService.getPiwikSiteForRepo(r.getId()));

        r.setCountryName(getCountryName(r.getCountryCode()));

        return r;

    }

    @Override

    public List<Repository> getRepositoriesOfUser(String page, String size) throws JSONException {

        String userEmail = ((OIDCAuthenticationToken) SecurityContextHolder.getContext().getAuthentication()).getUserInfo().getEmail();

        LOGGER.debug("Retrieving repositories of authenticated user : " + userEmail);

        Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles());

        return getRepositories(new ArrayList<>(repoIds));

    }

    @Override

    public List<Repository> getRepositoriesOfUser(String userEmail, String page, String size) throws JSONException {

        LOGGER.debug("Retrieving repositories of authenticated user : " + userEmail);

        Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail));

        return getRepositories(new ArrayList<>(repoIds));

    }

    @Override

    public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String page, String size) throws Exception {

        Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles());

        return getRepositoriesSnippets(new ArrayList<>(repoIds));

    }

    @Override

    public List<RepositorySnippet> getRepositoriesSnippetsOfUser(String userEmail, String page, String size) throws Exception {

        Collection<String> repoIds = roleMappingService.getRepoIdsByRoleIds(authorizationService.getUserRoles(userEmail));

        return getRepositoriesSnippets(new ArrayList<>(repoIds));

    }

    @Override

    public RepositorySnippet getRepositorySnippetById(String id) throws JSONException, ResourceNotFoundException {

        LOGGER.debug("Retrieving repositories with id : " + id);

        RepositorySnippet repo = null;

        UriComponents uriComponents = searchSnipperDatasource("0", "100");

        RequestFilter requestFilter = new RequestFilter();

        requestFilter.setId(id);

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

        JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

        if (jsonArray.length() == 0)

            throw new ResourceNotFoundException();

        repo = Converter.jsonToRepositorySnippetObject(jsonArray.getJSONObject(0));

        return repo;

    }

    @Override

    public Repository getRepositoryById(String id) throws JSONException, ResourceNotFoundException {

        LOGGER.debug("Retrieving repositories with id : " + id);

        Repository repo = null;

        UriComponents uriComponents = searchDatasource("0", "100");

        RequestFilter requestFilter = new RequestFilter();

        requestFilter.setId(id);

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

        JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

        if (jsonArray.length() == 0)

            throw new ResourceNotFoundException();

        repo = Converter.jsonToRepositoryObject(jsonArray.getJSONObject(0));

        return updateRepositoryInfo(repo);

    }

    @Override

    public List<AggregationDetails> getRepositoryAggregations(String id) throws JSONException {

        LOGGER.debug("Retrieving aggregations for repository with id : " + id);

        UriComponents uriComponents = getAggregationHistory(id);

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        JSONArray aggregationInfo = new JSONObject(rs).getJSONArray("aggregationInfo");

        List<AggregationDetails> aggregationHistory = new ArrayList<>(Converter.getAggregationHistoryFromJson(aggregationInfo));

        return aggregationHistory;

    }

    @Override

    public List<AggregationDetails> getRepositoryAggregations(String id, int from, int size) throws JSONException {

	List<AggregationDetails> res = getRepositoryAggregations(id);

	return res.subList(from, Math.min(from + size, res.size()));

    }

    @Override

    public Map<String, List<AggregationDetails>> getRepositoryAggregationsByYear(String id) throws JSONException {

        LOGGER.debug("Retrieving aggregations (by year) for repository with id : " + id);

        List<AggregationDetails> aggregationHistory = getRepositoryAggregations(id);

        Map<String, List<AggregationDetails>> aggregationByYear = new HashMap<>();

        return aggregationHistory.size() == 0 ? aggregationByYear : createYearMap(aggregationHistory);

    }

    private Map<String, List<AggregationDetails>> createYearMap(List<AggregationDetails> aggregationHistory) {

        aggregationHistory = aggregationHistory.stream()

                .sorted(Comparator.comparing(AggregationDetails::getDate).reversed())

                .collect(Collectors.toList());

        return aggregationHistory.stream()

                .collect(Collectors.groupingBy(AggregationDetails::getYear));

    }

    @Override

    public List<Repository> getRepositoriesByName(String name,

                                                  String page,

                                                  String size) throws JSONException {

        LOGGER.debug("Retrieving  repositories with official name : " + name);

        UriComponents uriComponents = searchDatasource("0", "100");

        RequestFilter requestFilter = new RequestFilter();

        requestFilter.setOfficialname(name);

        String rs = restTemplate.postForObject(uriComponents.toUri(), requestFilter, String.class);

        List<Repository> repos = Converter.jsonToRepositoryList(new JSONObject(rs));

        for (Repository r : repos)

            updateRepositoryInfo(r);

        return repos;

    }

    @Override

    public List<RepositoryInterface> getRepositoryInterface(String id) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/api/")

                .path("/{id}")

                .build().expand(id).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        return Converter.jsonToRepositoryInterfaceList(new JSONObject(rs));

    }

    @Override

    public Repository addRepository(String datatype, Repository repository) throws Exception {

        LOGGER.debug("storing " + datatype + " repository with id: " + repository.getId());

        repository.setCountryCode(countriesMap.get(repository.getCountryName()));

        repository.setActivationId(UUID.randomUUID().toString());

        repository.setCollectedFrom("infrastruct_::openaire");

        if (datatype.equals("journal")) {

            repository.setId("openaire____::issn" + repository.getIssn());

            repository.setNamespacePrefix("issn" + repository.getIssn());

            this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());

        } else if (datatype.equals("aggregator")) {

            repository.setId("openaire____::" + DigestUtils.md5Hex(repository.getOfficialName()));

            repository.setNamespacePrefix(DigestUtils.md5Hex(repository.getOfficialName()).substring(0, 12));

            this.storeRepository(repository, SecurityContextHolder.getContext().getAuthentication());

        } else {

            this.latentUpdate(repository, SecurityContextHolder.getContext().getAuthentication());

        }

        // TODO: move the following code elsewhere (creation and assignment of role to user) ??

        // Create new role

        String newRoleName = roleMappingService.getRoleIdByRepoId(repository.getId());

        Role newRole = new Role(newRoleName, repository.getOfficialName());

        Integer couId = null;

        try {

            couId = registryCalls.createRole(newRole);

        } catch (HttpClientErrorException e) {

            couId = registryCalls.getCouId(newRoleName);

            if (couId == null) {

                LOGGER.error(String.format("Could not create role '%s'", newRoleName), e);

            }

        } catch (Exception e) {

            LOGGER.error(String.format("Could not create role '%s'", newRoleName), e);

            throw e;

        }

        // Assign new role to the user that created it

        Integer coPersonId = registryCalls.getCoPersonIdByIdentifier();

        if (couId != null) {

            Integer role = registryCalls.getRoleId(coPersonId, couId);

            try {

                registryCalls.assignMemberRole(coPersonId, couId, role);

                // Add role to current user authorities

                authoritiesUpdater.addRole(roleMappingService.convertRepoIdToAuthority(repository.getId()));

            } catch (Exception e) {

                LOGGER.debug("Exception on assign role to user during add repository", e);

                throw e;

            }

        }

        return repository;

    }

    /* update method acting as add -> send email with registration topic/body*/

    private Repository latentUpdate(Repository repository, Authentication authentication) throws Exception {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/update/")

                .build()

                .encode();

        String json_repository = Converter.repositoryObjectToJson(repository);

        LOGGER.debug("JSON to add(update) -> " + json_repository);

        HttpEntity<String> httpEntity = new HttpEntity<>(json_repository, httpHeaders);

        ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);

        if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {

            try {

                emailUtils.sendUserRegistrationEmail(repository, authentication);

                emailUtils.sendAdminRegistrationEmail(repository, authentication);

            } catch (Exception e) {

                LOGGER.error("Error sending email", e);

            }

        } else

            LOGGER.error("Error storing repository: " + responseEntity.getBody().toString());

        return repository;

    }

    @Override

    public Repository updateRepository(Repository repository, Authentication authentication) throws Exception {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/update/")

                .build()

                .encode();

        String json_repository = Converter.repositoryObjectToJson(repository);

        LOGGER.debug("JSON to update -> " + json_repository);

        HttpEntity<String> httpEntity = new HttpEntity<String>(json_repository, httpHeaders);

        ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity

                , ResponseEntity.class);

        if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {

            try {

                emailUtils.sendUserUpdateRepositoryInfoEmail(repository, authentication);

                emailUtils.sendAdminUpdateRepositoryInfoEmail(repository, authentication);

            } catch (Exception e) {

                LOGGER.error("Error sending emails: " + e);

            }

        } else

            LOGGER.debug(responseEntity.getBody().toString());

        return repository;

    }

    private void storeRepository(Repository repository, Authentication authentication) throws Exception {

        Date utilDate = new Date();

        Timestamp date = new Timestamp(utilDate.getTime());

        repository.setDateOfCollection(date);

        repository.setAggregator("OPENAIRE");

        repository.setCountryCode(countriesMap.get(repository.getCountryName()));

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/add/")

                .build()

                .encode();

        String json_repository = Converter.repositoryObjectToJson(repository);

        HttpEntity<String> httpEntity = new HttpEntity<String>(json_repository, httpHeaders);

        ResponseEntity responseEntity = restTemplate.exchange(uriComponents.toUri(), HttpMethod.POST, httpEntity, ResponseEntity.class);

        if (responseEntity.getStatusCode().equals(HttpStatus.OK)) {

            try {

                emailUtils.sendUserRegistrationEmail(repository, authentication);

                emailUtils.sendAdminRegistrationEmail(repository, authentication);

            } catch (Exception e) {

                LOGGER.error("Error sending emails: " + e);

            }

        } else {

            LOGGER.debug(responseEntity.getBody().toString());

        }

    }

    @Override

    public void deleteRepositoryInterface(String id,

                                          String registeredBy) {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/api/")

                .path("/{id}")

                .build().expand(id).encode();

        LOGGER.debug(uriComponents.toUri());

        restTemplate.delete(uriComponents.toUri());

    }

    @Override

    public RepositoryInterface addRepositoryInterface(String datatype,

                                                      String repoId,

                                                      String registeredBy,

                                                      String comment, RepositoryInterface repositoryInterface) throws Exception {

        Repository e = this.getRepositoryById(repoId);

        repositoryInterface = createRepositoryInterface(e, repositoryInterface, datatype);

        String json_interface = Converter.repositoryInterfaceObjectToJson(e, repositoryInterface);

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/api/add/")

                .build()

                .encode();

        HttpEntity<String> httpEntity = new HttpEntity<>(json_interface, httpHeaders);

        restTemplate.postForObject(uriComponents.toUri(), httpEntity, String.class);

        try {

            emailUtils.sendAdminRegisterInterfaceEmail(e, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());

            emailUtils.sendUserRegisterInterfaceEmail(e, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());

        } catch (Exception ex) {

            LOGGER.error("Error sending emails: " + ex);

        }

        submitInterfaceValidation(e, registeredBy, repositoryInterface, false);

        return repositoryInterface;

    }

    @Override

    public RepositoryInterface updateRepositoryInterface(String repoId,

                                                         String registeredBy,

                                                         String comment, RepositoryInterface repositoryInterface) throws Exception {

        this.updateBaseUrl(repoId, repositoryInterface.getId(), repositoryInterface.getBaseUrl());

        this.updateCompliance(repoId, repositoryInterface.getId(), repositoryInterface.getCompliance());

        this.updateValidationSet(repoId, repositoryInterface.getId(), repositoryInterface.getAccessSet());

        Repository repository = this.getRepositoryById(repoId);

        try {

            try {

                emailUtils.sendAdminUpdateInterfaceEmail(repository, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());

                emailUtils.sendUserUpdateInterfaceEmail(repository, comment, repositoryInterface, SecurityContextHolder.getContext().getAuthentication());

            } catch (Exception e) {

                LOGGER.error("Error sending emails: " + e);

            }

        } catch (Exception e) {

            LOGGER.warn("Could not send emails", e);

        }

        submitInterfaceValidation(getRepositoryById(repoId), registeredBy, repositoryInterface, true);

        return repositoryInterface;

    }

    private void submitInterfaceValidation(Repository repo, String userEmail, RepositoryInterface iFace, boolean updateExisting) throws ValidatorServiceException {

        JobForValidation job = new JobForValidation();

        job.setActivationId(UUID.randomUUID().toString());

        job.setAdminEmails(Collections.singletonList(this.adminEmail));

        job.setBaseUrl(iFace.getBaseUrl());

        job.setDatasourceId(repo.getId());

        job.setDesiredCompatibilityLevel(iFace.getDesiredCompatibilityLevel());

        job.setInterfaceId(iFace.getId());

        job.setOfficialName(repo.getOfficialName());

        job.setRepoType(repo.getDatasourceType());

        job.setUserEmail(userEmail);

        job.setValidationSet((iFace.getAccessSet().isEmpty() ? "none" : iFace.getAccessSet()));

        job.setRecords(-1);

        job.setRegistration(!updateExisting);

        job.setUpdateExisting(updateExisting);

        this.validatorService.submitJobForValidation(job);

    }

    private RepositoryInterface createRepositoryInterface(Repository repo, RepositoryInterface iFace, String datatype) {

        iFace.setContentDescription("metadata");

        iFace.setCompliance("UNKNOWN");

        if (datatype.equals("re3data"))

            iFace.setAccessFormat("oai_datacite");

        else

            iFace.setAccessFormat("oai_dc");

        if (repo.getDatasourceClass() != null && !repo.getDatasourceClass().isEmpty())

            iFace.setTypology(repo.getDatasourceClass());

        else if (datatype.equalsIgnoreCase("journal"))

            iFace.setTypology("pubsrepository::journal");

        else if (datatype.equalsIgnoreCase("aggregator"))

            iFace.setTypology("aggregator::pubsrepository::unknown");

        else if (datatype.equalsIgnoreCase("opendoar"))

            iFace.setTypology("pubsrepository::unknown");

        else if (datatype.equalsIgnoreCase("re3data"))

            iFace.setTypology("datarepository::unknown");

        iFace.setRemovable(true);

        iFace.setAccessProtocol("oai");

        iFace.setMetadataIdentifierPath("//*[local-name()='header']/*[local-name()='identifier']");

        iFace.setId("api_________::" + repo.getId() + "::" + UUID.randomUUID().toString().substring(0, 8));

        if (iFace.getAccessSet() == null || iFace.getAccessSet().isEmpty()) {

            LOGGER.debug("set is empty: " + iFace.getAccessSet());

            iFace.removeAccessSet();

            iFace.setAccessSet("none");

        }

        return iFace;

    }

    @Override

    public List<String> getDnetCountries() {

        LOGGER.debug("Getting dnet-countries!");

        return Converter.readFile("countries.txt");

    }

    @Override

    public List<String> getTypologies() {

        return Converter.readFile("typologies.txt");

    }

    @Override

    public List<Timezone> getTimezones() {

        List<String> timezones = Converter.readFile("timezones.txt");

        return Converter.toTimezones(timezones);

    }

    @Override