D-Net

package eu.dnetlib.repo.manager.service.utils;

import eu.dnetlib.domain.data.PiwikInfo;

import eu.dnetlib.repo.manager.service.config.CascadingPropertyLoader;

import eu.dnetlib.utils.MailLibrary;

import org.apache.log4j.Logger;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.stereotype.Component;

import java.io.PrintWriter;

import java.io.StringWriter;

import java.io.Writer;

import java.util.ArrayList;

import java.util.List;

@Component

public class EmailUtilsImpl implements EmailUtils {

    private static Logger LOGGER = Logger.getLogger(EmailUtilsImpl.class);

    private List<String> specialRecipients = new ArrayList<String>();

    private boolean override = false, logonly = false;

    private String overrideEmail = null, from = null;

    @Autowired

    private MailLibrary mailLibrary;

    @Autowired

    private CascadingPropertyLoader pLoader;

    @Value("${services.repo-manager.baseUrl}")

    private String baseUrl;

    @Value("${services.repo-manager.adminEmail}")

    private String adminEmail;

    @Value("${services.repomanager.usagestats.adminEmail}")

    private String usageStatsAdminEmail;

    @Override

    public void reportException(Exception exception) {

        Writer writer = new StringWriter();

        PrintWriter printWriter = new PrintWriter(writer);

        exception.printStackTrace(printWriter);

        List<String> recipients = new ArrayList<String>();

        try {

            recipients.add(this.adminEmail);

            String message = "An exception has occurred:\n"+writer.toString();

            String subject = "Automatic Bug Report";

            this.sendMail(recipients, subject, message, false, null);

        } catch (Exception e) {

            LOGGER.error("Error sending error report", e);

        }

    }

    @Override

    public void sendAdministratorRequestToEnableMetrics(PiwikInfo piwikInfo) throws Exception {

        try {

            String subject = "[OpenAIRE-Usage Statistics] New request to enable usage statistics";

            String message = "Dear administrator,\n" +

                    "\n" +

                    "we have received a request to enable the OpenAIRE usage statistics for the following repository \n" +

                    "\n" +

                    "Repository - " + piwikInfo.getRepositoryName() + ", " + piwikInfo.getCountry() + " (" + piwikInfo.getRepositoryId() + ")\n" +

                    "Requestor - " + piwikInfo.getRequestorName() + ", " + piwikInfo.getRequestorEmail() + "\n" +

                    "Piwik ID - " + piwikInfo.getSiteId() + "\n" +

                    "Authentication token - " + piwikInfo.getAuthenticationToken() + "\n" +

                    "\n" +

                    "For more information about this request, go here: \n" +

                    this.baseUrl + "/#admin/metrics\n" +

                    "\n" +

                    "Best,\n" +

                    "The OpenAIRE team";

            this.sendMail(this.usageStatsAdminEmail, subject, message, false, null);

        } catch (Exception e) {

            LOGGER.error("Error while sending request to enable metrics email to administrator: " + this.usageStatsAdminEmail, e);

            throw e;

        }

    }

    @Override

    public void sendUserRequestToEnableMetrics(PiwikInfo piwikInfo) throws Exception {

        try {

            String subject = "[OpenAIRE-Usage Statistics] Your request to enable usage statistics";

            String message = "Dear " + piwikInfo.getRequestorName() + ",\n" +

                    "\n" +

                    "we have received your request to enable the OpenAIRE usage statistics for your repository\n" +

                    "\n" +

                    "Repository - " + piwikInfo.getRepositoryName() + ", " + piwikInfo.getCountry() + " (" + piwikInfo.getRepositoryId() + ")\n" +

                    "Piwik ID - " + piwikInfo.getSiteId() + "\n" +

                    "Authentication token - " + piwikInfo.getAuthenticationToken() + "\n" +

                    "\n" +

                    "In order to enable the usage statistics, you must install the OpenAIRE's tracking code in your repository software. " +

                    "OpenAIRE's usage statistics service tracking code is maintained on Github as a patch for various versions of DSpace " +

                    "(https://github.com/openaire/OpenAIRE-Piwik-DSpace) and as an Eprints plugin for version 3 " +

                    "(https://github.com/openaire/EPrints-OAPiwik). In case the platform is different from DSpace or EPrints please contact " +

                    "the OpenAIRE team in repositoryusagestats@openaire.eu in order to find a solution.\n" +

                    "\n" +

                    "For more information about your request and configuration details, go here: \n" +

                    this.baseUrl + "/#getImpact/instructions/" + piwikInfo.getRepositoryId() + "\n" +

                    "\n" +

                    "Once you have finished configuring your repository or if you have any questions, please notify the OpenAIRE team by sending \n" +

                    "an email to repositoryusagestats@openaire.eu\n" +

                    "\n" +

                    "Best,\n" +

                    "The OpenAIRE team";

            this.sendMail(piwikInfo.getRequestorEmail(), subject, message, false, null);

        } catch (Exception e) {

            LOGGER.error("Error while sending request to enable metrics email to user: " + piwikInfo.getRequestorEmail(), e);

            throw e;

        }

    }

    @Override

    public void sendAdministratorMetricsEnabled(PiwikInfo piwikInfo) throws Exception {

        try {

            String subject = "[OpenAIRE-Usage Statistics] Usage statistics have been enabled";

            String message = "Dear administrator,\n" +

                    "\n" +

                    "The installation and configuration of OpenAIRE's tracking code for the following repository " +

                    "has been completed and validated and the usage statistics have been enabled in OpenAIRE.\n" +

                    "\n" +

                    "Repository - " + piwikInfo.getRepositoryName() + ", " + piwikInfo.getCountry() + " (" + piwikInfo.getRepositoryId() + ")\n" +

                    "Requestor - " + piwikInfo.getRequestorName() + ", " + piwikInfo.getRequestorEmail() + "\n" +

                    "Piwik ID - " + piwikInfo.getSiteId() + "\n" +

                    "Authentication token - " + piwikInfo.getAuthenticationToken() + "\n" +

                    "\n" +

                    "Best,\n" +

                    "The OpenAIRE team";

            this.sendMail(piwikInfo.getRequestorEmail(), subject, message, false, null);

        } catch (Exception e) {

            LOGGER.error("Error while sending metrics enabled notification email to administator: " + this.usageStatsAdminEmail, e);

            throw e;

        }

    }

    @Override

    public void sendUserMetricsEnabled(PiwikInfo piwikInfo) throws Exception {

        try {

            String subject = "[OpenAIRE-Usage Statistics] Usage statistics have been enabled";

            String message = "Dear " + piwikInfo.getRequestorName() + ",\n" +

                    "\n" +

                    "The installation and configuration of OpenAIRE's tracking code for your repository \"" + piwikInfo.getRepositoryName() +

                    "\" has been completed and validated and the usage statistics have been enabled in OpenAIRE.\n" +

                    "\n" +

                    "You can preview the statistics in your repository's dashboard: \n" +

                    this.baseUrl + "/#getImpact/" + piwikInfo.getRepositoryId() + "\n" +

                    "\n" +

                    " For more information and questions, you can contact the openaire support team by sending an email to " +

                    "repositoryusagestats@openaire.eu\n" +

                    "\n" +

                    "Best,\n" +

                    "The OpenAIRE team";

            this.sendMail(piwikInfo.getRequestorEmail(), subject, message, false, null);

        } catch (Exception e) {

            LOGGER.error("Error while sending metrics enabled notification email to user: " + piwikInfo.getRequestorEmail(), e);

            throw e;

        }

    }

    private void sendMail(String email, String subject, String message, boolean sendToSpecial, List<String> repoAdminMails) throws Exception {

        ArrayList<String> to = new ArrayList<String>();

        to.add(email);

        this.sendMail(to,subject,message,sendToSpecial,repoAdminMails);

    }

    private void sendMail(List<String> recipients, String subject, String message, boolean sendToSpecial, List<String> repoAdminMails) throws Exception {

       /* try {

            if (sendToSpecial) {

                recipients.addAll(this.specialRecipients);

            }

            if (repoAdminMails != null)

                recipients.addAll(repoAdminMails);

            if (this.override) {

                recipients.clear();

                recipients.add(overrideEmail);

            }

            if (!logonly)

                mailLibrary.sendEmail(recipients.toArray(new String[]{}), subject, message);

            LOGGER.debug("Sending mail to Recipients: " + recipients + " Subject: " + subject + " Message: " + message);

        } catch (Exception e) {

            LOGGER.error("Error sending mail to Recipients: " + recipients + " Subject: " + subject + " Message: " + message, e);

            throw new Exception(e);

        }*/

    }

    private String getEmailProperty(String key) {

        return pLoader.getProperties().getProperty(key);

    }

    public void setSpecialRecipients(String specialRecipients) {

        String[] recps = specialRecipients.split(",");

        for (String recp : recps) {

            recp = recp.trim();

            this.specialRecipients.add(recp);

        }

    }

    public void setOverride(boolean override) {

        this.override = override;

    }

    public void setOverrideEmail(String overrideEmail) {

        this.overrideEmail = overrideEmail;

    }

    public String getFrom() {

        return from;

    }

    public void setFrom(String from) {

        this.from = from;

    }

    public boolean isLogonly() {

        return logonly;

    }

    public void setLogonly(boolean logonly) {

        this.logonly = logonly;

    }

}

package eu.dnetlib.repo.manager.service.utils;

import com.fasterxml.jackson.annotation.JsonInclude;

@JsonInclude(JsonInclude.Include.NON_NULL)

public class RequestFilter{

    private String registeredby = null;

    private String typology = null;

    private String country = null;

    private String id = null;

    private String officialname = null;

    public RequestFilter() {

    }

    public String getTypology() {

        return typology;

    }

    public void setTypology(String typology) {

        this.typology = typology;

    }

    public String getRegisteredby() {

        return registeredby;

    }

    public void setRegisteredby(String registeredby) {

        this.registeredby = registeredby;

    }

    public String getCountry() {

        return country;

    }

    public void setCountry(String country) {

        this.country = country;

    }

    public String getId() {

        return id;

    }

    public void setId(String id) {

        this.id = id;

    }

    public String getOfficialname() {

        return officialname;

    }

    public void setOfficialname(String officialname) {

        this.officialname = officialname;

    }

}

/*

package eu.dnetlib.repo.manager.service.utils;

import org.mitre.openid.connect.model.OIDCAuthenticationToken;

import org.springframework.security.core.Authentication;

import org.springframework.security.web.authentication.AuthenticationSuccessHandler;

import javax.servlet.ServletException;

import javax.servlet.http.Cookie;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import java.io.IOException;

public class FrontEndLinkURIAuthenticationSuccessHandler implements AuthenticationSuccessHandler {

    private String frontEndURI;

    @Override

    public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException {

        OIDCAuthenticationToken authOIDC = (OIDCAuthenticationToken) authentication;

        Cookie sessionCookie = new Cookie("currentUser", authOIDC.getSub());

        int expireSec = -1;

        sessionCookie.setMaxAge(expireSec);

        sessionCookie.setPath("/");

        response.addCookie(sessionCookie);

        response.sendRedirect(frontEndURI);

    }

    public String getFrontEndURI() {

        return frontEndURI;

    }

    public void setFrontEndURI(String frontEndURI) {

        this.frontEndURI = frontEndURI;

    }

}

*/

/*

package eu.dnetlib.repo.manager.service.utils;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.ComponentScan;

import org.springframework.context.annotation.Configuration;

import org.springframework.context.annotation.PropertySource;

import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;

import org.springframework.session.data.redis.config.annotation.web.http.EnableRedisHttpSession;

import org.springframework.session.web.http.CookieSerializer;

import org.springframework.session.web.http.DefaultCookieSerializer;

import javax.annotation.PostConstruct;

import java.util.logging.Logger;

@Configuration

@EnableRedisHttpSession

@PropertySource(value = { "classpath:eu/dnetlib/repo/manager/service/application.properties", "classpath:application.properties"} )

@ComponentScan(basePackages = "eu.dnetlib.repo.manager")

public class Config {

    private static Logger LOGGER = Logger.getLogger(String.valueOf(Config.class));

    @Value("${redis.host}")

    private String host;

    @Value("${redis.port:6379}")

    private String port;

    @Value("${redis.password:#{null}}")

    private String password;

    @PostConstruct

    private void init(){

        LOGGER.info(host);

    }

    @Bean

    public LettuceConnectionFactory connectionFactory() {

        LOGGER.info(String.format("Redis connection listens to %s:%s",host,port));

        LettuceConnectionFactory factory = new LettuceConnectionFactory(host,Integer.parseInt(port));

        if(password != null) factory.setPassword(password);

        return factory;

    }

    @Bean

    public CookieSerializer cookieSerializer() {

        DefaultCookieSerializer serializer = new DefaultCookieSerializer();

        serializer.setCookieName("SESSION"); // <1>

        serializer.setCookiePath("/"); // <2>

        return serializer;

    }

}

*/

package eu.dnetlib.repo.manager.service.utils;

import com.fasterxml.jackson.databind.ObjectMapper;

import eu.dnetlib.domain.data.Repository;

import eu.dnetlib.domain.data.RepositoryInterface;

import eu.dnetlib.repo.manager.shared.*;

import org.apache.log4j.Logger;

import org.json.JSONArray;

import org.json.JSONException;

import org.json.JSONObject;

import java.io.*;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.*;

public class Converter {

    private static final Logger LOGGER = Logger.getLogger(Converter.class);

    public static Repository jsonToRepositoryObject(JSONObject repositoryObject) throws JSONException {

        Repository repository = new Repository();

        LOGGER.debug("datasource response -> " + repositoryObject);

        JSONObject datasource = repositoryObject.getJSONObject("datasource");

        if( datasource.equals(null))

            return null;

        repository.setActivationId(datasource.get("activationId").toString());

        repository.setAggregator(datasource.get("aggregator").toString());

        repository.setCertificates(datasource.get("certificates").toString());

        repository.setCitationGuidelineUrl(datasource.get("citationguidelineurl").toString());

        repository.setCollectedFrom( datasource.get("collectedfrom").toString());

        repository.setContactEmail(datasource.get("contactemail").toString());

        if(repository.getContactEmail().equals("null"))

            repository.setContactEmail("");

        repository.setDatabaseAccessRestriction(datasource.get("databaseaccessrestriction").toString());

        repository.setDatabaseAccessType(datasource.get("databaseaccesstype").toString());

        repository.setDataUploadRestriction(datasource.get("datauploadrestriction").toString());

        repository.setDataUploadType(datasource.get("datauploadtype").toString());

        repository.setDateOfCollection(convertStringToDate( datasource.get("dateofcollection").toString()));

        repository.setDateOfValidation(convertStringToDate( datasource.get("dateofvalidation").toString()));

        repository.setDescription(datasource.get("description").toString());

        if(repository.getDescription().equals("null"))

            repository.setDescription("");

        repository.setEissn(datasource.get("eissn").toString());

        repository.setEnglishName( datasource.get("englishname").toString());

        if(repository.getEnglishName().equals("null"))

            repository.setEnglishName("");

        repository.setId(datasource.get("id").toString());

        repository.setIssn(datasource.get("issn").toString());

        repository.setOdLanguages(datasource.get("languages").toString());

        repository.setLatitude( toDouble(datasource.get("latitude").toString()));

        repository.setLissn(datasource.get("lissn").toString());

        repository.setLogoUrl(datasource.get("logourl").toString());

        if(repository.getLogoUrl().equals("null"))

            repository.setLogoUrl("");

        repository.setLongitude(toDouble(datasource.get("longitude").toString()));

        //datasource.get("managed");

        repository.setMissionStatementUrl(datasource.get("missionstatementurl").toString());

        repository.setNamespacePrefix(datasource.get("namespaceprefix").toString());

        repository.setOdContentTypes(datasource.get("od_contenttypes").toString());

        repository.setOfficialName(datasource.get("officialname").toString());

        if(repository.getOfficialName().equals("null"))

            repository.setOfficialName("");

        repository.setPidSystems(datasource.get("pidsystems").toString());

        //datasource.get("platform");

        repository.setProvenanceActionClass( datasource.get("provenanceaction").toString());

        repository.setQualityManagementKind(datasource.get("qualitymanagementkind").toString());

        repository.setRegisteredBy(datasource.get("registeredby").toString());

        if(Objects.equals(repository.getRegisteredBy(),"null"))

            repository.setRegistered(true);

        repository.setReleaseEndDate(convertStringToDate(datasource.get("releaseenddate").toString()));

        repository.setReleaseStartDate(convertStringToDate(datasource.get("releasestartdate").toString()));

        repository.setServiceProvider(Boolean.valueOf(datasource.get("serviceprovider").toString()));

        //datasource.get("subjects");

        Double timezone = toDouble(datasource.get("timezone").toString());

        repository.setTimezone(timezone!=null?timezone:0.0);

        repository.setTypology(datasource.get("platform").toString());

        repository.setVersioning(Boolean.valueOf(datasource.get("versioning").toString()));

        repository.setWebsiteUrl(datasource.get("websiteurl").toString());

        repository.setDatasourceClass(datasource.get("typology").toString());

        //TODO change organization to list

        repository.setOrganization( ((JSONArray)datasource.get("organizations")).getJSONObject(0).get("legalname").toString());

        String countryCode = ((JSONArray)datasource.get("organizations")).getJSONObject(0).get("country").toString();

        repository.setCountryCode(countryCode);

        String collectedFrom = datasource.get("collectedfrom").toString();

        //TODO check data consistency

        String type = "UNKNOWN";

        if (collectedFrom.equalsIgnoreCase("openaire____::opendoar")) {

            type = "opendoar";

        } else if (collectedFrom.equalsIgnoreCase("openaire____::re3data")) {

            type = "re3data";

        } else if (collectedFrom.equalsIgnoreCase("infrastruct_::openaire")) {

            type = "journal";

        }

        repository.setDatasourceType(type);

        return repository;

    }

    public static Date convertStringToDate(String date){

        if(Objects.equals(date, "null"))

            return null;

        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");

        try {

            return formatter.parse(date);

        } catch (ParseException e) {

            e.printStackTrace();

        }

        return null;

    }

    public static String convertDateToString(Date date){

        if(Objects.equals(date, null))

            return null;

        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");

        return formatter.format(date);

    }

    public static Double toDouble(String number){

        if(Objects.equals(number, "null"))

            return 0.0;

        else

            return Double.valueOf(number);

    }

    public static List<Repository> jsonToRepositoryList(JSONObject json) throws JSONException {

        List<Repository> resultSet = new ArrayList<>();

        JSONArray rs = json.getJSONArray("datasourceInfo");

        for(int i=0;i<rs.length();i++)

            resultSet.add(jsonToRepositoryObject( rs.getJSONObject(i)) );

        return resultSet;

    }

    public static List<RepositoryInterface> jsonToRepositoryInterfaceList(JSONArray rs) throws JSONException {

        List<RepositoryInterface> resultSet = new ArrayList<>();

        for(int i=0;i<rs.length();i++)

            resultSet.add(jsonToRepositoryInterfaceObject( rs.getJSONObject(i)) );

        return resultSet;

    }

    public static RepositoryInterface jsonToRepositoryInterfaceObject(JSONObject repositoryInterfaceObject) throws JSONException {

        RepositoryInterface repositoryInterface = new RepositoryInterface();

        repositoryInterface.setBaseUrl(repositoryInterfaceObject.get("baseurl").toString());

        repositoryInterface.setContentDescription(repositoryInterfaceObject.get("contentdescription").toString());

        repositoryInterface.setId(repositoryInterfaceObject.get("id").toString());

        repositoryInterface.setMetadataIdentifierPath(repositoryInterfaceObject.get("metadataIdentifierPath").toString());

        repositoryInterface.setAccessProtocol(repositoryInterfaceObject.get("protocol").toString());

        repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());

        repositoryInterface.setDesiredCompatibilityLevel(repositoryInterfaceObject.get("compatibility").toString());

        repositoryInterface.setActive(Boolean.parseBoolean(repositoryInterfaceObject.get("active").toString()));

        repositoryInterface.setRemovable(Boolean.parseBoolean(repositoryInterfaceObject.get("removable").toString()));

        repositoryInterface.setCompliance(repositoryInterfaceObject.get("compatibility").toString());

        Map<String, String> accessParams = new HashMap<>();

        Map<String, String> extraFields = new HashMap<>();

        ObjectMapper mapper = new ObjectMapper();

        JSONArray apiparams = repositoryInterfaceObject.getJSONArray("apiparam");

        for(int i=0;i<apiparams.length();i++)

            accessParams.put(apiparams.getJSONObject(i).getString("param"),apiparams.getJSONObject(i).getString("value"));

        return repositoryInterface;

    }

    public static String repositoryObjectToJson(Repository repository) throws JSONException {

        JSONObject jsonObject = new JSONObject();

        jsonObject.put("activationId",repository.getActivationId());

        jsonObject.put("aggregator",repository.getAggregator());

        jsonObject.put("certificates",repository.getCertificates());

        jsonObject.put("citationguidelineurl",repository.getCitationGuidelineUrl());

        jsonObject.put("collectedfrom",repository.getCollectedFrom());

        jsonObject.put("contactemail",repository.getContactEmail());

        jsonObject.put("databaseaccessrestriction",repository.getDatabaseAccessRestriction());

        jsonObject.put("databaseaccesstype",repository.getDatabaseAccessType());

        jsonObject.put("datauploadrestriction",repository.getDataUploadRestriction());

        jsonObject.put("datauploadtype",repository.getDataUploadType());

        jsonObject.put("dateofcollection",convertDateToString(repository.getDateOfCollection()));

        jsonObject.put("dateofvalidation",convertDateToString(repository.getDateOfValidation()));

        jsonObject.put("description",repository.getDescription());

        jsonObject.put("eissn",repository.getEissn());

        jsonObject.put("englishname",repository.getEnglishName());

        jsonObject.put("id",repository.getId());

        jsonObject.put("issn",repository.getIssn());

        jsonObject.put("languages",repository.getOdLanguages());

        jsonObject.put("latitude",repository.getLatitude().toString());

        jsonObject.put("lissn",repository.getLissn());

        jsonObject.put("logourl",repository.getLogoUrl());

        jsonObject.put("longitude",repository.getLongitude().toString());

        jsonObject.put("missionstatementurl",repository.getMissionStatementUrl());

        jsonObject.put("namespaceprefix",repository.getNamespacePrefix());

        jsonObject.put("od_contenttypes",repository.getOdContentTypes());

        jsonObject.put("officialname",repository.getOfficialName());

        jsonObject.put("pidsystems",repository.getPidSystems());

        jsonObject.put("provenanceaction",repository.getProvenanceActionClass());

        jsonObject.put("qualitymanagementkind",repository.getQualityManagementKind());

        jsonObject.put("registeredby",repository.getRegisteredBy());

        jsonObject.put("releaseenddate",convertDateToString(repository.getReleaseEndDate()));

        jsonObject.put("releasestartdate",convertDateToString(repository.getReleaseStartDate()));

        jsonObject.put("serviceprovider",repository.getServiceProvider());

        jsonObject.put("timezone",repository.getTimezone());

        jsonObject.put("typology",repository.getTypology());

        jsonObject.put("versioning",repository.getVersioning());

        jsonObject.put("websiteurl",repository.getWebsiteUrl());

        //datasource.get("managed");

        //datasource.get("platform");

        //datasource.get("subjects");

        return jsonObject.toString();

    }

    public static String repositoryInterfaceObjectToJson(Repository repository,RepositoryInterface repositoryInterface) throws JSONException {

        JSONObject jsonObject = new JSONObject();

        jsonObject.put("baseurl",repositoryInterface.getBaseUrl());

        jsonObject.put("contentdescription",repositoryInterface.getContentDescription());

        jsonObject.put("id",repositoryInterface.getId());

        jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());

        jsonObject.put("protocol",repositoryInterface.getAccessProtocol());

        jsonObject.put("typology",repositoryInterface.getTypology());

        jsonObject.put("compatibility",repositoryInterface.getDesiredCompatibilityLevel());

        jsonObject.put("datasource",repository.getId());

        jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());

        jsonObject.put("protocol",repositoryInterface.getAccessProtocol());

        jsonObject.put("removable",repositoryInterface.isRemovable());

        jsonObject.put("active",repositoryInterface.isActive());

        JSONArray apiparams = new JSONArray();

        for(String param: repositoryInterface.getAccessParams().keySet()){

            JSONObject jo = new JSONObject();

            jo.put("param",param);

            jo.put("value",repositoryInterface.getAccessParams().get(param));

            apiparams.put(jo);

        }

        jsonObject.put("apiparam",apiparams);

        jsonObject.put("lastCollectionDate",repositoryInterface.getLastCollectionDate());

        //jsonObject.put("lastCollectionMdid",repositoryInterface);

        //jsonObject.put("lastCollectionTotal");

        //jsonObject.put("lastDownloadDate");

//        jsonObject.put("lastDownloadMdid");

//        jsonObject.put("lastDownloadTotal");

//        jsonObject.put("lastValidationJob");

        //jsonObject.put("lastAggregationDate");

        //jsonObject.put("lastAggregationMdid");

        //jsonObject.put("lastAggregationTotal");

        return jsonObject.toString();

    }

    public static ArrayList<String> readFile(String filename) {

        String line;

        ArrayList<String> list = new ArrayList<String>();

        try {

            //InputStream in = Converter.class.getResourceAsStream("resources/eu/dnetlib/repo/manager/service/utils/"+filename);

            InputStream in = Converter.class.getClass().getResourceAsStream("/eu/**/" + filename);

            BufferedReader br = new BufferedReader(new InputStreamReader(in));

            while((line = br.readLine()) != null) {

                list.add(line.trim());

            }

            br.close();

        } catch (IOException e) {

            LOGGER.debug("Error opening file!");

            e.printStackTrace();

        }

        return list;

    }

    public static List<AggregationDetails> getAggregationHistoryFromJson(JSONObject repositoryObject) throws JSONException {

       /* if( repositoryObject.get("aggregationHistory").toString().equals("[]") ||

                repositoryObject.get("aggregationHistory")!= null)

            return null;*/

       if(repositoryObject.get("aggregationHistory").toString().equals("[]"))

           return null;

        JSONArray rs = new JSONArray(repositoryObject.get("aggregationHistory").toString());

        LOGGER.debug(rs.length());

        List<AggregationDetails> aggregationDetailsList = new ArrayList<>();

        for(int i=0;i<rs.length();i++)

            aggregationDetailsList.add(jsonToAggregationDetails(rs.getJSONObject(i)));

        return aggregationDetailsList;

    }

    private static AggregationDetails jsonToAggregationDetails(JSONObject aggregationObject) throws JSONException {

        AggregationDetails aggregationDetails = new AggregationDetails();

        aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString());

        if(aggregationObject.has("collectionMode"))

            aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString());

        aggregationDetails.setDate(convertStringToDate(aggregationObject.get("date").toString()));

        aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString()));

        return aggregationDetails;

    }

    public static AggregationDetails getLastCollectionFromJson(JSONObject repositoryObject) throws JSONException {

        if( repositoryObject.get("lastCollection").equals(null))

            return null;

        return jsonToAggregationDetails(repositoryObject.getJSONObject("lastCollection"));

    }

    public static AggregationDetails getLastTransformationFromJson(JSONObject repositoryObject) throws JSONException {

        if( repositoryObject.get("lastTransformation").equals(null))

            return null;

        return jsonToAggregationDetails(repositoryObject.getJSONObject("lastTransformation"));

    }

    public static List<Timezone> toTimezones(List<String> timezones) {

        List<Timezone> tmz = new ArrayList<>();

        for(String t : timezones){

            String[] s = t.split("\t");

            tmz.add(new Timezone(s[1],Double.parseDouble(s[0])));

        }

        return tmz;

    }

}

package eu.dnetlib.repo.manager.service.utils;

import eu.dnetlib.domain.data.PiwikInfo;

import eu.dnetlib.domain.functionality.UserProfile;

public interface EmailUtils {

    void reportException(Exception exception);

    void sendAdministratorRequestToEnableMetrics(PiwikInfo piwikInfo) throws Exception;

    void sendUserRequestToEnableMetrics(PiwikInfo piwikInfo) throws Exception;

    void sendAdministratorMetricsEnabled(PiwikInfo piwikInfo) throws Exception;

    void sendUserMetricsEnabled(PiwikInfo piwikInfo) throws Exception;

}

package eu.dnetlib.repo.manager.service.utils;

import org.apache.log4j.Logger;

import org.dom4j.io.DOMWriter;

import org.w3c.dom.Document;

import se.kb.oai.pmh.*;

import se.kb.oai.pmh.Set;

import javax.net.ssl.*;

import javax.xml.namespace.NamespaceContext;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import java.security.KeyManagementException;

import java.security.NoSuchAlgorithmException;

import java.security.cert.X509Certificate;

import java.util.*;

public class OaiTools {

	{

		disableSslVerification();

	}

	private static Logger LOGGER = Logger.getLogger(OaiTools.class);

	public static List<String> getSetsOfRepo(String baseUrl) throws Exception {

		try {

			LOGGER.debug("Getting sets of repository " + baseUrl);

			OaiPmhServer harvester = new OaiPmhServer(baseUrl);

			SetsList setList = harvester.listSets();

			ResumptionToken token = setList.getResumptionToken();

			List<Set> sets = new ArrayList<Set>();

			sets.addAll(setList.asList());

			while (token != null) {

				setList = harvester.listSets(token);

				token = setList.getResumptionToken();

				sets.addAll(setList.asList());

			}

			List<String> ret = new ArrayList<String>();

			for (Set set : sets) {

				ret.add(set.getSpec().trim());

			}

			if (ret.size() > 0 )

				Collections.sort(ret);

			return ret;

		} catch (Exception e) {

			LOGGER.error("Error getting sets of repository " + baseUrl, e);

			return new ArrayList<String>();

			//throw e;

		}

	}

	public static boolean identifyRepository(String baseUrl) throws Exception {

		LOGGER.debug("sending identify request to repo " + baseUrl);

		OaiPmhServer harvester = new OaiPmhServer(baseUrl);

		if (baseUrl.trim().isEmpty()) {

			return false;

		}

		try {

			Identification identification = harvester.identify();

			DOMWriter d4Writer = new DOMWriter();

			Document d = d4Writer.write(identification.getResponse());

			return verifyIdentify(d);

		} catch (Exception e) {

			LOGGER.debug("Error verifying identify response", e);

			throw e;

		}

	}

	private static boolean verifyIdentify(Document doc) throws XPathExpressionException {

		NamespaceContext ctx = new NamespaceContext() {

			public String getNamespaceURI(String prefix) {

				String uri;

				if (prefix.equals("oai"))

					uri = "http://www.openarchives.org/OAI/2.0/";

				else

					uri = null;

				return uri;

			}

			// Dummy implementation - not used!

			public Iterator<String> getPrefixes(String val) {

				return null;

			}

			// Dummy implemenation - not used!

			public String getPrefix(String uri) {

				return null;

			}

		};

		// Now the XPath expression

		String xpathStr = "//oai:OAI-PMH/oai:Identify";

		XPathFactory xpathFact = XPathFactory.newInstance();

		XPath xpath = xpathFact.newXPath();

		xpath.setNamespaceContext(ctx);

		String result = xpath.evaluate(xpathStr, doc);

		return (result != null && !result.equals(""));

	}

	private static void disableSslVerification() {

		try

		{

			LOGGER.debug("disabling ssl verification");

			// Create a trust manager that does not validate certificate chains

			TrustManager[] trustAllCerts = new TrustManager[] {new X509TrustManager() {

				public X509Certificate[] getAcceptedIssuers() {

					return null;

				}

				public void checkClientTrusted(X509Certificate[] certs, String authType) {

				}

				public void checkServerTrusted(X509Certificate[] certs, String authType) {

				}

			}

			};

			// Install the all-trusting trust manager

			SSLContext sc = SSLContext.getInstance("SSL");

			sc.init(null, trustAllCerts, new java.security.SecureRandom());

			HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());

			// Create all-trusting host name verifier

			HostnameVerifier allHostsValid = new HostnameVerifier() {

				public boolean verify(String hostname, SSLSession session) {

					return true;

				}

			};

			// Install the all-trusting host verifier

			HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);

		} catch (NoSuchAlgorithmException e) {

			LOGGER.error("disabling ssl verification", e);

		} catch (KeyManagementException e) {

			LOGGER.error("error while disabling ssl verification", e);

		}

	}

}

package eu.dnetlib.repo.manager.service.controllers;

import eu.dnetlib.api.functionality.ValidatorService;

import eu.dnetlib.api.functionality.ValidatorServiceException;

import eu.dnetlib.domain.functionality.validator.StoredJob;

import eu.dnetlib.repo.manager.shared.JobsOfUser;

import eu.dnetlib.repo.manager.shared.Constants;

import gr.uoa.di.driver.util.ServiceLocator;

import io.swagger.annotations.ApiParam;

import org.apache.log4j.Logger;

import org.json.JSONException;

import org.springframework.stereotype.Component;

import javax.annotation.Resource;

@Component

public class MonitorApiImpl implements MonitorApi {

    @Resource(name = "validatorServiceLocator")

    private ServiceLocator<ValidatorService> validatorServiceLocator;

    private ValidatorService getValidationService() {

        return this.validatorServiceLocator.getService();

    }

    public ServiceLocator<ValidatorService> getValidatorServiceLocator() {

        return validatorServiceLocator;

    }

    public void setValidatorServiceLocator(ServiceLocator<ValidatorService> validatorServiceLocator) {

        this.validatorServiceLocator = validatorServiceLocator;

    }

    private static final Logger LOGGER = Logger

            .getLogger(MonitorApiImpl.class);

    @Override

    public JobsOfUser getJobsOfUser(@ApiParam(value = "User email", required = true) String user,

                                    @ApiParam(value = "Equals to filter job type on validation history page", required = true) String jobType,

                                    @ApiParam(value = "Page number", required = true) String offset,

                                    @ApiParam(value = "Null value", required = true) String limit,

                                    @ApiParam(value = "Null value", required = true) String dateFrom,

                                    @ApiParam(value = "Null value", required = true) String dateTo,

                                    @ApiParam(value = "Equals to filter validation jobs", required = true) String validationStatus,

                                    @ApiParam(value = "Always true", required = true) String includeJobsTotal) throws JSONException, ValidatorServiceException {

        LOGGER.debug("Getting jobs of user : " + user);

        LOGGER.debug(user + "/" + jobType + "/" + offset + "/" + dateFrom + "/" + dateTo + "/" + validationStatus + "/" + includeJobsTotal);

        JobsOfUser retJobs = new JobsOfUser();

        retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),

                Integer.parseInt(limit), dateFrom, dateTo, validationStatus));

        if (Boolean.parseBoolean(includeJobsTotal)) {

            retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));

            retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));

            retJobs.setTotalJobsFailed(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_FAILED));

            retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType,Constants.VALIDATION_JOB_STATUS_ONGOING));

        }

        return retJobs;

    }

    private int getJobsTotalNumberOfUser(String user, String jobType, String validationStatus) throws ValidatorServiceException {

        return  getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);

    }

    @Override

    public int getJobsOfUserPerValidationStatus(String user,

                                                String jobType,

                                                String validationStatus) throws JSONException {

        LOGGER.debug("Getting job with validation status : " + validationStatus);

        try {

            return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);

        } catch (ValidatorServiceException e) {

            e.printStackTrace();

        }

        return 0;

    }

    @Override

    public StoredJob getJobSummary(String jobId,

                                   String groupBy) throws JSONException {

        LOGGER.debug("Getting job summary with id : " + jobId);

        try {

            return getValidationService().getStoredJob(Integer.parseInt(jobId), groupBy);

        } catch (ValidatorServiceException e) {

            e.printStackTrace();

        }

        return null;

    }

}

/*

package eu.dnetlib.repo.manager.service.controllers;

import io.swagger.annotations.Api;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestMethod;

import org.springframework.web.bind.annotation.RestController;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

@RestController

@RequestMapping(value = "/user")

@Api(description = "User API",  tags = {"user"})

public interface UserApi {

    @RequestMapping(value = "/login" , method = RequestMethod.GET)

    void login(HttpServletRequest req,

               HttpServletResponse resp);

}

*/

package eu.dnetlib.repo.manager.service.controllers;

import eu.dnetlib.domain.data.Repository;

import eu.dnetlib.domain.data.RepositoryInterface;

import eu.dnetlib.repo.manager.shared.*;

import io.swagger.annotations.Api;

import org.json.JSONException;

import org.springframework.http.MediaType;

import org.springframework.web.bind.annotation.*;

import java.util.List;

import java.util.Map;

@RestController

@RequestMapping(value = "/repository")

@Api(description = "Repository API",  tags = {"repository"})

public interface RepositoryApi {

    @RequestMapping(value = "/testAggregations", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> testAggregations() throws JSONException;

    @RequestMapping(value = "/getCountries", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Country[] getCountries() ;

    @RequestMapping(value = "/getRepositoriesByCountry/{country}/{mode}", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<Repository> getRepositoriesByCountry(String country, String mode, Boolean managed) throws JSONException;

    @RequestMapping(value = "/getRepositoriesOfUser/{userEmail}/{page}/{size}",method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<Repository> getRepositoriesOfUser(String userEmail,

                                           String page,

                                           String size) throws JSONException;

    @RequestMapping(value = "/getRepositoryById/{id}", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Repository getRepositoryById(String id) throws JSONException;

    @RequestMapping(value = "/getRepositoryAggregations/{id}", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Aggregations getRepositoryAggregations(String id) throws JSONException;

    @RequestMapping(value = "/getRepositoriesByName/{name}/{page}/{size}/", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<Repository> getRepositoriesByName(String name,

                                           String page,

                                           String size) throws JSONException;

    @RequestMapping(value = "/getRepositoryInterface/{id}", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<RepositoryInterface> getRepositoryInterface(String id) throws JSONException;

    @RequestMapping(value = "/addRepository", method = RequestMethod.POST,

            consumes = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    void addRepository(String datatype, Repository repository) throws Exception;

    @RequestMapping(value = "/deleteInterface", method = RequestMethod.DELETE)

    @ResponseBody

    void deleteRepositoryInterface(String id);

    @RequestMapping(value = "/addInterface", method = RequestMethod.POST,

            consumes = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    RepositoryInterface addRepositoryInterface(String datatype,

                                               String repoId,

                                               RepositoryInterface iFace) throws JSONException;

    @RequestMapping(value = "/getDnetCountries", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getDnetCountries();

    @RequestMapping(value = "/getTypologies", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getTypologies();

    @RequestMapping(value = "/getTimezones", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<Timezone> getTimezones();

    @RequestMapping(value = "/updateManagedStatus", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateManagedStatus(String id, String managed);

    @RequestMapping(value = "/updateEnglishName", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateEnglishName(String id, String englishName);

    @RequestMapping(value = "/updateLatitude", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateLatitude(String id, String latitude);

    @RequestMapping(value = "/updateLongitude", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateLongitude(String id, String longitude);

    @RequestMapping(value = "/updateOfficialName", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateOfficialName(String id, String officialName);

    @RequestMapping(value = "/updateTimezone", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateTimezone(String id, String timezone);

    @RequestMapping(value = "/updateTypology", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateTypology(String id, String typology);

    @RequestMapping(value = "/updateLogoUrl", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateLogoUrl(String id, String logoUrl);

    @RequestMapping(value = "/updatePlatform", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updatePlatform(String id, String platform);

    @RequestMapping(value = "/getUrlsOfUserRepos/{user_email}/{page}/{size}/",method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getUrlsOfUserRepos(String user_email,

                                    String page,

                                    String size) throws JSONException;

    @RequestMapping(value = "/getDatasourceVocabularies/{mode}",method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getDatasourceVocabularies(String mode);

    @RequestMapping(value = "/getCompatibilityClasses/{mode}",method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Map<String, String> getCompatibilityClasses(String mode);

    @RequestMapping(value = "/getDatasourceClasses/{mode}",method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Map<String, String> getDatasourceClasses(String mode);

    String getCountryName(String countryCode);

    @RequestMapping(value = "/getMetricsInfoForRepository/{repoId}",method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    MetricsInfo getMetricsInfoForRepository(String repoId) throws RepositoryServiceException;

}

/*

package eu.dnetlib.repo.manager.service.controllers;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.stereotype.Component;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

@Component

public class UserApiImpl implements UserApi {

    private static final org.apache.log4j.Logger LOGGER = org.apache.log4j.Logger

            .getLogger(UserApiImpl.class);

    @Value("${oidc.issuer}")

    private String oidc_issuer;

    @Override

    public void login(HttpServletRequest req,

                      HttpServletResponse resp) {

        LOGGER.debug(oidc_issuer);

        resp.setStatus(HttpServletResponse.SC_FOUND);

        resp.setHeader("Location", oidc_issuer);

    }

}

*/

package eu.dnetlib.repo.manager.service.controllers;

import com.fasterxml.jackson.databind.DeserializationFeature;

import com.fasterxml.jackson.databind.ObjectMapper;

import eu.dnetlib.domain.data.Repository;

import eu.dnetlib.domain.data.RepositoryInterface;

import eu.dnetlib.domain.enabling.Vocabulary;

import eu.dnetlib.repo.manager.service.utils.Converter;

import eu.dnetlib.repo.manager.shared.*;

import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader;

import org.apache.commons.codec.digest.DigestUtils;

import org.apache.log4j.Logger;

import org.json.JSONArray;

import org.json.JSONException;

import org.json.JSONObject;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.core.ParameterizedTypeReference;

import org.springframework.http.HttpEntity;

import org.springframework.http.HttpHeaders;

import org.springframework.http.HttpMethod;

import org.springframework.http.ResponseEntity;

import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;

import org.springframework.stereotype.Component;

import org.springframework.web.bind.annotation.PathVariable;

import org.springframework.web.bind.annotation.RequestBody;

import org.springframework.web.bind.annotation.RequestParam;

import org.springframework.web.client.RestClientException;

import org.springframework.web.client.RestTemplate;

import org.springframework.web.util.UriComponents;

import org.springframework.web.util.UriComponentsBuilder;

import javax.annotation.PostConstruct;

import javax.ws.rs.QueryParam;

import java.sql.Timestamp;

import java.text.Normalizer;

import java.util.*;

import java.util.concurrent.ConcurrentHashMap;

@Component

public class RepositoryApiImpl implements RepositoryApi {

    @Value("${api.baseAddress}")

    private String baseAddress;

    private RestTemplate restTemplate = null;

    private HttpHeaders httpHeaders;

    private final String[] vocabularyNames = {"dnet:countries", "dnet:datasource_typologies", "dnet:compatibilityLevel"};

    private static final Logger LOGGER = Logger.getLogger(RepositoryApiImpl.class);

    @Value("${services.repomanager.usageStatisticsDiagramsBaseURL}")

    private String usageStatisticsDiagramsBaseURL;

    @Value("${services.repomanager.usageStatisticsNumbersBaseURL}")

    private String usageStatisticsNumbersBaseURL;

    @Autowired

    private VocabularyLoader vocabularyLoader;

    @Autowired

    private PiWikApi piWikApi;

    private Map<String, Vocabulary> vocabularyMap = new ConcurrentHashMap<String, Vocabulary>();

    private Map<String, String> countriesMap = new HashMap<>();

    private Map<String, String> inverseCountriesMap = new HashMap<>();

    @PostConstruct

    private void init() {

        LOGGER.debug("Initialization method of repository api!");

        restTemplate = new RestTemplate();

        restTemplate.getMessageConverters().add(new MappingJackson2HttpMessageConverter());

        httpHeaders = new HttpHeaders();

        httpHeaders.set("Content-Type", "application/json");

        for (String vocName : vocabularyNames) {

            vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));

        }

        Country[] countries = getCountries();

        for (Country c : countries) {

            countriesMap.put(c.getName(), c.getCode());

            inverseCountriesMap.put(c.getCode(), c.getName());

        }

    }

    @Override

    public List<String> testAggregations() throws JSONException {

        int page = 0;

        int size = 1000;

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/list/")

                .path("/{page}/{size}/")

                .build().expand(page, size).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        List<String> ids = new ArrayList<>();

        while (!rs.equals("[]")) {

            ids.addAll(getIdsWithNonEmptyAggregations(rs));

            LOGGER.debug("Checked " + (page + 1) * size + " records!");

            page += 1;

            uriComponents = UriComponentsBuilder

                    .fromHttpUrl(baseAddress + "/ds/list/")

                    .path("/{page}/{size}/")

                    .build().expand(page, size).encode();

            rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        }

        return ids;

    }

    private List<String> getIdsWithNonEmptyAggregations(String rs) throws JSONException {

        JSONArray ids = new JSONArray(rs);

        List<String> agg_ids = new ArrayList<>();

        for (int i = 0; i < ids.length(); i++) {

            String id = ids.getString(i);

            Aggregations aggregations = getRepositoryAggregations(id);

            if (aggregations.getAggregationHistory() != null)

                agg_ids.add(id);

        }

        return agg_ids;

    }

    @Override

    public Country[] getCountries() {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/countries")

                .build().encode();

        return restTemplate.getForObject(uriComponents.toUri(), Country[].class);

    }

    @Override

    public List<Repository> getRepositoriesByCountry(@PathVariable("country") String country,

                                                     @PathVariable("mode") String mode,

                                                     @RequestParam(value = "managed",required=false) Boolean managed) throws JSONException {

        LOGGER.debug("Getting repositories by country!");

        int page = 0;

        int size = 100;

        String countryCode = countriesMap.get(country);

        String filterKey = "UNKNOWN";

        if (mode.equalsIgnoreCase("opendoar")) {

            filterKey = "openaire____::opendoar";

        } else if (mode.equalsIgnoreCase("re3data")) {

            filterKey = "openaire____::re3data";

        } else if (mode.equalsIgnoreCase("jour_aggr")) {

            filterKey = "infrastruct_::openaire";

        }

        LOGGER.debug("Country code equals : " + countryCode);

        LOGGER.debug("Filter mode equals : " + filterKey);

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/search/country/")

                .path("/{page}/{size}/")

                .queryParam("country", countryCode)

                .queryParam("managed", managed)

                .build().expand(page, size).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        JSONArray jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

        List<Repository> resultSet = new ArrayList<>();

        while (jsonArray.length() > 0 ) {

            List<Repository> rep = Converter.jsonToRepositoryList(new JSONObject(rs));

            Collection<Repository> repos = this.getRepositoriesByMode(filterKey, rep);

            resultSet.addAll(repos);

            page += 1;

            uriComponents = UriComponentsBuilder

                    .fromHttpUrl(baseAddress + "/ds/search/country/")

                    .path("/{page}/{size}/")

                    .queryParam("country", countryCode)

                    .queryParam("managed", managed)

                    .build().expand(page, size).encode();

            rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

            jsonArray = (JSONArray) new JSONObject(rs).get("datasourceInfo");

        }

        for (Repository r : resultSet)

            this.getRepositoryInfo(r);

        return resultSet;

    }