D-Net

package eu.manager.repo.manager.service.utils;

import eu.dnetlib.domain.data.Repository;

import eu.dnetlib.domain.data.RepositoryInterface;

import eu.manager.repo.manager.shared.AggregationDetails;

import eu.manager.repo.manager.shared.Timezone;

import com.google.gwt.core.client.impl.AsyncFragmentLoader.Logger;

import org.json.JSONArray;

import org.json.JSONException;

import org.json.JSONObject;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileReader;

import java.io.IOException;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.*;

public class Converter {

    private static final Logger LOGGER = Logger.getLogger(Converter.class);

    public static Repository jsonToRepositoryObject(JSONObject repositoryObject) throws JSONException {

        Repository repository = new Repository();

        if( repositoryObject.get("datasource").equals(null))

            return null;

        JSONObject datasource = (JSONObject) repositoryObject.get("datasource");

        repository.setActivationId(datasource.get("activationId").toString());

        repository.setAggregator(datasource.get("aggregator").toString());

        repository.setCertificates(datasource.get("certificates").toString());

        repository.setCitationGuidelineUrl(datasource.get("citationguidelineurl").toString());

        repository.setCollectedFrom( datasource.get("collectedfrom").toString());

        repository.setContactEmail(datasource.get("contactemail").toString());

        repository.setDatabaseAccessRestriction(datasource.get("databaseaccessrestriction").toString());

        repository.setDatabaseAccessType(datasource.get("databaseaccesstype").toString());

        repository.setDataUploadRestriction(datasource.get("datauploadrestriction").toString());

        repository.setDataUploadType(datasource.get("datauploadtype").toString());

        repository.setDateOfCollection(convertStringToDate( datasource.get("dateofcollection").toString()));

        repository.setDateOfValidation(convertStringToDate( datasource.get("dateofvalidation").toString()));

        repository.setDescription(datasource.get("description").toString());

        repository.setEissn(datasource.get("eissn").toString());

        repository.setEnglishName( datasource.get("englishname").toString());

        repository.setId(datasource.get("id").toString());

        repository.setIssn(datasource.get("issn").toString());

        repository.setOdLanguages(datasource.get("languages").toString());

        repository.setLatitude( toDouble(datasource.get("latitude").toString()));

        repository.setLissn(datasource.get("lissn").toString());

        repository.setLogoUrl(datasource.get("logourl").toString());

        repository.setLongitude(toDouble(datasource.get("longitude").toString()));

        //datasource.get("managed");

        repository.setMissionStatementUrl(datasource.get("missionstatementurl").toString());

        repository.setNamespacePrefix(datasource.get("namespaceprefix").toString());

        repository.setOdContentTypes(datasource.get("od_contenttypes").toString());

        repository.setOfficialName(datasource.get("officialname").toString());

        repository.setPidSystems(datasource.get("pidsystems").toString());

        //datasource.get("platform");

        repository.setProvenanceActionClass( datasource.get("provenanceaction").toString());

        repository.setQualityManagementKind(datasource.get("qualitymanagementkind").toString());

        repository.setRegisteredBy(datasource.get("registeredby").toString());

        repository.setReleaseEndDate(convertStringToDate(datasource.get("releaseenddate").toString()));

        repository.setReleaseStartDate(convertStringToDate(datasource.get("releasestartdate").toString()));

        repository.setServiceProvider(Boolean.valueOf(datasource.get("serviceprovider").toString()));

        //datasource.get("subjects");

        Double timezone = toDouble(datasource.get("timezone").toString());

        repository.setTimezone(timezone!=null?timezone:0.0);

        repository.setTypology(datasource.get("typology").toString());

        repository.setVersioning(Boolean.valueOf(datasource.get("versioning").toString()));

        repository.setWebsiteUrl(datasource.get("websiteurl").toString());

        return repository;

    }

    public static Date convertStringToDate(String date){

        if(Objects.equals(date, "null"))

            return null;

        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");

        try {

            return formatter.parse(date);

        } catch (ParseException e) {

            e.printStackTrace();

        }

        return null;

    }

    public static String convertDateToString(Date date){

        if(Objects.equals(date, "null"))

            return null;

        SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");

        return formatter.format(date);

    }

    public static Double toDouble(String number){

        if(Objects.equals(number, "null"))

            return null;

        else

            return Double.valueOf(number);

    }

    public static List<Repository> jsonToRepositoryList(JSONArray rs) throws JSONException {

        List<Repository> resultSet = new ArrayList<>();

        for(int i=0;i<rs.length();i++)

            resultSet.add(jsonToRepositoryObject( rs.getJSONObject(i)) );

        return resultSet;

    }

    public static RepositoryInterface jsonToRepositoryInterfaceObject(JSONObject repositoryInterfaceObject) throws JSONException {

        RepositoryInterface repositoryInterface = new RepositoryInterface();

        repositoryInterface.setBaseUrl(repositoryInterfaceObject.get("baseurl").toString());

        repositoryInterface.setContentDescription(repositoryInterfaceObject.get("contentdescription").toString());

        repositoryInterface.setId(repositoryInterfaceObject.get("id").toString());

        repositoryInterface.setMetadataIdentifierPath(repositoryInterfaceObject.get("metadataIdentifierPath").toString());

        repositoryInterface.setAccessProtocol(repositoryInterfaceObject.get("protocol").toString());

        repositoryInterface.setTypology(repositoryInterfaceObject.get("typology").toString());

        repositoryInterface.setDesiredCompatibilityLevel(repositoryInterfaceObject.get("compatibility").toString());

        repositoryInterface.setActive(Boolean.parseBoolean(repositoryInterfaceObject.get("active").toString()));

        repositoryInterface.setRemovable(Boolean.parseBoolean(repositoryInterfaceObject.get("removable").toString()));

        return repositoryInterface;

    }

    public static String repositoryObjectToJson(Repository repository) throws JSONException {

        JSONObject jsonObject = new JSONObject();

        jsonObject.put("activationId",repository.getActivationId());

        jsonObject.put("aggregator",repository.getAggregator());

        jsonObject.put("certificates",repository.getCertificates());

        jsonObject.put("citationguidelineurl",repository.getCitationGuidelineUrl());

        jsonObject.put("collectedfrom",repository.getCollectedFrom());

        jsonObject.put("contactemail",repository.getContactEmail());

        jsonObject.put("databaseaccessrestriction",repository.getDatabaseAccessRestriction());

        jsonObject.put("databaseaccesstype",repository.getDatabaseAccessType());

        jsonObject.put("datauploadrestriction",repository.getDataUploadRestriction());

        jsonObject.put("datauploadtype",repository.getDataUploadType());

        jsonObject.put("dateofcollection",convertDateToString(repository.getDateOfCollection()));

        jsonObject.put("dateofvalidation",convertDateToString(repository.getDateOfValidation()));

        jsonObject.put("description",repository.getDescription());

        jsonObject.put("eissn",repository.getEissn());

        jsonObject.put("englishname",repository.getEnglishName());

        jsonObject.put("id",repository.getId());

        jsonObject.put("issn",repository.getIssn());

        jsonObject.put("languages",repository.getOdLanguages());

        jsonObject.put("latitude",repository.getLatitude().toString());

        jsonObject.put("lissn",repository.getLissn());

        jsonObject.put("logourl",repository.getLogoUrl());

        jsonObject.put("longitude",repository.getLongitude().toString());

        jsonObject.put("missionstatementurl",repository.getMissionStatementUrl());

        jsonObject.put("namespaceprefix",repository.getNamespacePrefix());

        jsonObject.put("od_contenttypes",repository.getOdContentTypes());

        jsonObject.put("officialname",repository.getOfficialName());

        jsonObject.put("pidsystems",repository.getPidSystems());

        jsonObject.put("provenanceaction",repository.getProvenanceActionClass());

        jsonObject.put("qualitymanagementkind",repository.getQualityManagementKind());

        jsonObject.put("registeredby",repository.getRegisteredBy());

        jsonObject.put("releaseenddate",convertDateToString(repository.getReleaseEndDate()));

        jsonObject.put("releasestartdate",convertDateToString(repository.getReleaseStartDate()));

        jsonObject.put("serviceprovider",repository.getServiceProvider());

        jsonObject.put("timezone",repository.getTimezone());

        jsonObject.put("typology",repository.getTypology());

        jsonObject.put("versioning",repository.getVersioning());

        jsonObject.put("websiteurl",repository.getWebsiteUrl());

        //datasource.get("managed");

        //datasource.get("platform");

        //datasource.get("subjects");

        return jsonObject.toString();

    }

    public static String repositoryInterfaceObjectToJson(RepositoryInterface repositoryInterface) throws JSONException {

        JSONObject jsonObject = new JSONObject();

        jsonObject.put("baseurl",repositoryInterface.getBaseUrl());

        jsonObject.put("contentdescription",repositoryInterface.getContentDescription());

        jsonObject.put("id",repositoryInterface.getId());

        jsonObject.put("metadataIdentifierPath",repositoryInterface.getMetadataIdentifierPath());

        jsonObject.put("protocol",repositoryInterface.getAccessProtocol());

        jsonObject.put("typology",repositoryInterface.getTypology());

        jsonObject.put("compatibility",repositoryInterface.getDesiredCompatibilityLevel());

        //jsonObject.put("removable",repositoryInterface.getRemovable());

        //jsonObject.put("active",repositoryInterface.getActive());

        return jsonObject.toString();

    }

    public static ArrayList<String> readFile(String filename) {

        String line;

        ArrayList<String> list = new ArrayList<String>();

        try {

            File file = new File(Converter.class.getResource("/eu/dnetlib/repo/manager/service/utils/"+filename).getFile());

            BufferedReader br = new BufferedReader(new FileReader(file));

            while((line = br.readLine()) != null) {

                list.add(line.trim());

            }

            br.close();

        } catch (IOException e) {

            e.printStackTrace();

        }

        return list;

    }

    public static List<AggregationDetails> getAggregationHistoryFromJson(JSONObject repositoryObject) throws JSONException {

        if( repositoryObject.get("aggregationHistory").toString().equals("[]"))

            return null;

        JSONArray rs = new JSONArray(repositoryObject.get("aggregationHistory").toString());

        List<AggregationDetails> aggregationDetailsList = new ArrayList<>();

        for(int i=0;i<rs.length();i++)

            aggregationDetailsList.add(jsonToAggregationDetails(rs.getJSONObject(i)));

        return aggregationDetailsList;

    }

    private static AggregationDetails jsonToAggregationDetails(JSONObject aggregationObject) throws JSONException {

        AggregationDetails aggregationDetails = new AggregationDetails();

        aggregationDetails.setAggregationStage(aggregationObject.get("aggregationStage").toString());

        aggregationDetails.setCollectionMode(aggregationObject.get("collectionMode").toString());

        aggregationDetails.setDate(convertStringToDate(aggregationObject.get("date").toString()));

        aggregationDetails.setNumberOfRecords(Integer.parseInt(aggregationObject.get("numberOfRecords").toString()));

        return aggregationDetails;

    }

    public static AggregationDetails getLastCollectionFromJson(JSONObject repositoryObject) throws JSONException {

        if( repositoryObject.get("lastCollection").equals(null))

            return null;

        return jsonToAggregationDetails(repositoryObject.getJSONObject("lastCollection"));

    }

    public static AggregationDetails getLastTransformationFromJson(JSONObject repositoryObject) throws JSONException {

        if( repositoryObject.get("lastTransformation").equals(null))

            return null;

        return jsonToAggregationDetails(repositoryObject.getJSONObject("lastTransformation"));

    }

    public static List<Timezone> toTimezones(List<String> timezones) {

        List<Timezone> tmz = new ArrayList<>();

        for(String t : timezones){

            String[] s = t.split("\t");

            tmz.add(new Timezone(s[1],Double.parseDouble(s[0])));

        }

        return tmz;

    }

}

package eu.manager.repo.manager.service.utils;

import org.apache.log4j.Logger;

import org.dom4j.io.DOMWriter;

import org.w3c.dom.Document;

import se.kb.oai.pmh.*;

import se.kb.oai.pmh.Set;

import javax.net.ssl.*;

import javax.xml.namespace.NamespaceContext;

import javax.xml.xpath.XPath;

import javax.xml.xpath.XPathExpressionException;

import javax.xml.xpath.XPathFactory;

import java.security.KeyManagementException;

import java.security.NoSuchAlgorithmException;

import java.security.cert.X509Certificate;

import java.util.*;

public class OaiTools {

	{

		disableSslVerification();

	}

	private static Logger LOGGER = Logger.getLogger(OaiTools.class);

	public static List<String> getSetsOfRepo(String baseUrl) throws Exception {

		try {

			LOGGER.debug("Getting sets of repository " + baseUrl);

			OaiPmhServer harvester = new OaiPmhServer(baseUrl);

			SetsList setList = harvester.listSets();

			ResumptionToken token = setList.getResumptionToken();

			List<Set> sets = new ArrayList<Set>();

			sets.addAll(setList.asList());

			while (token != null) {

				setList = harvester.listSets(token);

				token = setList.getResumptionToken();

				sets.addAll(setList.asList());

			}

			List<String> ret = new ArrayList<String>();

			for (Set set : sets) {

				ret.add(set.getSpec().trim());

			}

			if (ret.size() > 0 )

				Collections.sort(ret);

			return ret;

		} catch (Exception e) {

			LOGGER.error("Error getting sets of repository " + baseUrl, e);

			return new ArrayList<String>();

			//throw e;

		}

	}

	public static boolean identifyRepository(String baseUrl) throws Exception {

		LOGGER.debug("sending identify request to repo " + baseUrl);

		OaiPmhServer harvester = new OaiPmhServer(baseUrl);

		if (baseUrl.trim().isEmpty()) {

			return false;

		}

		try {

			Identification identification = harvester.identify();

			DOMWriter d4Writer = new DOMWriter();

			Document d = d4Writer.write(identification.getResponse());

			return verifyIdentify(d);

		} catch (Exception e) {

			LOGGER.debug("Error verifying identify response", e);

			throw e;

		}

	}

	private static boolean verifyIdentify(Document doc) throws XPathExpressionException {

		NamespaceContext ctx = new NamespaceContext() {

			public String getNamespaceURI(String prefix) {

				String uri;

				if (prefix.equals("oai"))

					uri = "http://www.openarchives.org/OAI/2.0/";

				else

					uri = null;

				return uri;

			}

			// Dummy implementation - not used!

			public Iterator<String> getPrefixes(String val) {

				return null;

			}

			// Dummy implemenation - not used!

			public String getPrefix(String uri) {

				return null;

			}

		};

		// Now the XPath expression

		String xpathStr = "//oai:OAI-PMH/oai:Identify";

		XPathFactory xpathFact = XPathFactory.newInstance();

		XPath xpath = xpathFact.newXPath();

		xpath.setNamespaceContext(ctx);

		String result = xpath.evaluate(xpathStr, doc);

		return (result != null && !result.equals(""));

	}

	private static void disableSslVerification() {

		try

		{

			LOGGER.debug("disabling ssl verification");

			// Create a trust manager that does not validate certificate chains

			TrustManager[] trustAllCerts = new TrustManager[] {new X509TrustManager() {

				public X509Certificate[] getAcceptedIssuers() {

					return null;

				}

				public void checkClientTrusted(X509Certificate[] certs, String authType) {

				}

				public void checkServerTrusted(X509Certificate[] certs, String authType) {

				}

			}

			};

			// Install the all-trusting trust manager

			SSLContext sc = SSLContext.getInstance("SSL");

			sc.init(null, trustAllCerts, new java.security.SecureRandom());

			HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());

			// Create all-trusting host name verifier

			HostnameVerifier allHostsValid = new HostnameVerifier() {

				public boolean verify(String hostname, SSLSession session) {

					return true;

				}

			};

			// Install the all-trusting host verifier

			HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);

		} catch (NoSuchAlgorithmException e) {

			LOGGER.error("disabling ssl verification", e);

		} catch (KeyManagementException e) {

			LOGGER.error("error while disabling ssl verification", e);

		}

	}

}

package eu.manager.repo.manager.service.config;

import org.apache.commons.lang.ArrayUtils;

import org.apache.log4j.Logger;

import org.springframework.beans.BeansException;

import org.springframework.context.support.ClassPathXmlApplicationContext;

import org.springframework.web.context.ContextLoaderListener;

import org.springframework.web.context.WebApplicationContext;

import org.springframework.web.context.support.XmlWebApplicationContext;

import javax.servlet.ServletContext;

import java.util.Properties;

public class RepoManagerContextLoaderListener extends ContextLoaderListener {

	private static Logger logger = Logger.getLogger(RepoManagerContextLoaderListener.class);

	public RepoManagerContextLoaderListener() {

		super();

	}

	public RepoManagerContextLoaderListener(WebApplicationContext context) {

		super(context);

	}

	@Override

	protected WebApplicationContext createWebApplicationContext(

			ServletContext servletContext)

			throws BeansException {

		logger.debug("Creating web application context");

		Properties props = this.loadProperties();

		String repoMode = props.getProperty("services.validator.mode.repo");

		String userMode = props.getProperty("services.validator.mode.user");

		Boolean standaloneMode = Boolean.parseBoolean(props.getProperty("services.validator.mode.standalone"));

		logger.info("User mode: " + userMode);

		logger.info("Repo mode: " + repoMode);

		logger.info("Standalone mode: " + standaloneMode);

//		logger.info("Dnet workflow enabled: " + repoMode);

		XmlWebApplicationContext ctx = new XmlWebApplicationContext();

		ctx.setServletContext(servletContext);

		String userApiContext = null;

		if (userMode.equalsIgnoreCase("local"))

			userApiContext = "eu/dnetlib/validator/web/api/impls/users/springContext-validator-user-local.xml";

		else if (userMode.equalsIgnoreCase("ldap"))

			userApiContext = "eu/dnetlib/users/springContext-users-ldap.xml";

		String[] springContextCore = new  String[] {

				"classpath:META-INF/cxf/cxf.xml",

				"classpath:META-INF/cxf/cxf-extension-soap.xml",

				"classpath:META-INF/cxf/cxf-extension-jaxws.xml",

				"classpath:META-INF/cxf/cxf-servlet.xml",

				"classpath*:/cxf.xml",

				"classpath*:/eu/dnetlib/repos/ehcacher/springContext-repos-ehcacher.xml",

				"classpath*:/eu/dnetlib/clients/ws/springContext-locatorFactory.xml",

				"classpath*:/eu/dnetlib/soap/cxf/applicationContext-eprbuilders.xml",

//				"classpath*:/eu/dnetlib/validator/web/actions/springContext-validator-struts.xml",

//				"classpath*:/eu/dnetlib/validator/web/actions/springContext-validator-emailer.xml",

//				"classpath*:/eu/dnetlib/validator/web/config/springContext-validator.xml",

				"classpath*:/eu/dnetlib/repo/manager/server/config/springContext-repo-manager-config.xml",

//				"classpath*:/eu/dnetlib/validator/commons/dao/springContext-*.xml",

				"classpath*:/eu/dnetlib/repos/springContext-repos-" + repoMode + ".xml",

				"classpath*:/" + userApiContext

		};

		String[] springContextForStandalone = new String[] {

		};

		String[] springContextForIS = new String[] {

				"classpath*:/gr/uoa/di/driver/util/springContext-locators.xml",

				"classpath*:/gr/uoa/di/driver/app/springContext-lookupFactory.xml",

				"classpath*:/gr/uoa/di/driver/app/springContext-lookupClients.xml",

				"classpath*:/eu/dnetlib/enabling/hcm/springContext-hcmService.xml",

				"classpath*:/gr/uoa/di/driver/app/springContext-commons.xml",

				"classpath*:/gr/uoa/di/driver/app/springContext-registrator.xml"

		};

		if (standaloneMode) {

			logger.debug("Loading contexts for standalone mode");

			ctx.setConfigLocations((String[])ArrayUtils.addAll(springContextCore,springContextForStandalone));

		} else {

			logger.debug("Loading contexts for dnet");

			ctx.setConfigLocations((String[])ArrayUtils.addAll(springContextCore,springContextForIS));

		}

		ctx.refresh();

		logger.debug("done");

		return ctx;

	}	

	private Properties loadProperties() {

		ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext(new String[] {

				"classpath*:/eu/dnetlib/repo/manager/server/config/springContext-repo-manager-config.xml"

		});

		CascadingPropertyLoader pLoader = (CascadingPropertyLoader) ctx.getBean("propertyLoader");

		Properties props = pLoader.getProperties();

		ctx.destroy();

		ctx.close();

		return props;

	}

}

package eu.manager.repo.manager.service.config;

import org.springframework.beans.BeansException;

import org.springframework.beans.factory.InitializingBean;

import org.springframework.beans.factory.config.ConfigurableListableBeanFactory;

import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer;

import java.util.Properties;

/**

 * CascadingPropertyLoader loads a number of property files and mergers them together, so that the last properties

 * override the previous. It also supports property expansion like:

 * 

 * <code>

 *   something = 1

 *   somethingelse = 2

 *   test = ${something}/${somethingelse}

 *  </code>

 * 

 * <p>

 * And if you override something to XX, then test will become XX/2

 * </p>

 * 

 * 

 * @author marko

 * 

 */

public class CascadingPropertyLoader extends PropertyPlaceholderConfigurer implements InitializingBean {

	private Properties properties;

	public void afterPropertiesSet() throws Exception {

		this.properties = mergeProperties();

		// Convert the merged properties, if necessary.

		convertProperties(this.properties);

		logger.debug("Properties: " + properties);

	}

	@Override

	protected void processProperties(final ConfigurableListableBeanFactory beanFactoryToProcess, final Properties props) throws BeansException {

		super.processProperties(beanFactoryToProcess, props);

	}

	public Properties getProperties() {

		return properties;

	}

	public void setProperties(final Properties properties) {

		super.setProperties(properties);

		this.properties = properties;

	}

}

package eu.manager.repo.manager.service.controllers;

import eu.dnetlib.domain.functionality.validator.JobForValidation;

import eu.dnetlib.domain.functionality.validator.RuleSet;

import org.json.JSONException;

import org.springframework.http.MediaType;

import org.springframework.web.bind.annotation.*;

import java.util.List;

@RestController

@RequestMapping(value = "/validator")

public interface ValidatorApi {

    @RequestMapping(value = "/submitJobForValidation",method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE )

    @ResponseBody

    void submitJobForValidation(@RequestBody JobForValidation jobForValidation);

    @RequestMapping(value = "/reSubmitJobForValidation/",method = RequestMethod.POST, consumes = MediaType.APPLICATION_JSON_VALUE )

    @ResponseBody

    void reSubmitJobForValidation(@RequestBody String jobId) throws JSONException;

    @RequestMapping(value = "/getRuleSets/{mode}" , method = RequestMethod.GET,produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<RuleSet> getRuleSets(String mode);

    @RequestMapping(value = "/getSetsOfRepository" , method = RequestMethod.GET,produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getSetsOfRepository(@RequestBody String url);

}

package eu.manager.repo.manager.service.controllers;

import com.fasterxml.jackson.databind.ObjectMapper;

import com.google.gson.Gson;

import eu.dnetlib.domain.data.Repository;

import eu.manager.repo.manager.shared.broker.BrowseEntry;

import eu.manager.repo.manager.shared.broker.EventsPage;

import eu.manager.repo.manager.shared.BrokerException;

import eu.manager.repo.manager.shared.Tuple;

import eu.manager.repo.manager.shared.broker.AdvQueryObject;

import eu.manager.repo.manager.shared.broker.DatasourcesBroker;

import org.json.JSONException;

import org.json.JSONObject;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.core.ParameterizedTypeReference;

import org.springframework.http.HttpEntity;

import org.springframework.http.HttpMethod;

import org.springframework.http.ResponseEntity;

import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;

import org.springframework.stereotype.Component;

import org.springframework.util.LinkedMultiValueMap;

import org.springframework.util.MultiValueMap;

import org.springframework.web.bind.annotation.PathVariable;

import org.springframework.web.client.RestClientException;

import org.springframework.web.client.RestTemplate;

import org.springframework.web.util.UriComponentsBuilder;

import java.util.*;

@Component

public class BrokerApiImpl implements BrokerApi {

    @Autowired

    private RepositoryApiImpl repoAPI;

    @Value("${services.broker.url}:${services.broker.port}/${services.broker.api}${services.broker.openaire}")

    private String openairePath;

    @Override

    public DatasourcesBroker getDatasourcesOfUser(String params) throws JSONException {

        JSONObject json_params = new JSONObject(params);

        DatasourcesBroker ret = new DatasourcesBroker();

        String userEmail = json_params.getString("userEmail");

        boolean includeShared = json_params.getBoolean("includeShared");

        boolean includeByOthers = json_params.getBoolean("includeByOthers");

        try {

            ret.setDatasourcesOfUser(getDatasourcesOfUserType(getRepositoriesOfUser(userEmail)));

            if (includeShared) {

                //TODO whatever nikonas was saying

                List<String> sharedDatasourceIds = new ArrayList<String>();

                ret.setSharedDatasources(getDatasourcesOfUserType(getRepositoriesByIds(sharedDatasourceIds)));

            }

            if (includeByOthers) {

                ret.setDatasourcesOfOthers(getDatasourcesOfUserType(getRepositoriesOfUser(userEmail)));

            }

        } catch (BrokerException e) {

            e.printStackTrace();

        }

        return ret;

    }

    @Override

    public List<BrowseEntry> getTopicsForDatasource(@PathVariable("datasourceName")  String datasourceName) throws BrokerException {

        final String service = "/topicsForDatasource";

        UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(openairePath + service)

                .queryParam("ds", datasourceName);

        RestTemplate template = new RestTemplate();

        template.getMessageConverters().add(new MappingJackson2HttpMessageConverter());

        ResponseEntity<List<BrowseEntry>> resp;

        try {

            resp = template.exchange(

                    builder.build().encode().toUri(),

                    HttpMethod.GET,

                    null,

                    new ParameterizedTypeReference<List<BrowseEntry>>() {

                    });

        } catch (RestClientException e) {

            throw new BrokerException(e);

        }

        return resp.getBody();

    }

    @Override

    public EventsPage advancedShowEvents(String params) throws BrokerException, JSONException {

        JSONObject json_params = new JSONObject(params);

        long page = json_params.getLong("page");

        long pagesize = json_params.getLong("pagesize");

        JSONObject json_advQueryObject = json_params.getJSONObject("advQueryObject");

        ObjectMapper obj = new ObjectMapper();

        AdvQueryObject advQueryObject = new Gson().fromJson(json_advQueryObject.toString(),AdvQueryObject.class);

        final String service = "/events/{page}/{pageSize}";

        Map<String, Long> uriParams = new HashMap<>();

        uriParams.put("page", page);

        uriParams.put("pageSize", pagesize);

        UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(openairePath + service);

        MultiValueMap<String, String> headers = new LinkedMultiValueMap<>();

        headers.add("Content-Type", "application/json");

        advQueryObject.setPage(page);

        HttpEntity<AdvQueryObject> entity = new HttpEntity<>(advQueryObject, headers);

        RestTemplate template = new RestTemplate();

        template.getMessageConverters().add(new MappingJackson2HttpMessageConverter());

        ResponseEntity<EventsPage> resp;

        try {

            resp = template.exchange(

                    builder.buildAndExpand(uriParams).encode().toUri(),

                    HttpMethod.POST,

                    entity,

                    new ParameterizedTypeReference<EventsPage>() {

                    }

            );

        } catch (RestClientException e) {

            throw new BrokerException(e);

        }

        return resp.getBody();

    }

    private List<Tuple<BrowseEntry, String>> getDatasourcesOfUserType(List<Repository> repositories) throws BrokerException {

        List<Tuple<BrowseEntry, String>> entries = new ArrayList<>();

        for (Repository repo : repositories) {

            BrowseEntry temp = new BrowseEntry();

            temp.setValue(repo.getOfficialName());

            temp.setSize(new Long(0));

            for (BrowseEntry e : getTopicsForDatasource(repo.getOfficialName())) {

                temp.setSize(temp.getSize() + e.getSize());

            }

            Tuple<BrowseEntry, String> tup = new Tuple<>(temp, repo.getLogoUrl());

            entries.add(tup);

        }

        // sort the collection by the second field of the tuple which is size

        Collections.sort(entries, new Comparator<Tuple<BrowseEntry, String>>() {

            @Override

            public int compare(Tuple<BrowseEntry, String> e1, Tuple<BrowseEntry, String> e2) {

                return (int) (e2.getFirst().getSize().longValue() - e1.getFirst().getSize().longValue());

            }

        });

        return entries;

    }

    private List<Repository> getRepositoriesOfUser(String userEmail) throws JSONException {

        int page = 1;

        int size = 10;

        List<Repository> rs = null;

        List<Repository> resultSet = new ArrayList<>();

        while (true){

            rs = repoAPI.getRepositoriesOfUser(userEmail, String.valueOf(page), String.valueOf(size));

            if(rs.size() == 0) break;

            resultSet.addAll(rs);

        }

        return resultSet;

    }

    private List<Repository> getRepositoriesByIds(List<String> sharedDatasourceIds) {

        return null;

    }

}

package eu.manager.repo.manager.service.controllers;

import eu.dnetlib.api.functionality.ValidatorServiceException;

import eu.dnetlib.domain.functionality.validator.StoredJob;

import eu.manager.repo.manager.shared.JobsOfUser;

import org.json.JSONException;

import org.json.JSONObject;

import org.springframework.http.MediaType;

import org.springframework.web.bind.annotation.*;

@RestController

@RequestMapping(value = "/monitor")

public interface MonitorApi {

    @RequestMapping(value = "/getJobsOfUser" , method = RequestMethod.GET,consumes = MediaType.APPLICATION_JSON_VALUE,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    JobsOfUser getJobsOfUser(@RequestBody String user,

                             @RequestBody String jobType,

                             @RequestBody String offset,

                             @RequestBody String limit,

                             @RequestBody String dateFrom,

                             @RequestBody String dateTo,

                             @RequestBody String validationStatus,

                             @RequestBody String includeJobsTotal) throws JSONException, ValidatorServiceException;

    @RequestMapping(value = "/getJobsOfUserPerValidationStatus" , method = RequestMethod.GET,produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    int getJobsOfUserPerValidationStatus(@RequestBody String user,

                                         @RequestBody String jobType,

                                         @RequestBody String validationStatus) throws JSONException;

    @RequestMapping(value = "/getJobSummary" , method = RequestMethod.GET,produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    StoredJob getJobSummary(@RequestBody String jobId,

                            @RequestBody String groupBy) throws JSONException;

}

package eu.manager.repo.manager.service.controllers;

import eu.dnetlib.api.functionality.ValidatorServiceException;

import eu.dnetlib.domain.functionality.validator.StoredJob;

import eu.manager.repo.manager.service.utils.OaiTools;

import gr.uoa.di.driver.util.ServiceLocator;

import eu.dnetlib.domain.functionality.validator.JobForValidation;

import eu.dnetlib.domain.functionality.validator.RuleSet;

import eu.manager.repo.manager.shared.Constants;

import java.util.*;

import java.util.concurrent.ConcurrentHashMap;

import eu.dnetlib.api.functionality.ValidatorService;

import org.apache.log4j.Logger;

import org.json.JSONException;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Component;

import org.springframework.web.bind.annotation.PathVariable;

import javax.annotation.PostConstruct;

import javax.annotation.Resource;

@Component

public class ValidatorApiImpl implements ValidatorApi {

    @Autowired

    private MonitorApiImpl monitorApi;

    @Resource(name = "validatorServiceLocator")

    private ServiceLocator<ValidatorService> validatorServiceLocator;

    private ValidatorService getValidationService() {

        return this.validatorServiceLocator.getService();

    }

    public ServiceLocator<ValidatorService> getValidatorServiceLocator() {

        return validatorServiceLocator;

    }

    public void setValidatorServiceLocator(ServiceLocator<ValidatorService> validatorServiceLocator) {

        this.validatorServiceLocator = validatorServiceLocator;

    }

    private Map<String, List<RuleSet>> rulesetMap = new ConcurrentHashMap<String, List<RuleSet>>();

    private static final Logger LOGGER = Logger

            .getLogger(ValidatorApiImpl.class);

    @PostConstruct

    private void loadRules(){

        try {

            for (RuleSet ruleSet : getValidationService().getRuleSets()) {

                if (ruleSet.getVisibility() != null && ruleSet.getVisibility().contains("development")) {

                    String key = "";

                    if (ruleSet.getGuidelinesAcronym().matches("^openaire[1-9].0_data$"))

                        key = Constants.VALIDATION_MODE_DATA;

                    else if (ruleSet.getGuidelinesAcronym().matches("^openaire[1-9].0$") || ruleSet.getGuidelinesAcronym().equals("driver"))

                        key = Constants.VALIDATION_MODE_LITERATURE;

                    else if (ruleSet.getGuidelinesAcronym().matches("^openaire[1-9].0_cris$"))

                        key = Constants.VALIDATION_MODE_CRIS;

                    if (rulesetMap.containsKey(key))

                        rulesetMap.get(key).add(ruleSet);

                    else {

                        List<RuleSet> ruleSets = new ArrayList<RuleSet>();

                        ruleSets.add(ruleSet);

                        rulesetMap.put(key, ruleSets);

                    }

                }

            }

        } catch (ValidatorServiceException e) {

            e.printStackTrace();

        }

    }

    @Override

    public void submitJobForValidation(JobForValidation jobForValidation) {

        try {

            this.getValidationService().submitValidationJob(jobForValidation);

        } catch (ValidatorServiceException e) {

            e.printStackTrace();

        }

    }

    @Override

    public void reSubmitJobForValidation(String jobId) throws JSONException {

        StoredJob job = monitorApi.getJobSummary(jobId,"all");

        Set<Integer> contentRules = new HashSet<Integer>();

        Set<Integer> usageRules = new HashSet<Integer>();

        RuleSet ruleSet = null;

        for (List<RuleSet> ruleSets : this.rulesetMap.values()) {

            for (RuleSet rSet : ruleSets)

                if (rSet.getGuidelinesAcronym().equals(job.getDesiredCompatibilityLevel())) {

                    ruleSet = rSet;

                    break;

                }

        }

        for (int ruleId : job.getRules()) {

            if (ruleSet.getContentRulesIds().contains(ruleId))

                contentRules.add(ruleId);

            else if (ruleSet.getUsageRulesIds().contains(ruleId))

                usageRules.add(ruleId);

        }

        if (!contentRules.isEmpty())

            job.setSelectedContentRules(contentRules);

        if (!usageRules.isEmpty())

            job.setSelectedUsageRules(usageRules);

        this.submitJobForValidation(job);

    }

    @Override

    public List<RuleSet> getRuleSets(@PathVariable("mode") String mode) {

        return rulesetMap.get(mode);

    }

    @Override

    public List<String> getSetsOfRepository(String url) {

        try {

            return OaiTools.getSetsOfRepo(url);

        } catch (Exception e) {

            e.printStackTrace();

        }

        return null;

    }

}

package eu.manager.repo.manager.service.controllers;

import eu.dnetlib.api.functionality.ValidatorService;

import eu.dnetlib.api.functionality.ValidatorServiceException;

import eu.dnetlib.domain.functionality.validator.StoredJob;

import eu.manager.repo.manager.shared.JobsOfUser;

import eu.manager.repo.manager.shared.Constants;

import gr.uoa.di.driver.util.ServiceLocator;

import org.apache.log4j.Logger;

import org.json.JSONException;

import org.springframework.stereotype.Component;

import javax.annotation.Resource;

@Component

public class MonitorApiImpl implements MonitorApi {

    @Resource(name = "validatorServiceLocator")

    private ServiceLocator<ValidatorService> validatorServiceLocator;

    private ValidatorService getValidationService() {

        return this.validatorServiceLocator.getService();

    }

    public ServiceLocator<ValidatorService> getValidatorServiceLocator() {

        return validatorServiceLocator;

    }

    public void setValidatorServiceLocator(ServiceLocator<ValidatorService> validatorServiceLocator) {

        this.validatorServiceLocator = validatorServiceLocator;

    }

    private static final Logger LOGGER = Logger

            .getLogger(MonitorApiImpl.class);

    @Override

    public JobsOfUser getJobsOfUser(String user,

                                    String jobType,

                                    String offset,

                                    String limit,

                                    String dateFrom,

                                    String dateTo,

                                    String validationStatus,

                                    String includeJobsTotal) throws JSONException, ValidatorServiceException {

        if (jobType.equals(""))

            jobType = null;

        if(dateFrom.equals(""))

            dateFrom = null;

        if(dateTo.equals(""))

            dateTo = null;

        if(validationStatus.equals(""))

            validationStatus = null;

        JobsOfUser retJobs = new JobsOfUser();

        retJobs.setJobs(getValidationService().getStoredJobsNew(user, jobType, Integer.parseInt(offset),

                Integer.parseInt(limit), dateFrom, dateTo, validationStatus));

        if (Boolean.parseBoolean(includeJobsTotal)) {

            retJobs.setTotalJobs(this.getJobsTotalNumberOfUser(user, jobType, null));

            retJobs.setTotalJobsSuccessful(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_SUCCESSFUL));

            retJobs.setTotalJobsFailed(this.getJobsTotalNumberOfUser(user, jobType, Constants.VALIDATION_JOB_STATUS_FAILED));

            retJobs.setTotalJobsOngoing(this.getJobsTotalNumberOfUser(user, jobType,Constants.VALIDATION_JOB_STATUS_ONGOING));

        }

        return retJobs;

    }

    private int getJobsTotalNumberOfUser(String user, String jobType, String validationStatus) throws ValidatorServiceException {

        return  getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);

    }

    @Override

    public int getJobsOfUserPerValidationStatus(String user,

                                                String jobType,

                                                String validationStatus) throws JSONException {

        try {

            return getValidationService().getStoredJobsTotalNumberNew(user, jobType, validationStatus);

        } catch (ValidatorServiceException e) {

            e.printStackTrace();

        }

        return 0;

    }

    @Override

    public StoredJob getJobSummary(String jobId,

                                   String groupBy) throws JSONException {

        try {

            return getValidationService().getStoredJob(Integer.parseInt(jobId), groupBy);

        } catch (ValidatorServiceException e) {

            e.printStackTrace();

        }

        return null;

    }

}

package eu.manager.repo.manager.service.controllers;

import eu.dnetlib.domain.data.Repository;

import eu.dnetlib.domain.data.RepositoryInterface;

import eu.manager.repo.manager.shared.Aggregations;

import eu.manager.repo.manager.shared.Country;

import eu.manager.repo.manager.shared.Timezone;

import org.json.JSONException;

import org.springframework.http.MediaType;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.RequestMethod;

import org.springframework.web.bind.annotation.ResponseBody;

import org.springframework.web.bind.annotation.RestController;

import java.util.List;

import java.util.Set;

@RestController

@RequestMapping(value = "/repository")

public interface RepositoryApi {

    @RequestMapping(value = "/testAggregations", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Aggregations testAggregations() throws JSONException;

    @RequestMapping(value = "/getCountries", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Country[] getCountries() ;

    @RequestMapping(value = "/getRepositoriesByCountry/{country}", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Set<String> getRepositoriesByCountry(String country) throws JSONException;

    @RequestMapping(value = "/getRepositoriesOfUser/{userEmail}/{page}/{size}")

    @ResponseBody

    List<Repository> getRepositoriesOfUser( String userEmail,

                                        String page,

                                        String size) throws JSONException;

    @RequestMapping(value = "/getRepositoryById/{id}", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Repository getRepositoryById(String id) throws JSONException;

    @RequestMapping(value = "/getRepositoryAggregations/{id}", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    Aggregations getRepositoryAggregations(String id) throws JSONException;

    @RequestMapping(value = "/getRepositoriesByName/{name}/{page}/{size}/", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<Repository> getRepositoriesByName(String name,

                                          String page,

                                          String size) throws JSONException;

    @RequestMapping(value = "/getRepositoryInterface/{id}", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    RepositoryInterface getRepositoyInterface(String id) throws JSONException;

    @RequestMapping(value = "/addRepository", method = RequestMethod.POST,

            consumes = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String addRepository(Repository repository) throws JSONException;

    @RequestMapping(value = "/addInterface", method = RequestMethod.POST,

            consumes = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String addRepositoryInterface(RepositoryInterface repositoryInterface) throws JSONException;

    @RequestMapping(value = "/getDnetCountries", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getDnetCountries();

    @RequestMapping(value = "/getTypologies", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getTypologies();

    @RequestMapping(value = "/getTimezones", method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<Timezone> getTimezones();

    @RequestMapping(value = "/updateManagedStatus", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateManagedStatus(String id,String managed);

    @RequestMapping(value = "/updateEnglishName", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateEnglishName(String id,String englishName);

    @RequestMapping(value = "/updateLatitude", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateLatitude(String id,String latitude);

    @RequestMapping(value = "/updateLongitude", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateLongitude(String id,String longitude);

    @RequestMapping(value = "/updateOfficialName", method = RequestMethod.POST,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    String updateOfficialName(String id,String officialName);

    @RequestMapping(value = "/getUrlsOfUserRepos/{email}/{page}/{size}/",method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getUrlsOfUserRepos(String user_email,

                              String page,

                              String size) throws JSONException;

    @RequestMapping(value = "/getDatasourceVocabularies/{mode}",method = RequestMethod.GET,

            produces = MediaType.APPLICATION_JSON_VALUE)

    @ResponseBody

    List<String> getDatasourceVocabularies(String mode);

}

package eu.manager.repo.manager.service.controllers;

import eu.dnetlib.domain.enabling.Vocabulary;

import eu.manager.repo.manager.shared.Aggregations;

import eu.manager.repo.manager.shared.Country;

import eu.manager.repo.manager.shared.Timezone;

import eu.manager.repo.manager.service.utils.Converter;

import eu.dnetlib.domain.data.Repository;

import eu.dnetlib.domain.data.RepositoryInterface;

import gr.uoa.di.driver.enabling.vocabulary.VocabularyLoader;

import org.apache.log4j.Logger;

import org.json.JSONArray;

import org.json.JSONException;

import org.json.JSONObject;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;

import org.springframework.stereotype.Component;

import org.springframework.web.bind.annotation.PathVariable;

import org.springframework.web.bind.annotation.RequestBody;

import org.springframework.web.bind.annotation.RequestParam;

import org.springframework.web.client.RestTemplate;

import org.springframework.web.util.UriComponents;

import org.springframework.web.util.UriComponentsBuilder;

import javax.annotation.PostConstruct;

import java.util.*;

import java.util.concurrent.ConcurrentHashMap;

@Component

public class RepositoryApiImpl implements RepositoryApi {

    @Value("${api.baseAddress}")

    private String baseAddress;

    private RestTemplate restTemplate = null;

    private final String[] vocabularyNames = {"dnet:countries", "dnet:datasource_typologies", "dnet:compatibilityLevel"};

    private static final Logger LOGGER = Logger.getLogger(RepositoryApiImpl.class);

    @Autowired

    private VocabularyLoader vocabularyLoader;

    private Map<String, Vocabulary> vocabularyMap = new ConcurrentHashMap<String, Vocabulary>();

    @PostConstruct

    private void init(){

        restTemplate = new RestTemplate();

        restTemplate.getMessageConverters().add(new MappingJackson2HttpMessageConverter());

        for (String vocName : vocabularyNames) {

            vocabularyMap.put(vocName, vocabularyLoader.getVocabulary(vocName, Locale.ENGLISH, Locale.ROOT));

        }

    }

    @Override

    public Aggregations testAggregations() throws JSONException {

        int page = 1;

        int size = 1000;

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/list/")

                .path("/{page}/{size}/")

                .build().expand(page,size).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        while(!rs.equals("[]")){

            Aggregations aggregations = getFirstNonEmptyAggregation(rs);

            if(aggregations != null)

                return aggregations;

            LOGGER.debug("Checked " + page*size + " records!");

            page+=1;

            uriComponents = UriComponentsBuilder

                    .fromHttpUrl(baseAddress + "/ds/list/")

                    .path("/{page}/{size}/")

                    .build().expand(page,size).encode();

            rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        }

        return null;

    }

    private Aggregations getFirstNonEmptyAggregation(String rs) throws JSONException {

        JSONArray ids = new JSONArray(rs);

        for(int i=0;i<ids.length();i++){

            String id = ids.getString(i);

            Aggregations aggregations = getRepositoryAggregations(id);

            if(aggregations.getAggregationHistory() != null)

                return aggregations;

        }

        return null;

    }

    @Override

    public Country[] getCountries()  {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/countries")

                .build().encode();

        return restTemplate.getForObject(uriComponents.toUri(),Country[].class);

    }

    @Override

    public Set<String> getRepositoriesByCountry(@PathVariable("country") String country) throws JSONException {

        int page = 1;

        int size = 10;

        String mode = "openaire____::opendoar";

        UriComponents uriComponents = UriComponentsBuilder

                                            .fromHttpUrl(baseAddress + "/ds/search/country/")

                                            .path("/{page}/{size}/")

                                            .queryParam("country",country)

                                            .build().expand(page,size).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        Set<String> resultSet = new HashSet<>();

        while(!rs.equals("[]")){

            List<Repository> rep = Converter.jsonToRepositoryList(new JSONArray(rs));

            Collection<String> repos = this.getRepositoriesByMode(mode,rep);

            this.addRepos(resultSet,repos);

            page+=1;

            uriComponents = UriComponentsBuilder

                    .fromHttpUrl(baseAddress + "/ds/search/country/")

                    .path("/{page}/{size}/")

                    .queryParam("country",country)

                    .build().expand(page,size).encode();

            rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        }

        return resultSet;

    }

    private void addRepos(Set<String> resultSet, Collection<String> repos) {

        for(String s : repos)

            if(!resultSet.contains(s))

                resultSet.add(s);

    }

    private Collection<String> getRepositoriesByMode(String mode, List<Repository> rs) {

        List<String> reps = new ArrayList<>();

        for(Repository r : rs)

            if(r.getCollectedFrom().equals(mode))

                reps.add(r.getOfficialName());

        return reps;

    }

    @Override

    public List<Repository> getRepositoriesOfUser(@PathVariable("userEmail") String userEmail,

                                                  @PathVariable("page") String page,

                                                  @PathVariable("size") String size) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/search/email/")

                .path("/{page}/{size}/")

                .queryParam("contactemail",userEmail)

                .build().expand(page,size).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        return Converter.jsonToRepositoryList(new JSONArray(rs));

    }

    @Override

    public Repository getRepositoryById(@PathVariable("id") String id) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/get/")

                .path("/{id}/")

                .build().expand(id).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        return  Converter.jsonToRepositoryObject(new JSONObject(rs));

    }

    @Override

    public Aggregations getRepositoryAggregations(@PathVariable("id") String id) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/get/")

                .path("/{id}/")

                .build().expand(id).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        JSONObject repository = new JSONObject(rs);

        Aggregations aggregations = new Aggregations();

        aggregations.setAggregationHistory(Converter.getAggregationHistoryFromJson(repository));

        aggregations.setLastCollection(Converter.getLastCollectionFromJson(repository));

        aggregations.setLastTransformation(Converter.getLastTransformationFromJson(repository));

        return aggregations;

    }

    @Override

    public List<Repository> getRepositoriesByName(@PathVariable("name") String name,

                                                  @PathVariable("page") String page,

                                                  @PathVariable("size") String size) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/search/name/")

                .path("/{page}/{size}")

                .queryParam("name",name)

                .build().expand(page,size).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(), String.class);

        return Converter.jsonToRepositoryList(new JSONArray(rs));

    }

    @Override

    public RepositoryInterface getRepositoyInterface(@PathVariable("id") String id) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/service/")

                .path("/{id}/")

                .build().expand(id).encode();

        String rs = restTemplate.getForObject(uriComponents.toUri(),String.class);

        return  Converter.jsonToRepositoryInterfaceObject(new JSONArray(rs).getJSONObject(0));

    }

    @Override

    public String addRepository(@RequestBody Repository repository) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/add/")

                .build()

                .encode();

        return restTemplate.postForObject(uriComponents.toUri(),Converter.repositoryObjectToJson(repository),String.class);

    }

    @Override

    public String addRepositoryInterface(RepositoryInterface repositoryInterface) throws JSONException {

        UriComponents uriComponents = UriComponentsBuilder

                .fromHttpUrl(baseAddress + "/ds/service/add/")

                .build()

                .encode();

        return restTemplate.postForObject(uriComponents.toUri(), Converter.repositoryInterfaceObjectToJson(repositoryInterface),String.class);

    }

    @Override