D-Net

from elasticsearch import *

from elasticsearch_dsl import *

from os import path

import os

from eu.dnetlib.util import get_index_properties

from elasticsearch_dsl.response import Response

import logging

log = logging.getLogger("scholexplorer")

class ScholixConnector(object):    

    __instance = None

    def __new__(cls):

        if ScholixConnector.__instance is None:

            ScholixConnector.__instance = object.__new__(cls)        

            props = get_index_properties()

            index_name = props['api.index']+"_scholix"

            index_host = [x for x in props['es_index'].split(',')]   

            #connections.create_connection(hosts=index_host, timeout=1000)    

            ScholixConnector.__instance.connection_pool = ConnectionPool([(Elasticsearch(hosts=index_host, timeout=1000),{}) for x in range(10)]) 

            ScholixConnector.__instance.index_host = index_host

            ScholixConnector.__instance.index_name = index_name

        return ScholixConnector.__instance

    def create_pidType_query(self, value, start):

        args = {start + '.identifier.schema': value}

        return Q('nested', path=start + '.identifier', query=Q('bool', must=[Q('match', **args)]))

    def create_pid_query(self, value, start):

        args = {start + '.identifier.identifier': value.lower()}

        return Q('nested', path=start + '.identifier', query=Q('bool', must=[Q('match', **args)]))

    def create_typology_query(self, value, start):

        args = {start + '.objectType': value}

        return Q('nested', path=start, query=Q('bool', must=[Q('match', **args)]))

    def create_dataSource_query(self, value):

        args = {'linkprovider.name': value}

        return Q('nested', path='linkprovider', query=Q('bool', must=[Q('match', **args)]))

    def create_publisher_query(self, value, start):

        args = {start + '.publisher.name': value}

        q = Q('nested', path=start + '.publisher', query=Q('bool', must=[Q('match', **args)]))

        return Q('nested', path=start, query=q)

    def list_datasources(self, ds_name=None):

        search_object = Search(using=self.connection_pool.get_connection(), index=self.index_name).doc_type('scholix')

        if ds_name:

            search_object = search_object.query(self.create_dataSource_query(ds_name))

        else:

            search_object = search_object.query()

        search_object.aggs.bucket('all_datasources', 'nested', path='linkprovider').bucket('all_names', 'terms',

                                                                                           field='linkprovider.name',

                                                                                           size=100)

        response = search_object.execute()

        if ds_name:

            for item in response.aggs.all_datasources.all_names.buckets:

                if item.key == ds_name:

                    yield dict(name=item.key, totalRelationships=item.doc_count)

        else:

            for item in response.aggs.all_datasources.all_names.buckets:

                yield dict(name=item.key, totalRelationships=item.doc_count)    

    def list_publisher(self, start, pub_name=None):

        log.info("Started Index from host")

        search_object = Search(using=self.connection_pool.get_connection(), index=self.index_name)

        if pub_name:

            search_object = search_object.query(self.create_publisher_query(pub_name, start))

        else:

            search_object = search_object.query()

        search_object.aggs.bucket('all_targets', 'nested', path=start).bucket('all_t_pubs', 'nested',

                                                                              path=start + '.publisher').bucket(

            'all_pubs', 'terms',

            field=start + '.publisher.name',

            size=1000)

        response = search_object.execute()        

        for item in response.aggs.all_targets.all_t_pubs.all_pubs.buckets:

            if pub_name and item.key == pub_name:            

                yield dict(name=item.key, totalRelationships=item.doc_count)

            else:

                yield dict(name=item.key, totalRelationships=item.doc_count)

    def links(self, provider=None, s_pid=None, t_pid=None, s_publisher=None, t_publisher=None, s_pid_type=None,

              t_pid_type=None, target_Type=None, source_Type=None,page=0):       

        queries = []

        if provider:

            log.info("PROVIDER NOT NONE: {}".format(provider))

            queries.append(self.create_dataSource_query(provider))

        if s_pid:

            log.info("S_PID NOT NONE: {}".format(s_pid))

            queries.append(self.create_pid_query(s_pid, 'source'))

        if t_pid:

            queries.append(self.create_pid_query(t_pid, 'target'))

        if s_publisher:

            queries.append(self.create_publisher_query(s_publisher, 'source'))

        if t_publisher:

            queries.append(self.create_publisher_query(t_publisher, 'target'))

        if s_pid_type:

            queries.append(self.create_pidType_query(s_pid_type, 'source'))

        if t_pid_type:

            queries.append(self.create_pidType_query(t_pid_type, 'target'))

        if target_Type:

            if 'literature' == target_Type:

                target_Type = 'publication'

            queries.append(self.create_typology_query(target_Type,'target'))

        if source_Type:

            if 'literature' == source_Type:

                source_Type = 'publication'

            queries.append(self.create_typology_query(source_Type,'source'))

        q = None

        for item in queries:

            if not q:

                q = item

            else:

                q = q & item        

        log.debug("REQUEST CREATED {}".format(q))

        search_object = Search(using=self.connection_pool.get_connection(), index=self.index_name).doc_type('scholix').query(q)

        print (search_object.to_dict())

        log.debug("Page request size is {}".format(page))

        if page > 9999:

            return []

        return search_object[page:page + 100].execute()

    def realtionToPid(self, pid, pidType=None, datasource=None, typology=None, page=0):

        if pidType:

            query = self.create_pid_pidType_query(pidType.lower(), pid.lower())

        else:

            query = self.create_source_pid_query(pid.lower())

        filters = []

        if datasource and len(datasource):

            filters.append(self.create_dataSource_query(datasource))

        if typology and len(typology):

            filters.append(self.create_typology_query(typology,'target'))

        search_object = Search(using=self.connection_pool.get_connection(), index=self.index_name).doc_type('scholix').query(query)

        if len(filters):

            search_object = search_object.filter(Q('bool', must=filters))

            if page > 9999:

                return []

        return search_object[page:page + 100].execute()

    def realtionToTypology(self, typology, page=0):

        search_object = Search(using=self.connection_pool.get_connection(), index=self.index_name).doc_type('scholix').query(

            self.create_typology_query(typology,'target'))

        if page > 9999:

            return []

        return search_object[page:page + 100].execute()