D-Net

from elasticsearch import Elasticsearch

from elasticsearch_dsl import *

from jinja2 import Template

import json

import hashlib

import time

from lxml import etree

import zlib, base64

def md5sum(str_input):

    m = hashlib.md5()

    m.update(str_input.encode('utf-8'))

    return m.hexdigest()

def create_creator(item):

    creator = dict(name=item.get('name',''), identifier = None, affiliation= item.get('affiliation',''))

    for i in item.get('nameIdentifiers',[]):

        creator['identifier']= dict(schemename=i.get('nameIdentifierScheme',''), value=i.get('nameIdentifier',''))

    return creator

def create_record(x, t, matches):

    record = x.attributes.__dict__['_d_']

    dataset =dict(id= x.id, publisher = record.get('publisher'),publicationYear = record.get('publicationYear'), subjects=record.get('subjects',[]),

                  contributors= record.get('contributors',[]), dates = record.get('dates',[]), language= record.get('language'),

                  types = record.get('types'), identifiers = record.get('identifiers',[]), relatedIdentifiers= record.get('relatedIdentifiers',[]),

                  formats = record.get('formats',[]), version= record.get('version',''), sizes = record.get('sizes',[]),

                  rightsList = record.get('rightsList',[]), descriptions= record.get('descriptions', []),

                  geoLocations = record.get('geoLocations',[]), titles =[], creators=[])

    if record.get('titles'):

        dataset['titles'] = [dict(title=title['title'], lang=title.get('lang',None)) for title in record.get('titles',[]) if 'title' in title]

    if record.get('creators'):

        dataset['creators']= [create_creator(item) for item in record.get('creators',[])] 

    provider = x.relationships.client.data.id.upper()

    hosted = matches.get(provider, dict(openaire_id="openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", official_name="Unknown Repository"))

    return t.render(dataset=dataset, provider=provider, id= x.id, hosted=hosted, objIdentifier= md5sum(x.id))

def scan_index(timestamp=None, active=True, scroll_id=None, page_size= None):

    if not page_size:

        page_size = 1000

    es = Elasticsearch(hosts=['192.168.100.70','192.168.100.71','192.168.100.72', '192.168.100.73'], timeout=1000)  

    s = Search(using=es, index='datacite').query(Q('match', attributes__isActive=True))

    if timestamp:

        print ("APPLICO FILTRO ", timestamp)

        s= s.filter('range',timestamp={'gte': timestamp})

    t = Template(open('res/template.jinja').read())

    matches = json.loads(open('res/matches.json').read())

    total = s.execute().hits.total    

    result = dict(total=total, counter = 0, result=[], scroll_id = scroll_id)

    i = 0

    for item in s.params(size=1000).scan():

        record = dict(originalId = item.id, timestamp = item.timestamp)

        record['body'] = base64.b64encode(zlib.compress(create_record(item, t, matches).encode('utf-8'))).decode('ascii')

        record['id']= "datacite____::"+md5sum(item.id)

        result['result'].append(record)

        result['counter'] +=1

        if len(result['result']) == page_size:

            yield result

            del (result['result'])

            result['result'] = []

    yield result

def is_available():

    es = Elasticsearch(hosts=['192.168.100.70','192.168.100.71','192.168.100.72', '192.168.100.73'], timeout=1000)  

    s = Search(using=es, index='datacite').query(Q('match', attributes__isActive=True))

    return s.count() >0