Project

General

Profile

« Previous | Next » 

Revision 61369

fixed scholexplorer to use the new datamodel

View differences:

modules/dli-service-portal/branches/ES_7/static/partials/detail.html
84 84
                 <div class="row">
85 85
                <div class="panel panel-primary">
86 86
                    <div class="panel-body">
87
                        <div ng-repeat="id in item.localIdentifier">{{id.type }}:   <a href="{{ id.url }}">{{ id.id}}</a> </div>
87
                        <div ng-repeat="id in item.localIdentifier">{{id.schema }}:   <a href="{{ id.url }}">{{ id.identifier}}</a> </div>
88 88
                        <hr>
89 89
                        Sources
90 90
                        <ul>
modules/dli-service-portal/branches/ES_7/static/angular/queryController.js
235 235
            }
236 236
        }).error(function (data) {
237 237

  
238
            console.log(data)
239
            window.alert("ERROR")
238 240
        })
239 241

  
240 242
    }
modules/dli-service-portal/branches/ES_7/eu/dnetlib/es_connector.py
10 10
import os
11 11
from os import path
12 12

  
13

  
14 13
log = logging.getLogger("scholexplorer-portal")
15 14

  
16 15
pid_resolver = {
......
43 42
def resolveIdentifier(pid, pid_type):
44 43
    if pid_type != None:
45 44
        regex = r"\b(10[.][0-9]{4,}(?:[.][0-9]+)*/(?:(?![\"&\'<>])\S)+)\b"
46
        if re.match(regex,pid):
45
        if re.match(regex, pid):
47 46
            log.debug("It should be doi")
48 47
            pid_type = 'doi'
49 48
        if pid_type.lower() in pid_resolver:
......
52 51
            if pid_type.lower() == 'openaire':
53 52
                return "https://www.openaire.eu/search/publication?articleId=%s" % pid.replace('oai:dnet:', '')
54 53
            elif pid_type.lower() == 'url':
55
                return  pid
54
                return pid
56 55
            else:
57 56
                return "http://identifiers.org/%s:%s" % (pid_type, pid)
58 57
    return ""
59 58

  
60 59

  
61

  
62

  
63

  
64 60
def create_typology_filter(value):
65 61
    return Q('match', typology=value)
66 62

  
......
106 102
        self.index_name = props['api.index']
107 103

  
108 104
    def get_main_page_stats(self):
109
        stats = dict(total =int(Search(using=self.client, index=self.index_name+"_scholix").count()/2))
105
        stats = dict(total=int(Search(using=self.client, index=self.index_name + "_scholix").count() / 2))
110 106
        for item in ['dataset', 'publication']:
111
            s= Search(using=self.client, index=self.index_name+"_object").query(Q('match', typology=item))
107
            s = Search(using=self.client, index=self.index_name + "_object").query(Q('match', typology=item))
112 108
            stats[item] = s.count()
113 109
        return stats
114 110

  
115 111
    def query_by_id(self, id):
116
        s = Search(using=self.client, index=self.index_name+"_object")
112
        s = Search(using=self.client, index=self.index_name + "_object")
117 113
        s = s.query(create_pid_query(id))
118 114
        s.aggs.bucket('typologies', 'terms', field='typology')
119 115
        s.aggs.bucket('all_datasources', 'nested', path='datasources').bucket('all_names', 'terms',
......
162 158
                                        publishers=publishers), hits=hits)
163 159

  
164 160
    def simple_query(self, textual_query, start=None, end=None, user_filter=None):
165
        s = Search(using=self.client, index=self.index_name+"_object")
166
        if not textual_query  == '*':
161
        s = Search(using=self.client, index=self.index_name + "_object")
162
        if not textual_query == '*':
167 163
            q = Q("multi_match", query=textual_query, fields=['title', 'abstract'])
168 164
        else:
169 165
            q = Q()
......
200 196
            s = s[start:end]
201 197
        response = s.execute()
202 198

  
203
        
204

  
205 199
        hits = []
206 200

  
201
        print(f"index : {self.index_name}_object")
202
        print(response.hits.total)
203

  
207 204
        for index_result in response.hits:
208
            input_source = index_result.__dict__['_d_']           
209
            fixed_titles = []            
210
            for ids in input_source.get('localIdentifier', []):
211
                ds = resolveIdentifier(ids['id'], ids['type'])
212
                ids['url'] = ds
205
            input_source = index_result.__dict__['_d_']
206
            fixed_titles = []
207
            # for ids in input_source.get('localIdentifier', []):
208
            #     ds = resolveIdentifier(ids['id'], ids['type'])
209
            #     ids['url'] = ds
213 210

  
214
            if input_source.get('title', []) is not None:           
211
            if input_source.get('title', []) is not None:
215 212
                for t in input_source.get('title', []):
216 213
                    if len(t) > 0 and t[0] == '"' and t[-1] == '"':
217 214
                        fixed_titles.append(t[1:-1])
218 215
                    else:
219 216
                        fixed_titles.append(t)
220 217
            else:
221
                fixed_titles.append("title not available")            
218
                fixed_titles.append("title not available")
222 219
            input_source['title'] = fixed_titles
223 220
            hits.append(input_source)
224
        
221

  
225 222
        pid_types = []
226 223
        for tag in response.aggs.all_pids.all_types.buckets:
227 224
            pid_types.append(dict(key=tag.key, count=tag.doc_count))
......
248 245
        query_type = Q('nested', path='target', query=Q('bool', must=[Q('match', **args)]))
249 246
        args_id = {'source.dnetIdentifier': object_id}
250 247
        query_for_id = Q('nested', path='source', query=Q('bool', must=[Q('match', **args_id)]))
251
        s = Search(using=self.client).index(self.index_name+"_scholix").query(query_for_id & query_type)
248
        s = Search(using=self.client).index(self.index_name + "_scholix").query(query_for_id & query_type)
252 249
        if start:
253 250
            s = s[start:start + 10]
254
       
251

  
255 252
        response = s.execute()
256 253
        hits = []
257 254
        for index_hit in response.hits:
......
261 258
                for item in current_item['target']['identifier']:
262 259
                    c_it = item
263 260
                    c_it['url'] = resolveIdentifier(item['identifier'], item['schema'])
264
                    ids .append(c_it)
261
                    ids.append(c_it)
265 262
                current_item['target']['identifier'] = ids
266 263
            hits.append(current_item)
267 264

  
......
271 268
        if relation is None:
272 269
            return
273 270
        relSource = relation.get('source')
274
        collectedFrom = relSource.get('collectedFrom',[])
271
        collectedFrom = relSource.get('collectedFrom', [])
275 272
        if collectedFrom is not None:
276 273
            for coll in collectedFrom:
277 274
                for d in source['datasources']:
......
281 278

  
282 279
    def item_by_id(self, id, type=None, start=None):
283 280
        try:
284
            res = self.client.get(index=self.index_name+"_object",doc_type="_all", id=id, _source=True)
281
            res = self.client.get(index=self.index_name + "_object", doc_type="_all", id=id, _source=True)
285 282
            hits = []
286 283
            input_source = res['_source']
287 284
            fixed_titles = []
288
            for t in input_source.get('title',[]):
285
            for t in input_source.get('title', []):
289 286
                if len(t) > 0 and t[0] == '"' and t[-1] == '"':
290 287
                    fixed_titles.append(t[1:-1])
291 288
                else:
292 289
                    fixed_titles.append(t)
293 290
            input_source['title'] = fixed_titles
294 291

  
295
            for ids in input_source.get('localIdentifier', []):
296
                ds = resolveIdentifier(ids['id'], ids['type'])
297
                ids['url'] = ds
298 292
            related_publications = []
299 293
            related_dataset = []
300
            related_unknown = []            
294
            related_unknown = []
301 295

  
302 296
            rel_source = None
303 297
            if input_source.get('relatedPublications') > 0:
......
310 304
                else:
311 305
                    rel_source = {}
312 306

  
313

  
314

  
315 307
            if input_source.get('relatedDatasets') > 0:
316 308
                if 'dataset' == type:
317 309
                    related_dataset = self.related_type(id, 'dataset', start)
......
332 324
                             related_unknown=related_unknown))
333 325

  
334 326
            return DLIESResponse(total=1, hits=hits)
335
        except Exception as e:            
327
        except Exception as e:
336 328
            log.error("Error on getting item ")
337
            log.error(e)            
329
            log.error(e)
338 330
            log.error("on line %i" % sys.exc_info)
339 331
            return DLIESResponse()
modules/dli-service-portal/branches/ES_7/main.py
5 5
from starlette.responses import FileResponse
6 6
from eu.dnetlib.es_connector import DLIESConnector
7 7
import sys
8
from eu.dnetlib.metrics_utils import metrics 
8
from eu.dnetlib.metrics_utils import metrics
9 9
from eu.dnetlib.metricsMiddleware import PrometheusMiddleware
10 10

  
11 11
_CURDIR = os.path.dirname(os.path.abspath(__file__))
......
41 41

  
42 42
log = logging.getLogger("scholexplorer-portal")
43 43
log.setLevel(logging.INFO)
44
fh = logging.StreamHandler(sys.stdout) 
44
fh = logging.StreamHandler(sys.stdout)
45 45
fh.setLevel(logging.INFO)
46 46
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
47 47
fh.setFormatter(formatter)
......
66 66
    return dict(result=connector.get_main_page_stats())
67 67

  
68 68
@app.post('/api/queryPid/')
69
def query_pid(*, pid:str= Form(...)):       
70
    connector = DLIESConnector()    
69
def query_pid(*, pid:str= Form(...)):
70
    connector = DLIESConnector()
71 71
    return dict(result=connector.query_by_id(pid))
72 72

  
73 73

  
74 74

  
75 75
@app.post('/api/post/')
76 76
def query_post(*, start:int= Form(default=0), action:str=Form(default="query"), query:str=Form(...),filter:str=Form(default=None)):
77
    print("SONO QUIIII")
78

  
77 79
    filter_key = None
78
    result = {}  
80
    result = {}
79 81
    if filter:
80 82
        filter_key = filter
81 83
    if action == 'query' and query is not None:
82
        
84
        print(f"query is {query}")
83 85
        connector = DLIESConnector()
84 86
        try:
85 87
            result = connector.simple_query(query, start=start, user_filter=filter_key)
......
93 95
@app.get('/api/item/{identifier}')
94 96
@app.post('/api/item/{identifier}')
95 97
def get_item(identifier:str, object_type:str=Form(default=None), start:int=Form(default=0)):
96
    if identifier:        
98
    if identifier:
97 99
        connector = DLIESConnector()
98 100
        return dict(result=connector.item_by_id(identifier, object_type, start))
99 101
    else:

Also available in: Unified diff