Project

General

Profile

« Previous | Next » 

Revision 57285

return facets with more than the default 100 value

View differences:

modules/uoa-search/trunk/src/test/java/eu/dnetlib/data/search/utils/solr/SolrUtilsTest.java
35 35
    HttpServletRequest requestMock;
36 36
    RequestResponseHandler.Entity entity;
37 37
    List<String> refineFields = new ArrayList<String>();
38
    List<String> specialFacets = new ArrayList<String>();
38 39
    List<String> fieldQueries = new ArrayList<String>();
39 40
    List<String> fieldQueriesWithComma = new ArrayList<String>();
40 41
    CloudSolrClient mockSolrClient;
......
50 51

  
51 52
        refineFields = Arrays.asList(new String[]{"relfunderid", "relfundinglevel0_id", "relfundinglevel1_id", "relfundinglevel2_id"});
52 53

  
54
        specialFacets =  Arrays.asList(new String[]{"relfundinglevel0_id", "relfundinglevel1_id"});
55

  
53 56
        fieldQueries = Arrays.asList(new String[]{"(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)",
54 57
                "relfundinglevel0_id exact ec___::EC::SP1" , "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)"});
55 58

  
......
61 64

  
62 65
    @Test
63 66
    public void createEprQuery() {
64
        String query = SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, fieldQueries);
67
        String query = SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, specialFacets, fieldQueries);
65 68
        Assert.assertEquals("query=(oaftype=result)" +
66 69
                "&groupby=relfunderid,relfundinglevel0_id,relfundinglevel1_id,relfundinglevel2_id" +
70
                "&sf=relfundinglevel0_id,relfundinglevel1_id"+
67 71
                "&fq=(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT),relfundinglevel0_id exact ec___::EC::SP1," +
68 72
                        "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)", query);
69 73

  
70 74

  
75
        //empty refine, empty fq, empty fc
76
        query = SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<>(),new ArrayList<String>());
77
        Assert.assertEquals("query=(oaftype=result)" +
78
                "&groupby=" +
79
                "&sf="+
80
                "&fq=", query);
81

  
71 82
        //empty refine, empty fq
72
        query = SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<String>());
83
        query = SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), specialFacets,new ArrayList<String>());
73 84
        Assert.assertEquals("query=(oaftype=result)" +
74 85
                "&groupby=" +
86
                "&sf=relfundinglevel0_id,relfundinglevel1_id"+
75 87
                "&fq=", query);
76 88

  
77
        //empty fq
78
        query = SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, new ArrayList<String>());
89

  
90
        //empty fq, empty sf
91
        query = SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, new ArrayList<>(),  new ArrayList<String>());
79 92
        Assert.assertEquals("query=(oaftype=result)" +
80 93
                "&groupby=relfunderid,relfundinglevel0_id,relfundinglevel1_id,relfundinglevel2_id" +
94
                "&sf="+
81 95
                "&fq=", query);
82 96

  
83
        //empty refine
84
        query = SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), fieldQueries);
97
        //empty refine, empty sf
98
        query = SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<>(), fieldQueries);
85 99
        Assert.assertEquals("query=(oaftype=result)" +
86 100
                "&groupby=" +
101
                "&sf="+
87 102
                "&fq=(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT),relfundinglevel0_id exact ec___::EC::SP1," +
88 103
                "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)", query);
89 104

  
90
        //empty query
91
        query = SearchServiceImpl.createEprQuery("", new ArrayList<String>(), new ArrayList<String>());
105
        //empty refine, empty sf
106
        query = SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<>(), fieldQueries);
107
        Assert.assertEquals("query=(oaftype=result)" +
108
                "&groupby=" +
109
                "&sf="+
110
                "&fq=(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT),relfundinglevel0_id exact ec___::EC::SP1," +
111
                "(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)", query);
112

  
113
        //empty query, empty fc
114
        query = SearchServiceImpl.createEprQuery("", new ArrayList<String>(), new ArrayList<>(), new ArrayList<String>());
92 115
        Assert.assertEquals("query=" +
93 116
                "&groupby=" +
117
                "&sf="+
94 118
                "&fq=", query);
95 119

  
96 120
        //null values
97
        query = SearchServiceImpl.createEprQuery(null, null, null);
121
        query = SearchServiceImpl.createEprQuery(null, null, null,null);
98 122
        Assert.assertEquals("query=" +
99 123
                "&groupby=" +
124
                "&sf="+
100 125
                "&fq=", query);
101 126

  
102 127
    }
......
105 130
    public void extractQueryOptions() throws IOException, CQLParseException {
106 131
        CqlTranslator cqlTranslator = new CqlTranslatorImpl();
107 132

  
108
        NamedList<String> actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, fieldQueries));
133
        NamedList<String> actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, specialFacets, fieldQueries));
109 134
        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));
110 135
        Assert.assertEquals("__result", actual.get("fl"));
111 136
        Assert.assertEquals("true", actual.get("shards.tolerant"));
......
117 142
        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel1_id"));
118 143
        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel2_id"));
119 144
        Assert.assertFalse(actual.getAll("facet.field").contains("relfundinglevel3_id"));
145

  
146
        Assert.assertNotNull(actual.get("f.relfundinglevel0_id.facet.limit"));
147
        Assert.assertEquals(actual.get("f.relfundinglevel0_id.facet.limit"), "-1");
148
        Assert.assertNotNull(actual.get("f.relfundinglevel1_id.facet.limit"));
149
        Assert.assertEquals(actual.get("f.relfundinglevel1_id.facet.limit"), "-1");
150
        Assert.assertNull(actual.get("f.relfundinglevel3_id.facet.limit"));
151

  
120 152
        Assert.assertEquals(fieldQueries.size(), actual.getAll("fq").size());
121 153
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)")));
122 154
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("relfundinglevel0_id exact ec___::EC::SP1")));
123 155
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)")));
124 156

  
125
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, fieldQueriesWithComma));
157
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, specialFacets, fieldQueriesWithComma));
126 158
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfunderid exact ec___::EC) and (relfunderid exact wt___::WT)")));
127 159
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("relfundinglevel0_id exact \"ec___::EC::SP1,ec___::EC::SP2\"")));
128 160

  
129
        //empty refine, empty fq
130
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<String>()));
161
        //empty refine, empty fc, empty fq
162
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<>(), new ArrayList<String>()));
131 163
        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));
132 164
        Assert.assertEquals("__result", actual.get("fl"));
133 165
        Assert.assertNull(actual.get("q.op"));
......
140 172
        Assert.assertNull(actual.get("fq"));
141 173
        Assert.assertTrue(actual.getAll("fq").isEmpty());
142 174

  
143
        //empty fq
144
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, new ArrayList<String>()));
175
        //empty fq, empty fc
176
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, new ArrayList<>(), new ArrayList<String>()));
145 177
        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));
146 178
        Assert.assertEquals("__result", actual.get("fl"));
147 179
        Assert.assertNull(actual.get("q.op"));
......
154 186
        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel1_id"));
155 187
        Assert.assertTrue(actual.getAll("facet.field").contains("relfundinglevel2_id"));
156 188
        Assert.assertFalse(actual.getAll("facet.field").contains("relfundinglevel3_id"));
189
        Assert.assertNull(actual.get("f.relfundinglevel0_id.facet.limit"));
190
        Assert.assertNull(actual.get("f.relfundinglevel1_id.facet.limit"));
157 191
        Assert.assertNull(actual.get("fq"));
158 192
        Assert.assertTrue(actual.getAll("fq").isEmpty());
159 193

  
160
        //null fq
161
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, null));
194
        //null fc, null fq
195
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", refineFields, null, null));
162 196
        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));
163 197
        Assert.assertEquals("__result", actual.get("fl"));
164 198
        Assert.assertNull(actual.get("q.op"));
......
174 208
        Assert.assertNull(actual.get("fq"));
175 209
        Assert.assertTrue(actual.getAll("fq").isEmpty());
176 210

  
177
        //empty refine
178
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), fieldQueries));
211
        //empty refine, empty fc
212
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", new ArrayList<String>(), new ArrayList<>(), fieldQueries));
179 213
        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));
180 214
        Assert.assertEquals("__result", actual.get("fl"));
181 215
//        Assert.assertEquals("AND", actual.get("q.op"));
......
190 224
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("relfundinglevel0_id exact ec___::EC::SP1")));
191 225
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)")));
192 226

  
193
        //null refine
194
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", null, fieldQueries));
227
        //null refine, null fc
228
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", null, null, fieldQueries));
195 229
        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));
196 230
        Assert.assertEquals("__result", actual.get("fl"));
197 231
  //      Assert.assertEquals("AND", actual.get("q.op"));
......
206 240
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("relfundinglevel0_id exact ec___::EC::SP1")));
207 241
        Assert.assertTrue(actual.getAll("fq").contains(cqlTranslator.toLucene("(relfundinglevel1_id exact ec___::EC::SP1::VALUE1) or (relfundinglevel1_id exact ec___::EC::SP1::VALUE2)")));
208 242

  
209
        //null refine, null fq
210
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", null, null));
243
        //null refine, null fc, null fq
244
        actual = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery("(oaftype=result)", null, null, null));
211 245
        Assert.assertEquals(cqlTranslator.toLucene("(oaftype=result)"), actual.get("q"));
212 246
        Assert.assertEquals("__result", actual.get("fl"));
213 247
        Assert.assertNull(actual.get("q.op"));
......
223 257

  
224 258
    @Test(expected=CQLParseException.class)
225 259
    public void extractQueryOptionsEmptyQuery() throws IOException, CQLParseException {
226
        NamedList<String> actual  = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery(null, null, null));
260
        NamedList<String> actual  = SolrResultSetOptionsUtil.extractQueryOptions(SearchServiceImpl.createEprQuery(null, null, null, null));
227 261
    }
228 262

  
229 263
}
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/solr/SolrResultSetFactory.java
47 47
            return solrResultSets;
48 48

  
49 49
        } catch (Exception e) {
50
            logger.error("Error creating solr client", e);
50
            logger.warn("Error creating solr client", e);
51 51
        }
52 52

  
53 53
        logger.debug("NOT HERE!");
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/app/SearchServiceImpl.java
165 165
            refineFields = new ArrayList<String>(fields);
166 166
        }
167 167

  
168
        return newSearch(text, locale, refineFields, new ArrayList<String>(), page, size, "", sTransformer, oldRefineTransformer, true);
168
        return newSearch(text, locale, refineFields, null, new ArrayList<String>(), page, size, "", sTransformer, oldRefineTransformer, true);
169 169
    }
170 170

  
171 171
    @Override
......
200 200
        if (fields!=null) {
201 201
            refineFields = new ArrayList<String>(fields);
202 202
        }
203
        SearchResult searchResult = newSearch(queryText, locale, refineFields, new ArrayList<String>(), page, size, format, sTransformer, oldRefineTransformer, true);
203
        SearchResult searchResult = newSearch(queryText, locale, refineFields, new ArrayList<>(), new ArrayList<String>(), page, size, format, sTransformer, oldRefineTransformer, true);
204 204

  
205 205
        Formatter formatter = transformerFactory.getFormatter(format); // formatter cannot be returned as null
206 206
        try {
......
214 214
    }
215 215

  
216 216

  
217
    public SearchResult newSearch (String text, String locale, List<String> refinefields, List<String> fieldQueries,
217
    public SearchResult newSearch (String text, String locale, List<String> refinefields, List<String> specialFacets, List<String> fieldQueries,
218 218
                                   int from, int to, String format, Transformer transformer, Transformer oldRefineTransformer,
219 219
                                   boolean oldPaging) throws SearchServiceException {
220 220
        logger.info("newSearch > from: " + from + " to:" + to);
......
237 237
        try {
238 238
            //TODO see parser and maybe delete!
239 239
            //query = new CQLParser().parse(query).toCQL();
240
            String eprQuery = createEprQuery(query, refinefields, fieldQueries);
240
            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);
241 241

  
242 242
            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);
243 243

  
......
286 286
        return new SearchResult(query, Locale.getDefault().toString(), rs.size(), from, to, searchResults, browseResults, refinefields);
287 287
    }
288 288

  
289
    public void cursorSearch(String text, List<String> refinefields, List<String> fieldQueries,
289
    public void cursorSearch(String text, List<String> refinefields, List<String> specialFacets, List<String> fieldQueries,
290 290
                             String format, Transformer transformer, OutputStream os) throws SearchServiceException {
291 291

  
292 292
        long startTime = System.nanoTime();
......
304 304

  
305 305

  
306 306
        try {
307
            String eprQuery = createEprQuery(query, refinefields, fieldQueries);
307
            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);
308 308
            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);
309 309

  
310 310
            if (epr == null) {
......
365 365
        }
366 366
    }
367 367

  
368
    public static String createEprQuery(String query, List<String> refineFields, List<String> fieldQueries) {
368
    public static String createEprQuery(String query, List<String> refineFields, List<String> specialFacets, List<String> fieldQueries) {
369 369
        StringBuffer queryBuffer = new StringBuffer();
370 370
        queryBuffer.append("query=");
371 371

  
......
375 375
        StringBuffer fqBuffer = new StringBuffer();
376 376
        fqBuffer.append("&fq=");
377 377

  
378
        StringBuffer sfBuffer = new StringBuffer();
379
        sfBuffer.append("&sf=");
380

  
378 381
        if (query != null) { //TODO consider exception?
379 382
            queryBuffer.append(query);
380 383
        }
......
388 391
            }
389 392
        }
390 393

  
394
        if(specialFacets != null) {
395
            for (Iterator<String> iterator = specialFacets.iterator(); iterator.hasNext(); ) {
396
                sfBuffer.append(iterator.next());
397
                if (iterator.hasNext()) {
398
                    sfBuffer.append(",");
399
                }
400
            }
401
        }
402
       // logger.debug("special buffer " + sfBuffer.toString());
403

  
391 404
        if(fieldQueries != null) {
392 405
            for (Iterator<String> iterator = fieldQueries.iterator(); iterator.hasNext(); ) {
393 406
                fqBuffer.append(iterator.next());
......
397 410
            }
398 411
        }
399 412

  
400
        return queryBuffer.append(facetsBuffer.toString()).append(fqBuffer.toString()).toString();
413
        return queryBuffer.append(facetsBuffer.toString()).append(sfBuffer.toString()).append(fqBuffer.toString()).toString();
401 414
    }
402 415

  
403 416

  
......
406 419
    //I did not want to pass a parameter since I do not know if we are going to keep it. This is for a tech meeting showcase.
407 420
    //If we want to keep this I need to redesign.
408 421

  
409
    public SearchResult newSearchWithoutFieldQueries (String text, String locale, List<String> refinefields, List<String> fieldQueries,
410
                                                      int from, int to, String format, Transformer transformer, Transformer oldRefineTransformer,
411
                                                      boolean oldPaging) throws SearchServiceException {
422
    public SearchResult newSearchWithoutFieldQueries (String text, String locale, List<String> refinefields, List<String> specialFacets,
423
                                                      List<String> fieldQueries, int from, int to, String format, Transformer transformer,
424
                                                      Transformer oldRefineTransformer, boolean oldPaging) throws SearchServiceException {
412 425
        logger.info("non filtered search for...  > from: " + from + " to:" + to);
413 426
        long startTime = System.nanoTime();
414 427

  
......
426 439
        try {
427 440
            //TODO see parser and maybe delete!
428 441
            //query = new CQLParser().parse(query).toCQL();
429
            String eprQuery = createEprQuery(query, refinefields, fieldQueries);
442
            String eprQuery = createEprQuery(query, refinefields, specialFacets, fieldQueries);
430 443

  
431 444
            epr = index.getBrowsingStatistics(eprQuery, "all", mdFormat, indexLayout);
432 445

  
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/utils/solr/SolrResultSetOptionsUtil.java
9 9
import org.z3950.zing.cql.CQLParseException;
10 10

  
11 11
import java.io.IOException;
12
import java.util.Arrays;
13
import java.util.Collections;
12 14

  
13 15
/**
14 16
 * Created by kiatrop on 21/11/2016.
......
22 24
        NamedList<String> queryOpts = new NamedList<String>();
23 25
        String[] queryParts = eprQuery.split("&groupby=");
24 26

  
25
        logger.info("QUERY: " + queryParts[0].replace("query=",""));
27
        logger.debug("QUERY: " + queryParts[0].replace("query=",""));
26 28
        TranslatedQuery translatedQuery = translator.getTranslatedQuery(queryParts[0].replace("query=",""));
27 29
        queryOpts.add("q", translatedQuery.asLucene());
28 30
        queryOpts.add("fl", "__result");
......
33 35
        }
34 36

  
35 37
        if (queryParts.length > 1) {
38

  
39
            //facetParts groupby + sf
36 40
            String[] facetParts = queryParts[1].split("&fq=");
37 41

  
42
            logger.debug("facet parts" + Arrays.asList(facetParts));
43

  
38 44
            String[] refineParts = null;
45
            String[] refineFields = null;
46
            String[] specialFacets = null;
39 47
            String[] facetQueries = null;
40 48

  
41 49
            if (facetParts != null && facetParts.length > 0) {
42 50
                if (!facetParts[0].isEmpty()) {
43
                    refineParts = facetParts[0].split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1);
51
                    refineParts = facetParts[0].split("&sf=");
52
                    logger.debug("refine parts " + Arrays.asList(refineParts));
53

  
54
                    if (refineParts != null && refineParts.length > 0) {
55
                        refineFields = refineParts[0].split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1);
56
                    }
57

  
58
                    if(refineParts.length > 1){
59
                        refineParts[1].replace("&sf=","");
60
                        specialFacets = refineParts[1].split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1);
61
                    }
44 62
                }
45 63

  
46 64
                if (facetParts.length > 1){
47 65
                    facetQueries = facetParts[1].split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1);
48 66
                }
49 67

  
50
                if (facetParts[0].isEmpty() && facetParts.length > 1) {
51
                    facetQueries = facetParts[1].split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1);
52
                }
53 68
            }
54 69

  
55
            if (refineParts != null && refineParts.length > 0) {
70
/*            logger.debug("refine fields " + Arrays.asList(refineFields));
71
            logger.debug("special facets " + Arrays.asList(specialFacets));
72
            logger.debug("facet queries " + Arrays.asList(facetParts));*/
73

  
74
            if (refineFields != null && refineFields.length > 0) {
56 75
                queryOpts.add("facet", "true");
57 76
                queryOpts.add("facet.mincount", "1");
58
                queryOpts.add("facet.threads", refineParts.length + "");
59
                for (String field : refineParts) {
77
                queryOpts.add("facet.threads", refineFields.length + "");
78
                for (String field : refineFields) {
60 79
                    queryOpts.add("facet.field", field);
61 80
                }
62 81
            }
63 82

  
83
            if (specialFacets != null && specialFacets.length > 0) {
84
                for (String field : specialFacets) {
85
                    queryOpts.add("f."+field+".facet.limit", "-1");
86
                }
87
            }
88
            
89

  
64 90
            if (facetQueries!=null && facetQueries.length > 0) {
65 91
                //queryOpts.add("q.op", "AND"); TODO is it correct?
66 92

  
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/utils/vocabulary/IndexVocabularyLoader.java
43 43
					((IndexVocabulary)vocabulary).getTransformer(),
44 44
					"vocabulary", null, 1, REQUEST_SIZE);
45 45

  
46
			int totalResultNumber = fsr.getResultsNumber();
46
			if (fsr !=null) {
47
				int totalResultNumber = fsr.getResultsNumber();
47 48

  
48
			//make sure that every result is loaded
49
			if (totalResultNumber > REQUEST_SIZE ) {
50
				fsr = searchServiceLocator.getService().search(
51
						((IndexVocabulary)vocabulary).getQuery(),
52
						((IndexVocabulary)vocabulary).getTransformer(),
53
						"vocabulary", null, 1, totalResultNumber);
49
				//make sure that every result is loaded
50
				if (totalResultNumber > REQUEST_SIZE) {
51

  
52
					if (fsr != null) {
53
						fsr = searchServiceLocator.getService().search(
54
								((IndexVocabulary) vocabulary).getQuery(),
55
								((IndexVocabulary) vocabulary).getTransformer(),
56
								"vocabulary", null, 1, totalResultNumber);
57
					}
58

  
59
					xml = fsr.getFormattedResult();
60
				}
54 61
			}
55 62

  
56
			xml = fsr.getFormattedResult();
57

  
58 63
		} catch (SearchServiceException sse) {
59 64
			logger.error("Error getting Vocabulary xml", sse);
60 65
		}
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/web/api/SearchApiService.java
49 49
                                      @DefaultValue("0") @QueryParam("page") final int offset,
50 50
                                      @DefaultValue("10") @QueryParam("size") final int limit,
51 51
                                      @QueryParam("refine") @DefaultValue("false") final boolean refine,
52
                                      @QueryParam("fields") final List<String> fields,
52
                                      @QueryParam("fields") final List<String> refineFields,
53
                                      @QueryParam("sf") final List<String> specialFacets,
53 54
                                      @QueryParam("fq") final List<String> fieldQueries,
54 55
                                      @QueryParam("format") final String format,
55 56
                                      @QueryParam("type") final String type,
56 57
                                      @Context final HttpServletRequest request) {
57 58

  
58
        return getResponseByEntity(query, extractEntity(type), offset, limit, format, request, refine, fields, fieldQueries);
59
        return getResponseByEntity(query, extractEntity(type), offset, limit, format, request, refine, refineFields, specialFacets, fieldQueries);
59 60
    }
60 61

  
61 62
    @GET
......
68 69
                                      @DefaultValue("0") @QueryParam("page") final int offset,
69 70
                                      @DefaultValue("10") @QueryParam("size") final int limit,
70 71
                                      @QueryParam("refine") @DefaultValue("false") final boolean refine,
71
                                      @QueryParam("fields") final List<String> fields,
72
                                      @QueryParam("fields") final List<String> refineFields,
73
                                      @QueryParam("sf") final List<String> specialFacets,
72 74
                                      @QueryParam("fq") final List<String> fieldQueries,
73 75
                                      @QueryParam("format") final String format,
74 76
                                      @Context final HttpServletRequest request) {
......
77 79
        //TODO do not allow q and keywords both - SWITCH back if needed
78 80
        //String simpleQuery = buildSimpleSearchQuery(RequestResponseHandler.Entity.PUBLICATION, keywords) ;
79 81
        String simpleQuery = buildSearchRequest(RequestResponseHandler.Entity.PUBLICATION, keywords, doi,sortBy, fieldQueries);
80
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.PUBLICATION, offset, limit, format, request, refine, fields, fieldQueries);
82
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.PUBLICATION, offset, limit, format, request, refine, refineFields, specialFacets, fieldQueries);
81 83

  
82 84
    }
83 85

  
......
117 119
                                  @DefaultValue("0") @QueryParam("page") int offset,
118 120
                                  @DefaultValue("10") @QueryParam("size") int limit,
119 121
                                  @QueryParam("refine") @DefaultValue("false") boolean refine,
120
                                  @QueryParam("fields") final List<String> fields,
122
                                  @QueryParam("fields") final List<String> refineFields,
123
                                  @QueryParam("sf") final List<String> specialFacets,
121 124
                                  @QueryParam("fq") final List<String> fieldQueries,
122 125
                                  @QueryParam("format") final String format,
123 126
                                  @Context HttpServletRequest request) {
124 127

  
125 128
        String simpleQuery = buildSearchRequest(RequestResponseHandler.Entity.DATASET, keywords, doi, sortBy, fieldQueries);
126
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.DATASET, offset, limit, format, request, refine, fields, fieldQueries);
129
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.DATASET, offset, limit, format, request, refine, refineFields, specialFacets, fieldQueries);
127 130
    }
128 131

  
129 132
    @GET
......
160 163
                                  @DefaultValue("0") @QueryParam("page") int offset,
161 164
                                  @DefaultValue("10") @QueryParam("size") int limit,
162 165
                                  @QueryParam("refine") @DefaultValue("false") boolean refine,
163
                                  @QueryParam("fields") final List<String> fields,
166
                                  @QueryParam("fields") final List<String> refineFields,
167
                                  @QueryParam("sf") final List<String> specialFacets,
164 168
                                  @QueryParam("fq") final List<String> fieldQueries,
165 169
                                  @QueryParam("format") final String format,
166 170
                                  @Context HttpServletRequest request) {
167 171

  
168 172
        String simpleQuery = buildSearchRequest(RequestResponseHandler.Entity.SOFTWARE, keywords, doi, sortBy, fieldQueries);
169
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.SOFTWARE, offset, limit, format, request, refine, fields, fieldQueries);
173
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.SOFTWARE, offset, limit, format, request, refine, refineFields, specialFacets, fieldQueries);
170 174
    }
171 175

  
172 176
    @GET
......
202 206
                                  @DefaultValue("0") @QueryParam("page") int offset,
203 207
                                  @DefaultValue("10") @QueryParam("size") int limit,
204 208
                                  @QueryParam("refine") @DefaultValue("false") boolean refine,
205
                                  @QueryParam("fields") final List<String> fields,
206
                                  @QueryParam("fq") final List<String> fieldQueries,
209
                                  @QueryParam("fields") final List<String> refineFields,
210
                                  @QueryParam("sf") final List<String> specialFacets,
211
                                   @QueryParam("fq") final List<String> fieldQueries,
207 212
                                  @QueryParam("format") final String format,
208 213
                                  @Context HttpServletRequest request) {
209 214

  
210 215
        String simpleQuery = buildSearchRequest(RequestResponseHandler.Entity.OTHER, keywords, doi, sortBy, fieldQueries);
211
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.OTHER, offset, limit, format, request, refine, fields, fieldQueries);
216
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.OTHER, offset, limit, format, request, refine, refineFields, specialFacets, fieldQueries);
212 217
    }
213 218

  
214 219
    @GET
......
241 246
                                  @DefaultValue("0") @QueryParam("page") int offset,
242 247
                                  @DefaultValue("10") @QueryParam("size") int limit,
243 248
                                  @QueryParam("refine") @DefaultValue("false") boolean refine,
244
                                  @QueryParam("fields") final List<String> fields,
249
                                  @QueryParam("fields") final List<String> refineFields,
250
                                  @QueryParam("sf") final List<String> specialFacets,
245 251
                                  @QueryParam("fq") final List<String> fieldQueries,
246 252
                                  @QueryParam("format") final String format,
247 253
                                  @Context HttpServletRequest request) {
248 254

  
249 255
        String simpleQuery = buildSearchRequest(RequestResponseHandler.Entity.PROJECT, keywords, fieldQueries);
250
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.PROJECT, offset, limit, format, request, refine, fields, fieldQueries);
256
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.PROJECT, offset, limit, format, request, refine, refineFields, specialFacets, fieldQueries);
251 257
    }
252 258

  
253 259

  
......
286 292
                                             @Context HttpServletRequest request)  {
287 293

  
288 294
        String fullQuery = builtEntity2EntityRelationQuery(RequestResponseHandler.Entity.PUBLICATION, "relprojectid", projectId, fieldQueries);
289
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.PUBLICATION, offset, limit, format, request, false, null, fieldQueries);
295
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.PUBLICATION, offset, limit, format, request, false, null, null,fieldQueries);
290 296
    }
291 297

  
292 298
    @GET
......
313 319
                                             @Context HttpServletRequest request)  {
314 320

  
315 321
        String fullQuery = builtEntity2EntityRelationQuery(RequestResponseHandler.Entity.DATASET, "relprojectid", projectId, fieldQueries);
316
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.DATASET, offset, limit, format, request, false, null, fieldQueries);
322
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.DATASET, offset, limit, format, request, false, null, null, fieldQueries);
317 323
    }
318 324

  
319 325

  
......
359 365
                                         @Context HttpServletRequest request)  {
360 366

  
361 367
        String fullQuery = builtEntity2EntityRelationQuery(RequestResponseHandler.Entity.SOFTWARE, "relprojectid", projectId, fieldQueries);
362
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.SOFTWARE, offset, limit, format, request, false, null, fieldQueries);
368
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.SOFTWARE, offset, limit, format, request, false, null, null,fieldQueries);
363 369
    }
364 370

  
365 371
    @GET
......
388 394
                                         @Context HttpServletRequest request)  {
389 395

  
390 396
        String fullQuery = builtEntity2EntityRelationQuery(RequestResponseHandler.Entity.OTHER, "relprojectid", projectId, fieldQueries);
391
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.OTHER, offset, limit, format, request, false, null, fieldQueries);
397
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.OTHER, offset, limit, format, request, false, null, null, fieldQueries);
392 398
    }
393 399

  
394 400
    @GET
......
399 405
                                     @DefaultValue("0") @QueryParam("page") final int offset,
400 406
                                     @DefaultValue("10") @QueryParam("size") final int limit,
401 407
                                     @QueryParam("refine") @DefaultValue("false") final boolean refine,
402
                                     @QueryParam("fields") final List<String> fields,
408
                                     @QueryParam("fields") final List<String> refineFields,
403 409
                                     @QueryParam("fq") final List<String> fieldQueries,
410
                                     @QueryParam("sf") final List<String> specialFacets,
404 411
                                     @QueryParam("format") final String format,
405 412
                                     @Context final HttpServletRequest request) {
406 413

  
407 414
        String simpleQuery = buildSearchRequest(RequestResponseHandler.Entity.DATASOURCE, keywords, fieldQueries);
408
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.DATASOURCE, offset, limit, format, request, refine, fields, fieldQueries);
415
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.DATASOURCE, offset, limit, format, request, refine, refineFields, specialFacets, fieldQueries);
409 416
    }
410 417

  
411 418
    @GET
......
440 447
                                       @DefaultValue("0") @QueryParam("page") final int offset,
441 448
                                       @DefaultValue("10") @QueryParam("size") final int limit,
442 449
                                       @QueryParam("refine") @DefaultValue("false") final boolean refine,
443
                                       @QueryParam("fields") final List<String> fields,
450
                                       @QueryParam("fields") final List<String> refineFields,
451
                                       @QueryParam("sf") final List<String> specialFacets,
444 452
                                       @QueryParam("fq") final List<String> fieldQueries,
445 453
                                       @QueryParam("format") final String format,
446 454
                                       @Context final HttpServletRequest request) {
447 455

  
448 456
        String simpleQuery = buildSearchRequest(RequestResponseHandler.Entity.ORGANIZATION, keywords, fieldQueries);
449
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.ORGANIZATION, offset, limit, format, request, refine, fields, fieldQueries);
457
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.ORGANIZATION, offset, limit, format, request, refine, refineFields, specialFacets, fieldQueries);
450 458
    }
451 459

  
452 460

  
......
485 493
                                                 @QueryParam("fq") final List<String> fieldQueries,
486 494
                                                 @Context HttpServletRequest request)  {
487 495
        String fullQuery = builtEntity2EntityRelationQuery(RequestResponseHandler.Entity.DATASOURCE, "relorganizationid", organizationid, fieldQueries);
488
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.DATASOURCE, offset, limit, format, request, false, null, fieldQueries);
496
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.DATASOURCE, offset, limit, format, request, false, null, null, fieldQueries);
489 497
    }
490 498

  
491 499
    @GET
......
512 520
                                              @QueryParam("fq") final List<String> fieldQueries,
513 521
                                              @Context HttpServletRequest request)  {
514 522
        String fullQuery = builtEntity2EntityRelationQuery(RequestResponseHandler.Entity.PUBLICATION, "relorganizationid", organizationid, fieldQueries);
515
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.PUBLICATION, offset, limit, format, request, false, null, fieldQueries);
523
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.PUBLICATION, offset, limit, format, request, false, null, null, fieldQueries);
516 524
    }
517 525

  
518 526
    @GET
......
540 548
                                                  @QueryParam("fq") final List<String> fieldQueries,
541 549
                                                  @Context HttpServletRequest request)  {
542 550
        String fullQuery = builtEntity2EntityRelationQuery(RequestResponseHandler.Entity.DATASET, "relorganizationid", organizationid, fieldQueries);
543
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.DATASET, offset, limit, format, request, false, null, fieldQueries);
551
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.DATASET, offset, limit, format, request, false, null, null,fieldQueries);
544 552
    }
545 553

  
546 554
    @GET
......
568 576
                                                 @QueryParam("fq") final List<String> fieldQueries,
569 577
                                                 @Context HttpServletRequest request)  {
570 578
        String fullQuery = builtEntity2EntityRelationQuery(RequestResponseHandler.Entity.PROJECT, "relorganizationid", organizationid, fieldQueries);
571
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.PROJECT, offset, limit, format, request, false, null, fieldQueries);
579
        return getResponseByEntity(fullQuery, RequestResponseHandler.Entity.PROJECT, offset, limit, format, request, false, null, null, fieldQueries);
572 580
    }
573 581

  
574 582
    @GET
......
592 600
                                @DefaultValue("0") @QueryParam("page") final int offset,
593 601
                                @DefaultValue("10") @QueryParam("size") final int limit,
594 602
                                @QueryParam("refine") @DefaultValue("false") final boolean refine,
595
                                @QueryParam("fields") final List<String> fields,
603
                                @QueryParam("fields") final List<String> refineFields,
596 604
                                @QueryParam("fq") final List<String> fieldQueries,
597 605
                                @QueryParam("format") final String format,
598 606
                                @Context final HttpServletRequest request) {
599 607

  
600 608
        String simpleQuery = buildSearchRequest(RequestResponseHandler.Entity.PERSON, keywords, fieldQueries);
601
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.PERSON, offset, limit, format, request, refine, fields, fieldQueries);
609
        return getResponseByEntity(simpleQuery, RequestResponseHandler.Entity.PERSON, offset, limit, format, request, refine, refineFields, fieldQueries);
602 610
    }
603 611

  
604 612

  
......
679 687
    @Timed(value = "http.server.request.duration", extraTags = {"referer", "portal", "uri", "/rest/v2/api/reports"}, longTask = false)
680 688
    public Response fetchReport(@QueryParam("q") String keywords,
681 689
                                @QueryParam("refine") @DefaultValue("false") boolean refine,
682
                                @QueryParam("fields") final List<String> fields,
690
                                @QueryParam("fields") final List<String> refineFields,
683 691
                                @QueryParam("fq") final List<String> fieldQueries,
684 692
                                @QueryParam("type") final String type,
685 693
                                @QueryParam("format") final String format,
......
703 711
        logger.debug("fieldQueries " + fieldQueries);
704 712

  
705 713
        try {
706
            SearchResult sr = ((SearchServiceImpl)searchService).newSearch(fullQuery, Locale.getDefault().toString(), fields, fieldQueries,0, 0, format, transformer,null,false);
714
            SearchResult sr = ((SearchServiceImpl)searchService).newSearch(fullQuery, Locale.getDefault().toString(), refineFields, null, fieldQueries,0, 0, format, transformer,null,false);
707 715
            logger.debug("Total number of results " + sr.getTotal());
708 716

  
709 717

  
......
720 728
                    os.write(CSVResponseFormat.appendTitle(entity, special).getBytes());
721 729
                }
722 730
                try {
723
                    ((SearchServiceImpl)searchService).cursorSearch(fullQuery, fields, fieldQueries, format, transformer, os);
731
                    ((SearchServiceImpl)searchService).cursorSearch(fullQuery, refineFields, null, fieldQueries, format, transformer, os);
724 732

  
725 733
                } catch (SearchServiceException sse) {
726 734
                    os.write(("Fail to return report. " + sse.getMessage()).getBytes());
......
946 954
        String responseFormat = extractResponseFormat(format);
947 955

  
948 956
        try {
949
            SearchResult searchResult = ((SearchServiceImpl)searchService).newSearch(query, Locale.getDefault().toString(), null, fieldQueries, 0, 0, responseFormat, null, null, false);
957
            SearchResult searchResult = ((SearchServiceImpl)searchService).newSearch(query, Locale.getDefault().toString(), null, null, fieldQueries, 0, 0, responseFormat, null, null, false);
950 958
            return Response.status(Response.Status.OK).entity(APIResponseFormatter.createCountMeta(request, query, searchResult.getTotal(), responseFormat)).type(responseFormat).build();
951 959

  
952 960
        } catch (SearchServiceException sse) {
......
968 976
                        type(responseType).build();
969 977
            }
970 978

  
971
            SearchResult searchResult = ((SearchServiceImpl)searchService).newSearch(fullQuery, Locale.getDefault().toString(), null, fieldQueries, 0, 1, responseType, null, null, false);
979
            SearchResult searchResult = ((SearchServiceImpl)searchService).newSearch(fullQuery, Locale.getDefault().toString(), null, null, fieldQueries, 0, 1, responseType, null, null, false);
972 980

  
973 981
            if (searchResult.getSearchResults() == null || searchResult.getSearchResults().isEmpty() ||
974 982
                    searchResult.getSearchResults().size() == 0 || searchResult.getSearchResults().get(0) == null) {
......
984 992
        }
985 993
    }
986 994

  
987
    private Response getResponseByEntity(String fullQuery, RequestResponseHandler.Entity entity, int offset, int limit, String format, HttpServletRequest request, boolean refine, List<String> refineFields, List<String> fieldQueries) {
995
    private Response getResponseByEntity(String fullQuery, RequestResponseHandler.Entity entity, int offset, int limit, String format, HttpServletRequest request, boolean refine, List<String> refineFields, List<String> specialFacets, List<String> fieldQueries) {
988 996
        long startTime = System.nanoTime();
989 997
        Transformer transformer = extractTransformer(format, entity);
990 998
        boolean special = isSpecialFormat(format);
......
998 1006
        }
999 1007

  
1000 1008
        try {
1001
            SearchResult solrResults = ((SearchServiceImpl)searchService).newSearch(fullQuery, Locale.getDefault().toString(), refineFields, fieldQueries, offset, limit, responseType, transformer, null, false);
1009
            SearchResult solrResults = ((SearchServiceImpl)searchService).newSearch(fullQuery, Locale.getDefault().toString(), refineFields, specialFacets, fieldQueries, offset, limit, responseType, transformer, null, false);
1002 1010
            long estimatedTime = System.nanoTime() - startTime;
1003 1011
            logger.debug("Publications search time " + estimatedTime/1000000 +  " milliseconds");
1004 1012
            return Response.status(Response.Status.OK).entity(APIResponseFormatter.createEntitiesResponse(request, entity, fullQuery, fieldQueries, solrResults, refine, responseType, special, hasTitle)).type(responseType).build();
......
1107 1115
                        type(responseType).build();
1108 1116
            }
1109 1117

  
1110
            SearchResult searchResults = ((SearchServiceImpl)searchService).newSearchWithoutFieldQueries(fullQuery, Locale.getDefault().toString(), null, fieldQueries, 0, limit, responseType, null, null, false);
1118
            SearchResult searchResults = ((SearchServiceImpl)searchService).newSearchWithoutFieldQueries(fullQuery, Locale.getDefault().toString(), null, null, fieldQueries, 0, limit, responseType, null, null, false);
1111 1119

  
1112 1120
            if (searchResults.getSearchResults() == null || searchResults.getSearchResults().isEmpty() ||
1113 1121
                    searchResults.getSearchResults().size() == 0 || searchResults.getSearchResults().get(0) == null) {
modules/uoa-search/trunk/src/main/java/eu/dnetlib/data/search/web/api/SearchRequestController.java
313 313
            throw new Exception("Fail to execute search.", e);
314 314
        }
315 315

  
316
        if (formattedSearchResult == null) {
317
            throw new Exception("Fail to execute search.");
318
        }
319

  
316 320
        PrintWriter writer = response.getWriter();
317 321
        writer.append(formattedSearchResult.getFormattedResult());
318 322
        writer.close();

Also available in: Unified diff