Revision 42184
Added by Michele Artini about 8 years ago
| MongoQueryParser.java | ||
|---|---|---|
| 1 | 1 |
package eu.dnetlib.oai.parser; |
| 2 | 2 |
|
| 3 | 3 |
import java.io.IOException; |
| 4 |
import java.time.LocalDateTime; |
|
| 5 |
import java.time.format.DateTimeFormatter; |
|
| 4 | 6 |
import java.util.List; |
| 5 | 7 |
|
| 6 |
import com.google.common.collect.Lists; |
|
| 7 |
import com.mongodb.BasicDBObject; |
|
| 8 |
import com.mongodb.BasicDBObjectBuilder; |
|
| 9 |
import eu.dnetlib.data.information.oai.publisher.OaiPublisherRuntimeException; |
|
| 10 |
import eu.dnetlib.data.information.oai.publisher.conf.OAIConfigurationReader; |
|
| 11 |
import eu.dnetlib.functionality.index.parse.Relation; |
|
| 12 |
import eu.dnetlib.functionality.index.parse.Relations; |
|
| 13 |
import org.apache.commons.lang.StringUtils; |
|
| 8 |
import org.apache.commons.lang3.StringUtils; |
|
| 14 | 9 |
import org.apache.commons.logging.Log; |
| 15 | 10 |
import org.apache.commons.logging.LogFactory; |
| 16 | 11 |
import org.bson.conversions.Bson; |
| 17 | 12 |
import org.bson.types.ObjectId; |
| 18 |
import org.joda.time.DateTime; |
|
| 19 |
import org.joda.time.format.DateTimeFormat; |
|
| 20 |
import org.joda.time.format.DateTimeFormatter; |
|
| 21 |
import org.joda.time.format.ISODateTimeFormat; |
|
| 22 |
import org.z3950.zing.cql.*; |
|
| 13 |
import org.z3950.zing.cql.CQLAndNode; |
|
| 14 |
import org.z3950.zing.cql.CQLBooleanNode; |
|
| 15 |
import org.z3950.zing.cql.CQLNode; |
|
| 16 |
import org.z3950.zing.cql.CQLNotNode; |
|
| 17 |
import org.z3950.zing.cql.CQLOrNode; |
|
| 18 |
import org.z3950.zing.cql.CQLParseException; |
|
| 19 |
import org.z3950.zing.cql.CQLParser; |
|
| 20 |
import org.z3950.zing.cql.CQLTermNode; |
|
| 23 | 21 |
|
| 22 |
import com.google.common.collect.Lists; |
|
| 23 |
import com.mongodb.BasicDBObject; |
|
| 24 |
import com.mongodb.BasicDBObjectBuilder; |
|
| 25 |
|
|
| 26 |
import eu.dnetlib.index.parse.Relation; |
|
| 27 |
import eu.dnetlib.index.parse.Relations; |
|
| 28 |
import eu.dnetlib.oai.conf.OAIConfigurationReader; |
|
| 29 |
import eu.dnetlib.rmi.provision.OaiPublisherRuntimeException; |
|
| 30 |
|
|
| 24 | 31 |
/** |
| 25 | 32 |
* Instances of this class parse query string into mongo DBObject query. |
| 26 | 33 |
* |
| ... | ... | |
| 33 | 40 |
/** |
| 34 | 41 |
* Parses the given query string into a mongo DBObject. |
| 35 | 42 |
* |
| 36 |
* @param query String to parse |
|
| 43 |
* @param query |
|
| 44 |
* String to parse |
|
| 37 | 45 |
* @return DBObject corresponding to the query string |
| 38 | 46 |
*/ |
| 39 | 47 |
public Bson parse(final String query) {
|
| 40 | 48 |
log.debug("PARSING: " + query);
|
| 41 |
if (StringUtils.isBlank(query)) return new BasicDBObject();
|
|
| 42 |
Bson parsed = toMongo(query); |
|
| 49 |
if (StringUtils.isBlank(query)) { return new BasicDBObject(); }
|
|
| 50 |
final Bson parsed = toMongo(query);
|
|
| 43 | 51 |
log.debug(parsed); |
| 44 | 52 |
return parsed; |
| 45 | 53 |
} |
| 46 | 54 |
|
| 47 | 55 |
private Bson toMongo(final String query) {
|
| 48 |
CQLParser parser = new CQLParser(); |
|
| 56 |
final CQLParser parser = new CQLParser();
|
|
| 49 | 57 |
CQLNode root; |
| 50 | 58 |
try {
|
| 51 | 59 |
root = parser.parse(query); |
| 52 | 60 |
return this.toMongo(root); |
| 53 |
} catch (CQLParseException e) {
|
|
| 61 |
} catch (final CQLParseException e) {
|
|
| 54 | 62 |
throw new OaiPublisherRuntimeException(e); |
| 55 |
} catch (IOException e) {
|
|
| 63 |
} catch (final IOException e) {
|
|
| 56 | 64 |
throw new OaiPublisherRuntimeException(e); |
| 57 | 65 |
} |
| 58 | 66 |
} |
| 59 | 67 |
|
| 60 | 68 |
private Bson toMongo(final CQLNode node) {
|
| 61 |
if (node instanceof CQLTermNode) return doTranslate((CQLTermNode) node);
|
|
| 62 |
if (node instanceof CQLBooleanNode) return doTranslate((CQLBooleanNode) node);
|
|
| 69 |
if (node instanceof CQLTermNode) { return doTranslate((CQLTermNode) node); }
|
|
| 70 |
if (node instanceof CQLBooleanNode) { return doTranslate((CQLBooleanNode) node); }
|
|
| 63 | 71 |
|
| 64 | 72 |
throw new RuntimeException("error choice for CQLNode " + node.getClass());
|
| 65 | 73 |
} |
| 66 | 74 |
|
| 67 | 75 |
private Bson doTranslate(final CQLTermNode termNode) {
|
| 68 |
if (termNode.getTerm().equals("*")) return new BasicDBObject();
|
|
| 69 |
String relation = termNode.getRelation().getBase(); |
|
| 70 |
Relation rel = Relations.get(relation); |
|
| 76 |
if (termNode.getTerm().equals("*")) { return new BasicDBObject(); }
|
|
| 77 |
final String relation = termNode.getRelation().getBase();
|
|
| 78 |
final Relation rel = Relations.get(relation);
|
|
| 71 | 79 |
return this.handleRelationNode(rel, termNode); |
| 72 | 80 |
} |
| 73 | 81 |
|
| 74 | 82 |
private Bson handleRelationNode(final Relation rel, final CQLTermNode termNode) {
|
| 75 |
BasicDBObject mongoQueryObject = new BasicDBObject(); |
|
| 76 |
String term = termNode.getTerm(); |
|
| 77 |
String indexName = termNode.getIndex(); |
|
| 83 |
final BasicDBObject mongoQueryObject = new BasicDBObject();
|
|
| 84 |
final String term = termNode.getTerm();
|
|
| 85 |
final String indexName = termNode.getIndex();
|
|
| 78 | 86 |
Object termObj = term; |
| 79 | 87 |
if (indexName.equals("_id")) {
|
| 80 | 88 |
termObj = new ObjectId(term); |
| 81 | 89 |
} else {
|
| 82 | 90 |
if (indexName.equals(OAIConfigurationReader.DATESTAMP_FIELD) || indexName.equals(OAIConfigurationReader.LAST_COLLECTION_DATE_FIELD)) {
|
| 83 | 91 |
// termObj = this.parseDate(term); |
| 84 |
OAIDate termDate = this.parseDate(term); |
|
| 92 |
final OAIDate termDate = this.parseDate(term);
|
|
| 85 | 93 |
return handleDateRelationNode(indexName, rel, termDate); |
| 86 | 94 |
} |
| 87 | 95 |
} |
| ... | ... | |
| 130 | 138 |
* @return |
| 131 | 139 |
*/ |
| 132 | 140 |
private Bson handleDateRelationNode(final String indexName, final Relation rel, final OAIDate date) {
|
| 133 |
BasicDBObject mongoQueryObject = new BasicDBObject(); |
|
| 134 |
DateTime fromDate = date.date; |
|
| 141 |
final BasicDBObject mongoQueryObject = new BasicDBObject(); |
|
| 142 |
|
|
| 143 |
LocalDateTime fromDate = date.date; |
|
| 135 | 144 |
switch (rel) {
|
| 136 | 145 |
case EQUAL: |
| 137 | 146 |
case EXACT: |
| 138 | 147 |
if (date.onlyDate) {
|
| 139 |
DateTime endDate = date.date.plusDays(1); |
|
| 140 |
mongoQueryObject.put(indexName, BasicDBObjectBuilder.start("$gte", fromDate.toDate()).append("$lt", endDate.toDate()).get());
|
|
| 148 |
|
|
| 149 |
final LocalDateTime endDate = date.date.plusDays(1); |
|
| 150 |
mongoQueryObject.put(indexName, BasicDBObjectBuilder.start("$gte", fromDate.toLocalDate()).append("$lt", endDate.toLocalDate()).get());
|
|
| 141 | 151 |
} else {
|
| 142 |
DateTime endDate = date.date.plusSeconds(1); |
|
| 143 |
mongoQueryObject.put(indexName, BasicDBObjectBuilder.start("$gte", fromDate.toDate()).append("$lt", endDate.toDate()).get());
|
|
| 152 |
final LocalDateTime endDate = date.date.plusSeconds(1);
|
|
| 153 |
mongoQueryObject.put(indexName, BasicDBObjectBuilder.start("$gte", fromDate.toLocalDate()).append("$lt", endDate.toLocalDate()).get());
|
|
| 144 | 154 |
} |
| 145 | 155 |
break; |
| 146 | 156 |
case NOT: |
| 147 |
mongoQueryObject.put(indexName, new BasicDBObject("$ne", fromDate.toDate()));
|
|
| 157 |
mongoQueryObject.put(indexName, new BasicDBObject("$ne", fromDate.toLocalDate()));
|
|
| 148 | 158 |
break; |
| 149 | 159 |
case GT: |
| 150 |
mongoQueryObject.put(indexName, new BasicDBObject("$gt", fromDate.toDate()));
|
|
| 160 |
mongoQueryObject.put(indexName, new BasicDBObject("$gt", fromDate.toLocalDate()));
|
|
| 151 | 161 |
break; |
| 152 | 162 |
case GTE: |
| 153 |
mongoQueryObject.put(indexName, new BasicDBObject("$gte", fromDate.toDate()));
|
|
| 163 |
mongoQueryObject.put(indexName, new BasicDBObject("$gte", fromDate.toLocalDate()));
|
|
| 154 | 164 |
break; |
| 155 | 165 |
case LT: |
| 156 |
mongoQueryObject.put(indexName, new BasicDBObject("$lt", fromDate.toDate()));
|
|
| 166 |
mongoQueryObject.put(indexName, new BasicDBObject("$lt", fromDate.toLocalDate()));
|
|
| 157 | 167 |
break; |
| 158 | 168 |
case LTE: |
| 159 | 169 |
/* |
| 160 |
If the request is date <= YYYY-MM-DD then we need to change the date. |
|
| 161 |
The parseDate returned YYYY-MM-DDT00:00:00Z, but we need YYYY-MM-DDT23:59:59Z. |
|
| 162 |
To simplify we can add one day and perform < instead of <=. |
|
| 170 |
* If the request is date <= YYYY-MM-DD then we need to change the date. The parseDate returned YYYY-MM-DDT00:00:00Z, but we |
|
| 171 |
* need YYYY-MM-DDT23:59:59Z. To simplify we can add one day and perform < instead of <=. |
|
| 163 | 172 |
*/ |
| 164 | 173 |
if (date.onlyDate) {
|
| 165 | 174 |
fromDate = date.date.plusDays(1); |
| 166 |
mongoQueryObject.put(indexName, new BasicDBObject("$lt", fromDate.toDate()));
|
|
| 175 |
mongoQueryObject.put(indexName, new BasicDBObject("$lt", fromDate.toLocalDate()));
|
|
| 167 | 176 |
} else {
|
| 168 |
mongoQueryObject.put(indexName, new BasicDBObject("$lte", fromDate.toDate()));
|
|
| 177 |
mongoQueryObject.put(indexName, new BasicDBObject("$lte", fromDate.toLocalDate()));
|
|
| 169 | 178 |
} |
| 170 | 179 |
break; |
| 171 | 180 |
default: |
| ... | ... | |
| 175 | 184 |
} |
| 176 | 185 |
|
| 177 | 186 |
private Bson doTranslate(final CQLBooleanNode node) {
|
| 178 |
if (node instanceof CQLAndNode) return getBooleanQuery("$and", node);
|
|
| 179 |
if (node instanceof CQLOrNode) return getBooleanQuery("$or", node);
|
|
| 180 |
if (node instanceof CQLNotNode) return getNotQuery((CQLNotNode) node);
|
|
| 187 |
if (node instanceof CQLAndNode) { return getBooleanQuery("$and", node); }
|
|
| 188 |
if (node instanceof CQLOrNode) { return getBooleanQuery("$or", node); }
|
|
| 189 |
if (node instanceof CQLNotNode) { return getNotQuery((CQLNotNode) node); }
|
|
| 181 | 190 |
throw new RuntimeException("error choice for CQLBooleanNode " + node.getClass());
|
| 182 | 191 |
} |
| 183 | 192 |
|
| 184 | 193 |
private Bson getBooleanQuery(final String mongoOperator, final CQLBooleanNode node) {
|
| 185 |
Bson left = this.toMongo(node.left); |
|
| 186 |
Bson right = this.toMongo(node.right); |
|
| 187 |
BasicDBObject opQuery = new BasicDBObject(); |
|
| 188 |
List<Bson> termList = Lists.newArrayList(left, right); |
|
| 194 |
final Bson left = this.toMongo(node.left);
|
|
| 195 |
final Bson right = this.toMongo(node.right);
|
|
| 196 |
final BasicDBObject opQuery = new BasicDBObject();
|
|
| 197 |
final List<Bson> termList = Lists.newArrayList(left, right);
|
|
| 189 | 198 |
opQuery.put(mongoOperator, termList); |
| 190 | 199 |
return opQuery; |
| 191 | 200 |
} |
| 192 | 201 |
|
| 193 | 202 |
private Bson getNotQuery(final CQLNotNode node) {
|
| 194 |
Bson left = this.toMongo(node.left); |
|
| 195 |
Bson right = this.toMongo(node.right); |
|
| 196 |
Bson notRight = new BasicDBObject("$not", right);
|
|
| 197 |
BasicDBObject andQuery = new BasicDBObject(); |
|
| 198 |
List<Bson> termList = Lists.newArrayList(left, notRight); |
|
| 203 |
final Bson left = this.toMongo(node.left);
|
|
| 204 |
final Bson right = this.toMongo(node.right);
|
|
| 205 |
final Bson notRight = new BasicDBObject("$not", right);
|
|
| 206 |
final BasicDBObject andQuery = new BasicDBObject();
|
|
| 207 |
final List<Bson> termList = Lists.newArrayList(left, notRight);
|
|
| 199 | 208 |
andQuery.put("$and", termList);
|
| 200 | 209 |
return andQuery; |
| 201 | 210 |
} |
| 202 | 211 |
|
| 203 | 212 |
private OAIDate parseDate(final String date) {
|
| 204 |
DateTimeFormatter dateNoTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd").withZoneUTC();
|
|
| 205 |
DateTimeFormatter iso8601NoMsTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ssZ").withZoneUTC();
|
|
| 206 |
DateTimeFormatter iso8601Formatter = ISODateTimeFormat.dateTime().withZoneUTC(); |
|
| 213 |
|
|
| 214 |
final DateTimeFormatter dateNoTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
|
| 215 |
final DateTimeFormatter iso8601NoMsTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ");
|
|
| 216 |
final DateTimeFormatter iso8601Formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ");
|
|
| 207 | 217 |
OAIDate res = null; |
| 208 | 218 |
try {
|
| 209 | 219 |
log.debug("Using default " + iso8601Formatter.getClass());
|
| 210 |
DateTime dt = iso8601Formatter.parseDateTime(date); |
|
| 220 |
|
|
| 221 |
final LocalDateTime dt = LocalDateTime.parse(date, iso8601Formatter); |
|
| 211 | 222 |
res = new OAIDate(dt, false); |
| 212 |
} catch (Exception e) {
|
|
| 223 |
} catch (final Exception e) {
|
|
| 213 | 224 |
try {
|
| 214 | 225 |
log.debug("Switching to ISO with no millisecond date formatter: yyyy-MM-dd'T'HH:mm:ssZ");
|
| 215 |
DateTime dt = iso8601NoMsTimeFormatter.parseDateTime(date);
|
|
| 226 |
final LocalDateTime dt = LocalDateTime.parse(date, iso8601NoMsTimeFormatter);
|
|
| 216 | 227 |
res = new OAIDate(dt, false); |
| 217 |
} catch (Exception ex) {
|
|
| 228 |
} catch (final Exception ex) {
|
|
| 218 | 229 |
log.debug("Switching to simple date formatter: yyyy-MM-dd");
|
| 219 |
DateTime dt = dateNoTimeFormatter.parseDateTime(date);
|
|
| 230 |
final LocalDateTime dt = LocalDateTime.parse(date, dateNoTimeFormatter);
|
|
| 220 | 231 |
res = new OAIDate(dt, true); |
| 221 | 232 |
} |
| 222 | 233 |
} |
| ... | ... | |
| 225 | 236 |
|
| 226 | 237 |
class OAIDate {
|
| 227 | 238 |
|
| 228 |
DateTime date; |
|
| 239 |
LocalDateTime date;
|
|
| 229 | 240 |
boolean onlyDate; |
| 230 | 241 |
|
| 231 |
OAIDate(final DateTime date, final boolean onlyDate) {
|
|
| 242 |
OAIDate(final LocalDateTime date, final boolean onlyDate) {
|
|
| 232 | 243 |
this.date = date; |
| 233 | 244 |
this.onlyDate = onlyDate; |
| 234 | 245 |
} |
Also available in: Unified diff
oai import