2010-10-20 22:45:52 +02:00
|
|
|
/*
|
|
|
|
|
2011-06-24 01:42:12 +02:00
|
|
|
Copyright 2010,2011. Google Inc.
|
2010-10-20 22:45:52 +02:00
|
|
|
All rights reserved.
|
|
|
|
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
|
|
modification, are permitted provided that the following conditions are
|
|
|
|
met:
|
|
|
|
|
|
|
|
* Redistributions of source code must retain the above copyright
|
|
|
|
notice, this list of conditions and the following disclaimer.
|
|
|
|
* Redistributions in binary form must reproduce the above
|
|
|
|
copyright notice, this list of conditions and the following disclaimer
|
|
|
|
in the documentation and/or other materials provided with the
|
|
|
|
distribution.
|
|
|
|
* Neither the name of Google Inc. nor the names of its
|
|
|
|
contributors may be used to endorse or promote products derived from
|
|
|
|
this software without specific prior written permission.
|
|
|
|
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
2010-11-20 13:11:37 +01:00
|
|
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
2010-10-20 22:45:52 +02:00
|
|
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
|
|
*/
|
|
|
|
|
2010-10-07 07:19:35 +02:00
|
|
|
package com.google.refine.grel;
|
2010-05-05 01:24:48 +02:00
|
|
|
|
|
|
|
import java.util.LinkedList;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.regex.Pattern;
|
|
|
|
|
2010-09-22 19:04:10 +02:00
|
|
|
import com.google.refine.expr.Evaluable;
|
|
|
|
import com.google.refine.expr.ParsingException;
|
|
|
|
import com.google.refine.expr.functions.arrays.ArgsToArray;
|
2010-10-07 07:19:35 +02:00
|
|
|
import com.google.refine.grel.Scanner.NumberToken;
|
|
|
|
import com.google.refine.grel.Scanner.RegexToken;
|
|
|
|
import com.google.refine.grel.Scanner.Token;
|
|
|
|
import com.google.refine.grel.Scanner.TokenType;
|
|
|
|
import com.google.refine.grel.ast.ControlCallExpr;
|
|
|
|
import com.google.refine.grel.ast.FieldAccessorExpr;
|
|
|
|
import com.google.refine.grel.ast.FunctionCallExpr;
|
|
|
|
import com.google.refine.grel.ast.LiteralExpr;
|
|
|
|
import com.google.refine.grel.ast.OperatorCallExpr;
|
|
|
|
import com.google.refine.grel.ast.VariableExpr;
|
2010-05-05 01:24:48 +02:00
|
|
|
|
|
|
|
public class Parser {
|
|
|
|
protected Scanner _scanner;
|
|
|
|
protected Token _token;
|
|
|
|
protected Evaluable _root;
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
public Parser(String s) throws ParsingException {
|
|
|
|
this(s, 0, s.length());
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
public Parser(String s, int from, int to) throws ParsingException {
|
|
|
|
_scanner = new Scanner(s, from, to);
|
|
|
|
_token = _scanner.next(true);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
_root = parseExpression();
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
public Evaluable getExpression() {
|
|
|
|
return _root;
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
protected void next(boolean regexPossible) {
|
|
|
|
_token = _scanner.next(regexPossible);
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
protected ParsingException makeException(String desc) {
|
|
|
|
int index = _token != null ? _token.start : _scanner.getIndex();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
return new ParsingException("Parsing error at offset " + index + ": " + desc);
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
/**
|
|
|
|
* <expression> := <sub-expression>
|
|
|
|
* | <expression> [ "<" "<=" ">" ">=" "==" "!=" ] <sub-expression>
|
|
|
|
*/
|
|
|
|
protected Evaluable parseExpression() throws ParsingException {
|
|
|
|
Evaluable sub = parseSubExpression();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
|
|
|
while (_token != null &&
|
|
|
|
_token.type == TokenType.Operator &&
|
2010-05-05 01:24:48 +02:00
|
|
|
">=<==!=".indexOf(_token.text) >= 0) {
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
String op = _token.text;
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
next(true);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
Evaluable sub2 = parseSubExpression();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op);
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
return sub;
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
/**
|
|
|
|
* <sub-expression> := <term>
|
|
|
|
* | <sub-expression> [ "+" "-" ] <term>
|
|
|
|
*/
|
|
|
|
protected Evaluable parseSubExpression() throws ParsingException {
|
|
|
|
Evaluable sub = parseTerm();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
|
|
|
while (_token != null &&
|
|
|
|
_token.type == TokenType.Operator &&
|
2010-05-05 01:24:48 +02:00
|
|
|
"+-".indexOf(_token.text) >= 0) {
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
String op = _token.text;
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
next(true);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2011-06-24 01:42:12 +02:00
|
|
|
Evaluable sub2 = parseTerm();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op);
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
return sub;
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
/**
|
|
|
|
* <term> := <factor>
|
2010-11-20 13:11:37 +01:00
|
|
|
* | <term> [ "*" "/" "%" ] <factor>
|
2010-05-05 01:24:48 +02:00
|
|
|
*/
|
|
|
|
protected Evaluable parseTerm() throws ParsingException {
|
|
|
|
Evaluable factor = parseFactor();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
|
|
|
while (_token != null &&
|
|
|
|
_token.type == TokenType.Operator &&
|
|
|
|
"*/%".indexOf(_token.text) >= 0) {
|
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
String op = _token.text;
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
next(true);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
Evaluable factor2 = parseFactor();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
factor = new OperatorCallExpr(new Evaluable[] { factor, factor2 }, op);
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
return factor;
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
/**
|
2010-11-20 13:11:37 +01:00
|
|
|
* <term> := <term-start> ( <path-segment> )*
|
2010-05-05 01:24:48 +02:00
|
|
|
* <term-start> :=
|
|
|
|
* <string> | <number> | - <number> | <regex> | <identifier> |
|
|
|
|
* <identifier> ( <expression-list> )
|
2010-11-20 13:11:37 +01:00
|
|
|
*
|
2010-05-05 01:24:48 +02:00
|
|
|
* <path-segment> := "[" <expression-list> "]"
|
|
|
|
* | "." <identifier>
|
|
|
|
* | "." <identifier> "(" <expression-list> ")"
|
2010-11-20 13:11:37 +01:00
|
|
|
*
|
2010-05-05 01:24:48 +02:00
|
|
|
*/
|
|
|
|
protected Evaluable parseFactor() throws ParsingException {
|
|
|
|
if (_token == null) {
|
|
|
|
throw makeException("Expecting something more at end of expression");
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
Evaluable eval = null;
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (_token.type == TokenType.String) {
|
|
|
|
eval = new LiteralExpr(_token.text);
|
|
|
|
next(false);
|
|
|
|
} else if (_token.type == TokenType.Regex) {
|
|
|
|
RegexToken t = (RegexToken) _token;
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
try {
|
|
|
|
Pattern pattern = Pattern.compile(_token.text, t.caseInsensitive ? Pattern.CASE_INSENSITIVE : 0);
|
|
|
|
eval = new LiteralExpr(pattern);
|
|
|
|
next(false);
|
|
|
|
} catch (Exception e) {
|
|
|
|
throw makeException("Bad regular expression (" + e.getMessage() + ")");
|
|
|
|
}
|
|
|
|
} else if (_token.type == TokenType.Number) {
|
|
|
|
eval = new LiteralExpr(((NumberToken)_token).value);
|
|
|
|
next(false);
|
|
|
|
} else if (_token.type == TokenType.Operator && _token.text.equals("-")) { // unary minus?
|
|
|
|
next(true);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (_token != null && _token.type == TokenType.Number) {
|
2010-09-28 06:21:44 +02:00
|
|
|
Number n = ((NumberToken)_token).value;
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-09-28 06:21:44 +02:00
|
|
|
eval = new LiteralExpr(n instanceof Long ? -n.longValue() : -n.doubleValue());
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
next(false);
|
|
|
|
} else {
|
|
|
|
throw makeException("Bad negative number");
|
|
|
|
}
|
|
|
|
} else if (_token.type == TokenType.Identifier) {
|
|
|
|
String text = _token.text;
|
|
|
|
next(false);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (_token == null || _token.type != TokenType.Delimiter || !_token.text.equals("(")) {
|
2010-05-25 01:49:18 +02:00
|
|
|
eval = "null".equals(text) ? new LiteralExpr(null) : new VariableExpr(text);
|
2010-11-20 13:11:37 +01:00
|
|
|
} else if( "PI".equals(text) ) {
|
|
|
|
eval = new LiteralExpr(Math.PI);
|
|
|
|
next(false);
|
2010-05-05 01:24:48 +02:00
|
|
|
} else {
|
|
|
|
Function f = ControlFunctionRegistry.getFunction(text);
|
|
|
|
Control c = ControlFunctionRegistry.getControl(text);
|
|
|
|
if (f == null && c == null) {
|
|
|
|
throw makeException("Unknown function or control named " + text);
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
next(true); // swallow (
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
List<Evaluable> args = parseExpressionList(")");
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (c != null) {
|
|
|
|
Evaluable[] argsA = makeArray(args);
|
|
|
|
String errorMessage = c.checkArguments(argsA);
|
|
|
|
if (errorMessage != null) {
|
|
|
|
throw makeException(errorMessage);
|
|
|
|
}
|
|
|
|
eval = new ControlCallExpr(argsA, c);
|
|
|
|
} else {
|
|
|
|
eval = new FunctionCallExpr(makeArray(args), f);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else if (_token.type == TokenType.Delimiter && _token.text.equals("(")) {
|
|
|
|
next(true);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
eval = parseExpression();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(")")) {
|
|
|
|
next(false);
|
|
|
|
} else {
|
|
|
|
throw makeException("Missing )");
|
|
|
|
}
|
2010-05-20 19:49:20 +02:00
|
|
|
} else if (_token.type == TokenType.Delimiter && _token.text.equals("[")) { // [ ... ] array
|
2011-08-02 22:26:32 +02:00
|
|
|
next(true); // swallow [
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-20 19:49:20 +02:00
|
|
|
List<Evaluable> args = parseExpressionList("]");
|
|
|
|
|
|
|
|
eval = new FunctionCallExpr(makeArray(args), new ArgsToArray());
|
2010-05-05 01:24:48 +02:00
|
|
|
} else {
|
|
|
|
throw makeException("Missing number, string, identifier, regex, or parenthesized expression");
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
while (_token != null) {
|
|
|
|
if (_token.type == TokenType.Operator && _token.text.equals(".")) {
|
|
|
|
next(false); // swallow .
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (_token == null || _token.type != TokenType.Identifier) {
|
|
|
|
throw makeException("Missing function name");
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
String identifier = _token.text;
|
|
|
|
next(false);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals("(")) {
|
|
|
|
next(true); // swallow (
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
Function f = ControlFunctionRegistry.getFunction(identifier);
|
|
|
|
if (f == null) {
|
|
|
|
throw makeException("Unknown function " + identifier);
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
List<Evaluable> args = parseExpressionList(")");
|
|
|
|
args.add(0, eval);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
eval = new FunctionCallExpr(makeArray(args), f);
|
|
|
|
} else {
|
|
|
|
eval = new FieldAccessorExpr(eval, identifier);
|
|
|
|
}
|
|
|
|
} else if (_token.type == TokenType.Delimiter && _token.text.equals("[")) {
|
|
|
|
next(true); // swallow [
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
List<Evaluable> args = parseExpressionList("]");
|
|
|
|
args.add(0, eval);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
eval = new FunctionCallExpr(makeArray(args), ControlFunctionRegistry.getFunction("get"));
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
return eval;
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
/**
|
|
|
|
* <expression-list> := <empty>
|
|
|
|
* | <expression> ( "," <expression> )*
|
2010-11-20 13:11:37 +01:00
|
|
|
*
|
2010-05-05 01:24:48 +02:00
|
|
|
*/
|
|
|
|
protected List<Evaluable> parseExpressionList(String closingDelimiter) throws ParsingException {
|
|
|
|
List<Evaluable> l = new LinkedList<Evaluable>();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
|
|
|
if (_token != null &&
|
2010-05-05 01:24:48 +02:00
|
|
|
(_token.type != TokenType.Delimiter || !_token.text.equals(closingDelimiter))) {
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
while (_token != null) {
|
|
|
|
Evaluable eval = parseExpression();
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
l.add(eval);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(",")) {
|
|
|
|
next(true); // swallow comma, loop back for more
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(closingDelimiter)) {
|
|
|
|
next(false); // swallow closing delimiter
|
|
|
|
} else {
|
|
|
|
throw makeException("Missing " + closingDelimiter);
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
return l;
|
|
|
|
}
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
protected Evaluable[] makeArray(List<Evaluable> l) {
|
|
|
|
Evaluable[] a = new Evaluable[l.size()];
|
|
|
|
l.toArray(a);
|
2010-11-20 13:11:37 +01:00
|
|
|
|
2010-05-05 01:24:48 +02:00
|
|
|
return a;
|
|
|
|
}
|
|
|
|
}
|