From 23b9e313b8a1fb2fb1d039c62fd18b56ce38b95f Mon Sep 17 00:00:00 2001 From: David Huynh Date: Wed, 27 Jan 2010 22:27:22 +0000 Subject: [PATCH] Implemented expression parser. git-svn-id: http://google-refine.googlecode.com/svn/trunk@10 7d457c2a-affb-35e4-300a-418c747d4874 --- .../browsing/accessors/CellAccessor.java | 2 +- .../browsing/accessors/DecoratedValue.java | 11 + .../accessors/ReconFeatureCellAccessor.java | 2 +- .../browsing/accessors/ReconTypeAccessor.java | 2 +- .../browsing/accessors/ValueCellAccessor.java | 2 +- .../facets/CellAccessorNominalRowGrouper.java | 26 +- .../gridlock/browsing/facets/ListFacet.java | 19 +- .../browsing/facets/NominalFacetChoice.java | 12 +- .../filters/CellAccessorEqualRowFilter.java | 2 +- .../metaweb/gridlock/commands/Command.java | 22 +- .../commands/DoTextTransformCommand.java | 117 ++++----- .../gridlock/expr/FieldAccessorExpr.java | 4 + .../gridlock/expr/FunctionCallExpr.java | 13 + .../metaweb/gridlock/expr/LiteralExpr.java | 6 + .../gridlock/expr/OperatorCallExpr.java | 66 +++++ .../com/metaweb/gridlock/expr/Parser.java | 242 ++++++++++++++++++ .../com/metaweb/gridlock/expr/Scanner.java | 228 +++++++++++++++++ .../metaweb/gridlock/expr/VariableExpr.java | 6 + .../gridlock/expr/functions/Slice.java | 64 +++++ .../webapp/scripts/project/data-table-view.js | 8 +- 20 files changed, 756 insertions(+), 98 deletions(-) create mode 100644 src/main/java/com/metaweb/gridlock/browsing/accessors/DecoratedValue.java create mode 100644 src/main/java/com/metaweb/gridlock/expr/OperatorCallExpr.java create mode 100644 src/main/java/com/metaweb/gridlock/expr/Parser.java create mode 100644 src/main/java/com/metaweb/gridlock/expr/Scanner.java create mode 100644 src/main/java/com/metaweb/gridlock/expr/functions/Slice.java diff --git a/src/main/java/com/metaweb/gridlock/browsing/accessors/CellAccessor.java b/src/main/java/com/metaweb/gridlock/browsing/accessors/CellAccessor.java index 4f5c66fe6..8ada172b8 100644 --- a/src/main/java/com/metaweb/gridlock/browsing/accessors/CellAccessor.java +++ b/src/main/java/com/metaweb/gridlock/browsing/accessors/CellAccessor.java @@ -3,5 +3,5 @@ package com.metaweb.gridlock.browsing.accessors; import com.metaweb.gridlock.model.Cell; public interface CellAccessor { - public Object[] get(Cell cell); + public Object[] get(Cell cell, boolean decorated); } diff --git a/src/main/java/com/metaweb/gridlock/browsing/accessors/DecoratedValue.java b/src/main/java/com/metaweb/gridlock/browsing/accessors/DecoratedValue.java new file mode 100644 index 000000000..d39f50bc6 --- /dev/null +++ b/src/main/java/com/metaweb/gridlock/browsing/accessors/DecoratedValue.java @@ -0,0 +1,11 @@ +package com.metaweb.gridlock.browsing.accessors; + +public class DecoratedValue { + final public Object value; + final public String label; + + public DecoratedValue(Object value, String label) { + this.value = value; + this.label = label; + } +} diff --git a/src/main/java/com/metaweb/gridlock/browsing/accessors/ReconFeatureCellAccessor.java b/src/main/java/com/metaweb/gridlock/browsing/accessors/ReconFeatureCellAccessor.java index 085509f99..6dd48bec1 100644 --- a/src/main/java/com/metaweb/gridlock/browsing/accessors/ReconFeatureCellAccessor.java +++ b/src/main/java/com/metaweb/gridlock/browsing/accessors/ReconFeatureCellAccessor.java @@ -10,7 +10,7 @@ public class ReconFeatureCellAccessor implements CellAccessor { } @Override - public Object[] get(Cell cell) { + public Object[] get(Cell cell, boolean decorated) { if (cell.recon != null) { return new Object[] { cell.recon.features.get(_name) }; } diff --git a/src/main/java/com/metaweb/gridlock/browsing/accessors/ReconTypeAccessor.java b/src/main/java/com/metaweb/gridlock/browsing/accessors/ReconTypeAccessor.java index 9d64318a3..962417de6 100644 --- a/src/main/java/com/metaweb/gridlock/browsing/accessors/ReconTypeAccessor.java +++ b/src/main/java/com/metaweb/gridlock/browsing/accessors/ReconTypeAccessor.java @@ -5,7 +5,7 @@ import com.metaweb.gridlock.model.ReconCandidate; public class ReconTypeAccessor implements CellAccessor { @Override - public Object[] get(Cell cell) { + public Object[] get(Cell cell, boolean decorated) { if (cell.recon != null && cell.recon.candidates.size() > 0) { ReconCandidate c = cell.recon.candidates.get(0); return c.typeIDs; diff --git a/src/main/java/com/metaweb/gridlock/browsing/accessors/ValueCellAccessor.java b/src/main/java/com/metaweb/gridlock/browsing/accessors/ValueCellAccessor.java index c3816be8a..977720350 100644 --- a/src/main/java/com/metaweb/gridlock/browsing/accessors/ValueCellAccessor.java +++ b/src/main/java/com/metaweb/gridlock/browsing/accessors/ValueCellAccessor.java @@ -4,7 +4,7 @@ import com.metaweb.gridlock.model.Cell; public class ValueCellAccessor implements CellAccessor { @Override - public Object[] get(Cell cell) { + public Object[] get(Cell cell, boolean decorated) { if (cell.value != null) { return new Object[] { cell.value }; } diff --git a/src/main/java/com/metaweb/gridlock/browsing/facets/CellAccessorNominalRowGrouper.java b/src/main/java/com/metaweb/gridlock/browsing/facets/CellAccessorNominalRowGrouper.java index ce0145fe1..5e0d3bc76 100644 --- a/src/main/java/com/metaweb/gridlock/browsing/facets/CellAccessorNominalRowGrouper.java +++ b/src/main/java/com/metaweb/gridlock/browsing/facets/CellAccessorNominalRowGrouper.java @@ -5,6 +5,7 @@ import java.util.Map; import com.metaweb.gridlock.browsing.RowVisitor; import com.metaweb.gridlock.browsing.accessors.CellAccessor; +import com.metaweb.gridlock.browsing.accessors.DecoratedValue; import com.metaweb.gridlock.model.Cell; import com.metaweb.gridlock.model.Row; @@ -12,7 +13,7 @@ public class CellAccessorNominalRowGrouper implements RowVisitor { final protected CellAccessor _accessor; final protected int _cellIndex; - final public Map groups = new HashMap(); + final public Map choices = new HashMap(); public CellAccessorNominalRowGrouper(CellAccessor accessor, int cellIndex) { _accessor = accessor; @@ -24,18 +25,23 @@ public class CellAccessorNominalRowGrouper implements RowVisitor { if (_cellIndex < row.cells.size()) { Cell cell = row.cells.get(_cellIndex); if (cell != null) { - Object[] values = _accessor.get(cell); + Object[] values = _accessor.get(cell, true); if (values != null && values.length > 0) { - for (Object v : values) { - if (v != null) { - if (groups.containsKey(v)) { - groups.get(v).count++; + for (Object value : values) { + if (value != null) { + DecoratedValue dValue = + value instanceof DecoratedValue ? + (DecoratedValue) value : + new DecoratedValue(value, value.toString()); + + Object v = dValue.value; + if (choices.containsKey(value)) { + choices.get(value).count++; } else { - NominalFacetChoice group = new NominalFacetChoice(); - group.value = v; - group.count = 1; + NominalFacetChoice choice = new NominalFacetChoice(dValue, v); + choice.count = 1; - groups.put(v, group); + choices.put(v, choice); } } } diff --git a/src/main/java/com/metaweb/gridlock/browsing/facets/ListFacet.java b/src/main/java/com/metaweb/gridlock/browsing/facets/ListFacet.java index 95eb637eb..ff16cf258 100644 --- a/src/main/java/com/metaweb/gridlock/browsing/facets/ListFacet.java +++ b/src/main/java/com/metaweb/gridlock/browsing/facets/ListFacet.java @@ -1,13 +1,18 @@ package com.metaweb.gridlock.browsing.facets; +import java.util.LinkedList; +import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import com.metaweb.gridlock.browsing.FilteredRows; import com.metaweb.gridlock.browsing.filters.RowFilter; public class ListFacet implements Facet { + final protected List _choices = new LinkedList(); @Override public JSONObject getJSON(Properties options) throws JSONException { @@ -15,6 +20,16 @@ public class ListFacet implements Facet { return null; } + @Override + public void initializeFromJSON(JSONObject o) throws JSONException { + JSONArray a = o.getJSONArray("choices"); + int length = a.length(); + + for (int i = 0; i < length; i++) { + + } + } + @Override public RowFilter getRowFilter() { // TODO Auto-generated method stub @@ -22,9 +37,9 @@ public class ListFacet implements Facet { } @Override - public void initializeFromJSON(JSONObject o) throws JSONException { + public void computeChoices(FilteredRows filteredRows) { // TODO Auto-generated method stub - + } } diff --git a/src/main/java/com/metaweb/gridlock/browsing/facets/NominalFacetChoice.java b/src/main/java/com/metaweb/gridlock/browsing/facets/NominalFacetChoice.java index 18e71ab60..4d804316d 100644 --- a/src/main/java/com/metaweb/gridlock/browsing/facets/NominalFacetChoice.java +++ b/src/main/java/com/metaweb/gridlock/browsing/facets/NominalFacetChoice.java @@ -1,6 +1,14 @@ package com.metaweb.gridlock.browsing.facets; +import com.metaweb.gridlock.browsing.accessors.DecoratedValue; + public class NominalFacetChoice { - public Object value; - public int count; + final public DecoratedValue decoratedValue; + final public Object value; + public int count; + + public NominalFacetChoice(DecoratedValue decoratedValue, Object value) { + this.decoratedValue = decoratedValue; + this.value = value; + } } diff --git a/src/main/java/com/metaweb/gridlock/browsing/filters/CellAccessorEqualRowFilter.java b/src/main/java/com/metaweb/gridlock/browsing/filters/CellAccessorEqualRowFilter.java index 267b8190a..5ba15ae5f 100644 --- a/src/main/java/com/metaweb/gridlock/browsing/filters/CellAccessorEqualRowFilter.java +++ b/src/main/java/com/metaweb/gridlock/browsing/filters/CellAccessorEqualRowFilter.java @@ -20,7 +20,7 @@ public class CellAccessorEqualRowFilter implements RowFilter { if (_cellIndex < row.cells.size()) { Cell cell = row.cells.get(_cellIndex); if (cell != null) { - Object[] values = _accessor.get(cell); + Object[] values = _accessor.get(cell, false); if (values != null && values.length > 0) { for (Object v : values) { for (Object match : _matches) { diff --git a/src/main/java/com/metaweb/gridlock/commands/Command.java b/src/main/java/com/metaweb/gridlock/commands/Command.java index 1e708512e..d79e66ceb 100644 --- a/src/main/java/com/metaweb/gridlock/commands/Command.java +++ b/src/main/java/com/metaweb/gridlock/commands/Command.java @@ -5,7 +5,9 @@ import java.io.InputStreamReader; import java.io.LineNumberReader; import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.io.Reader; +import java.io.StringWriter; import java.util.Properties; import javax.servlet.ServletException; @@ -68,12 +70,28 @@ public abstract class Command { } protected void respondJSON(HttpServletResponse response, JSONObject o) throws IOException { + response.setHeader("Content-Type", "application/json"); respond(response, o.toString()); } protected void respondException(HttpServletResponse response, Exception e) throws IOException { - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - e.printStackTrace(response.getWriter()); + try { + JSONObject o = new JSONObject(); + o.put("code", "error"); + o.put("message", e.getMessage()); + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + pw.flush(); + sw.flush(); + + o.put("stack", sw.toString()); + + respondJSON(response, o); + } catch (JSONException e1) { + e.printStackTrace(response.getWriter()); + } } protected void redirect(HttpServletResponse response, String url) throws IOException { diff --git a/src/main/java/com/metaweb/gridlock/commands/DoTextTransformCommand.java b/src/main/java/com/metaweb/gridlock/commands/DoTextTransformCommand.java index 5ed77ff26..9b60d8dba 100644 --- a/src/main/java/com/metaweb/gridlock/commands/DoTextTransformCommand.java +++ b/src/main/java/com/metaweb/gridlock/commands/DoTextTransformCommand.java @@ -9,24 +9,14 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONTokener; import com.metaweb.gridlock.expr.Evaluable; -import com.metaweb.gridlock.expr.FieldAccessorExpr; -import com.metaweb.gridlock.expr.Function; -import com.metaweb.gridlock.expr.FunctionCallExpr; -import com.metaweb.gridlock.expr.LiteralExpr; -import com.metaweb.gridlock.expr.VariableExpr; -import com.metaweb.gridlock.expr.functions.Replace; -import com.metaweb.gridlock.expr.functions.ToLowercase; -import com.metaweb.gridlock.expr.functions.ToTitlecase; -import com.metaweb.gridlock.expr.functions.ToUppercase; +import com.metaweb.gridlock.expr.Parser; import com.metaweb.gridlock.history.CellChange; import com.metaweb.gridlock.history.HistoryEntry; import com.metaweb.gridlock.history.MassCellChange; import com.metaweb.gridlock.model.Cell; +import com.metaweb.gridlock.model.Column; import com.metaweb.gridlock.model.Project; import com.metaweb.gridlock.model.Row; import com.metaweb.gridlock.process.QuickHistoryEntryProcess; @@ -40,72 +30,53 @@ public class DoTextTransformCommand extends Command { Project project = getProject(request); int cellIndex = Integer.parseInt(request.getParameter("cell")); + String columnName = null; + for (Column column : project.columnModel.columns) { + if (column.cellIndex == cellIndex) { + columnName = column.headerLabel; + break; + } + } + String expression = request.getParameter("expression"); - // HACK: quick hack before we implement a parser - - Evaluable eval = null; - if (expression.startsWith("replace(this.value,")) { - // HACK: huge hack + try { + Evaluable eval = new Parser(expression).getExpression(); + //System.out.println("--- " + eval.toString()); + Properties bindings = new Properties(); + List cellChanges = new ArrayList(project.rows.size()); - String s = "[" + expression.substring( - "replace(this.value,".length(), expression.length() - 1) + "]"; - - try { - JSONTokener t = new JSONTokener(s); - JSONArray a = (JSONArray) t.nextValue(); - - eval = new FunctionCallExpr(new Evaluable[] { - new FieldAccessorExpr(new VariableExpr("this"), "value"), - new LiteralExpr(a.get(0)), - new LiteralExpr(a.get(1)) - }, new Replace()); - - } catch (JSONException e) { - } - } else { - Function f = null; - if (expression.equals("toUppercase(this.value)")) { - f = new ToUppercase(); - } else if (expression.equals("toLowercase(this.value)")) { - f = new ToLowercase(); - } else if (expression.equals("toTitlecase(this.value)")) { - f = new ToTitlecase(); - } - - eval = new FunctionCallExpr(new Evaluable[] { - new FieldAccessorExpr(new VariableExpr("this"), "value") - }, f); - } - - Properties bindings = new Properties(); - List cellChanges = new ArrayList(project.rows.size()); - - for (int r = 0; r < project.rows.size(); r++) { - Row row = project.rows.get(r); - if (cellIndex < row.cells.size()) { - Cell cell = row.cells.get(cellIndex); - if (cell.value == null) { - continue; + for (int r = 0; r < project.rows.size(); r++) { + Row row = project.rows.get(r); + if (cellIndex < row.cells.size()) { + Cell cell = row.cells.get(cellIndex); + if (cell.value == null) { + continue; + } + + bindings.put("this", cell); + bindings.put("value", cell.value); + + Cell newCell = new Cell(); + newCell.value = eval.evaluate(bindings); + newCell.recon = cell.recon; + + CellChange cellChange = new CellChange(r, cellIndex, cell, newCell); + cellChanges.add(cellChange); } - - bindings.put("this", cell); - - Cell newCell = new Cell(); - newCell.value = eval.evaluate(bindings); - newCell.recon = cell.recon; - - CellChange cellChange = new CellChange(r, cellIndex, cell, newCell); - cellChanges.add(cellChange); } + + MassCellChange massCellChange = new MassCellChange(cellChanges); + HistoryEntry historyEntry = new HistoryEntry( + project, "Text transform on " + columnName + ": " + expression, massCellChange); + + boolean done = project.processManager.queueProcess( + new QuickHistoryEntryProcess(project, historyEntry)); + + respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }"); + + } catch (Exception e) { + respondException(response, e); } - - MassCellChange massCellChange = new MassCellChange(cellChanges); - HistoryEntry historyEntry = new HistoryEntry(project, "Text transform: " + expression, massCellChange); - - boolean done = project.processManager.queueProcess( - new QuickHistoryEntryProcess(project, historyEntry)); - - respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }"); } } diff --git a/src/main/java/com/metaweb/gridlock/expr/FieldAccessorExpr.java b/src/main/java/com/metaweb/gridlock/expr/FieldAccessorExpr.java index 6e5a53587..e76520ba4 100644 --- a/src/main/java/com/metaweb/gridlock/expr/FieldAccessorExpr.java +++ b/src/main/java/com/metaweb/gridlock/expr/FieldAccessorExpr.java @@ -20,4 +20,8 @@ public class FieldAccessorExpr implements Evaluable { return null; } + @Override + public String toString() { + return _inner.toString() + "." + _fieldName; + } } diff --git a/src/main/java/com/metaweb/gridlock/expr/FunctionCallExpr.java b/src/main/java/com/metaweb/gridlock/expr/FunctionCallExpr.java index a62b387cf..96169ef7e 100644 --- a/src/main/java/com/metaweb/gridlock/expr/FunctionCallExpr.java +++ b/src/main/java/com/metaweb/gridlock/expr/FunctionCallExpr.java @@ -20,4 +20,17 @@ public class FunctionCallExpr implements Evaluable { return _function.call(bindings, args); } + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + + for (Evaluable ev : _args) { + if (sb.length() > 0) { + sb.append(", "); + } + sb.append(ev.toString()); + } + + return _function.getClass().getSimpleName() + "(" + sb.toString() + ")"; + } } diff --git a/src/main/java/com/metaweb/gridlock/expr/LiteralExpr.java b/src/main/java/com/metaweb/gridlock/expr/LiteralExpr.java index 02d6875e4..ea304cf14 100644 --- a/src/main/java/com/metaweb/gridlock/expr/LiteralExpr.java +++ b/src/main/java/com/metaweb/gridlock/expr/LiteralExpr.java @@ -2,6 +2,8 @@ package com.metaweb.gridlock.expr; import java.util.Properties; +import org.json.JSONObject; + public class LiteralExpr implements Evaluable { final protected Object _value; @@ -14,4 +16,8 @@ public class LiteralExpr implements Evaluable { return _value; } + @Override + public String toString() { + return _value instanceof String ? JSONObject.quote((String) _value) : _value.toString(); + } } diff --git a/src/main/java/com/metaweb/gridlock/expr/OperatorCallExpr.java b/src/main/java/com/metaweb/gridlock/expr/OperatorCallExpr.java new file mode 100644 index 000000000..020cc92be --- /dev/null +++ b/src/main/java/com/metaweb/gridlock/expr/OperatorCallExpr.java @@ -0,0 +1,66 @@ +package com.metaweb.gridlock.expr; + +import java.util.Properties; + +public class OperatorCallExpr implements Evaluable { + final protected Evaluable[] _args; + final protected String _op; + + public OperatorCallExpr(Evaluable[] args, String op) { + _args = args; + _op = op; + } + + @Override + public Object evaluate(Properties bindings) { + Object[] args = new Object[_args.length]; + for (int i = 0; i < _args.length; i++) { + args[i] = _args[i].evaluate(bindings); + } + + if ("+".equals(_op)) { + if (args.length == 2) { + if (args[0] instanceof Number && args[1] instanceof Number) { + return ((Number) args[0]).doubleValue() + ((Number) args[1]).doubleValue(); + } else { + return args[0].toString() + args[1].toString(); + } + } + } else if ("-".equals(_op)) { + if (args.length == 2) { + if (args[0] instanceof Number && args[1] instanceof Number) { + return ((Number) args[0]).doubleValue() - ((Number) args[1]).doubleValue(); + } + } + } else if ("*".equals(_op)) { + if (args.length == 2) { + if (args[0] instanceof Number && args[1] instanceof Number) { + return ((Number) args[0]).doubleValue() * ((Number) args[1]).doubleValue(); + } + } + } else if ("/".equals(_op)) { + if (args.length == 2) { + if (args[0] instanceof Number && args[1] instanceof Number) { + return ((Number) args[0]).doubleValue() / ((Number) args[1]).doubleValue(); + } + } + } + return null; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + + for (Evaluable ev : _args) { + if (sb.length() > 0) { + sb.append(' '); + sb.append(_op); + sb.append(' '); + } + sb.append(ev.toString()); + } + + return sb.toString(); + } +} diff --git a/src/main/java/com/metaweb/gridlock/expr/Parser.java b/src/main/java/com/metaweb/gridlock/expr/Parser.java new file mode 100644 index 000000000..a6e1abedf --- /dev/null +++ b/src/main/java/com/metaweb/gridlock/expr/Parser.java @@ -0,0 +1,242 @@ +package com.metaweb.gridlock.expr; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import com.metaweb.gridlock.expr.Scanner.NumberToken; +import com.metaweb.gridlock.expr.Scanner.Token; +import com.metaweb.gridlock.expr.Scanner.TokenType; +import com.metaweb.gridlock.expr.functions.Replace; +import com.metaweb.gridlock.expr.functions.Slice; +import com.metaweb.gridlock.expr.functions.ToLowercase; +import com.metaweb.gridlock.expr.functions.ToTitlecase; +import com.metaweb.gridlock.expr.functions.ToUppercase; + +public class Parser { + protected Scanner _scanner; + protected Token _token; + protected Evaluable _root; + + static public Map functionTable = new HashMap(); + static { + functionTable.put("toUppercase", new ToUppercase()); + functionTable.put("toLowercase", new ToLowercase()); + functionTable.put("toTitlecase", new ToTitlecase()); + functionTable.put("slice", new Slice()); + functionTable.put("substring", new Slice()); + functionTable.put("replace", new Replace()); + } + + public Parser(String s) throws Exception { + this(s, 0, s.length()); + } + + public Parser(String s, int from, int to) throws Exception { + _scanner = new Scanner(s, from, to); + _token = _scanner.next(); + + _root = parseExpression(); + } + + public Evaluable getExpression() { + return _root; + } + + protected void next() { + _token = _scanner.next(); + } + + protected Exception makeException(String desc) { + int index = _token != null ? _token.start : _scanner.getIndex(); + + return new Exception("Parsing error at offset " + index + ": " + desc); + } + + protected Evaluable parseExpression() throws Exception { + Evaluable sub = parseSubExpression(); + + while (_token != null && + _token.type == TokenType.Operator && + ">=<==!=".indexOf(_token.text) >= 0) { + + String op = _token.text; + + next(); + + Evaluable sub2 = parseSubExpression(); + + sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op); + } + + return sub; + } + + protected Evaluable parseSubExpression() throws Exception { + Evaluable sub = parseTerm(); + + while (_token != null && + _token.type == TokenType.Operator && + "+-".indexOf(_token.text) >= 0) { + + String op = _token.text; + + next(); + + Evaluable sub2 = parseSubExpression(); + + sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op); + } + + return sub; + } + + protected Evaluable parseTerm() throws Exception { + Evaluable factor = parseFactor(); + + while (_token != null && + _token.type == TokenType.Operator && + "*/".indexOf(_token.text) >= 0) { + + String op = _token.text; + + next(); + + Evaluable factor2 = parseFactor(); + + factor = new OperatorCallExpr(new Evaluable[] { factor, factor2 }, op); + } + + return factor; + } + + protected Evaluable parseFactor() throws Exception { + if (_token == null) { + throw makeException("Expression ends too early"); + } + + Evaluable eval = null; + + if (_token.type == TokenType.String) { + eval = new LiteralExpr(_token.text); + next(); + } else if (_token.type == TokenType.Number) { + eval = new LiteralExpr(((NumberToken)_token).value); + next(); + } else if (_token.type == TokenType.Operator && _token.text.equals("-")) { // unary minus? + next(); + + if (_token != null && _token.type == TokenType.Number) { + eval = new LiteralExpr(-((NumberToken)_token).value); + next(); + } else { + throw makeException("Bad negative number"); + } + } else if (_token.type == TokenType.Identifier) { + String text = _token.text; + next(); + + if (_token == null || _token.type != TokenType.Delimiter || !_token.text.equals("(")) { + eval = new VariableExpr(text); + } else { + Function f = functionTable.get(text); + if (f == null) { + throw makeException("Unknown function " + text); + } + + next(); // swallow ( + + List args = parseExpressionList(")"); + + eval = new FunctionCallExpr(makeArray(args), f); + } + } else if (_token.type == TokenType.Delimiter && _token.text.equals("(")) { + next(); + + eval = parseExpression(); + + if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(")")) { + next(); + } else { + throw makeException("Missing )"); + } + } else { + throw makeException("Missing number, string, identifier, or parenthesized expression"); + } + + while (_token != null) { + if (_token.type == TokenType.Operator && _token.text.equals(".")) { + next(); // swallow . + + if (_token == null || _token.type != TokenType.Identifier) { + throw makeException("Missing function name"); + } + + String identifier = _token.text; + Function f = functionTable.get(identifier); + if (f == null) { + throw makeException("Unknown function " + identifier); + } + next(); + + if (_token == null || _token.type != TokenType.Delimiter || !_token.text.equals("(")) { + throw makeException("Missing ("); + } + next(); + + List args = parseExpressionList(")"); + args.add(0, eval); + + eval = new FunctionCallExpr(makeArray(args), f); + + } else if (_token.type == TokenType.Delimiter && _token.text.equals("[")) { + next(); // swallow [ + + List args = parseExpressionList("]"); + args.add(0, eval); + + eval = new FunctionCallExpr(makeArray(args), functionTable.get("slice")); + } else { + break; + } + } + + return eval; + } + + protected List parseExpressionList(String closingDelimiter) throws Exception { + List l = new LinkedList(); + + if (_token != null && + (_token.type != TokenType.Delimiter || !_token.text.equals(closingDelimiter))) { + + while (_token != null) { + Evaluable eval = parseExpression(); + + l.add(eval); + + if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(",")) { + next(); // swallow comma, loop back for more + } else { + break; + } + } + } + + if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(closingDelimiter)) { + next(); // swallow closing delimiter + } else { + throw makeException("Missing " + closingDelimiter); + } + + return l; + } + + protected Evaluable[] makeArray(List l) { + Evaluable[] a = new Evaluable[l.size()]; + l.toArray(a); + + return a; + } +} diff --git a/src/main/java/com/metaweb/gridlock/expr/Scanner.java b/src/main/java/com/metaweb/gridlock/expr/Scanner.java new file mode 100644 index 000000000..3bc8c10d4 --- /dev/null +++ b/src/main/java/com/metaweb/gridlock/expr/Scanner.java @@ -0,0 +1,228 @@ +package com.metaweb.gridlock.expr; + +public class Scanner { + static public enum TokenType { + Error, + Delimiter, + Operator, + Identifier, + Number, + String + } + + static public class Token { + final public int start; + final public int end; + final public TokenType type; + final public String text; + + Token(int start, int end, TokenType type, String text) { + this.start = start; + this.end = end; + this.type = type; + this.text = text; + } + } + + static public class ErrorToken extends Token { + final public String detail; // error detail + + public ErrorToken(int start, int end, String text, String detail) { + super(start, end, TokenType.Error, text); + this.detail = detail; + } + } + + static public class NumberToken extends Token { + final public double value; + + public NumberToken(int start, int end, String text, double value) { + super(start, end, TokenType.Number, text); + this.value = value; + } + } + + protected String _text; + protected int _index; + protected int _limit; + + public Scanner(String s) { + this(s, 0, s.length()); + } + + public Scanner(String s, int from, int to) { + _text = s; + _index = from; + _limit = to; + } + + public int getIndex() { + return _index; + } + + public Token next() { + // skip whitespace + while (_index < _limit && Character.isWhitespace(_text.charAt(_index))) { + _index++; + } + if (_index == _limit) { + return null; + } + + char c = _text.charAt(_index); + int start = _index; + String detail = null; + + if (Character.isDigit(c)) { // number literal + double value = 0; + + while (_index < _limit && Character.isDigit(c = _text.charAt(_index))) { + value = value * 10 + (c - '0'); + _index++; + } + + if (_index < _limit && c == '.') { + _index++; + + double division = 1; + while (_index < _limit && Character.isDigit(c = _text.charAt(_index))) { + value = value * 10 + (c - '0'); + division *= 10; + _index++; + } + + value /= division; + } + + // TODO: support exponent e notation + + return new NumberToken( + start, + _index, + _text.substring(start, _index), + value + ); + } else if (c == '"' || c == '\'') { + /* + * String Literal + */ + + StringBuffer sb = new StringBuffer(); + char delimiter = c; + + _index++; // skip opening delimiter + + while (_index < _limit) { + c = _text.charAt(_index); + if (c == delimiter) { + _index++; // skip closing delimiter + + return new Token( + start, + _index, + TokenType.String, + sb.toString() + ); + } else if (c == '\\') { + _index++; // skip escaping marker + if (_index < _limit) { + sb.append(_text.charAt(_index)); + } + } else { + sb.append(c); + } + _index++; + } + + detail = "String not properly closed"; + // fall through + + } else if (Character.isLetter(c)) { // identifier + while (_index < _limit && Character.isLetterOrDigit(_text.charAt(_index))) { + _index++; + } + + return new Token( + start, + _index, + TokenType.Identifier, + _text.substring(start, _index) + ); + } else if ("+-*/.".indexOf(c) >= 0) { // operator + _index++; + + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } else if ("()[],".indexOf(c) >= 0) { // delimiter + _index++; + + return new Token( + start, + _index, + TokenType.Delimiter, + _text.substring(start, _index) + ); + } else if (c == '!' && _index < _limit - 1 && _text.charAt(_index + 1) == '=') { + _index += 2; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } else if (c == '<') { + if (_index < _limit - 1 && + (_text.charAt(_index + 1) == '=' || + _text.charAt(_index + 1) == '>')) { + + _index += 2; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } else { + _index++; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } + } else if (">=".indexOf(c) >= 0) { // operator + if (_index < _limit - 1 && _text.charAt(_index + 1) == '=') { + _index += 2; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } else { + _index++; + return new Token( + start, + _index, + TokenType.Operator, + _text.substring(start, _index) + ); + } + } else { + _index++; + detail = "Unrecognized symbol"; + } + + return new ErrorToken( + start, + _index, + _text.substring(start, _index), + detail + ); + } +} diff --git a/src/main/java/com/metaweb/gridlock/expr/VariableExpr.java b/src/main/java/com/metaweb/gridlock/expr/VariableExpr.java index aea24d896..19fe55eef 100644 --- a/src/main/java/com/metaweb/gridlock/expr/VariableExpr.java +++ b/src/main/java/com/metaweb/gridlock/expr/VariableExpr.java @@ -2,6 +2,8 @@ package com.metaweb.gridlock.expr; import java.util.Properties; +import org.json.JSONObject; + public class VariableExpr implements Evaluable { final protected String _name; @@ -14,4 +16,8 @@ public class VariableExpr implements Evaluable { return bindings.get(_name); } + @Override + public String toString() { + return _name; + } } diff --git a/src/main/java/com/metaweb/gridlock/expr/functions/Slice.java b/src/main/java/com/metaweb/gridlock/expr/functions/Slice.java new file mode 100644 index 000000000..332e313e1 --- /dev/null +++ b/src/main/java/com/metaweb/gridlock/expr/functions/Slice.java @@ -0,0 +1,64 @@ +package com.metaweb.gridlock.expr.functions; + +import java.lang.reflect.Array; +import java.util.Properties; + +import com.metaweb.gridlock.expr.Function; + +public class Slice implements Function { + + @Override + public Object call(Properties bindings, Object[] args) { + if (args.length > 1 && args.length <= 3) { + Object v = args[0]; + Object from = args[1]; + Object to = args.length == 3 ? args[2] : null; + + if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) { + if (v instanceof Array) { + Object[] a = (Object[]) v; + int start = ((Number) from).intValue(); + int end = to != null && to instanceof Number ? + ((Number) to).intValue() : a.length; + + if (start < 0) { + start = a.length - start; + } + start = Math.min(a.length, Math.max(0, start)); + + if (end < 0) { + end = a.length - end; + } + end = Math.min(a.length, Math.max(start, end)); + + Object[] a2 = new Object[end - start]; + System.arraycopy(a, start, a2, 0, end - start); + + return a2; + } else { + String s = (v instanceof String ? (String) v : v.toString()); + + int start = ((Number) from).intValue(); + if (start < 0) { + start = s.length() - start; + } + start = Math.min(s.length(), Math.max(0, start)); + + if (to != null && to instanceof Number) { + int end = ((Number) to).intValue(); + if (end < 0) { + end = s.length() - end; + } + end = Math.min(s.length(), Math.max(start, end)); + + return s.substring(start, end); + } else { + return s.substring(start); + } + } + } + } + return null; + } + +} diff --git a/src/main/webapp/scripts/project/data-table-view.js b/src/main/webapp/scripts/project/data-table-view.js index 40373d0a0..d98a5921e 100644 --- a/src/main/webapp/scripts/project/data-table-view.js +++ b/src/main/webapp/scripts/project/data-table-view.js @@ -218,21 +218,21 @@ DataTableView.prototype._createMenuForColumnHeader = function(column, index, elm submenu: [ { label: "To Titlecase", - click: function() { self._doTextTransform(column, "toTitlecase(this.value)"); } + click: function() { self._doTextTransform(column, "toTitlecase(value)"); } }, { label: "To Uppercase", - click: function() { self._doTextTransform(column, "toUppercase(this.value)"); } + click: function() { self._doTextTransform(column, "toUppercase(value)"); } }, { label: "To Lowercase", - click: function() { self._doTextTransform(column, "toLowercase(this.value)"); } + click: function() { self._doTextTransform(column, "toLowercase(value)"); } }, {}, { label: "Custom Expression ...", click: function() { - var expression = window.prompt("Enter expression", 'replace(this.value,"","")'); + var expression = window.prompt("Enter expression", 'replace(value, "", "")'); if (expression != null) { self._doTextTransform(column, expression); }