Implemented expression parser.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@10 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridlock/browsing/accessors/CellAccessor.java

@@ -3,5 +3,5 @@ package com.metaweb.gridlock.browsing.accessors;
 import com.metaweb.gridlock.model.Cell;
 
 public interface CellAccessor {
-	public Object[] get(Cell cell);
+	public Object[] get(Cell cell, boolean decorated);
 }

src/main/java/com/metaweb/gridlock/browsing/accessors/DecoratedValue.java

@@ -0,0 +1,11 @@
+package com.metaweb.gridlock.browsing.accessors;
+
+public class DecoratedValue {
+	final public Object value;
+	final public String	label;
+	
+	public DecoratedValue(Object value, String label) {
+		this.value = value;
+		this.label = label;
+	}
+}

src/main/java/com/metaweb/gridlock/browsing/accessors/ReconFeatureCellAccessor.java

@@ -10,7 +10,7 @@ public class ReconFeatureCellAccessor implements CellAccessor {
 	}
 	
 	@Override
-	public Object[] get(Cell cell) {
+	public Object[] get(Cell cell, boolean decorated) {
 		if (cell.recon != null) {
 			return new Object[] { cell.recon.features.get(_name) };
 		}

src/main/java/com/metaweb/gridlock/browsing/accessors/ReconTypeAccessor.java

@@ -5,7 +5,7 @@ import com.metaweb.gridlock.model.ReconCandidate;
 
 public class ReconTypeAccessor implements CellAccessor {
 	@Override
-	public Object[] get(Cell cell) {
+	public Object[] get(Cell cell, boolean decorated) {
 		if (cell.recon != null && cell.recon.candidates.size() > 0) {
 			ReconCandidate c = cell.recon.candidates.get(0);
 			return c.typeIDs;

src/main/java/com/metaweb/gridlock/browsing/accessors/ValueCellAccessor.java

@@ -4,7 +4,7 @@ import com.metaweb.gridlock.model.Cell;
 
 public class ValueCellAccessor implements CellAccessor {
 	@Override
-	public Object[] get(Cell cell) {
+	public Object[] get(Cell cell, boolean decorated) {
 		if (cell.value != null) {
 			return new Object[] { cell.value };
 		}

src/main/java/com/metaweb/gridlock/browsing/facets/CellAccessorNominalRowGrouper.java

@@ -5,6 +5,7 @@ import java.util.Map;
 
 import com.metaweb.gridlock.browsing.RowVisitor;
 import com.metaweb.gridlock.browsing.accessors.CellAccessor;
+import com.metaweb.gridlock.browsing.accessors.DecoratedValue;
 import com.metaweb.gridlock.model.Cell;
 import com.metaweb.gridlock.model.Row;
 
@@ -12,7 +13,7 @@ public class CellAccessorNominalRowGrouper implements RowVisitor {
 	final protected CellAccessor 	_accessor;
 	final protected int 			_cellIndex;
 	
-	final public Map<Object, NominalFacetChoice> groups = new HashMap<Object, NominalFacetChoice>();
+	final public Map<Object, NominalFacetChoice> choices = new HashMap<Object, NominalFacetChoice>();
 	
 	public CellAccessorNominalRowGrouper(CellAccessor accessor, int cellIndex) {
 		_accessor = accessor;
@@ -24,18 +25,23 @@ public class CellAccessorNominalRowGrouper implements RowVisitor {
 		if (_cellIndex < row.cells.size()) {
 			Cell cell = row.cells.get(_cellIndex);
 			if (cell != null) {
-				Object[] values = _accessor.get(cell);
+				Object[] values = _accessor.get(cell, true);
 				if (values != null && values.length > 0) {
-					for (Object v : values) {
+					for (Object value : values) {
-						if (v != null) {
+						if (value != null) {
-							if (groups.containsKey(v)) {
+							DecoratedValue dValue = 
-								groups.get(v).count++;
+								value instanceof DecoratedValue ?
-							} else {
+									(DecoratedValue) value : 
-								NominalFacetChoice group = new NominalFacetChoice();
+									new DecoratedValue(value, value.toString());
-								group.value = v;
-								group.count = 1;
 							
-								groups.put(v, group);
+							Object v = dValue.value;
+							if (choices.containsKey(value)) {
+								choices.get(value).count++;
+							} else {
+								NominalFacetChoice choice = new NominalFacetChoice(dValue, v);
+								choice.count = 1;
+								
+								choices.put(v, choice);
 							}
 						}
 					}

src/main/java/com/metaweb/gridlock/browsing/facets/ListFacet.java

@@ -1,13 +1,18 @@
 package com.metaweb.gridlock.browsing.facets;
 
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import com.metaweb.gridlock.browsing.FilteredRows;
 import com.metaweb.gridlock.browsing.filters.RowFilter;
 
 public class ListFacet implements Facet {
+	final protected List<Object> _choices = new LinkedList<Object>();
 
 	@Override
 	public JSONObject getJSON(Properties options) throws JSONException {
@@ -15,6 +20,16 @@ public class ListFacet implements Facet {
 		return null;
 	}
 
+	@Override
+	public void initializeFromJSON(JSONObject o) throws JSONException {
+		JSONArray a = o.getJSONArray("choices");
+		int length = a.length();
+		
+		for (int i = 0; i < length; i++) {
+			
+		}
+	}
+
 	@Override
 	public RowFilter getRowFilter() {
 		// TODO Auto-generated method stub
@@ -22,7 +37,7 @@ public class ListFacet implements Facet {
 	}
 
 	@Override
-	public void initializeFromJSON(JSONObject o) throws JSONException {
+	public void computeChoices(FilteredRows filteredRows) {
 		// TODO Auto-generated method stub
 		
 	}

src/main/java/com/metaweb/gridlock/browsing/facets/NominalFacetChoice.java

@@ -1,6 +1,14 @@
 package com.metaweb.gridlock.browsing.facets;
 
+import com.metaweb.gridlock.browsing.accessors.DecoratedValue;
+
 public class NominalFacetChoice {
-	public Object	value;
+	final public DecoratedValue	decoratedValue;
+	final public Object			value;
 	public int					count;
+	
+	public NominalFacetChoice(DecoratedValue decoratedValue, Object value) {
+		this.decoratedValue = decoratedValue;
+		this.value = value;
+	}
 }

src/main/java/com/metaweb/gridlock/browsing/filters/CellAccessorEqualRowFilter.java

@@ -20,7 +20,7 @@ public class CellAccessorEqualRowFilter implements RowFilter {
 		if (_cellIndex < row.cells.size()) {
 			Cell cell = row.cells.get(_cellIndex);
 			if (cell != null) {
-				Object[] values = _accessor.get(cell);
+				Object[] values = _accessor.get(cell, false);
 				if (values != null && values.length > 0) {
 					for (Object v : values) {
 						for (Object match : _matches) {

src/main/java/com/metaweb/gridlock/commands/Command.java

@@ -5,7 +5,9 @@ import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
 import java.io.Reader;
+import java.io.StringWriter;
 import java.util.Properties;
 
 import javax.servlet.ServletException;
@@ -68,13 +70,29 @@ public abstract class Command {
     }
     
     protected void respondJSON(HttpServletResponse response, JSONObject o) throws IOException {
+    	response.setHeader("Content-Type", "application/json");
     	respond(response, o.toString());
     }
     
     protected void respondException(HttpServletResponse response, Exception e) throws IOException {
-    	response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+    	try {
+        	JSONObject o = new JSONObject();
+        	o.put("code", "error");
+			o.put("message", e.getMessage());
+			
+			StringWriter sw = new StringWriter();
+			PrintWriter pw = new PrintWriter(sw);
+			e.printStackTrace(pw);
+			pw.flush();
+			sw.flush();
+			
+			o.put("stack", sw.toString());
+			
+			respondJSON(response, o);
+		} catch (JSONException e1) {
 	   		e.printStackTrace(response.getWriter());
 		}
+    }
     
     protected void redirect(HttpServletResponse response, String url) throws IOException {
     	response.setStatus(HttpServletResponse.SC_OK);

src/main/java/com/metaweb/gridlock/commands/DoTextTransformCommand.java

@@ -9,24 +9,14 @@ import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONTokener;
 
 import com.metaweb.gridlock.expr.Evaluable;
-import com.metaweb.gridlock.expr.FieldAccessorExpr;
+import com.metaweb.gridlock.expr.Parser;
-import com.metaweb.gridlock.expr.Function;
-import com.metaweb.gridlock.expr.FunctionCallExpr;
-import com.metaweb.gridlock.expr.LiteralExpr;
-import com.metaweb.gridlock.expr.VariableExpr;
-import com.metaweb.gridlock.expr.functions.Replace;
-import com.metaweb.gridlock.expr.functions.ToLowercase;
-import com.metaweb.gridlock.expr.functions.ToTitlecase;
-import com.metaweb.gridlock.expr.functions.ToUppercase;
 import com.metaweb.gridlock.history.CellChange;
 import com.metaweb.gridlock.history.HistoryEntry;
 import com.metaweb.gridlock.history.MassCellChange;
 import com.metaweb.gridlock.model.Cell;
+import com.metaweb.gridlock.model.Column;
 import com.metaweb.gridlock.model.Project;
 import com.metaweb.gridlock.model.Row;
 import com.metaweb.gridlock.process.QuickHistoryEntryProcess;
@@ -40,44 +30,19 @@ public class DoTextTransformCommand extends Command {
 		Project project = getProject(request);
 		int cellIndex = Integer.parseInt(request.getParameter("cell"));
 		
+		String columnName = null;
+		for (Column column : project.columnModel.columns) {
+			if (column.cellIndex == cellIndex) {
+				columnName = column.headerLabel;
+				break;
+			}
+		}
+		
 		String expression = request.getParameter("expression");
 		
-		// HACK: quick hack before we implement a parser
-		
-		Evaluable eval = null;
-		if (expression.startsWith("replace(this.value,")) {
-			// HACK: huge hack
-			
-			String s = "[" + expression.substring(
-					"replace(this.value,".length(), expression.length() - 1) + "]";
-			
 		try {
-		    	JSONTokener t = new JSONTokener(s);
+			Evaluable eval = new Parser(expression).getExpression();
-		    	JSONArray a = (JSONArray) t.nextValue();
+			//System.out.println("--- " + eval.toString());
-		    	
-				eval = new FunctionCallExpr(new Evaluable[] {
-						new FieldAccessorExpr(new VariableExpr("this"), "value"),
-						new LiteralExpr(a.get(0)),
-						new LiteralExpr(a.get(1))
-					}, new Replace());
-				
-			} catch (JSONException e) {
-			}
-		} else {
-			Function f = null;
-			if (expression.equals("toUppercase(this.value)")) {
-				f = new ToUppercase();
-			} else if (expression.equals("toLowercase(this.value)")) {
-				f = new ToLowercase();
-			} else if (expression.equals("toTitlecase(this.value)")) {
-				f = new ToTitlecase();
-			}
-		
-			eval = new FunctionCallExpr(new Evaluable[] {
-				new FieldAccessorExpr(new VariableExpr("this"), "value")
-			}, f);
-		}
-		
 			Properties bindings = new Properties();
 			List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
 			
@@ -90,6 +55,7 @@ public class DoTextTransformCommand extends Command {
 					}
 					
 					bindings.put("this", cell);
+					bindings.put("value", cell.value);
 					
 					Cell newCell = new Cell();
 					newCell.value = eval.evaluate(bindings);
@@ -101,11 +67,16 @@ public class DoTextTransformCommand extends Command {
 			}
 			
 			MassCellChange massCellChange = new MassCellChange(cellChanges);
-		HistoryEntry historyEntry = new HistoryEntry(project, "Text transform: " + expression, massCellChange);
+			HistoryEntry historyEntry = new HistoryEntry(
+				project, "Text transform on " + columnName + ": " + expression, massCellChange);
 			
 			boolean done = project.processManager.queueProcess(
 					new QuickHistoryEntryProcess(project, historyEntry));
 			
 			respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
+			
+		} catch (Exception e) {
+			respondException(response, e);
+		}
 	}
 }

src/main/java/com/metaweb/gridlock/expr/FieldAccessorExpr.java

@@ -20,4 +20,8 @@ public class FieldAccessorExpr implements Evaluable {
 		return null;
 	}
 
+	@Override
+	public String toString() {
+		return _inner.toString() + "." + _fieldName;
+	}
 }

src/main/java/com/metaweb/gridlock/expr/FunctionCallExpr.java

@@ -20,4 +20,17 @@ public class FunctionCallExpr implements Evaluable {
 		return _function.call(bindings, args);
 	}
 
+	@Override
+	public String toString() {
+		StringBuffer sb = new StringBuffer();
+		
+		for (Evaluable ev : _args) {
+			if (sb.length() > 0) {
+				sb.append(", ");
+			}
+			sb.append(ev.toString());
+		}
+		
+		return _function.getClass().getSimpleName() + "(" + sb.toString() + ")";
+	}
 }

src/main/java/com/metaweb/gridlock/expr/LiteralExpr.java

@@ -2,6 +2,8 @@ package com.metaweb.gridlock.expr;
 
 import java.util.Properties;
 
+import org.json.JSONObject;
+
 public class LiteralExpr implements Evaluable {
 	final protected Object _value;
 	
@@ -14,4 +16,8 @@ public class LiteralExpr implements Evaluable {
 		return _value;
 	}
 
+	@Override
+	public String toString() {
+		return _value instanceof String ? JSONObject.quote((String) _value) : _value.toString();
+	}
 }

src/main/java/com/metaweb/gridlock/expr/OperatorCallExpr.java

@@ -0,0 +1,66 @@
+package com.metaweb.gridlock.expr;
+
+import java.util.Properties;
+
+public class OperatorCallExpr implements Evaluable {
+	final protected Evaluable[] _args;
+	final protected String		_op;
+	
+	public OperatorCallExpr(Evaluable[] args, String op) {
+		_args = args;
+		_op = op;
+	}
+	                          
+	@Override
+	public Object evaluate(Properties bindings) {
+		Object[] args = new Object[_args.length];
+		for (int i = 0; i < _args.length; i++) {
+			args[i] = _args[i].evaluate(bindings);
+		}
+		
+		if ("+".equals(_op)) {
+			if (args.length == 2) {
+				if (args[0] instanceof Number && args[1] instanceof Number) {
+					return ((Number) args[0]).doubleValue() + ((Number) args[1]).doubleValue();
+				} else {
+					return args[0].toString() + args[1].toString();
+				}
+			}
+		} else if ("-".equals(_op)) {
+			if (args.length == 2) {
+				if (args[0] instanceof Number && args[1] instanceof Number) {
+					return ((Number) args[0]).doubleValue() - ((Number) args[1]).doubleValue();
+				}
+			}
+		} else if ("*".equals(_op)) {
+			if (args.length == 2) {
+				if (args[0] instanceof Number && args[1] instanceof Number) {
+					return ((Number) args[0]).doubleValue() * ((Number) args[1]).doubleValue();
+				}
+			}
+		} else if ("/".equals(_op)) {
+			if (args.length == 2) {
+				if (args[0] instanceof Number && args[1] instanceof Number) {
+					return ((Number) args[0]).doubleValue() / ((Number) args[1]).doubleValue();
+				}
+			}
+		}
+		return null;
+	}
+
+	@Override
+	public String toString() {
+		StringBuffer sb = new StringBuffer();
+		
+		for (Evaluable ev : _args) {
+			if (sb.length() > 0) {
+				sb.append(' ');
+				sb.append(_op);
+				sb.append(' ');
+			}
+			sb.append(ev.toString());
+		}
+		
+		return sb.toString();
+	}
+}

src/main/java/com/metaweb/gridlock/expr/Parser.java

@@ -0,0 +1,242 @@
+package com.metaweb.gridlock.expr;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import com.metaweb.gridlock.expr.Scanner.NumberToken;
+import com.metaweb.gridlock.expr.Scanner.Token;
+import com.metaweb.gridlock.expr.Scanner.TokenType;
+import com.metaweb.gridlock.expr.functions.Replace;
+import com.metaweb.gridlock.expr.functions.Slice;
+import com.metaweb.gridlock.expr.functions.ToLowercase;
+import com.metaweb.gridlock.expr.functions.ToTitlecase;
+import com.metaweb.gridlock.expr.functions.ToUppercase;
+
+public class Parser {
+	protected Scanner 	_scanner;
+	protected Token 	_token;
+	protected Evaluable _root;
+	
+	static public Map<String, Function> functionTable = new HashMap<String, Function>();
+	static {
+		functionTable.put("toUppercase", new ToUppercase());
+		functionTable.put("toLowercase", new ToLowercase());
+		functionTable.put("toTitlecase", new ToTitlecase());
+		functionTable.put("slice", new Slice());
+		functionTable.put("substring", new Slice());
+		functionTable.put("replace", new Replace());
+	}
+	
+	public Parser(String s) throws Exception {
+		this(s, 0, s.length());
+	}
+	
+	public Parser(String s, int from, int to) throws Exception {
+		_scanner = new Scanner(s, from, to);
+		_token = _scanner.next();
+		
+		_root = parseExpression();
+	}
+	
+	public Evaluable getExpression() {
+		return _root;
+	}
+	
+	protected void next() {
+		_token = _scanner.next();
+	}
+	
+	protected Exception makeException(String desc) {
+		int index = _token != null ? _token.start : _scanner.getIndex();
+		
+		return new Exception("Parsing error at offset " + index + ": " + desc);
+	}
+	
+	protected Evaluable parseExpression() throws Exception {
+		Evaluable sub = parseSubExpression();
+		
+		while (_token != null && 
+				_token.type == TokenType.Operator && 
+				">=<==!=".indexOf(_token.text) >= 0) {
+			
+			String op = _token.text;
+			
+			next();
+			
+			Evaluable sub2 = parseSubExpression();
+			
+			sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op);
+		}
+		
+		return sub;
+	}
+	
+	protected Evaluable parseSubExpression() throws Exception {
+		Evaluable sub = parseTerm();
+		
+		while (_token != null && 
+				_token.type == TokenType.Operator && 
+				"+-".indexOf(_token.text) >= 0) {
+			
+			String op = _token.text;
+			
+			next();
+			
+			Evaluable sub2 = parseSubExpression();
+			
+			sub = new OperatorCallExpr(new Evaluable[] { sub, sub2 }, op);
+		}
+		
+		return sub;
+	}
+	
+	protected Evaluable parseTerm() throws Exception {
+		Evaluable factor = parseFactor();
+		
+		while (_token != null && 
+				_token.type == TokenType.Operator && 
+				"*/".indexOf(_token.text) >= 0) {
+			
+			String op = _token.text;
+			
+			next();
+			
+			Evaluable factor2 = parseFactor();
+			
+			factor = new OperatorCallExpr(new Evaluable[] { factor, factor2 }, op);
+		}
+		
+		return factor;
+	}
+	
+	protected Evaluable parseFactor() throws Exception {
+		if (_token == null) {
+			throw makeException("Expression ends too early");
+		}
+		
+		Evaluable eval = null;
+		
+		if (_token.type == TokenType.String) {
+			eval = new LiteralExpr(_token.text);
+			next();
+		} else if (_token.type == TokenType.Number) {
+			eval = new LiteralExpr(((NumberToken)_token).value);
+			next();
+		} else if (_token.type == TokenType.Operator && _token.text.equals("-")) { // unary minus?
+			next();
+			
+			if (_token != null && _token.type == TokenType.Number) {
+				eval = new LiteralExpr(-((NumberToken)_token).value);
+				next();
+			} else {
+				throw makeException("Bad negative number");
+			}
+		} else if (_token.type == TokenType.Identifier) {
+			String text = _token.text;
+			next();
+			
+			if (_token == null || _token.type != TokenType.Delimiter || !_token.text.equals("(")) {
+				eval = new VariableExpr(text);
+			} else {
+				Function f = functionTable.get(text);
+				if (f == null) {
+					throw makeException("Unknown function " + text);
+				}
+				
+				next(); // swallow (
+				
+				List<Evaluable> args = parseExpressionList(")");
+				
+				eval = new FunctionCallExpr(makeArray(args), f);
+			}
+		} else if (_token.type == TokenType.Delimiter && _token.text.equals("(")) {
+			next();
+			
+			eval = parseExpression();
+			
+			if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(")")) {
+				next();
+			} else {
+				throw makeException("Missing )");
+			}
+		} else {
+			throw makeException("Missing number, string, identifier, or parenthesized expression");
+		}
+		
+		while (_token != null) {
+			if (_token.type == TokenType.Operator && _token.text.equals(".")) {
+				next(); // swallow .
+				
+				if (_token == null || _token.type != TokenType.Identifier) {
+					throw makeException("Missing function name");
+				}
+				
+				String identifier = _token.text;
+				Function f = functionTable.get(identifier);
+				if (f == null) {
+					throw makeException("Unknown function " + identifier);
+				}
+				next();
+				
+				if (_token == null || _token.type != TokenType.Delimiter || !_token.text.equals("(")) {
+					throw makeException("Missing (");
+				}
+				next();
+				
+				List<Evaluable> args = parseExpressionList(")");
+				args.add(0, eval);
+				
+				eval = new FunctionCallExpr(makeArray(args), f);
+				
+			} else if (_token.type == TokenType.Delimiter && _token.text.equals("[")) {
+				next(); // swallow [
+				
+				List<Evaluable> args = parseExpressionList("]");
+				args.add(0, eval);
+				
+				eval = new FunctionCallExpr(makeArray(args), functionTable.get("slice"));
+			} else {
+				break;
+			}
+		}
+		
+		return eval;
+	}
+	
+	protected List<Evaluable> parseExpressionList(String closingDelimiter) throws Exception {
+		List<Evaluable> l = new LinkedList<Evaluable>();
+		
+		if (_token != null && 
+			(_token.type != TokenType.Delimiter || !_token.text.equals(closingDelimiter))) {
+			
+			while (_token != null) {
+				Evaluable eval = parseExpression();
+				
+				l.add(eval);
+				
+				if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(",")) {
+					next(); // swallow comma, loop back for more
+				} else {
+					break;
+				}
+			}
+		}
+		
+		if (_token != null && _token.type == TokenType.Delimiter && _token.text.equals(closingDelimiter)) {
+			next(); // swallow closing delimiter
+		} else {
+			throw makeException("Missing " + closingDelimiter);
+		}
+		
+		return l;
+	}
+	
+	protected Evaluable[] makeArray(List<Evaluable> l) {
+		Evaluable[] a = new Evaluable[l.size()];
+		l.toArray(a);
+		
+		return a;
+	}
+}

src/main/java/com/metaweb/gridlock/expr/Scanner.java

@@ -0,0 +1,228 @@
+package com.metaweb.gridlock.expr;
+
+public class Scanner {
+	static public enum TokenType {
+		Error,
+		Delimiter,
+		Operator,
+		Identifier,
+		Number,
+		String
+	}
+	
+	static public class Token {
+		final public int		start;
+		final public int		end;
+		final public TokenType	type;
+		final public String		text;
+		
+		Token(int start, int end, TokenType type, String text) {
+			this.start = start;
+			this.end = end;
+			this.type = type;
+			this.text = text;
+		}
+	}
+	
+	static public class ErrorToken extends Token {
+		final public String		detail; // error detail
+		
+		public ErrorToken(int start, int end, String text, String detail) {
+			super(start, end, TokenType.Error, text);
+			this.detail = detail;
+		}
+	}
+	
+	static public class NumberToken extends Token {
+		final public double value;
+		
+		public NumberToken(int start, int end, String text, double value) {
+			super(start, end, TokenType.Number, text);
+			this.value = value;
+		}
+	}
+	
+	protected String 	_text;
+	protected int		_index;
+	protected int		_limit;
+	
+	public Scanner(String s) {
+		this(s, 0, s.length());
+	}
+	
+	public Scanner(String s, int from, int to) {
+		_text = s;
+		_index = from;
+		_limit = to;
+	}
+	
+	public int getIndex() {
+		return _index;
+	}
+	
+	public Token next() {
+		// skip whitespace
+		while (_index < _limit && Character.isWhitespace(_text.charAt(_index))) {
+			_index++;
+		}
+		if (_index == _limit) {
+			return null;
+		}
+		
+		char c = _text.charAt(_index);
+		int start = _index;
+		String detail = null;
+		
+		if (Character.isDigit(c)) { // number literal
+			double value = 0;
+			
+			while (_index < _limit && Character.isDigit(c = _text.charAt(_index))) {
+				value = value * 10 + (c - '0');
+				_index++;
+			}
+			
+			if (_index < _limit && c == '.') {
+				_index++;
+				
+				double division = 1;
+				while (_index < _limit && Character.isDigit(c = _text.charAt(_index))) {
+					value = value * 10 + (c - '0');
+					division *= 10;
+					_index++;
+				}
+				
+				value /= division;
+			}
+			
+			// TODO: support exponent e notation
+			
+			return new NumberToken(
+				start, 
+				_index, 
+				_text.substring(start, _index),
+				value
+			);
+		} else if (c == '"' || c == '\'') { 
+			/*
+			 *  String Literal
+			 */
+			
+			StringBuffer sb = new StringBuffer();
+			char delimiter = c;
+			
+			_index++; // skip opening delimiter
+			
+			while (_index < _limit) {
+				c = _text.charAt(_index);
+				if (c == delimiter) {
+					_index++; // skip closing delimiter
+					
+					return new Token(
+						start, 
+						_index, 
+						TokenType.String, 
+						sb.toString()
+					);
+				} else if (c == '\\') {
+					_index++; // skip escaping marker
+					if (_index < _limit) {
+						sb.append(_text.charAt(_index));
+					}
+				} else {
+					sb.append(c);
+				}
+				_index++;
+			}
+			
+			detail = "String not properly closed";
+			// fall through
+			
+		} else if (Character.isLetter(c)) { // identifier
+			while (_index < _limit && Character.isLetterOrDigit(_text.charAt(_index))) {
+				_index++;
+			}
+			
+			return new Token(
+				start, 
+				_index, 
+				TokenType.Identifier, 
+				_text.substring(start, _index)
+			);
+		} else if ("+-*/.".indexOf(c) >= 0) { // operator
+			_index++;
+			
+			return new Token(
+				start, 
+				_index, 
+				TokenType.Operator, 
+				_text.substring(start, _index)
+			);
+		} else if ("()[],".indexOf(c) >= 0) { // delimiter
+			_index++;
+			
+			return new Token(
+				start, 
+				_index, 
+				TokenType.Delimiter, 
+				_text.substring(start, _index)
+			);
+		} else if (c == '!' && _index < _limit - 1 && _text.charAt(_index + 1) == '=') {
+			_index += 2;
+			return new Token(
+				start, 
+				_index, 
+				TokenType.Operator, 
+				_text.substring(start, _index)
+			);
+		} else if (c == '<') {
+			if (_index < _limit - 1 && 
+					(_text.charAt(_index + 1) == '=' || 
+					 _text.charAt(_index + 1) == '>')) {
+				
+				_index += 2;
+				return new Token(
+					start, 
+					_index, 
+					TokenType.Operator, 
+					_text.substring(start, _index)
+				);
+			} else {
+				_index++;
+				return new Token(
+					start, 
+					_index, 
+					TokenType.Operator, 
+					_text.substring(start, _index)
+				);
+			}
+		} else if (">=".indexOf(c) >= 0) { // operator
+			if (_index < _limit - 1 && _text.charAt(_index + 1) == '=') {
+				_index += 2;
+				return new Token(
+					start, 
+					_index, 
+					TokenType.Operator, 
+					_text.substring(start, _index)
+				);
+			} else {
+				_index++;
+				return new Token(
+					start, 
+					_index, 
+					TokenType.Operator, 
+					_text.substring(start, _index)
+				);
+			}
+		} else {
+			_index++;
+			detail = "Unrecognized symbol";
+		}
+		
+		return new ErrorToken(
+			start, 
+			_index, 
+			_text.substring(start, _index),
+			detail
+		);
+	}
+}

src/main/java/com/metaweb/gridlock/expr/VariableExpr.java

@@ -2,6 +2,8 @@ package com.metaweb.gridlock.expr;
 
 import java.util.Properties;
 
+import org.json.JSONObject;
+
 public class VariableExpr implements Evaluable {
 	final protected String _name;
 	
@@ -14,4 +16,8 @@ public class VariableExpr implements Evaluable {
 		return bindings.get(_name);
 	}
 
+	@Override
+	public String toString() {
+		return _name;
+	}
 }

src/main/java/com/metaweb/gridlock/expr/functions/Slice.java

@@ -0,0 +1,64 @@
+package com.metaweb.gridlock.expr.functions;
+
+import java.lang.reflect.Array;
+import java.util.Properties;
+
+import com.metaweb.gridlock.expr.Function;
+
+public class Slice implements Function {
+
+	@Override
+	public Object call(Properties bindings, Object[] args) {
+		if (args.length > 1 && args.length <= 3) {
+			Object v = args[0];
+			Object from = args[1];
+			Object to = args.length == 3 ? args[2] : null;
+			
+			if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) {
+				if (v instanceof Array) {
+					Object[] a = (Object[]) v;
+					int start = ((Number) from).intValue();
+					int end = to != null && to instanceof Number ? 
+							((Number) to).intValue() : a.length;
+								
+					if (start < 0) {
+						start = a.length - start;
+					}
+					start = Math.min(a.length, Math.max(0, start));
+					
+					if (end < 0) {
+						end = a.length - end;
+					}
+					end = Math.min(a.length, Math.max(start, end));
+					
+					Object[] a2 = new Object[end - start];
+					System.arraycopy(a, start, a2, 0, end - start);
+					
+					return a2;
+				} else {
+					String s = (v instanceof String ? (String) v : v.toString());
+					
+					int start = ((Number) from).intValue();
+					if (start < 0) {
+						start = s.length() - start;
+					}
+					start = Math.min(s.length(), Math.max(0, start));
+					
+					if (to != null && to instanceof Number) {
+						int end = ((Number) to).intValue();
+						if (end < 0) {
+							end = s.length() - end;
+						}
+						end = Math.min(s.length(), Math.max(start, end));
+						
+						return s.substring(start, end);
+					} else {
+						return s.substring(start);
+					}
+				}
+			}
+		}
+		return null;
+	}
+
+}

src/main/webapp/scripts/project/data-table-view.js

@@ -218,21 +218,21 @@ DataTableView.prototype._createMenuForColumnHeader = function(column, index, elm
             submenu: [
                 {
                     label: "To Titlecase",
-                    click: function() { self._doTextTransform(column, "toTitlecase(this.value)"); }
+                    click: function() { self._doTextTransform(column, "toTitlecase(value)"); }
                 },
                 {
                     label: "To Uppercase",
-                    click: function() { self._doTextTransform(column, "toUppercase(this.value)"); }
+                    click: function() { self._doTextTransform(column, "toUppercase(value)"); }
                 },
                 {
                     label: "To Lowercase",
-                    click: function() { self._doTextTransform(column, "toLowercase(this.value)"); }
+                    click: function() { self._doTextTransform(column, "toLowercase(value)"); }
                 },
                 {},
                 {
                     label: "Custom Expression ...",
                     click: function() {
-                        var expression = window.prompt("Enter expression", 'replace(this.value,"","")');
+                        var expression = window.prompt("Enter expression", 'replace(value, "", "")');
                         if (expression != null) {
                             self._doTextTransform(column, expression);
                         }