Added JSON support in GEL.

Added GEL functions: escape, parseJson, hasField.
Fixed bug in preference store: expression history was still not loaded properly.
Integers are now rendered without decimals in the expression preview dialogs.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1145 7d457c2a-affb-35e4-300a-418c747d4874

main/src/com/google/gridworks/browsing/filters/ExpressionEqualRowFilter.java

@@ -3,6 +3,9 @@ package com.google.gridworks.browsing.filters;
 import java.util.Collection;
 import java.util.Properties;
 
+import org.json.JSONArray;
+import org.json.JSONException;
+
 import com.google.gridworks.browsing.RowFilter;
 import com.google.gridworks.expr.Evaluable;
 import com.google.gridworks.expr.ExpressionUtils;
@@ -75,6 +78,20 @@ public class ExpressionEqualRowFilter implements RowFilter {
                     }
                 }
                 return false;
+            } else if (value instanceof JSONArray) {
+                JSONArray a = (JSONArray) value;
+                int l = a.length();
+                
+                for (int i = 0; i < l; i++) {
+                    try {
+                        if (testValue(a.get(i))) {
+                            return true;
+                        }
+                    } catch (JSONException e) {
+                        // ignore
+                    }
+                }
+                return false;
             } // else, fall through
         }
         
@@ -104,6 +121,20 @@ public class ExpressionEqualRowFilter implements RowFilter {
                     }
                 }
                 return true;
+            } else if (value instanceof JSONArray) {
+                JSONArray a = (JSONArray) value;
+                int l = a.length();
+                
+                for (int i = 0; i < l; i++) {
+                    try {
+                        if (testValue(a.get(i))) {
+                            return false;
+                        }
+                    } catch (JSONException e) {
+                        // ignore
+                    }
+                }
+                return true;
             } // else, fall through
         }
         

main/src/com/google/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java

@@ -3,6 +3,9 @@ package com.google.gridworks.browsing.filters;
 import java.util.Collection;
 import java.util.Properties;
 
+import org.json.JSONArray;
+import org.json.JSONException;
+
 import com.google.gridworks.browsing.RowFilter;
 import com.google.gridworks.browsing.util.RowEvaluable;
 import com.google.gridworks.expr.ExpressionUtils;
@@ -56,6 +59,20 @@ abstract public class ExpressionNumberComparisonRowFilter implements RowFilter {
                     }
                 }
                 return false;
+            } else if (value instanceof JSONArray) {
+                JSONArray a = (JSONArray) value;
+                int l = a.length();
+                
+                for (int i = 0; i < l; i++) {
+                    try {
+                        if (checkValue(a.get(i))) {
+                            return true;
+                        }
+                    } catch (JSONException e) {
+                        // ignore
+                    }
+                }
+                return false;
             } // else, fall through
         }
         

main/src/com/google/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java

@@ -1,7 +1,11 @@
 package com.google.gridworks.browsing.filters;
 
+import java.util.Collection;
 import java.util.Properties;
 
+import org.json.JSONArray;
+import org.json.JSONException;
+
 import com.google.gridworks.browsing.RowFilter;
 import com.google.gridworks.expr.Evaluable;
 import com.google.gridworks.expr.ExpressionUtils;
@@ -39,6 +43,27 @@ abstract public class ExpressionStringComparisonRowFilter implements RowFilter {
                         return true;
                     }
                 }
+            } else if (value instanceof Collection<?>) {
+                for (Object v : ExpressionUtils.toObjectCollection(value)) {
+                    if (checkValue(v.toString())) {
+                        return true;
+                    }
+                }
+                return false;
+            } else if (value instanceof JSONArray) {
+                JSONArray a = (JSONArray) value;
+                int l = a.length();
+                
+                for (int i = 0; i < l; i++) {
+                    try {
+                        if (checkValue(a.get(i).toString())) {
+                            return true;
+                        }
+                    } catch (JSONException e) {
+                        // ignore
+                    }
+                }
+                return false;
             } else {
                 if (checkValue(value instanceof String ? ((String) value) : value.toString())) {
                     return true;

main/src/com/google/gridworks/commands/expr/PreviewExpressionCommand.java

@@ -149,6 +149,10 @@ public class PreviewExpressionCommand extends Command {
                     sb.append("[object Cell]");
                 } else if (v instanceof WrappedRow) {
                     sb.append("[object Row]");
+                } else if (v instanceof JSONObject) {
+                   sb.append(((JSONObject) v).toString());
+                } else if (v instanceof JSONArray) {
+                    sb.append(((JSONArray) v).toString());
                 } else if (ExpressionUtils.isArray(v)) {
                     Object[] a = (Object[]) v;
                     sb.append("[ ");
@@ -185,6 +189,13 @@ public class PreviewExpressionCommand extends Command {
                     } else {
                         sb.append((String) v);
                     }
+                } else if (v instanceof Double || v instanceof Float) {
+                    Number n = (Number) v;
+                    if (n.doubleValue() - n.longValue() == 0.0) {
+                        sb.append(n.longValue());
+                    } else {
+                        sb.append(n.doubleValue());
+                    }
                 } else {
                     sb.append(v.toString());
                 }

main/src/com/google/gridworks/expr/ExpressionUtils.java

@@ -9,6 +9,9 @@ import java.util.List;
 import java.util.Properties;
 import java.util.Set;
 
+import org.json.JSONArray;
+import org.json.JSONObject;
+
 import com.google.gridworks.model.Cell;
 import com.google.gridworks.model.Project;
 import com.google.gridworks.model.Row;
@@ -108,9 +111,15 @@ public class ExpressionUtils {
     }
     
     static public Serializable wrapStorable(Object v) {
-        return isStorable(v) ? 
+        if (v instanceof JSONArray) {
-            (Serializable) v : 
+            return ((JSONArray) v).toString();
-            new EvalError(v.getClass().getSimpleName() + " value not storable");
+        } else if (v instanceof JSONObject) {
+            return ((JSONObject) v).toString();
+        } else {
+            return isStorable(v) ? 
+                (Serializable) v : 
+                new EvalError(v.getClass().getSimpleName() + " value not storable");
+        }
     }
     
     static public boolean isArray(Object v) {

main/src/com/google/gridworks/expr/functions/Get.java

@@ -3,7 +3,9 @@ package com.google.gridworks.expr.functions;
 import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
+import org.json.JSONObject;
 import org.json.JSONWriter;
 
 import com.google.gridworks.expr.ExpressionUtils;
@@ -22,14 +24,26 @@ public class Get implements Function {
             if (v != null && from != null) {
                 if (v instanceof HasFields && from instanceof String) {
                     return ((HasFields) v).getField((String) from, bindings);
+                } else if (v instanceof JSONObject && from instanceof String) {
+                    try {
+                        return ((JSONObject) v).get((String) from);
+                    } catch (JSONException e) {
+                        // ignore; will return null
+                    }
                 } else {
                     if (from instanceof Number && (to == null || to instanceof Number)) {
-                        if (v.getClass().isArray() || v instanceof List<?> || v instanceof HasFieldsList) {
+                        if (v.getClass().isArray() || 
+                            v instanceof List<?> || 
+                            v instanceof HasFieldsList || 
+                            v instanceof JSONArray) {
+                            
                             int length = 0;
                             if (v.getClass().isArray()) { 
                                 length = ((Object[]) v).length;
                             } else if (v instanceof HasFieldsList) {
                                 length = ((HasFieldsList) v).length();
+                            } else if (v instanceof JSONArray) {
+                                length = ((JSONArray) v).length();
                             } else {
                                 length = ExpressionUtils.toObjectList(v).size();
                             }
@@ -45,6 +59,12 @@ public class Get implements Function {
                                     return ((Object[]) v)[start];
                                 } else if (v instanceof HasFieldsList) {
                                     return ((HasFieldsList) v).get(start);
+                                } else if (v instanceof JSONArray) {
+                                    try {
+                                        return ((JSONArray) v).get(start);
+                                    } catch (JSONException e) {
+                                        // ignore; will return null
+                                    }
                                 } else {
                                     return ExpressionUtils.toObjectList(v).get(start);
                                 }
@@ -65,6 +85,19 @@ public class Get implements Function {
                                         return a2;
                                     } else if (v instanceof HasFieldsList) {
                                         return ((HasFieldsList) v).getSubList(start, end);
+                                    } else if (v instanceof JSONArray) {
+                                        JSONArray a = (JSONArray) v;
+                                        Object[] a2 = new Object[end - start];
+                                        
+                                        for (int i = 0; i < a2.length; i++) {
+                                            try {
+                                                a2[i] = a.get(start + i);
+                                            } catch (JSONException e) {
+                                                // ignore
+                                            }
+                                        }
+                                        
+                                        return a2;
                                     } else {
                                         return ExpressionUtils.toObjectList(v).subList(start, end);
                                     }

main/src/com/google/gridworks/expr/functions/HasField.java

@@ -0,0 +1,44 @@
+package com.google.gridworks.expr.functions;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.gridworks.expr.HasFields;
+import com.google.gridworks.gel.Function;
+
+public class HasField implements Function {
+
+    public Object call(Properties bindings, Object[] args) {
+        if (args.length > 1 && args.length <= 2) {
+            Object v = args[0];
+            Object f = args[1];
+            
+            if (v != null && f != null && f instanceof String) {
+                String name = (String) f;
+                if (v instanceof HasFields) {
+                    return ((HasFields) v).getField(name, bindings) != null;
+                } else if (v instanceof JSONObject) {
+                    try {
+                        return ((JSONObject) v).get(name) != null;
+                    } catch (JSONException e) {
+                        // ignore; will return false
+                    }
+                }
+            }
+        }
+        return false;
+    }
+
+    public void write(JSONWriter writer, Properties options)
+        throws JSONException {
+    
+        writer.object();
+        writer.key("description"); writer.value("Returns whether o has field name");
+        writer.key("params"); writer.value("o, string name");
+        writer.key("returns"); writer.value("boolean");
+        writer.endObject();
+    }
+}

main/src/com/google/gridworks/expr/functions/Length.java

@@ -3,10 +3,12 @@ package com.google.gridworks.expr.functions;
 import java.util.Collection;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
 import com.google.gridworks.expr.EvalError;
+import com.google.gridworks.expr.HasFieldsList;
 import com.google.gridworks.gel.ControlFunctionRegistry;
 import com.google.gridworks.gel.Function;
 
@@ -22,6 +24,10 @@ public class Length implements Function {
                     return a.length;
                 } else if (v instanceof Collection<?>) {
                     return ((Collection<?>) v).size();
+                } else if (v instanceof HasFieldsList) {
+                    return ((HasFieldsList) v).length();
+                } else if (v instanceof JSONArray) {
+                    return ((JSONArray) v).length();
                 } else {
                     String s = (v instanceof String ? (String) v : v.toString());
                     return s.length();

main/src/com/google/gridworks/expr/functions/Slice.java

@@ -3,6 +3,7 @@ package com.google.gridworks.expr.functions;
 import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
@@ -19,12 +20,14 @@ public class Slice implements Function {
             Object to = (args.length == 3) ? args[2] : null;
             
             if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) {
-                if (v.getClass().isArray() || v instanceof List<?> || v instanceof HasFieldsList) {
+                if (v.getClass().isArray() || v instanceof List<?> || v instanceof HasFieldsList || v instanceof JSONArray) {
                     int length = 0;
                     if (v.getClass().isArray()) { 
                         length = ((Object[]) v).length;
                     } else if (v instanceof HasFieldsList) {
                         length = ((HasFieldsList) v).length();
+                    } else if (v instanceof JSONArray) {
+                        length = ((JSONArray) v).length();
                     } else {
                         length = ExpressionUtils.toObjectList(v).size();
                     }
@@ -50,6 +53,19 @@ public class Slice implements Function {
                         return a2;
                     } else if (v instanceof HasFieldsList) {
                         return ((HasFieldsList) v).getSubList(start, end);
+                    } else if (v instanceof JSONArray) {
+                        JSONArray a = (JSONArray) v;
+                        Object[] a2 = new Object[end - start];
+                        
+                        for (int i = 0; i < a2.length; i++) {
+                            try {
+                                a2[i] = a.get(start + i);
+                            } catch (JSONException e) {
+                                // ignore
+                            }
+                        }
+                        
+                        return a2;
                     } else {
                         return ExpressionUtils.toObjectList(v).subList(start, end);
                     }

main/src/com/google/gridworks/expr/functions/arrays/Join.java

@@ -3,6 +3,7 @@ package com.google.gridworks.expr.functions.arrays;
 import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
@@ -18,33 +19,48 @@ public class Join implements Function {
             Object v = args[0];
             Object s = args[1];
             
-            if (v != null && (v.getClass().isArray() || v instanceof List<?>) &&
+            if (v != null && s != null && s instanceof String) {
-                s != null && s instanceof String) {
-                
                 String separator = (String) s;
                 
-                StringBuffer sb = new StringBuffer();
+                if (v.getClass().isArray() || v instanceof List<?> || v instanceof JSONArray) {
-                if (v.getClass().isArray()) {
+                    StringBuffer sb = new StringBuffer();
-                    for (Object o : (Object[]) v) {
+                    if (v.getClass().isArray()) {
-                        if (o != null) {
+                        for (Object o : (Object[]) v) {
+                            if (o != null) {
+                                if (sb.length() > 0) {
+                                    sb.append(separator);
+                                }
+                                sb.append(o.toString());
+                            }
+                        }
+                    } else if (v instanceof JSONArray) {
+                        JSONArray a = (JSONArray) v;
+                        int l = a.length();
+                        
+                        for (int i = 0; i < l; i++) {
                             if (sb.length() > 0) {
                                 sb.append(separator);
                             }
-                            sb.append(o.toString());
+                            try {
-                        }
+                                sb.append(a.get(i).toString());
-                    }
+                            } catch (JSONException e) {
-                } else {
+                                return new EvalError(ControlFunctionRegistry.getFunctionName(this) + 
-                    for (Object o : ExpressionUtils.toObjectList(v)) {
+                                    " cannot retrieve element " + i + " of array");
-                        if (o != null) {
+                            }
-                            if (sb.length() > 0) {
+                        }
-                                sb.append(separator);
+                    } else {
+                        for (Object o : ExpressionUtils.toObjectList(v)) {
+                            if (o != null) {
+                                if (sb.length() > 0) {
+                                    sb.append(separator);
+                                }
+                                sb.append(o.toString());
                             }
-                            sb.append(o.toString());
                         }
                     }
+                    
+                    return sb.toString();
                 }
-                
-                return sb.toString();
             }
         }
         return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array and a string");

main/src/com/google/gridworks/expr/functions/arrays/Reverse.java

@@ -3,6 +3,7 @@ package com.google.gridworks.expr.functions.arrays;
 import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
@@ -10,6 +11,7 @@ import com.google.gridworks.expr.EvalError;
 import com.google.gridworks.expr.ExpressionUtils;
 import com.google.gridworks.gel.ControlFunctionRegistry;
 import com.google.gridworks.gel.Function;
+import com.google.gridworks.util.JSONUtilities;
 
 public class Reverse implements Function {
 
@@ -17,24 +19,35 @@ public class Reverse implements Function {
         if (args.length == 1) {
             Object v = args[0];
             
-            if (v != null && (v.getClass().isArray() || v instanceof List<?>)) {
+            if (v != null) {
-                int length = v.getClass().isArray() ? 
+                if (v instanceof JSONArray) {
-                        ((Object[]) v).length :
+                    try {
-                        ExpressionUtils.toObjectList(v).size();
+                        v = JSONUtilities.toArray((JSONArray) v);
-                
+                    } catch (JSONException e) {
-                Object[] r = new Object[length];
+                        return new EvalError(ControlFunctionRegistry.getFunctionName(this) +
-                if (v.getClass().isArray()) {
+                                " fails to process a JSON array: " + e.getMessage());
-                    Object[] a = (Object[]) v;
-                    for (int i = 0; i < length; i++) {
-                        r[i] = a[r.length - i - 1];
-                    }
-                } else {
-                    List<Object> a = ExpressionUtils.toObjectList(v);
-                    for (int i = 0; i < length; i++) {
-                        r[i] = a.get(r.length - i - 1);
                     }
                 }
-                return r;
+                
+                if (v.getClass().isArray() || v instanceof List<?>) {
+                    int length = v.getClass().isArray() ? 
+                            ((Object[]) v).length :
+                            ExpressionUtils.toObjectList(v).size();
+                    
+                    Object[] r = new Object[length];
+                    if (v.getClass().isArray()) {
+                        Object[] a = (Object[]) v;
+                        for (int i = 0; i < length; i++) {
+                            r[i] = a[r.length - i - 1];
+                        }
+                    } else {
+                        List<Object> a = ExpressionUtils.toObjectList(v);
+                        for (int i = 0; i < length; i++) {
+                            r[i] = a.get(r.length - i - 1);
+                        }
+                    }
+                    return r;
+                }
             }
         }
         return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array");

main/src/com/google/gridworks/expr/functions/arrays/Sort.java

@@ -5,12 +5,14 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
 import com.google.gridworks.expr.EvalError;
 import com.google.gridworks.gel.ControlFunctionRegistry;
 import com.google.gridworks.gel.Function;
+import com.google.gridworks.util.JSONUtilities;
 
 public class Sort implements Function {
 
@@ -27,6 +29,17 @@ public class Sort implements Function {
                     Arrays.sort(r, 0, r.length);
                     
                     return r;
+                } else if (v instanceof JSONArray) {
+                    try {
+                        Object[] r = JSONUtilities.toArray((JSONArray) v);
+                        
+                        Arrays.sort(r, 0, r.length);
+                        
+                        return r;
+                    } catch (JSONException e) {
+                        return new EvalError(ControlFunctionRegistry.getFunctionName(this) +
+                                " fails to process a JSON array: " + e.getMessage());
+                    }
                 } else if (v instanceof List<?>) {
                     List<? extends Comparable<Object>> a = (List<? extends Comparable<Object>>) v;
                     Collections.sort(a);

main/src/com/google/gridworks/expr/functions/strings/Escape.java

@@ -0,0 +1,53 @@
+package com.google.gridworks.expr.functions.strings;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.util.Properties;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.gridworks.expr.EvalError;
+import com.google.gridworks.gel.ControlFunctionRegistry;
+import com.google.gridworks.gel.Function;
+
+public class Escape implements Function {
+
+    public Object call(Properties bindings, Object[] args) {
+        if (args.length == 2) {
+            Object o1 = args[0];
+            Object o2 = args[1];
+            if (o1 != null && o2 != null && o1 instanceof String && o2 instanceof String) {
+                String s = (String) o1;
+                String mode = ((String) o2).toLowerCase();
+                if ("html".equals(mode)) {
+                    return StringEscapeUtils.escapeHtml(s);
+                } else if ("xml".equals(mode)) {
+                    return StringEscapeUtils.escapeXml(s);
+                } else if ("csv".equals(mode)) {
+                    return StringEscapeUtils.escapeCsv(s);
+                } else if ("javascript".equals(mode)) {
+                    return StringEscapeUtils.escapeJavaScript(s);
+                } else if ("url".equals(mode)) {
+                    try {
+                        return URLEncoder.encode(s,"UTF-8");
+                    } catch (UnsupportedEncodingException e) {}
+                } else {
+                    return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " does not recognize mode '" + mode + "'.");
+                }
+            }
+        }
+        return null;
+    }
+    
+    public void write(JSONWriter writer, Properties options)
+        throws JSONException {
+    
+        writer.object();
+        writer.key("description"); writer.value("Escapes a string depending on the given escaping mode.");
+        writer.key("params"); writer.value("string s, string mode ['html','xml','csv','url','javascript']");
+        writer.key("returns"); writer.value("string");
+        writer.endObject();
+    }
+}

main/src/com/google/gridworks/expr/functions/strings/ParseJson.java

@@ -0,0 +1,39 @@
+package com.google.gridworks.expr.functions.strings;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONTokener;
+import org.json.JSONWriter;
+
+import com.google.gridworks.expr.EvalError;
+import com.google.gridworks.gel.ControlFunctionRegistry;
+import com.google.gridworks.gel.Function;
+
+public class ParseJson implements Function {
+
+    public Object call(Properties bindings, Object[] args) {
+        if (args.length >= 1) {
+            Object o1 = args[0];
+            if (o1 != null) {
+                try {
+                    return new JSONTokener(o1.toString()).nextValue();
+                } catch (JSONException e) {
+                    return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " failed: " + e.getMessage());
+                }
+            }
+        }
+        return null;
+    }
+
+    
+    public void write(JSONWriter writer, Properties options)
+        throws JSONException {
+    
+        writer.object();
+        writer.key("description"); writer.value("Parses a string as JSON");
+        writer.key("params"); writer.value("string s");
+        writer.key("returns"); writer.value("JSON object");
+        writer.endObject();
+    }
+}

main/src/com/google/gridworks/gel/ControlFunctionRegistry.java

@@ -8,6 +8,7 @@ import java.util.Map.Entry;
 import com.google.gridworks.expr.functions.Cross;
 import com.google.gridworks.expr.functions.FacetCount;
 import com.google.gridworks.expr.functions.Get;
+import com.google.gridworks.expr.functions.HasField;
 import com.google.gridworks.expr.functions.Jsonize;
 import com.google.gridworks.expr.functions.Length;
 import com.google.gridworks.expr.functions.Slice;
@@ -39,6 +40,7 @@ import com.google.gridworks.expr.functions.strings.Chomp;
 import com.google.gridworks.expr.functions.strings.Contains;
 import com.google.gridworks.expr.functions.strings.Diff;
 import com.google.gridworks.expr.functions.strings.EndsWith;
+import com.google.gridworks.expr.functions.strings.Escape;
 import com.google.gridworks.expr.functions.strings.Fingerprint;
 import com.google.gridworks.expr.functions.strings.IndexOf;
 import com.google.gridworks.expr.functions.strings.LastIndexOf;
@@ -46,6 +48,7 @@ import com.google.gridworks.expr.functions.strings.MD5;
 import com.google.gridworks.expr.functions.strings.Match;
 import com.google.gridworks.expr.functions.strings.NGram;
 import com.google.gridworks.expr.functions.strings.NGramFingerprint;
+import com.google.gridworks.expr.functions.strings.ParseJson;
 import com.google.gridworks.expr.functions.strings.Partition;
 import com.google.gridworks.expr.functions.strings.Phonetic;
 import com.google.gridworks.expr.functions.strings.RPartition;
@@ -128,6 +131,7 @@ public class ControlFunctionRegistry {
         registerFunction("toLowercase", new ToLowercase());
         registerFunction("toTitlecase", new ToTitlecase());
 
+        registerFunction("hasField", new HasField());
         registerFunction("get", new Get());
         registerFunction("slice", new Slice());
         registerFunction("substring", new Slice());
@@ -142,6 +146,7 @@ public class ControlFunctionRegistry {
         registerFunction("trim", new Trim());
         registerFunction("strip", new Trim());
         registerFunction("contains", new Contains());
+        registerFunction("escape", new Escape());
         registerFunction("unescape", new Unescape());
         registerFunction("length", new Length());
         registerFunction("sha1", new SHA1());
@@ -155,6 +160,7 @@ public class ControlFunctionRegistry {
         registerFunction("phonetic", new Phonetic());
         registerFunction("reinterpret", new Reinterpret());
         registerFunction("jsonize", new Jsonize());
+        registerFunction("parseJson", new ParseJson());
         registerFunction("ngram", new NGram());
         registerFunction("match", new Match());
 

main/src/com/google/gridworks/gel/ast/FieldAccessorExpr.java

@@ -2,6 +2,9 @@ package com.google.gridworks.gel.ast;
 
 import java.util.Properties;
 
+import org.json.JSONException;
+import org.json.JSONObject;
+
 import com.google.gridworks.expr.EvalError;
 import com.google.gridworks.expr.Evaluable;
 import com.google.gridworks.expr.ExpressionUtils;
@@ -29,6 +32,12 @@ public class FieldAccessorExpr implements Evaluable {
             return new EvalError("Cannot retrieve field from null");
         } else if (o instanceof HasFields) {
             return ((HasFields) o).getField(_fieldName, bindings);
+        } else if (o instanceof JSONObject) {
+            try {
+                return ((JSONObject) o).get(_fieldName);
+            } catch (JSONException e) {
+                return new EvalError("Object does not have any field, including " + _fieldName);
+            }
         } else {
             return new EvalError("Object does not have any field, including " + _fieldName);
         }

main/src/com/google/gridworks/gel/controls/Filter.java

@@ -5,6 +5,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
@@ -30,7 +31,7 @@ public class Filter implements Control {
         Object o = args[0].evaluate(bindings);
         if (ExpressionUtils.isError(o)) {
             return o;
-        } else if (!ExpressionUtils.isArrayOrCollection(o)) {
+        } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) {
             return new EvalError("First argument is not an array");
         }
         
@@ -49,7 +50,26 @@ public class Filter implements Control {
                     
                     Object r = args[2].evaluate(bindings);
                     if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
-                    	results.add(v);
+                        results.add(v);
+                    }
+                }
+            } else if (o instanceof JSONArray) {
+                JSONArray a = (JSONArray) o;
+                int l = a.length();
+                
+                results = new ArrayList<Object>(l);
+                for (int i = 0; i < l; i++) {
+                    try {
+                        Object v = a.get(i);
+                        
+                        bindings.put(name, v);
+                        
+                        Object r = args[2].evaluate(bindings);
+                        if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
+                            results.add(v);
+                        }
+                    } catch (JSONException e) {
+                        results.add(new EvalError(e.getMessage()));
                     }
                 }
             } else {

main/src/com/google/gridworks/gel/controls/ForEach.java

@@ -5,6 +5,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
@@ -30,7 +31,7 @@ public class ForEach implements Control {
         Object o = args[0].evaluate(bindings);
         if (ExpressionUtils.isError(o)) {
             return o;
-        } else if (!ExpressionUtils.isArrayOrCollection(o)) {
+        } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) {
             return new EvalError("First argument to forEach is not an array");
         }
         
@@ -51,6 +52,24 @@ public class ForEach implements Control {
                     
                     results.add(r);
                 }
+            } else if (o instanceof JSONArray) {
+                JSONArray a = (JSONArray) o;
+                int l = a.length();
+                
+                results = new ArrayList<Object>(l);
+                for (int i = 0; i < l; i++) {
+                    try {
+                        Object v = a.get(i);
+                        
+                        bindings.put(name, v);
+                        
+                        Object r = args[2].evaluate(bindings);
+                        
+                        results.add(r);
+                    } catch (JSONException e) {
+                        results.add(new EvalError(e.getMessage()));
+                    }
+                }
             } else {
                 Collection<Object> collection = ExpressionUtils.toObjectCollection(o);
                 

main/src/com/google/gridworks/gel/controls/ForEachIndex.java

@@ -4,6 +4,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
@@ -32,7 +33,7 @@ public class ForEachIndex implements Control {
         Object o = args[0].evaluate(bindings);
         if (ExpressionUtils.isError(o)) {
             return o;
-        } else if (!ExpressionUtils.isArrayOrCollection(o)) {
+        } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) {
             return new EvalError("First argument to forEach is not an array");
         }
         
@@ -50,15 +51,34 @@ public class ForEachIndex implements Control {
                 results = new ArrayList<Object>(values.length);
                 
                 for (int i = 0; i < values.length; i++) {
-                	Object v = values[i];
+                    Object v = values[i];
-                	
+                    
-                	bindings.put(indexName, i);
+                    bindings.put(indexName, i);
                     bindings.put(elementName, v);
                     
                     Object r = args[3].evaluate(bindings);
                     
                     results.add(r);
                 }
+            } else if (o instanceof JSONArray) {
+                JSONArray a = (JSONArray) o;
+                int l = a.length();
+                
+                results = new ArrayList<Object>(l);
+                for (int i = 0; i < l; i++) {
+                    try {
+                        Object v = a.get(i);
+                        
+                        bindings.put(indexName, i);
+                        bindings.put(elementName, v);
+                        
+                        Object r = args[3].evaluate(bindings);
+                        
+                        results.add(r);
+                    } catch (JSONException e) {
+                        results.add(new EvalError(e.getMessage()));
+                    }
+                }
             } else {
                 List<Object> list = ExpressionUtils.toObjectList(o);
                 

main/src/com/google/gridworks/preference/PreferenceStore.java

@@ -62,8 +62,10 @@ public class PreferenceStore implements Jsonizable {
             Iterator<String> i = entries.keys();
             while (i.hasNext()) {
                 String key = i.next();
-                Object o = entries.get(key);
+                if (!entries.isNull(key)) {
-                _prefs.put(key, loadObject(o));
+                    Object o = entries.get(key);
+                    _prefs.put(key, loadObject(o));
+                }
             }
         }
     }

main/src/com/google/gridworks/util/JSONUtilities.java

@@ -129,4 +129,15 @@ public class JSONUtilities {
     		obj.put(key, value.toString());
     	}
     }
+    
+    static public Object[] toArray(JSONArray a) throws JSONException {
+        int l = a.length();
+        
+        Object[] a2 = new Object[l];
+        for (int i = 0; i < l; i++) {
+            a2[i] = a.get(i);
+        }
+        
+        return a2;
+    }
 }