From d1a66e2e63e7457db7823d481489aa5046259bb2 Mon Sep 17 00:00:00 2001 From: David Huynh Date: Sat, 7 Aug 2010 22:57:48 +0000 Subject: [PATCH] Added JSON support in GEL. Added GEL functions: escape, parseJson, hasField. Fixed bug in preference store: expression history was still not loaded properly. Integers are now rendered without decimals in the expression preview dialogs. git-svn-id: http://google-refine.googlecode.com/svn/trunk@1145 7d457c2a-affb-35e4-300a-418c747d4874 --- .../filters/ExpressionEqualRowFilter.java | 31 +++++++++++ .../ExpressionNumberComparisonRowFilter.java | 17 ++++++ .../ExpressionStringComparisonRowFilter.java | 25 +++++++++ .../expr/PreviewExpressionCommand.java | 11 ++++ .../gridworks/expr/ExpressionUtils.java | 15 ++++-- .../google/gridworks/expr/functions/Get.java | 35 +++++++++++- .../gridworks/expr/functions/HasField.java | 44 +++++++++++++++ .../gridworks/expr/functions/Length.java | 6 +++ .../gridworks/expr/functions/Slice.java | 18 ++++++- .../gridworks/expr/functions/arrays/Join.java | 52 +++++++++++------- .../expr/functions/arrays/Reverse.java | 45 ++++++++++------ .../gridworks/expr/functions/arrays/Sort.java | 13 +++++ .../expr/functions/strings/Escape.java | 53 +++++++++++++++++++ .../expr/functions/strings/ParseJson.java | 39 ++++++++++++++ .../gel/ControlFunctionRegistry.java | 6 +++ .../gridworks/gel/ast/FieldAccessorExpr.java | 9 ++++ .../google/gridworks/gel/controls/Filter.java | 24 ++++++++- .../gridworks/gel/controls/ForEach.java | 21 +++++++- .../gridworks/gel/controls/ForEachIndex.java | 28 ++++++++-- .../gridworks/preference/PreferenceStore.java | 6 ++- .../google/gridworks/util/JSONUtilities.java | 11 ++++ 21 files changed, 461 insertions(+), 48 deletions(-) create mode 100644 main/src/com/google/gridworks/expr/functions/HasField.java create mode 100644 main/src/com/google/gridworks/expr/functions/strings/Escape.java create mode 100644 main/src/com/google/gridworks/expr/functions/strings/ParseJson.java diff --git a/main/src/com/google/gridworks/browsing/filters/ExpressionEqualRowFilter.java b/main/src/com/google/gridworks/browsing/filters/ExpressionEqualRowFilter.java index b3774e44f..3a52b5cf5 100644 --- a/main/src/com/google/gridworks/browsing/filters/ExpressionEqualRowFilter.java +++ b/main/src/com/google/gridworks/browsing/filters/ExpressionEqualRowFilter.java @@ -3,6 +3,9 @@ package com.google.gridworks.browsing.filters; import java.util.Collection; import java.util.Properties; +import org.json.JSONArray; +import org.json.JSONException; + import com.google.gridworks.browsing.RowFilter; import com.google.gridworks.expr.Evaluable; import com.google.gridworks.expr.ExpressionUtils; @@ -75,6 +78,20 @@ public class ExpressionEqualRowFilter implements RowFilter { } } return false; + } else if (value instanceof JSONArray) { + JSONArray a = (JSONArray) value; + int l = a.length(); + + for (int i = 0; i < l; i++) { + try { + if (testValue(a.get(i))) { + return true; + } + } catch (JSONException e) { + // ignore + } + } + return false; } // else, fall through } @@ -104,6 +121,20 @@ public class ExpressionEqualRowFilter implements RowFilter { } } return true; + } else if (value instanceof JSONArray) { + JSONArray a = (JSONArray) value; + int l = a.length(); + + for (int i = 0; i < l; i++) { + try { + if (testValue(a.get(i))) { + return false; + } + } catch (JSONException e) { + // ignore + } + } + return true; } // else, fall through } diff --git a/main/src/com/google/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java b/main/src/com/google/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java index b9e354329..6fe3d5bda 100644 --- a/main/src/com/google/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java +++ b/main/src/com/google/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java @@ -3,6 +3,9 @@ package com.google.gridworks.browsing.filters; import java.util.Collection; import java.util.Properties; +import org.json.JSONArray; +import org.json.JSONException; + import com.google.gridworks.browsing.RowFilter; import com.google.gridworks.browsing.util.RowEvaluable; import com.google.gridworks.expr.ExpressionUtils; @@ -56,6 +59,20 @@ abstract public class ExpressionNumberComparisonRowFilter implements RowFilter { } } return false; + } else if (value instanceof JSONArray) { + JSONArray a = (JSONArray) value; + int l = a.length(); + + for (int i = 0; i < l; i++) { + try { + if (checkValue(a.get(i))) { + return true; + } + } catch (JSONException e) { + // ignore + } + } + return false; } // else, fall through } diff --git a/main/src/com/google/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java b/main/src/com/google/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java index 0bb391691..41f013d3f 100644 --- a/main/src/com/google/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java +++ b/main/src/com/google/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java @@ -1,7 +1,11 @@ package com.google.gridworks.browsing.filters; +import java.util.Collection; import java.util.Properties; +import org.json.JSONArray; +import org.json.JSONException; + import com.google.gridworks.browsing.RowFilter; import com.google.gridworks.expr.Evaluable; import com.google.gridworks.expr.ExpressionUtils; @@ -39,6 +43,27 @@ abstract public class ExpressionStringComparisonRowFilter implements RowFilter { return true; } } + } else if (value instanceof Collection) { + for (Object v : ExpressionUtils.toObjectCollection(value)) { + if (checkValue(v.toString())) { + return true; + } + } + return false; + } else if (value instanceof JSONArray) { + JSONArray a = (JSONArray) value; + int l = a.length(); + + for (int i = 0; i < l; i++) { + try { + if (checkValue(a.get(i).toString())) { + return true; + } + } catch (JSONException e) { + // ignore + } + } + return false; } else { if (checkValue(value instanceof String ? ((String) value) : value.toString())) { return true; diff --git a/main/src/com/google/gridworks/commands/expr/PreviewExpressionCommand.java b/main/src/com/google/gridworks/commands/expr/PreviewExpressionCommand.java index 54dad5fb7..3aca66301 100644 --- a/main/src/com/google/gridworks/commands/expr/PreviewExpressionCommand.java +++ b/main/src/com/google/gridworks/commands/expr/PreviewExpressionCommand.java @@ -149,6 +149,10 @@ public class PreviewExpressionCommand extends Command { sb.append("[object Cell]"); } else if (v instanceof WrappedRow) { sb.append("[object Row]"); + } else if (v instanceof JSONObject) { + sb.append(((JSONObject) v).toString()); + } else if (v instanceof JSONArray) { + sb.append(((JSONArray) v).toString()); } else if (ExpressionUtils.isArray(v)) { Object[] a = (Object[]) v; sb.append("[ "); @@ -185,6 +189,13 @@ public class PreviewExpressionCommand extends Command { } else { sb.append((String) v); } + } else if (v instanceof Double || v instanceof Float) { + Number n = (Number) v; + if (n.doubleValue() - n.longValue() == 0.0) { + sb.append(n.longValue()); + } else { + sb.append(n.doubleValue()); + } } else { sb.append(v.toString()); } diff --git a/main/src/com/google/gridworks/expr/ExpressionUtils.java b/main/src/com/google/gridworks/expr/ExpressionUtils.java index 8a3cdd784..95ca6b6da 100644 --- a/main/src/com/google/gridworks/expr/ExpressionUtils.java +++ b/main/src/com/google/gridworks/expr/ExpressionUtils.java @@ -9,6 +9,9 @@ import java.util.List; import java.util.Properties; import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; + import com.google.gridworks.model.Cell; import com.google.gridworks.model.Project; import com.google.gridworks.model.Row; @@ -108,9 +111,15 @@ public class ExpressionUtils { } static public Serializable wrapStorable(Object v) { - return isStorable(v) ? - (Serializable) v : - new EvalError(v.getClass().getSimpleName() + " value not storable"); + if (v instanceof JSONArray) { + return ((JSONArray) v).toString(); + } else if (v instanceof JSONObject) { + return ((JSONObject) v).toString(); + } else { + return isStorable(v) ? + (Serializable) v : + new EvalError(v.getClass().getSimpleName() + " value not storable"); + } } static public boolean isArray(Object v) { diff --git a/main/src/com/google/gridworks/expr/functions/Get.java b/main/src/com/google/gridworks/expr/functions/Get.java index 2bac8b637..fd358186d 100644 --- a/main/src/com/google/gridworks/expr/functions/Get.java +++ b/main/src/com/google/gridworks/expr/functions/Get.java @@ -3,7 +3,9 @@ package com.google.gridworks.expr.functions; import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; +import org.json.JSONObject; import org.json.JSONWriter; import com.google.gridworks.expr.ExpressionUtils; @@ -22,14 +24,26 @@ public class Get implements Function { if (v != null && from != null) { if (v instanceof HasFields && from instanceof String) { return ((HasFields) v).getField((String) from, bindings); + } else if (v instanceof JSONObject && from instanceof String) { + try { + return ((JSONObject) v).get((String) from); + } catch (JSONException e) { + // ignore; will return null + } } else { if (from instanceof Number && (to == null || to instanceof Number)) { - if (v.getClass().isArray() || v instanceof List || v instanceof HasFieldsList) { + if (v.getClass().isArray() || + v instanceof List || + v instanceof HasFieldsList || + v instanceof JSONArray) { + int length = 0; if (v.getClass().isArray()) { length = ((Object[]) v).length; } else if (v instanceof HasFieldsList) { length = ((HasFieldsList) v).length(); + } else if (v instanceof JSONArray) { + length = ((JSONArray) v).length(); } else { length = ExpressionUtils.toObjectList(v).size(); } @@ -45,6 +59,12 @@ public class Get implements Function { return ((Object[]) v)[start]; } else if (v instanceof HasFieldsList) { return ((HasFieldsList) v).get(start); + } else if (v instanceof JSONArray) { + try { + return ((JSONArray) v).get(start); + } catch (JSONException e) { + // ignore; will return null + } } else { return ExpressionUtils.toObjectList(v).get(start); } @@ -65,6 +85,19 @@ public class Get implements Function { return a2; } else if (v instanceof HasFieldsList) { return ((HasFieldsList) v).getSubList(start, end); + } else if (v instanceof JSONArray) { + JSONArray a = (JSONArray) v; + Object[] a2 = new Object[end - start]; + + for (int i = 0; i < a2.length; i++) { + try { + a2[i] = a.get(start + i); + } catch (JSONException e) { + // ignore + } + } + + return a2; } else { return ExpressionUtils.toObjectList(v).subList(start, end); } diff --git a/main/src/com/google/gridworks/expr/functions/HasField.java b/main/src/com/google/gridworks/expr/functions/HasField.java new file mode 100644 index 000000000..0741de3de --- /dev/null +++ b/main/src/com/google/gridworks/expr/functions/HasField.java @@ -0,0 +1,44 @@ +package com.google.gridworks.expr.functions; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.gridworks.expr.HasFields; +import com.google.gridworks.gel.Function; + +public class HasField implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length > 1 && args.length <= 2) { + Object v = args[0]; + Object f = args[1]; + + if (v != null && f != null && f instanceof String) { + String name = (String) f; + if (v instanceof HasFields) { + return ((HasFields) v).getField(name, bindings) != null; + } else if (v instanceof JSONObject) { + try { + return ((JSONObject) v).get(name) != null; + } catch (JSONException e) { + // ignore; will return false + } + } + } + } + return false; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Returns whether o has field name"); + writer.key("params"); writer.value("o, string name"); + writer.key("returns"); writer.value("boolean"); + writer.endObject(); + } +} diff --git a/main/src/com/google/gridworks/expr/functions/Length.java b/main/src/com/google/gridworks/expr/functions/Length.java index 12dd450b4..124ccc404 100644 --- a/main/src/com/google/gridworks/expr/functions/Length.java +++ b/main/src/com/google/gridworks/expr/functions/Length.java @@ -3,10 +3,12 @@ package com.google.gridworks.expr.functions; import java.util.Collection; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONWriter; import com.google.gridworks.expr.EvalError; +import com.google.gridworks.expr.HasFieldsList; import com.google.gridworks.gel.ControlFunctionRegistry; import com.google.gridworks.gel.Function; @@ -22,6 +24,10 @@ public class Length implements Function { return a.length; } else if (v instanceof Collection) { return ((Collection) v).size(); + } else if (v instanceof HasFieldsList) { + return ((HasFieldsList) v).length(); + } else if (v instanceof JSONArray) { + return ((JSONArray) v).length(); } else { String s = (v instanceof String ? (String) v : v.toString()); return s.length(); diff --git a/main/src/com/google/gridworks/expr/functions/Slice.java b/main/src/com/google/gridworks/expr/functions/Slice.java index 6f00a6918..b097f1c58 100644 --- a/main/src/com/google/gridworks/expr/functions/Slice.java +++ b/main/src/com/google/gridworks/expr/functions/Slice.java @@ -3,6 +3,7 @@ package com.google.gridworks.expr.functions; import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONWriter; @@ -19,12 +20,14 @@ public class Slice implements Function { Object to = (args.length == 3) ? args[2] : null; if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) { - if (v.getClass().isArray() || v instanceof List || v instanceof HasFieldsList) { + if (v.getClass().isArray() || v instanceof List || v instanceof HasFieldsList || v instanceof JSONArray) { int length = 0; if (v.getClass().isArray()) { length = ((Object[]) v).length; } else if (v instanceof HasFieldsList) { length = ((HasFieldsList) v).length(); + } else if (v instanceof JSONArray) { + length = ((JSONArray) v).length(); } else { length = ExpressionUtils.toObjectList(v).size(); } @@ -50,6 +53,19 @@ public class Slice implements Function { return a2; } else if (v instanceof HasFieldsList) { return ((HasFieldsList) v).getSubList(start, end); + } else if (v instanceof JSONArray) { + JSONArray a = (JSONArray) v; + Object[] a2 = new Object[end - start]; + + for (int i = 0; i < a2.length; i++) { + try { + a2[i] = a.get(start + i); + } catch (JSONException e) { + // ignore + } + } + + return a2; } else { return ExpressionUtils.toObjectList(v).subList(start, end); } diff --git a/main/src/com/google/gridworks/expr/functions/arrays/Join.java b/main/src/com/google/gridworks/expr/functions/arrays/Join.java index a301a3f62..81c6db10c 100644 --- a/main/src/com/google/gridworks/expr/functions/arrays/Join.java +++ b/main/src/com/google/gridworks/expr/functions/arrays/Join.java @@ -3,6 +3,7 @@ package com.google.gridworks.expr.functions.arrays; import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONWriter; @@ -18,33 +19,48 @@ public class Join implements Function { Object v = args[0]; Object s = args[1]; - if (v != null && (v.getClass().isArray() || v instanceof List) && - s != null && s instanceof String) { - + if (v != null && s != null && s instanceof String) { String separator = (String) s; - StringBuffer sb = new StringBuffer(); - if (v.getClass().isArray()) { - for (Object o : (Object[]) v) { - if (o != null) { + if (v.getClass().isArray() || v instanceof List || v instanceof JSONArray) { + StringBuffer sb = new StringBuffer(); + if (v.getClass().isArray()) { + for (Object o : (Object[]) v) { + if (o != null) { + if (sb.length() > 0) { + sb.append(separator); + } + sb.append(o.toString()); + } + } + } else if (v instanceof JSONArray) { + JSONArray a = (JSONArray) v; + int l = a.length(); + + for (int i = 0; i < l; i++) { if (sb.length() > 0) { sb.append(separator); } - sb.append(o.toString()); - } - } - } else { - for (Object o : ExpressionUtils.toObjectList(v)) { - if (o != null) { - if (sb.length() > 0) { - sb.append(separator); + try { + sb.append(a.get(i).toString()); + } catch (JSONException e) { + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + + " cannot retrieve element " + i + " of array"); + } + } + } else { + for (Object o : ExpressionUtils.toObjectList(v)) { + if (o != null) { + if (sb.length() > 0) { + sb.append(separator); + } + sb.append(o.toString()); } - sb.append(o.toString()); } } + + return sb.toString(); } - - return sb.toString(); } } return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array and a string"); diff --git a/main/src/com/google/gridworks/expr/functions/arrays/Reverse.java b/main/src/com/google/gridworks/expr/functions/arrays/Reverse.java index 0a1c1670c..48cc991a3 100644 --- a/main/src/com/google/gridworks/expr/functions/arrays/Reverse.java +++ b/main/src/com/google/gridworks/expr/functions/arrays/Reverse.java @@ -3,6 +3,7 @@ package com.google.gridworks.expr.functions.arrays; import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONWriter; @@ -10,6 +11,7 @@ import com.google.gridworks.expr.EvalError; import com.google.gridworks.expr.ExpressionUtils; import com.google.gridworks.gel.ControlFunctionRegistry; import com.google.gridworks.gel.Function; +import com.google.gridworks.util.JSONUtilities; public class Reverse implements Function { @@ -17,24 +19,35 @@ public class Reverse implements Function { if (args.length == 1) { Object v = args[0]; - if (v != null && (v.getClass().isArray() || v instanceof List)) { - int length = v.getClass().isArray() ? - ((Object[]) v).length : - ExpressionUtils.toObjectList(v).size(); - - Object[] r = new Object[length]; - if (v.getClass().isArray()) { - Object[] a = (Object[]) v; - for (int i = 0; i < length; i++) { - r[i] = a[r.length - i - 1]; - } - } else { - List a = ExpressionUtils.toObjectList(v); - for (int i = 0; i < length; i++) { - r[i] = a.get(r.length - i - 1); + if (v != null) { + if (v instanceof JSONArray) { + try { + v = JSONUtilities.toArray((JSONArray) v); + } catch (JSONException e) { + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + + " fails to process a JSON array: " + e.getMessage()); } } - return r; + + if (v.getClass().isArray() || v instanceof List) { + int length = v.getClass().isArray() ? + ((Object[]) v).length : + ExpressionUtils.toObjectList(v).size(); + + Object[] r = new Object[length]; + if (v.getClass().isArray()) { + Object[] a = (Object[]) v; + for (int i = 0; i < length; i++) { + r[i] = a[r.length - i - 1]; + } + } else { + List a = ExpressionUtils.toObjectList(v); + for (int i = 0; i < length; i++) { + r[i] = a.get(r.length - i - 1); + } + } + return r; + } } } return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array"); diff --git a/main/src/com/google/gridworks/expr/functions/arrays/Sort.java b/main/src/com/google/gridworks/expr/functions/arrays/Sort.java index 682f4188c..d590ad706 100644 --- a/main/src/com/google/gridworks/expr/functions/arrays/Sort.java +++ b/main/src/com/google/gridworks/expr/functions/arrays/Sort.java @@ -5,12 +5,14 @@ import java.util.Collections; import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONWriter; import com.google.gridworks.expr.EvalError; import com.google.gridworks.gel.ControlFunctionRegistry; import com.google.gridworks.gel.Function; +import com.google.gridworks.util.JSONUtilities; public class Sort implements Function { @@ -27,6 +29,17 @@ public class Sort implements Function { Arrays.sort(r, 0, r.length); return r; + } else if (v instanceof JSONArray) { + try { + Object[] r = JSONUtilities.toArray((JSONArray) v); + + Arrays.sort(r, 0, r.length); + + return r; + } catch (JSONException e) { + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + + " fails to process a JSON array: " + e.getMessage()); + } } else if (v instanceof List) { List> a = (List>) v; Collections.sort(a); diff --git a/main/src/com/google/gridworks/expr/functions/strings/Escape.java b/main/src/com/google/gridworks/expr/functions/strings/Escape.java new file mode 100644 index 000000000..eb7b7cfd7 --- /dev/null +++ b/main/src/com/google/gridworks/expr/functions/strings/Escape.java @@ -0,0 +1,53 @@ +package com.google.gridworks.expr.functions.strings; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.Properties; + +import org.apache.commons.lang.StringEscapeUtils; +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.gridworks.expr.EvalError; +import com.google.gridworks.gel.ControlFunctionRegistry; +import com.google.gridworks.gel.Function; + +public class Escape implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length == 2) { + Object o1 = args[0]; + Object o2 = args[1]; + if (o1 != null && o2 != null && o1 instanceof String && o2 instanceof String) { + String s = (String) o1; + String mode = ((String) o2).toLowerCase(); + if ("html".equals(mode)) { + return StringEscapeUtils.escapeHtml(s); + } else if ("xml".equals(mode)) { + return StringEscapeUtils.escapeXml(s); + } else if ("csv".equals(mode)) { + return StringEscapeUtils.escapeCsv(s); + } else if ("javascript".equals(mode)) { + return StringEscapeUtils.escapeJavaScript(s); + } else if ("url".equals(mode)) { + try { + return URLEncoder.encode(s,"UTF-8"); + } catch (UnsupportedEncodingException e) {} + } else { + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " does not recognize mode '" + mode + "'."); + } + } + } + return null; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Escapes a string depending on the given escaping mode."); + writer.key("params"); writer.value("string s, string mode ['html','xml','csv','url','javascript']"); + writer.key("returns"); writer.value("string"); + writer.endObject(); + } +} diff --git a/main/src/com/google/gridworks/expr/functions/strings/ParseJson.java b/main/src/com/google/gridworks/expr/functions/strings/ParseJson.java new file mode 100644 index 000000000..1baae3be1 --- /dev/null +++ b/main/src/com/google/gridworks/expr/functions/strings/ParseJson.java @@ -0,0 +1,39 @@ +package com.google.gridworks.expr.functions.strings; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONTokener; +import org.json.JSONWriter; + +import com.google.gridworks.expr.EvalError; +import com.google.gridworks.gel.ControlFunctionRegistry; +import com.google.gridworks.gel.Function; + +public class ParseJson implements Function { + + public Object call(Properties bindings, Object[] args) { + if (args.length >= 1) { + Object o1 = args[0]; + if (o1 != null) { + try { + return new JSONTokener(o1.toString()).nextValue(); + } catch (JSONException e) { + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " failed: " + e.getMessage()); + } + } + } + return null; + } + + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("description"); writer.value("Parses a string as JSON"); + writer.key("params"); writer.value("string s"); + writer.key("returns"); writer.value("JSON object"); + writer.endObject(); + } +} diff --git a/main/src/com/google/gridworks/gel/ControlFunctionRegistry.java b/main/src/com/google/gridworks/gel/ControlFunctionRegistry.java index 4ea300637..8e83098bc 100644 --- a/main/src/com/google/gridworks/gel/ControlFunctionRegistry.java +++ b/main/src/com/google/gridworks/gel/ControlFunctionRegistry.java @@ -8,6 +8,7 @@ import java.util.Map.Entry; import com.google.gridworks.expr.functions.Cross; import com.google.gridworks.expr.functions.FacetCount; import com.google.gridworks.expr.functions.Get; +import com.google.gridworks.expr.functions.HasField; import com.google.gridworks.expr.functions.Jsonize; import com.google.gridworks.expr.functions.Length; import com.google.gridworks.expr.functions.Slice; @@ -39,6 +40,7 @@ import com.google.gridworks.expr.functions.strings.Chomp; import com.google.gridworks.expr.functions.strings.Contains; import com.google.gridworks.expr.functions.strings.Diff; import com.google.gridworks.expr.functions.strings.EndsWith; +import com.google.gridworks.expr.functions.strings.Escape; import com.google.gridworks.expr.functions.strings.Fingerprint; import com.google.gridworks.expr.functions.strings.IndexOf; import com.google.gridworks.expr.functions.strings.LastIndexOf; @@ -46,6 +48,7 @@ import com.google.gridworks.expr.functions.strings.MD5; import com.google.gridworks.expr.functions.strings.Match; import com.google.gridworks.expr.functions.strings.NGram; import com.google.gridworks.expr.functions.strings.NGramFingerprint; +import com.google.gridworks.expr.functions.strings.ParseJson; import com.google.gridworks.expr.functions.strings.Partition; import com.google.gridworks.expr.functions.strings.Phonetic; import com.google.gridworks.expr.functions.strings.RPartition; @@ -128,6 +131,7 @@ public class ControlFunctionRegistry { registerFunction("toLowercase", new ToLowercase()); registerFunction("toTitlecase", new ToTitlecase()); + registerFunction("hasField", new HasField()); registerFunction("get", new Get()); registerFunction("slice", new Slice()); registerFunction("substring", new Slice()); @@ -142,6 +146,7 @@ public class ControlFunctionRegistry { registerFunction("trim", new Trim()); registerFunction("strip", new Trim()); registerFunction("contains", new Contains()); + registerFunction("escape", new Escape()); registerFunction("unescape", new Unescape()); registerFunction("length", new Length()); registerFunction("sha1", new SHA1()); @@ -155,6 +160,7 @@ public class ControlFunctionRegistry { registerFunction("phonetic", new Phonetic()); registerFunction("reinterpret", new Reinterpret()); registerFunction("jsonize", new Jsonize()); + registerFunction("parseJson", new ParseJson()); registerFunction("ngram", new NGram()); registerFunction("match", new Match()); diff --git a/main/src/com/google/gridworks/gel/ast/FieldAccessorExpr.java b/main/src/com/google/gridworks/gel/ast/FieldAccessorExpr.java index e042a9c77..4863d432b 100644 --- a/main/src/com/google/gridworks/gel/ast/FieldAccessorExpr.java +++ b/main/src/com/google/gridworks/gel/ast/FieldAccessorExpr.java @@ -2,6 +2,9 @@ package com.google.gridworks.gel.ast; import java.util.Properties; +import org.json.JSONException; +import org.json.JSONObject; + import com.google.gridworks.expr.EvalError; import com.google.gridworks.expr.Evaluable; import com.google.gridworks.expr.ExpressionUtils; @@ -29,6 +32,12 @@ public class FieldAccessorExpr implements Evaluable { return new EvalError("Cannot retrieve field from null"); } else if (o instanceof HasFields) { return ((HasFields) o).getField(_fieldName, bindings); + } else if (o instanceof JSONObject) { + try { + return ((JSONObject) o).get(_fieldName); + } catch (JSONException e) { + return new EvalError("Object does not have any field, including " + _fieldName); + } } else { return new EvalError("Object does not have any field, including " + _fieldName); } diff --git a/main/src/com/google/gridworks/gel/controls/Filter.java b/main/src/com/google/gridworks/gel/controls/Filter.java index d0d61f2c7..d77532b64 100644 --- a/main/src/com/google/gridworks/gel/controls/Filter.java +++ b/main/src/com/google/gridworks/gel/controls/Filter.java @@ -5,6 +5,7 @@ import java.util.Collection; import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONWriter; @@ -30,7 +31,7 @@ public class Filter implements Control { Object o = args[0].evaluate(bindings); if (ExpressionUtils.isError(o)) { return o; - } else if (!ExpressionUtils.isArrayOrCollection(o)) { + } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { return new EvalError("First argument is not an array"); } @@ -49,7 +50,26 @@ public class Filter implements Control { Object r = args[2].evaluate(bindings); if (r instanceof Boolean && ((Boolean) r).booleanValue()) { - results.add(v); + results.add(v); + } + } + } else if (o instanceof JSONArray) { + JSONArray a = (JSONArray) o; + int l = a.length(); + + results = new ArrayList(l); + for (int i = 0; i < l; i++) { + try { + Object v = a.get(i); + + bindings.put(name, v); + + Object r = args[2].evaluate(bindings); + if (r instanceof Boolean && ((Boolean) r).booleanValue()) { + results.add(v); + } + } catch (JSONException e) { + results.add(new EvalError(e.getMessage())); } } } else { diff --git a/main/src/com/google/gridworks/gel/controls/ForEach.java b/main/src/com/google/gridworks/gel/controls/ForEach.java index d1d0797a7..653dd1389 100644 --- a/main/src/com/google/gridworks/gel/controls/ForEach.java +++ b/main/src/com/google/gridworks/gel/controls/ForEach.java @@ -5,6 +5,7 @@ import java.util.Collection; import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONWriter; @@ -30,7 +31,7 @@ public class ForEach implements Control { Object o = args[0].evaluate(bindings); if (ExpressionUtils.isError(o)) { return o; - } else if (!ExpressionUtils.isArrayOrCollection(o)) { + } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { return new EvalError("First argument to forEach is not an array"); } @@ -51,6 +52,24 @@ public class ForEach implements Control { results.add(r); } + } else if (o instanceof JSONArray) { + JSONArray a = (JSONArray) o; + int l = a.length(); + + results = new ArrayList(l); + for (int i = 0; i < l; i++) { + try { + Object v = a.get(i); + + bindings.put(name, v); + + Object r = args[2].evaluate(bindings); + + results.add(r); + } catch (JSONException e) { + results.add(new EvalError(e.getMessage())); + } + } } else { Collection collection = ExpressionUtils.toObjectCollection(o); diff --git a/main/src/com/google/gridworks/gel/controls/ForEachIndex.java b/main/src/com/google/gridworks/gel/controls/ForEachIndex.java index 4d24672f0..e0b296e55 100644 --- a/main/src/com/google/gridworks/gel/controls/ForEachIndex.java +++ b/main/src/com/google/gridworks/gel/controls/ForEachIndex.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONWriter; @@ -32,7 +33,7 @@ public class ForEachIndex implements Control { Object o = args[0].evaluate(bindings); if (ExpressionUtils.isError(o)) { return o; - } else if (!ExpressionUtils.isArrayOrCollection(o)) { + } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { return new EvalError("First argument to forEach is not an array"); } @@ -50,15 +51,34 @@ public class ForEachIndex implements Control { results = new ArrayList(values.length); for (int i = 0; i < values.length; i++) { - Object v = values[i]; - - bindings.put(indexName, i); + Object v = values[i]; + + bindings.put(indexName, i); bindings.put(elementName, v); Object r = args[3].evaluate(bindings); results.add(r); } + } else if (o instanceof JSONArray) { + JSONArray a = (JSONArray) o; + int l = a.length(); + + results = new ArrayList(l); + for (int i = 0; i < l; i++) { + try { + Object v = a.get(i); + + bindings.put(indexName, i); + bindings.put(elementName, v); + + Object r = args[3].evaluate(bindings); + + results.add(r); + } catch (JSONException e) { + results.add(new EvalError(e.getMessage())); + } + } } else { List list = ExpressionUtils.toObjectList(o); diff --git a/main/src/com/google/gridworks/preference/PreferenceStore.java b/main/src/com/google/gridworks/preference/PreferenceStore.java index d60f0933f..9d3c41c09 100644 --- a/main/src/com/google/gridworks/preference/PreferenceStore.java +++ b/main/src/com/google/gridworks/preference/PreferenceStore.java @@ -62,8 +62,10 @@ public class PreferenceStore implements Jsonizable { Iterator i = entries.keys(); while (i.hasNext()) { String key = i.next(); - Object o = entries.get(key); - _prefs.put(key, loadObject(o)); + if (!entries.isNull(key)) { + Object o = entries.get(key); + _prefs.put(key, loadObject(o)); + } } } } diff --git a/main/src/com/google/gridworks/util/JSONUtilities.java b/main/src/com/google/gridworks/util/JSONUtilities.java index 1051a79af..ca8110816 100644 --- a/main/src/com/google/gridworks/util/JSONUtilities.java +++ b/main/src/com/google/gridworks/util/JSONUtilities.java @@ -129,4 +129,15 @@ public class JSONUtilities { obj.put(key, value.toString()); } } + + static public Object[] toArray(JSONArray a) throws JSONException { + int l = a.length(); + + Object[] a2 = new Object[l]; + for (int i = 0; i < l; i++) { + a2[i] = a.get(i); + } + + return a2; + } }