From 6d05631a07c975bce7d12249661d46ffea36373b Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Mon, 19 Nov 2018 13:31:24 +0000 Subject: [PATCH] Migrate GREL functions and controls to Jackson --- .../google/refine/expr/ExpressionUtils.java | 13 ++-- .../com/google/refine/expr/functions/Get.java | 40 ++++-------- .../refine/expr/functions/HasField.java | 12 +--- .../google/refine/expr/functions/Jsonize.java | 31 ++------- .../google/refine/expr/functions/Length.java | 7 +- .../google/refine/expr/functions/Slice.java | 20 ++---- .../refine/expr/functions/arrays/Join.java | 10 +-- .../refine/expr/functions/arrays/Reverse.java | 13 +--- .../refine/expr/functions/arrays/Sort.java | 21 ++---- .../refine/expr/functions/arrays/Uniques.java | 13 +--- .../expr/functions/strings/ParseJson.java | 9 ++- .../refine/expr/util/JsonValueConverter.java | 37 +++++++++++ .../refine/grel/ast/FieldAccessorExpr.java | 17 +---- .../google/refine/grel/controls/Filter.java | 39 +++++------ .../google/refine/grel/controls/ForEach.java | 37 +++++------ .../refine/grel/controls/ForEachIndex.java | 33 ++++------ .../com/google/refine/util/JSONUtilities.java | 12 ++++ .../expr/util/JsonValueConverterTests.java | 64 +++++++++++++++++++ 18 files changed, 220 insertions(+), 208 deletions(-) create mode 100644 main/src/com/google/refine/expr/util/JsonValueConverter.java create mode 100644 main/tests/server/src/com/google/refine/tests/expr/util/JsonValueConverterTests.java diff --git a/main/src/com/google/refine/expr/ExpressionUtils.java b/main/src/com/google/refine/expr/ExpressionUtils.java index f44b0cd52..95179dcfe 100644 --- a/main/src/com/google/refine/expr/ExpressionUtils.java +++ b/main/src/com/google/refine/expr/ExpressionUtils.java @@ -41,9 +41,8 @@ import java.util.List; import java.util.Properties; import java.util.Set; -import org.json.JSONArray; -import org.json.JSONObject; - +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.model.Cell; import com.google.refine.model.Project; import com.google.refine.model.Row; @@ -150,10 +149,10 @@ public class ExpressionUtils { } static public Serializable wrapStorable(Object v) { - if (v instanceof JSONArray) { - return ((JSONArray) v).toString(); - } else if (v instanceof JSONObject) { - return ((JSONObject) v).toString(); + if (v instanceof ArrayNode) { + return ((ArrayNode) v).toString(); + } else if (v instanceof ObjectNode) { + return ((ObjectNode) v).toString(); } else { return isStorable(v) ? (Serializable) v : diff --git a/main/src/com/google/refine/expr/functions/Get.java b/main/src/com/google/refine/expr/functions/Get.java index d3dad40fd..4ee7866f9 100644 --- a/main/src/com/google/refine/expr/functions/Get.java +++ b/main/src/com/google/refine/expr/functions/Get.java @@ -36,13 +36,12 @@ package com.google.refine.expr.functions; import java.util.List; import java.util.Properties; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.HasFields; import com.google.refine.expr.HasFieldsList; +import com.google.refine.expr.util.JsonValueConverter; import com.google.refine.grel.Function; public class Get implements Function { @@ -57,26 +56,22 @@ public class Get implements Function { if (v != null && from != null) { if (v instanceof HasFields && from instanceof String) { return ((HasFields) v).getField((String) from, bindings); - } else if (v instanceof JSONObject && from instanceof String) { - try { - return ((JSONObject) v).get((String) from); - } catch (JSONException e) { - // ignore; will return null - } + } else if (v instanceof ObjectNode && from instanceof String) { + return JsonValueConverter.convert(((ObjectNode) v).get((String) from)); } else { if (from instanceof Number && (to == null || to instanceof Number)) { if (v.getClass().isArray() || v instanceof List || v instanceof HasFieldsList || - v instanceof JSONArray) { + v instanceof ArrayNode) { int length = 0; if (v.getClass().isArray()) { length = ((Object[]) v).length; } else if (v instanceof HasFieldsList) { length = ((HasFieldsList) v).length(); - } else if (v instanceof JSONArray) { - length = ((JSONArray) v).length(); + } else if (v instanceof ArrayNode) { + length = ((ArrayNode) v).size(); } else { length = ExpressionUtils.toObjectList(v).size(); } @@ -92,12 +87,8 @@ public class Get implements Function { return ((Object[]) v)[start]; } else if (v instanceof HasFieldsList) { return ((HasFieldsList) v).get(start); - } else if (v instanceof JSONArray) { - try { - return ((JSONArray) v).get(start); - } catch (JSONException e) { - // ignore; will return null - } + } else if (v instanceof ArrayNode) { + return JsonValueConverter.convert(((ArrayNode) v).get(start)); } else { return ExpressionUtils.toObjectList(v).get(start); } @@ -118,18 +109,13 @@ public class Get implements Function { return a2; } else if (v instanceof HasFieldsList) { return ((HasFieldsList) v).getSubList(start, end); - } else if (v instanceof JSONArray) { - JSONArray a = (JSONArray) v; + } else if (v instanceof ArrayNode) { + ArrayNode a = (ArrayNode) v; Object[] a2 = new Object[end - start]; for (int i = 0; i < a2.length; i++) { - try { - a2[i] = a.get(start + i); - } catch (JSONException e) { - // ignore - } + a2[i] = JsonValueConverter.convert(a.get(start + i)); } - return a2; } else { return ExpressionUtils.toObjectList(v).subList(start, end); diff --git a/main/src/com/google/refine/expr/functions/HasField.java b/main/src/com/google/refine/expr/functions/HasField.java index 8fb0cb282..3e466ca2f 100644 --- a/main/src/com/google/refine/expr/functions/HasField.java +++ b/main/src/com/google/refine/expr/functions/HasField.java @@ -35,9 +35,7 @@ package com.google.refine.expr.functions; import java.util.Properties; -import org.json.JSONException; -import org.json.JSONObject; - +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.refine.expr.HasFields; import com.google.refine.grel.Function; @@ -53,12 +51,8 @@ public class HasField implements Function { String name = (String) f; if (v instanceof HasFields) { return ((HasFields) v).getField(name, bindings) != null; - } else if (v instanceof JSONObject) { - try { - return ((JSONObject) v).get(name) != null; - } catch (JSONException e) { - // ignore; will return false - } + } else if (v instanceof ObjectNode) { + return ((ObjectNode) v).has(name); } } } diff --git a/main/src/com/google/refine/expr/functions/Jsonize.java b/main/src/com/google/refine/expr/functions/Jsonize.java index 1fdd02dbf..f9081fe12 100644 --- a/main/src/com/google/refine/expr/functions/Jsonize.java +++ b/main/src/com/google/refine/expr/functions/Jsonize.java @@ -33,15 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.expr.functions; -import java.util.Collection; -import java.util.Map; +import java.io.IOException; import java.util.Properties; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - import com.google.refine.grel.Function; +import com.google.refine.util.ParsingUtilities; public class Jsonize implements Function { @@ -49,27 +45,8 @@ public class Jsonize implements Function { public Object call(Properties bindings, Object[] args) { if (args.length >= 1) { try { - Object o1 = args[0]; - if (o1 == null) { - return "null"; - } else if (o1 instanceof Number) { - return JSONObject.numberToString((Number) o1); - } else if (o1 instanceof Boolean) { - return o1.toString(); - } else if (o1 instanceof JSONObject) { - return ((JSONObject) o1).toString(); - } else if (o1 instanceof JSONArray) { - return ((JSONArray) o1).toString(); - } else if (o1 instanceof Map) { - return new JSONObject((Map) o1).toString(); - } else if (o1 instanceof Collection) { - return new JSONArray((Collection) o1).toString(); - } else if (o1.getClass().isArray()) { - return new JSONArray(o1).toString(); - } else { - return JSONObject.quote(o1.toString()); - } - } catch (JSONException e) { + return ParsingUtilities.mapper.writeValueAsString(args[0]); + } catch (IOException e) { throw new RuntimeException(e); } } diff --git a/main/src/com/google/refine/expr/functions/Length.java b/main/src/com/google/refine/expr/functions/Length.java index e01b45eaf..ddf58f38b 100644 --- a/main/src/com/google/refine/expr/functions/Length.java +++ b/main/src/com/google/refine/expr/functions/Length.java @@ -36,8 +36,7 @@ package com.google.refine.expr.functions; import java.util.Collection; import java.util.Properties; -import org.json.JSONArray; - +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.EvalError; import com.google.refine.expr.HasFieldsList; import com.google.refine.grel.ControlFunctionRegistry; @@ -58,8 +57,8 @@ public class Length implements Function { return ((Collection) v).size(); } else if (v instanceof HasFieldsList) { return ((HasFieldsList) v).length(); - } else if (v instanceof JSONArray) { - return ((JSONArray) v).length(); + } else if (v instanceof ArrayNode) { + return ((ArrayNode) v).size(); } else { String s = (v instanceof String ? (String) v : v.toString()); return s.length(); diff --git a/main/src/com/google/refine/expr/functions/Slice.java b/main/src/com/google/refine/expr/functions/Slice.java index b757337ce..cb689203e 100644 --- a/main/src/com/google/refine/expr/functions/Slice.java +++ b/main/src/com/google/refine/expr/functions/Slice.java @@ -36,9 +36,7 @@ package com.google.refine.expr.functions; import java.util.List; import java.util.Properties; -import org.json.JSONArray; -import org.json.JSONException; - +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.HasFieldsList; import com.google.refine.grel.Function; @@ -53,14 +51,14 @@ public class Slice implements Function { Object to = (args.length == 3) ? args[2] : null; if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) { - if (v.getClass().isArray() || v instanceof List || v instanceof HasFieldsList || v instanceof JSONArray) { + if (v.getClass().isArray() || v instanceof List || v instanceof HasFieldsList || v instanceof ArrayNode) { int length = 0; if (v.getClass().isArray()) { length = ((Object[]) v).length; } else if (v instanceof HasFieldsList) { length = ((HasFieldsList) v).length(); - } else if (v instanceof JSONArray) { - length = ((JSONArray) v).length(); + } else if (v instanceof ArrayNode) { + length = ((ArrayNode) v).size(); } else { length = ExpressionUtils.toObjectList(v).size(); } @@ -86,16 +84,12 @@ public class Slice implements Function { return a2; } else if (v instanceof HasFieldsList) { return ((HasFieldsList) v).getSubList(start, end); - } else if (v instanceof JSONArray) { - JSONArray a = (JSONArray) v; + } else if (v instanceof ArrayNode) { + ArrayNode a = (ArrayNode) v; Object[] a2 = new Object[end - start]; for (int i = 0; i < a2.length; i++) { - try { - a2[i] = a.get(start + i); - } catch (JSONException e) { - // ignore - } + a2[i] = a.get(start + i); } return a2; diff --git a/main/src/com/google/refine/expr/functions/arrays/Join.java b/main/src/com/google/refine/expr/functions/arrays/Join.java index fb3fad9a4..64b571d32 100644 --- a/main/src/com/google/refine/expr/functions/arrays/Join.java +++ b/main/src/com/google/refine/expr/functions/arrays/Join.java @@ -36,9 +36,9 @@ package com.google.refine.expr.functions.arrays; import java.util.List; import java.util.Properties; -import org.json.JSONArray; import org.json.JSONException; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.EvalError; import com.google.refine.expr.ExpressionUtils; import com.google.refine.grel.ControlFunctionRegistry; @@ -55,7 +55,7 @@ public class Join implements Function { if (v != null && s != null && s instanceof String) { String separator = (String) s; - if (v.getClass().isArray() || v instanceof List || v instanceof JSONArray) { + if (v.getClass().isArray() || v instanceof List || v instanceof ArrayNode) { StringBuffer sb = new StringBuffer(); if (v.getClass().isArray()) { for (Object o : (Object[]) v) { @@ -66,9 +66,9 @@ public class Join implements Function { sb.append(o.toString()); } } - } else if (v instanceof JSONArray) { - JSONArray a = (JSONArray) v; - int l = a.length(); + } else if (v instanceof ArrayNode) { + ArrayNode a = (ArrayNode) v; + int l = a.size(); for (int i = 0; i < l; i++) { if (sb.length() > 0) { diff --git a/main/src/com/google/refine/expr/functions/arrays/Reverse.java b/main/src/com/google/refine/expr/functions/arrays/Reverse.java index eedc0195e..383e7763c 100644 --- a/main/src/com/google/refine/expr/functions/arrays/Reverse.java +++ b/main/src/com/google/refine/expr/functions/arrays/Reverse.java @@ -36,9 +36,7 @@ package com.google.refine.expr.functions.arrays; import java.util.List; import java.util.Properties; -import org.json.JSONArray; -import org.json.JSONException; - +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.EvalError; import com.google.refine.expr.ExpressionUtils; import com.google.refine.grel.ControlFunctionRegistry; @@ -53,13 +51,8 @@ public class Reverse implements Function { Object v = args[0]; if (v != null) { - if (v instanceof JSONArray) { - try { - v = JSONUtilities.toArray((JSONArray) v); - } catch (JSONException e) { - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + - " fails to process a JSON array: " + e.getMessage()); - } + if (v instanceof ArrayNode) { + v = JSONUtilities.toArray((ArrayNode) v); } if (v.getClass().isArray() || v instanceof List) { diff --git a/main/src/com/google/refine/expr/functions/arrays/Sort.java b/main/src/com/google/refine/expr/functions/arrays/Sort.java index 090a79968..e0f0436ed 100644 --- a/main/src/com/google/refine/expr/functions/arrays/Sort.java +++ b/main/src/com/google/refine/expr/functions/arrays/Sort.java @@ -38,9 +38,7 @@ import java.util.Collections; import java.util.List; import java.util.Properties; -import org.json.JSONArray; -import org.json.JSONException; - +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.EvalError; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; @@ -62,17 +60,12 @@ public class Sort implements Function { Arrays.sort(r, 0, r.length); return r; - } else if (v instanceof JSONArray) { - try { - Object[] r = JSONUtilities.toArray((JSONArray) v); - - Arrays.sort(r, 0, r.length); - - return r; - } catch (JSONException e) { - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + - " fails to process a JSON array: " + e.getMessage()); - } + } else if (v instanceof ArrayNode) { + Object[] r = JSONUtilities.toArray((ArrayNode) v); + + Arrays.sort(r, 0, r.length); + + return r; } else if (v instanceof List) { List> a = (List>) v; Collections.sort(a); diff --git a/main/src/com/google/refine/expr/functions/arrays/Uniques.java b/main/src/com/google/refine/expr/functions/arrays/Uniques.java index 0ee2c50ef..ed78bd370 100644 --- a/main/src/com/google/refine/expr/functions/arrays/Uniques.java +++ b/main/src/com/google/refine/expr/functions/arrays/Uniques.java @@ -38,9 +38,7 @@ import java.util.List; import java.util.Properties; import java.util.Set; -import org.json.JSONArray; -import org.json.JSONException; - +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.EvalError; import com.google.refine.expr.ExpressionUtils; import com.google.refine.grel.ControlFunctionRegistry; @@ -55,13 +53,8 @@ public class Uniques implements Function { Object v = args[0]; if (v != null) { - if (v instanceof JSONArray) { - try { - v = JSONUtilities.toArray((JSONArray) v); - } catch (JSONException e) { - return new EvalError(ControlFunctionRegistry.getFunctionName(this) + - " fails to process a JSON array: " + e.getMessage()); - } + if (v instanceof ArrayNode) { + v = JSONUtilities.toArray((ArrayNode) v); } if (v.getClass().isArray() || v instanceof List) { diff --git a/main/src/com/google/refine/expr/functions/strings/ParseJson.java b/main/src/com/google/refine/expr/functions/strings/ParseJson.java index 3e2725282..c858ea0bd 100644 --- a/main/src/com/google/refine/expr/functions/strings/ParseJson.java +++ b/main/src/com/google/refine/expr/functions/strings/ParseJson.java @@ -33,14 +33,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.expr.functions.strings; +import java.io.IOException; import java.util.Properties; -import org.json.JSONException; -import org.json.JSONTokener; - import com.google.refine.expr.EvalError; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; +import com.google.refine.util.ParsingUtilities; public class ParseJson implements Function { @@ -50,8 +49,8 @@ public class ParseJson implements Function { Object o1 = args[0]; if (o1 != null) { try { - return new JSONTokener(o1.toString()).nextValue(); - } catch (JSONException e) { + return ParsingUtilities.mapper.readTree(o1.toString()); + } catch (IOException e) { return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " failed: " + e.getMessage()); } } diff --git a/main/src/com/google/refine/expr/util/JsonValueConverter.java b/main/src/com/google/refine/expr/util/JsonValueConverter.java new file mode 100644 index 000000000..995136b79 --- /dev/null +++ b/main/src/com/google/refine/expr/util/JsonValueConverter.java @@ -0,0 +1,37 @@ +package com.google.refine.expr.util; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * Converts the a JSON value + * @author antonin + * + */ +public class JsonValueConverter { + + public static Object convert(JsonNode value) { + if (value == null) { + return null; + } + if (value.isObject()) { + return value; + } else if (value.isArray()) { + return value; + } else if (value.isBigDecimal() || value.isDouble() || value.isFloat()) { + return value.asDouble(); + } else if (value.isBigInteger()) { + return value.asLong(); + } else if (value.isInt()) { + return value.asInt(); + } else if (value.isBinary() || value.isTextual()) { + return value.asText(); + } else if (value.isBoolean()) { + return value.asBoolean(); + } else if (value.isNull()) { + return null; + } else { + return null; + } + } + +} diff --git a/main/src/com/google/refine/grel/ast/FieldAccessorExpr.java b/main/src/com/google/refine/grel/ast/FieldAccessorExpr.java index 73f45c7c1..e942e9023 100644 --- a/main/src/com/google/refine/grel/ast/FieldAccessorExpr.java +++ b/main/src/com/google/refine/grel/ast/FieldAccessorExpr.java @@ -41,6 +41,7 @@ import com.google.common.collect.Lists; import com.google.refine.expr.Evaluable; import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.HasFields; +import com.google.refine.expr.util.JsonValueConverter; /** * An abstract syntax tree node encapsulating a field accessor, @@ -67,21 +68,7 @@ public class FieldAccessorExpr implements Evaluable { return ((HasFields) o).getField(_fieldName, bindings); } else if (o instanceof ObjectNode) { JsonNode value = ((ObjectNode) o).get(_fieldName); - if (value.isArray()) { - return Lists.newArrayList(value.elements()); - } else if (value.isBigDecimal() || value.isDouble() || value.isFloat()) { - return value.asDouble(); - } else if (value.isBigInteger() || value.isInt()) { - return value.asLong(); - } else if (value.isBinary() || value.isTextual()) { - return value.asText(); - } else if (value.isBoolean()) { - return value.asBoolean(); - } else if (value.isNull()) { - return null; - } else { - return null; - } + return JsonValueConverter.convert(value); } else { return null; } diff --git a/main/src/com/google/refine/grel/controls/Filter.java b/main/src/com/google/refine/grel/controls/Filter.java index a2bb4000d..36a679649 100644 --- a/main/src/com/google/refine/grel/controls/Filter.java +++ b/main/src/com/google/refine/grel/controls/Filter.java @@ -38,12 +38,11 @@ import java.util.Collection; import java.util.List; import java.util.Properties; -import org.json.JSONArray; -import org.json.JSONException; - +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.EvalError; import com.google.refine.expr.Evaluable; import com.google.refine.expr.ExpressionUtils; +import com.google.refine.expr.util.JsonValueConverter; import com.google.refine.grel.Control; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ast.VariableExpr; @@ -65,7 +64,7 @@ public class Filter implements Control { Object o = args[0].evaluate(bindings); if (ExpressionUtils.isError(o)) { return o; - } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { + } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) { return new EvalError("First argument is not an array"); } @@ -91,27 +90,23 @@ public class Filter implements Control { results.add(v); } } - } else if (o instanceof JSONArray) { - JSONArray a = (JSONArray) o; - int l = a.length(); + } else if (o instanceof ArrayNode) { + ArrayNode a = (ArrayNode) o; + int l = a.size(); results = new ArrayList(l); for (int i = 0; i < l; i++) { - try { - Object v = a.get(i); - - if (v != null) { - bindings.put(name, v); - } else { - bindings.remove(name); - } - - Object r = args[2].evaluate(bindings); - if (r instanceof Boolean && ((Boolean) r).booleanValue()) { - results.add(v); - } - } catch (JSONException e) { - results.add(new EvalError(e.getMessage())); + Object v = JsonValueConverter.convert(a.get(i)); + + if (v != null) { + bindings.put(name, v); + } else { + bindings.remove(name); + } + + Object r = args[2].evaluate(bindings); + if (r instanceof Boolean && ((Boolean) r).booleanValue()) { + results.add(v); } } } else { diff --git a/main/src/com/google/refine/grel/controls/ForEach.java b/main/src/com/google/refine/grel/controls/ForEach.java index 493c18619..0103d60d0 100644 --- a/main/src/com/google/refine/grel/controls/ForEach.java +++ b/main/src/com/google/refine/grel/controls/ForEach.java @@ -38,12 +38,11 @@ import java.util.Collection; import java.util.List; import java.util.Properties; -import org.json.JSONArray; -import org.json.JSONException; - +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.EvalError; import com.google.refine.expr.Evaluable; import com.google.refine.expr.ExpressionUtils; +import com.google.refine.expr.util.JsonValueConverter; import com.google.refine.grel.Control; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ast.VariableExpr; @@ -65,7 +64,7 @@ public class ForEach implements Control { Object o = args[0].evaluate(bindings); if (ExpressionUtils.isError(o)) { return o; - } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { + } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) { return new EvalError("First argument to forEach is not an array"); } @@ -90,27 +89,23 @@ public class ForEach implements Control { results.add(r); } - } else if (o instanceof JSONArray) { - JSONArray a = (JSONArray) o; - int l = a.length(); + } else if (o instanceof ArrayNode) { + ArrayNode a = (ArrayNode) o; + int l = a.size(); results = new ArrayList(l); for (int i = 0; i < l; i++) { - try { - Object v = a.get(i); - - if (v != null) { - bindings.put(name, v); - } else { - bindings.remove(name); - } - - Object r = args[2].evaluate(bindings); - - results.add(r); - } catch (JSONException e) { - results.add(new EvalError(e.getMessage())); + Object v = JsonValueConverter.convert(a.get(i)); + + if (v != null) { + bindings.put(name, v); + } else { + bindings.remove(name); } + + Object r = args[2].evaluate(bindings); + + results.add(r); } } else { Collection collection = ExpressionUtils.toObjectCollection(o); diff --git a/main/src/com/google/refine/grel/controls/ForEachIndex.java b/main/src/com/google/refine/grel/controls/ForEachIndex.java index 5535a19f6..2c4df3164 100644 --- a/main/src/com/google/refine/grel/controls/ForEachIndex.java +++ b/main/src/com/google/refine/grel/controls/ForEachIndex.java @@ -37,12 +37,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; -import org.json.JSONArray; -import org.json.JSONException; - +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.refine.expr.EvalError; import com.google.refine.expr.Evaluable; import com.google.refine.expr.ExpressionUtils; +import com.google.refine.expr.util.JsonValueConverter; import com.google.refine.grel.Control; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ast.VariableExpr; @@ -67,7 +66,7 @@ public class ForEachIndex implements Control { Object o = args[0].evaluate(bindings); if (ExpressionUtils.isError(o)) { return o; - } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { + } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) { return new EvalError("First argument to forEach is not an array"); } @@ -94,24 +93,20 @@ public class ForEachIndex implements Control { results.add(r); } - } else if (o instanceof JSONArray) { - JSONArray a = (JSONArray) o; - int l = a.length(); + } else if (o instanceof ArrayNode) { + ArrayNode a = (ArrayNode) o; + int l = a.size(); results = new ArrayList(l); for (int i = 0; i < l; i++) { - try { - Object v = a.get(i); - - bindings.put(indexName, i); - bindings.put(elementName, v); - - Object r = args[3].evaluate(bindings); - - results.add(r); - } catch (JSONException e) { - results.add(new EvalError(e.getMessage())); - } + Object v = JsonValueConverter.convert(a.get(i)); + + bindings.put(indexName, i); + bindings.put(elementName, v); + + Object r = args[3].evaluate(bindings); + + results.add(r); } } else { List list = ExpressionUtils.toObjectList(o); diff --git a/main/src/com/google/refine/util/JSONUtilities.java b/main/src/com/google/refine/util/JSONUtilities.java index 9f8433087..84dd576d8 100644 --- a/main/src/com/google/refine/util/JSONUtilities.java +++ b/main/src/com/google/refine/util/JSONUtilities.java @@ -50,6 +50,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; +import com.google.refine.expr.util.JsonValueConverter; public class JSONUtilities { @@ -477,4 +478,15 @@ public class JSONUtilities { public static JSONObject objectNodeToJsonNode(ObjectNode fieldJsonObj) { return new JSONObject(fieldJsonObj.toString()); } + + public static Object[] toArray(ArrayNode v) { + if (v == null) { + return null; + } + Object[] result = new Object[v.size()]; + for (int i = 0; i != v.size(); i++) { + result[i] = JsonValueConverter.convert(v.get(i)); + } + return result; + } } diff --git a/main/tests/server/src/com/google/refine/tests/expr/util/JsonValueConverterTests.java b/main/tests/server/src/com/google/refine/tests/expr/util/JsonValueConverterTests.java new file mode 100644 index 000000000..ed666b8e6 --- /dev/null +++ b/main/tests/server/src/com/google/refine/tests/expr/util/JsonValueConverterTests.java @@ -0,0 +1,64 @@ +package com.google.refine.tests.expr.util; + +import static org.testng.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; + +import org.testng.annotations.Test; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.refine.expr.util.JsonValueConverter; +import com.google.refine.util.ParsingUtilities; + +public class JsonValueConverterTests { + + private void fieldEquals(String json, Object expectedValue) { + try { + ObjectNode n = (ObjectNode) ParsingUtilities.mapper.readTree(json); + assertEquals(expectedValue, JsonValueConverter.convert(n.get("foo"))); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testConvertJsonObject() throws IOException { + fieldEquals("{\"foo\":{\"ob\":\"ject\"}}", ParsingUtilities.mapper.readTree("{\"ob\":\"ject\"}")); + } + + @Test + public void testConvertJsonArray() throws IOException { + fieldEquals("{\"foo\":[1,2]}", ParsingUtilities.mapper.readTree("[1,2]")); + } + + @Test + public void testConvertInt() { + fieldEquals("{\"foo\":3}", 3); + } + + @Test + public void testConvertFloat() { + fieldEquals("{\"foo\":3.14}", 3.14); + } + + @Test + public void testConvertBool() { + fieldEquals("{\"foo\":true}", true); + } + + @Test + public void testConvertNull() { + fieldEquals("{\"foo\":null}", null); + } + + @Test + public void testConvertString() { + fieldEquals("{\"foo\":\"bar\"}", "bar"); + } + + @Test + public void testConvertNoField() { + fieldEquals("{}", null); + } +}