s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate GREL functions and controls to Jackson

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-11-19 13:31:24 +00:00
parent 5a4a79028f
commit 6d05631a07
18 changed files with 220 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
main/src/com/google/refine/expr/ExpressionUtils.java
View File

@ -41,9 +41,8 @@ import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONObject; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Row; import com.google.refine.model.Row;
@ -150,10 +149,10 @@ public class ExpressionUtils {
} }
static public Serializable wrapStorable(Object v) { static public Serializable wrapStorable(Object v) {
if (v instanceof JSONArray) { if (v instanceof ArrayNode) {
return ((JSONArray) v).toString(); return ((ArrayNode) v).toString();
} else if (v instanceof JSONObject) { } else if (v instanceof ObjectNode) {
return ((JSONObject) v).toString(); return ((ObjectNode) v).toString();
} else { } else {
return isStorable(v) ? return isStorable(v) ?
(Serializable) v : (Serializable) v :

40
main/src/com/google/refine/expr/functions/Get.java
View File

@ -36,13 +36,12 @@ package com.google.refine.expr.functions;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONException; import com.fasterxml.jackson.databind.node.ObjectNode;
import org.json.JSONObject;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.HasFields; import com.google.refine.expr.HasFields;
import com.google.refine.expr.HasFieldsList; import com.google.refine.expr.HasFieldsList;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.grel.Function; import com.google.refine.grel.Function;
public class Get implements Function { public class Get implements Function {
@ -57,26 +56,22 @@ public class Get implements Function {
if (v != null && from != null) { if (v != null && from != null) {
if (v instanceof HasFields && from instanceof String) { if (v instanceof HasFields && from instanceof String) {
return ((HasFields) v).getField((String) from, bindings); return ((HasFields) v).getField((String) from, bindings);
} else if (v instanceof JSONObject && from instanceof String) { } else if (v instanceof ObjectNode && from instanceof String) {
try { return JsonValueConverter.convert(((ObjectNode) v).get((String) from));
return ((JSONObject) v).get((String) from);
} catch (JSONException e) {
// ignore; will return null
}
} else { } else {
if (from instanceof Number && (to == null || to instanceof Number)) { if (from instanceof Number && (to == null || to instanceof Number)) {
if (v.getClass().isArray() || if (v.getClass().isArray() ||
v instanceof List<?> || v instanceof List<?> ||
v instanceof HasFieldsList || v instanceof HasFieldsList ||
v instanceof JSONArray) { v instanceof ArrayNode) {
int length = 0; int length = 0;
if (v.getClass().isArray()) { if (v.getClass().isArray()) {
length = ((Object[]) v).length; length = ((Object[]) v).length;
} else if (v instanceof HasFieldsList) { } else if (v instanceof HasFieldsList) {
length = ((HasFieldsList) v).length(); length = ((HasFieldsList) v).length();
} else if (v instanceof JSONArray) { } else if (v instanceof ArrayNode) {
length = ((JSONArray) v).length(); length = ((ArrayNode) v).size();
} else { } else {
length = ExpressionUtils.toObjectList(v).size(); length = ExpressionUtils.toObjectList(v).size();
} }
@ -92,12 +87,8 @@ public class Get implements Function {
return ((Object[]) v)[start]; return ((Object[]) v)[start];
} else if (v instanceof HasFieldsList) { } else if (v instanceof HasFieldsList) {
return ((HasFieldsList) v).get(start); return ((HasFieldsList) v).get(start);
} else if (v instanceof JSONArray) { } else if (v instanceof ArrayNode) {
try { return JsonValueConverter.convert(((ArrayNode) v).get(start));
return ((JSONArray) v).get(start);
} catch (JSONException e) {
// ignore; will return null
}
} else { } else {
return ExpressionUtils.toObjectList(v).get(start); return ExpressionUtils.toObjectList(v).get(start);
} }
@ -118,18 +109,13 @@ public class Get implements Function {
return a2; return a2;
} else if (v instanceof HasFieldsList) { } else if (v instanceof HasFieldsList) {
return ((HasFieldsList) v).getSubList(start, end); return ((HasFieldsList) v).getSubList(start, end);
} else if (v instanceof JSONArray) { } else if (v instanceof ArrayNode) {
JSONArray a = (JSONArray) v; ArrayNode a = (ArrayNode) v;
Object[] a2 = new Object[end - start]; Object[] a2 = new Object[end - start];
for (int i = 0; i < a2.length; i++) { for (int i = 0; i < a2.length; i++) {
try { a2[i] = JsonValueConverter.convert(a.get(start + i));
a2[i] = a.get(start + i);
} catch (JSONException e) {
// ignore
}
} }
return a2; return a2;
} else { } else {
return ExpressionUtils.toObjectList(v).subList(start, end); return ExpressionUtils.toObjectList(v).subList(start, end);

12
main/src/com/google/refine/expr/functions/HasField.java
View File

@ -35,9 +35,7 @@ package com.google.refine.expr.functions;
import java.util.Properties; import java.util.Properties;
import org.json.JSONException; import com.fasterxml.jackson.databind.node.ObjectNode;
import org.json.JSONObject;
import com.google.refine.expr.HasFields; import com.google.refine.expr.HasFields;
import com.google.refine.grel.Function; import com.google.refine.grel.Function;
@ -53,12 +51,8 @@ public class HasField implements Function {
String name = (String) f; String name = (String) f;
if (v instanceof HasFields) { if (v instanceof HasFields) {
return ((HasFields) v).getField(name, bindings) != null; return ((HasFields) v).getField(name, bindings) != null;
} else if (v instanceof JSONObject) { } else if (v instanceof ObjectNode) {
try { return ((ObjectNode) v).has(name);
return ((JSONObject) v).get(name) != null;
} catch (JSONException e) {
// ignore; will return false
}
} }
} }
} }

31
main/src/com/google/refine/expr/functions/Jsonize.java
View File

@ -33,15 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions; package com.google.refine.expr.functions;
import java.util.Collection; import java.io.IOException;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.grel.Function; import com.google.refine.grel.Function;
import com.google.refine.util.ParsingUtilities;
public class Jsonize implements Function { public class Jsonize implements Function {
@ -49,27 +45,8 @@ public class Jsonize implements Function {
public Object call(Properties bindings, Object[] args) { public Object call(Properties bindings, Object[] args) {
if (args.length >= 1) { if (args.length >= 1) {
try { try {
Object o1 = args[0]; return ParsingUtilities.mapper.writeValueAsString(args[0]);
if (o1 == null) { } catch (IOException e) {
return "null";
} else if (o1 instanceof Number) {
return JSONObject.numberToString((Number) o1);
} else if (o1 instanceof Boolean) {
return o1.toString();
} else if (o1 instanceof JSONObject) {
return ((JSONObject) o1).toString();
} else if (o1 instanceof JSONArray) {
return ((JSONArray) o1).toString();
} else if (o1 instanceof Map) {
return new JSONObject((Map<?,?>) o1).toString();
} else if (o1 instanceof Collection) {
return new JSONArray((Collection<?>) o1).toString();
} else if (o1.getClass().isArray()) {
return new JSONArray(o1).toString();
} else {
return JSONObject.quote(o1.toString());
}
} catch (JSONException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }

7
main/src/com/google/refine/expr/functions/Length.java
View File

@ -36,8 +36,7 @@ package com.google.refine.expr.functions;
import java.util.Collection; import java.util.Collection;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.expr.HasFieldsList; import com.google.refine.expr.HasFieldsList;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
@ -58,8 +57,8 @@ public class Length implements Function {
return ((Collection<?>) v).size(); return ((Collection<?>) v).size();
} else if (v instanceof HasFieldsList) { } else if (v instanceof HasFieldsList) {
return ((HasFieldsList) v).length(); return ((HasFieldsList) v).length();
} else if (v instanceof JSONArray) { } else if (v instanceof ArrayNode) {
return ((JSONArray) v).length(); return ((ArrayNode) v).size();
} else { } else {
String s = (v instanceof String ? (String) v : v.toString()); String s = (v instanceof String ? (String) v : v.toString());
return s.length(); return s.length();

20
main/src/com/google/refine/expr/functions/Slice.java
View File

@ -36,9 +36,7 @@ package com.google.refine.expr.functions;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONException;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.HasFieldsList; import com.google.refine.expr.HasFieldsList;
import com.google.refine.grel.Function; import com.google.refine.grel.Function;
@ -53,14 +51,14 @@ public class Slice implements Function {
Object to = (args.length == 3) ? args[2] : null; Object to = (args.length == 3) ? args[2] : null;
if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) { if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) {
if (v.getClass().isArray() || v instanceof List<?> || v instanceof HasFieldsList || v instanceof JSONArray) { if (v.getClass().isArray() || v instanceof List<?> || v instanceof HasFieldsList || v instanceof ArrayNode) {
int length = 0; int length = 0;
if (v.getClass().isArray()) { if (v.getClass().isArray()) {
length = ((Object[]) v).length; length = ((Object[]) v).length;
} else if (v instanceof HasFieldsList) { } else if (v instanceof HasFieldsList) {
length = ((HasFieldsList) v).length(); length = ((HasFieldsList) v).length();
} else if (v instanceof JSONArray) { } else if (v instanceof ArrayNode) {
length = ((JSONArray) v).length(); length = ((ArrayNode) v).size();
} else { } else {
length = ExpressionUtils.toObjectList(v).size(); length = ExpressionUtils.toObjectList(v).size();
} }
@ -86,16 +84,12 @@ public class Slice implements Function {
return a2; return a2;
} else if (v instanceof HasFieldsList) { } else if (v instanceof HasFieldsList) {
return ((HasFieldsList) v).getSubList(start, end); return ((HasFieldsList) v).getSubList(start, end);
} else if (v instanceof JSONArray) { } else if (v instanceof ArrayNode) {
JSONArray a = (JSONArray) v; ArrayNode a = (ArrayNode) v;
Object[] a2 = new Object[end - start]; Object[] a2 = new Object[end - start];
for (int i = 0; i < a2.length; i++) { for (int i = 0; i < a2.length; i++) {
try { a2[i] = a.get(start + i);
a2[i] = a.get(start + i);
} catch (JSONException e) {
// ignore
}
} }
return a2; return a2;

10
main/src/com/google/refine/expr/functions/arrays/Join.java
View File

@ -36,9 +36,9 @@ package com.google.refine.expr.functions.arrays;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
@ -55,7 +55,7 @@ public class Join implements Function {
if (v != null && s != null && s instanceof String) { if (v != null && s != null && s instanceof String) {
String separator = (String) s; String separator = (String) s;
if (v.getClass().isArray() || v instanceof List<?> || v instanceof JSONArray) { if (v.getClass().isArray() || v instanceof List<?> || v instanceof ArrayNode) {
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
if (v.getClass().isArray()) { if (v.getClass().isArray()) {
for (Object o : (Object[]) v) { for (Object o : (Object[]) v) {
@ -66,9 +66,9 @@ public class Join implements Function {
sb.append(o.toString()); sb.append(o.toString());
} }
} }
} else if (v instanceof JSONArray) { } else if (v instanceof ArrayNode) {
JSONArray a = (JSONArray) v; ArrayNode a = (ArrayNode) v;
int l = a.length(); int l = a.size();
for (int i = 0; i < l; i++) { for (int i = 0; i < l; i++) {
if (sb.length() > 0) { if (sb.length() > 0) {

13
main/src/com/google/refine/expr/functions/arrays/Reverse.java
View File

@ -36,9 +36,7 @@ package com.google.refine.expr.functions.arrays;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONException;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
@ -53,13 +51,8 @@ public class Reverse implements Function {
Object v = args[0]; Object v = args[0];
if (v != null) { if (v != null) {
if (v instanceof JSONArray) { if (v instanceof ArrayNode) {
try { v = JSONUtilities.toArray((ArrayNode) v);
v = JSONUtilities.toArray((JSONArray) v);
} catch (JSONException e) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) +
" fails to process a JSON array: " + e.getMessage());
}
} }
if (v.getClass().isArray() || v instanceof List<?>) { if (v.getClass().isArray() || v instanceof List<?>) {

17
main/src/com/google/refine/expr/functions/arrays/Sort.java
View File

@ -38,9 +38,7 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONException;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function; import com.google.refine.grel.Function;
@ -62,17 +60,12 @@ public class Sort implements Function {
Arrays.sort(r, 0, r.length); Arrays.sort(r, 0, r.length);
return r; return r;
} else if (v instanceof JSONArray) { } else if (v instanceof ArrayNode) {
try { Object[] r = JSONUtilities.toArray((ArrayNode) v);
Object[] r = JSONUtilities.toArray((JSONArray) v);
Arrays.sort(r, 0, r.length); Arrays.sort(r, 0, r.length);
return r; return r;
} catch (JSONException e) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) +
" fails to process a JSON array: " + e.getMessage());
}
} else if (v instanceof List<?>) { } else if (v instanceof List<?>) {
List<? extends Comparable<Object>> a = (List<? extends Comparable<Object>>) v; List<? extends Comparable<Object>> a = (List<? extends Comparable<Object>>) v;
Collections.sort(a); Collections.sort(a);

13
main/src/com/google/refine/expr/functions/arrays/Uniques.java
View File

@ -38,9 +38,7 @@ import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONException;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
@ -55,13 +53,8 @@ public class Uniques implements Function {
Object v = args[0]; Object v = args[0];
if (v != null) { if (v != null) {
if (v instanceof JSONArray) { if (v instanceof ArrayNode) {
try { v = JSONUtilities.toArray((ArrayNode) v);
v = JSONUtilities.toArray((JSONArray) v);
} catch (JSONException e) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) +
" fails to process a JSON array: " + e.getMessage());
}
} }
if (v.getClass().isArray() || v instanceof List<?>) { if (v.getClass().isArray() || v instanceof List<?>) {

9
main/src/com/google/refine/expr/functions/strings/ParseJson.java
View File

@ -33,14 +33,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions.strings; package com.google.refine.expr.functions.strings;
import java.io.IOException;
import java.util.Properties; import java.util.Properties;
import org.json.JSONException;
import org.json.JSONTokener;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function; import com.google.refine.grel.Function;
import com.google.refine.util.ParsingUtilities;
public class ParseJson implements Function { public class ParseJson implements Function {
@ -50,8 +49,8 @@ public class ParseJson implements Function {
Object o1 = args[0]; Object o1 = args[0];
if (o1 != null) { if (o1 != null) {
try { try {
return new JSONTokener(o1.toString()).nextValue(); return ParsingUtilities.mapper.readTree(o1.toString());
} catch (JSONException e) { } catch (IOException e) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " failed: " + e.getMessage()); return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " failed: " + e.getMessage());
} }
} }

37
main/src/com/google/refine/expr/util/JsonValueConverter.java Normal file
View File

@ -0,0 +1,37 @@
package com.google.refine.expr.util;
import com.fasterxml.jackson.databind.JsonNode;
/**
* Converts the a JSON value
* @author antonin
*
*/
public class JsonValueConverter {
public static Object convert(JsonNode value) {
if (value == null) {
return null;
}
if (value.isObject()) {
return value;
} else if (value.isArray()) {
return value;
} else if (value.isBigDecimal() || value.isDouble() || value.isFloat()) {
return value.asDouble();
} else if (value.isBigInteger()) {
return value.asLong();
} else if (value.isInt()) {
return value.asInt();
} else if (value.isBinary() || value.isTextual()) {
return value.asText();
} else if (value.isBoolean()) {
return value.asBoolean();
} else if (value.isNull()) {
return null;
} else {
return null;
}
}
}

17
main/src/com/google/refine/grel/ast/FieldAccessorExpr.java
View File

@ -41,6 +41,7 @@ import com.google.common.collect.Lists;
import com.google.refine.expr.Evaluable; import com.google.refine.expr.Evaluable;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.HasFields; import com.google.refine.expr.HasFields;
import com.google.refine.expr.util.JsonValueConverter;
/** /**
* An abstract syntax tree node encapsulating a field accessor, * An abstract syntax tree node encapsulating a field accessor,
@ -67,21 +68,7 @@ public class FieldAccessorExpr implements Evaluable {
return ((HasFields) o).getField(_fieldName, bindings); return ((HasFields) o).getField(_fieldName, bindings);
} else if (o instanceof ObjectNode) { } else if (o instanceof ObjectNode) {
JsonNode value = ((ObjectNode) o).get(_fieldName); JsonNode value = ((ObjectNode) o).get(_fieldName);
if (value.isArray()) { return JsonValueConverter.convert(value);
return Lists.newArrayList(value.elements());
} else if (value.isBigDecimal() || value.isDouble() || value.isFloat()) {
return value.asDouble();
} else if (value.isBigInteger() || value.isInt()) {
return value.asLong();
} else if (value.isBinary() || value.isTextual()) {
return value.asText();
} else if (value.isBoolean()) {
return value.asBoolean();
} else if (value.isNull()) {
return null;
} else {
return null;
}
} else { } else {
return null; return null;
} }

35
main/src/com/google/refine/grel/controls/Filter.java
View File

@ -38,12 +38,11 @@ import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONException;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.expr.Evaluable; import com.google.refine.expr.Evaluable;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.grel.Control; import com.google.refine.grel.Control;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.ast.VariableExpr; import com.google.refine.grel.ast.VariableExpr;
@ -65,7 +64,7 @@ public class Filter implements Control {
Object o = args[0].evaluate(bindings); Object o = args[0].evaluate(bindings);
if (ExpressionUtils.isError(o)) { if (ExpressionUtils.isError(o)) {
return o; return o;
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) {
return new EvalError("First argument is not an array"); return new EvalError("First argument is not an array");
} }
@ -91,27 +90,23 @@ public class Filter implements Control {
results.add(v); results.add(v);
} }
} }
} else if (o instanceof JSONArray) { } else if (o instanceof ArrayNode) {
JSONArray a = (JSONArray) o; ArrayNode a = (ArrayNode) o;
int l = a.length(); int l = a.size();
results = new ArrayList<Object>(l); results = new ArrayList<Object>(l);
for (int i = 0; i < l; i++) { for (int i = 0; i < l; i++) {
try { Object v = JsonValueConverter.convert(a.get(i));
Object v = a.get(i);
if (v != null) { if (v != null) {
bindings.put(name, v); bindings.put(name, v);
} else { } else {
bindings.remove(name); bindings.remove(name);
} }
Object r = args[2].evaluate(bindings); Object r = args[2].evaluate(bindings);
if (r instanceof Boolean && ((Boolean) r).booleanValue()) { if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
results.add(v); results.add(v);
}
} catch (JSONException e) {
results.add(new EvalError(e.getMessage()));
} }
} }
} else { } else {

35
main/src/com/google/refine/grel/controls/ForEach.java
View File

@ -38,12 +38,11 @@ import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONException;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.expr.Evaluable; import com.google.refine.expr.Evaluable;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.grel.Control; import com.google.refine.grel.Control;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.ast.VariableExpr; import com.google.refine.grel.ast.VariableExpr;
@ -65,7 +64,7 @@ public class ForEach implements Control {
Object o = args[0].evaluate(bindings); Object o = args[0].evaluate(bindings);
if (ExpressionUtils.isError(o)) { if (ExpressionUtils.isError(o)) {
return o; return o;
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) {
return new EvalError("First argument to forEach is not an array"); return new EvalError("First argument to forEach is not an array");
} }
@ -90,27 +89,23 @@ public class ForEach implements Control {
results.add(r); results.add(r);
} }
} else if (o instanceof JSONArray) { } else if (o instanceof ArrayNode) {
JSONArray a = (JSONArray) o; ArrayNode a = (ArrayNode) o;
int l = a.length(); int l = a.size();
results = new ArrayList<Object>(l); results = new ArrayList<Object>(l);
for (int i = 0; i < l; i++) { for (int i = 0; i < l; i++) {
try { Object v = JsonValueConverter.convert(a.get(i));
Object v = a.get(i);
if (v != null) { if (v != null) {
bindings.put(name, v); bindings.put(name, v);
} else { } else {
bindings.remove(name); bindings.remove(name);
}
Object r = args[2].evaluate(bindings);
results.add(r);
} catch (JSONException e) {
results.add(new EvalError(e.getMessage()));
} }
Object r = args[2].evaluate(bindings);
results.add(r);
} }
} else { } else {
Collection<Object> collection = ExpressionUtils.toObjectCollection(o); Collection<Object> collection = ExpressionUtils.toObjectCollection(o);

27
main/src/com/google/refine/grel/controls/ForEachIndex.java
View File

@ -37,12 +37,11 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.json.JSONArray; import com.fasterxml.jackson.databind.node.ArrayNode;
import org.json.JSONException;
import com.google.refine.expr.EvalError; import com.google.refine.expr.EvalError;
import com.google.refine.expr.Evaluable; import com.google.refine.expr.Evaluable;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.grel.Control; import com.google.refine.grel.Control;
import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.ast.VariableExpr; import com.google.refine.grel.ast.VariableExpr;
@ -67,7 +66,7 @@ public class ForEachIndex implements Control {
Object o = args[0].evaluate(bindings); Object o = args[0].evaluate(bindings);
if (ExpressionUtils.isError(o)) { if (ExpressionUtils.isError(o)) {
return o; return o;
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) { } else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) {
return new EvalError("First argument to forEach is not an array"); return new EvalError("First argument to forEach is not an array");
} }
@ -94,24 +93,20 @@ public class ForEachIndex implements Control {
results.add(r); results.add(r);
} }
} else if (o instanceof JSONArray) { } else if (o instanceof ArrayNode) {
JSONArray a = (JSONArray) o; ArrayNode a = (ArrayNode) o;
int l = a.length(); int l = a.size();
results = new ArrayList<Object>(l); results = new ArrayList<Object>(l);
for (int i = 0; i < l; i++) { for (int i = 0; i < l; i++) {
try { Object v = JsonValueConverter.convert(a.get(i));
Object v = a.get(i);
bindings.put(indexName, i); bindings.put(indexName, i);
bindings.put(elementName, v); bindings.put(elementName, v);
Object r = args[3].evaluate(bindings); Object r = args[3].evaluate(bindings);
results.add(r); results.add(r);
} catch (JSONException e) {
results.add(new EvalError(e.getMessage()));
}
} }
} else { } else {
List<Object> list = ExpressionUtils.toObjectList(o); List<Object> list = ExpressionUtils.toObjectList(o);

12
main/src/com/google/refine/util/JSONUtilities.java
View File

@ -50,6 +50,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.refine.expr.util.JsonValueConverter;
public class JSONUtilities { public class JSONUtilities {
@ -477,4 +478,15 @@ public class JSONUtilities {
public static JSONObject objectNodeToJsonNode(ObjectNode fieldJsonObj) { public static JSONObject objectNodeToJsonNode(ObjectNode fieldJsonObj) {
return new JSONObject(fieldJsonObj.toString()); return new JSONObject(fieldJsonObj.toString());
} }
public static Object[] toArray(ArrayNode v) {
if (v == null) {
return null;
}
Object[] result = new Object[v.size()];
for (int i = 0; i != v.size(); i++) {
result[i] = JsonValueConverter.convert(v.get(i));
}
return result;
}
} }

64
main/tests/server/src/com/google/refine/tests/expr/util/JsonValueConverterTests.java Normal file
View File

@ -0,0 +1,64 @@
package com.google.refine.tests.expr.util;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.util.Arrays;
import org.testng.annotations.Test;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.util.ParsingUtilities;
public class JsonValueConverterTests {
private void fieldEquals(String json, Object expectedValue) {
try {
ObjectNode n = (ObjectNode) ParsingUtilities.mapper.readTree(json);
assertEquals(expectedValue, JsonValueConverter.convert(n.get("foo")));
} catch (IOException e) {
e.printStackTrace();
}
}
@Test
public void testConvertJsonObject() throws IOException {
fieldEquals("{\"foo\":{\"ob\":\"ject\"}}", ParsingUtilities.mapper.readTree("{\"ob\":\"ject\"}"));
}
@Test
public void testConvertJsonArray() throws IOException {
fieldEquals("{\"foo\":[1,2]}", ParsingUtilities.mapper.readTree("[1,2]"));
}
@Test
public void testConvertInt() {
fieldEquals("{\"foo\":3}", 3);
}
@Test
public void testConvertFloat() {
fieldEquals("{\"foo\":3.14}", 3.14);
}
@Test
public void testConvertBool() {
fieldEquals("{\"foo\":true}", true);
}
@Test
public void testConvertNull() {
fieldEquals("{\"foo\":null}", null);
}
@Test
public void testConvertString() {
fieldEquals("{\"foo\":\"bar\"}", "bar");
}
@Test
public void testConvertNoField() {
fieldEquals("{}", null);
}
}