Migrate GREL functions and controls to Jackson

This commit is contained in:
Antonin Delpeuch 2018-11-19 13:31:24 +00:00
parent 5a4a79028f
commit 6d05631a07
18 changed files with 220 additions and 208 deletions

View File

@ -41,9 +41,8 @@ import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
@ -150,10 +149,10 @@ public class ExpressionUtils {
}
static public Serializable wrapStorable(Object v) {
if (v instanceof JSONArray) {
return ((JSONArray) v).toString();
} else if (v instanceof JSONObject) {
return ((JSONObject) v).toString();
if (v instanceof ArrayNode) {
return ((ArrayNode) v).toString();
} else if (v instanceof ObjectNode) {
return ((ObjectNode) v).toString();
} else {
return isStorable(v) ?
(Serializable) v :

View File

@ -36,13 +36,12 @@ package com.google.refine.expr.functions;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.HasFields;
import com.google.refine.expr.HasFieldsList;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.grel.Function;
public class Get implements Function {
@ -57,26 +56,22 @@ public class Get implements Function {
if (v != null && from != null) {
if (v instanceof HasFields && from instanceof String) {
return ((HasFields) v).getField((String) from, bindings);
} else if (v instanceof JSONObject && from instanceof String) {
try {
return ((JSONObject) v).get((String) from);
} catch (JSONException e) {
// ignore; will return null
}
} else if (v instanceof ObjectNode && from instanceof String) {
return JsonValueConverter.convert(((ObjectNode) v).get((String) from));
} else {
if (from instanceof Number && (to == null || to instanceof Number)) {
if (v.getClass().isArray() ||
v instanceof List<?> ||
v instanceof HasFieldsList ||
v instanceof JSONArray) {
v instanceof ArrayNode) {
int length = 0;
if (v.getClass().isArray()) {
length = ((Object[]) v).length;
} else if (v instanceof HasFieldsList) {
length = ((HasFieldsList) v).length();
} else if (v instanceof JSONArray) {
length = ((JSONArray) v).length();
} else if (v instanceof ArrayNode) {
length = ((ArrayNode) v).size();
} else {
length = ExpressionUtils.toObjectList(v).size();
}
@ -92,12 +87,8 @@ public class Get implements Function {
return ((Object[]) v)[start];
} else if (v instanceof HasFieldsList) {
return ((HasFieldsList) v).get(start);
} else if (v instanceof JSONArray) {
try {
return ((JSONArray) v).get(start);
} catch (JSONException e) {
// ignore; will return null
}
} else if (v instanceof ArrayNode) {
return JsonValueConverter.convert(((ArrayNode) v).get(start));
} else {
return ExpressionUtils.toObjectList(v).get(start);
}
@ -118,18 +109,13 @@ public class Get implements Function {
return a2;
} else if (v instanceof HasFieldsList) {
return ((HasFieldsList) v).getSubList(start, end);
} else if (v instanceof JSONArray) {
JSONArray a = (JSONArray) v;
} else if (v instanceof ArrayNode) {
ArrayNode a = (ArrayNode) v;
Object[] a2 = new Object[end - start];
for (int i = 0; i < a2.length; i++) {
try {
a2[i] = a.get(start + i);
} catch (JSONException e) {
// ignore
a2[i] = JsonValueConverter.convert(a.get(start + i));
}
}
return a2;
} else {
return ExpressionUtils.toObjectList(v).subList(start, end);

View File

@ -35,9 +35,7 @@ package com.google.refine.expr.functions;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.expr.HasFields;
import com.google.refine.grel.Function;
@ -53,12 +51,8 @@ public class HasField implements Function {
String name = (String) f;
if (v instanceof HasFields) {
return ((HasFields) v).getField(name, bindings) != null;
} else if (v instanceof JSONObject) {
try {
return ((JSONObject) v).get(name) != null;
} catch (JSONException e) {
// ignore; will return false
}
} else if (v instanceof ObjectNode) {
return ((ObjectNode) v).has(name);
}
}
}

View File

@ -33,15 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions;
import java.util.Collection;
import java.util.Map;
import java.io.IOException;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import com.google.refine.grel.Function;
import com.google.refine.util.ParsingUtilities;
public class Jsonize implements Function {
@ -49,27 +45,8 @@ public class Jsonize implements Function {
public Object call(Properties bindings, Object[] args) {
if (args.length >= 1) {
try {
Object o1 = args[0];
if (o1 == null) {
return "null";
} else if (o1 instanceof Number) {
return JSONObject.numberToString((Number) o1);
} else if (o1 instanceof Boolean) {
return o1.toString();
} else if (o1 instanceof JSONObject) {
return ((JSONObject) o1).toString();
} else if (o1 instanceof JSONArray) {
return ((JSONArray) o1).toString();
} else if (o1 instanceof Map) {
return new JSONObject((Map<?,?>) o1).toString();
} else if (o1 instanceof Collection) {
return new JSONArray((Collection<?>) o1).toString();
} else if (o1.getClass().isArray()) {
return new JSONArray(o1).toString();
} else {
return JSONObject.quote(o1.toString());
}
} catch (JSONException e) {
return ParsingUtilities.mapper.writeValueAsString(args[0]);
} catch (IOException e) {
throw new RuntimeException(e);
}
}

View File

@ -36,8 +36,7 @@ package com.google.refine.expr.functions;
import java.util.Collection;
import java.util.Properties;
import org.json.JSONArray;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.HasFieldsList;
import com.google.refine.grel.ControlFunctionRegistry;
@ -58,8 +57,8 @@ public class Length implements Function {
return ((Collection<?>) v).size();
} else if (v instanceof HasFieldsList) {
return ((HasFieldsList) v).length();
} else if (v instanceof JSONArray) {
return ((JSONArray) v).length();
} else if (v instanceof ArrayNode) {
return ((ArrayNode) v).size();
} else {
String s = (v instanceof String ? (String) v : v.toString());
return s.length();

View File

@ -36,9 +36,7 @@ package com.google.refine.expr.functions;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.HasFieldsList;
import com.google.refine.grel.Function;
@ -53,14 +51,14 @@ public class Slice implements Function {
Object to = (args.length == 3) ? args[2] : null;
if (v != null && from != null && from instanceof Number && (to == null || to instanceof Number)) {
if (v.getClass().isArray() || v instanceof List<?> || v instanceof HasFieldsList || v instanceof JSONArray) {
if (v.getClass().isArray() || v instanceof List<?> || v instanceof HasFieldsList || v instanceof ArrayNode) {
int length = 0;
if (v.getClass().isArray()) {
length = ((Object[]) v).length;
} else if (v instanceof HasFieldsList) {
length = ((HasFieldsList) v).length();
} else if (v instanceof JSONArray) {
length = ((JSONArray) v).length();
} else if (v instanceof ArrayNode) {
length = ((ArrayNode) v).size();
} else {
length = ExpressionUtils.toObjectList(v).size();
}
@ -86,16 +84,12 @@ public class Slice implements Function {
return a2;
} else if (v instanceof HasFieldsList) {
return ((HasFieldsList) v).getSubList(start, end);
} else if (v instanceof JSONArray) {
JSONArray a = (JSONArray) v;
} else if (v instanceof ArrayNode) {
ArrayNode a = (ArrayNode) v;
Object[] a2 = new Object[end - start];
for (int i = 0; i < a2.length; i++) {
try {
a2[i] = a.get(start + i);
} catch (JSONException e) {
// ignore
}
}
return a2;

View File

@ -36,9 +36,9 @@ package com.google.refine.expr.functions.arrays;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.grel.ControlFunctionRegistry;
@ -55,7 +55,7 @@ public class Join implements Function {
if (v != null && s != null && s instanceof String) {
String separator = (String) s;
if (v.getClass().isArray() || v instanceof List<?> || v instanceof JSONArray) {
if (v.getClass().isArray() || v instanceof List<?> || v instanceof ArrayNode) {
StringBuffer sb = new StringBuffer();
if (v.getClass().isArray()) {
for (Object o : (Object[]) v) {
@ -66,9 +66,9 @@ public class Join implements Function {
sb.append(o.toString());
}
}
} else if (v instanceof JSONArray) {
JSONArray a = (JSONArray) v;
int l = a.length();
} else if (v instanceof ArrayNode) {
ArrayNode a = (ArrayNode) v;
int l = a.size();
for (int i = 0; i < l; i++) {
if (sb.length() > 0) {

View File

@ -36,9 +36,7 @@ package com.google.refine.expr.functions.arrays;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.grel.ControlFunctionRegistry;
@ -53,13 +51,8 @@ public class Reverse implements Function {
Object v = args[0];
if (v != null) {
if (v instanceof JSONArray) {
try {
v = JSONUtilities.toArray((JSONArray) v);
} catch (JSONException e) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) +
" fails to process a JSON array: " + e.getMessage());
}
if (v instanceof ArrayNode) {
v = JSONUtilities.toArray((ArrayNode) v);
}
if (v.getClass().isArray() || v instanceof List<?>) {

View File

@ -38,9 +38,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
@ -62,17 +60,12 @@ public class Sort implements Function {
Arrays.sort(r, 0, r.length);
return r;
} else if (v instanceof JSONArray) {
try {
Object[] r = JSONUtilities.toArray((JSONArray) v);
} else if (v instanceof ArrayNode) {
Object[] r = JSONUtilities.toArray((ArrayNode) v);
Arrays.sort(r, 0, r.length);
return r;
} catch (JSONException e) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) +
" fails to process a JSON array: " + e.getMessage());
}
} else if (v instanceof List<?>) {
List<? extends Comparable<Object>> a = (List<? extends Comparable<Object>>) v;
Collections.sort(a);

View File

@ -38,9 +38,7 @@ import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.grel.ControlFunctionRegistry;
@ -55,13 +53,8 @@ public class Uniques implements Function {
Object v = args[0];
if (v != null) {
if (v instanceof JSONArray) {
try {
v = JSONUtilities.toArray((JSONArray) v);
} catch (JSONException e) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) +
" fails to process a JSON array: " + e.getMessage());
}
if (v instanceof ArrayNode) {
v = JSONUtilities.toArray((ArrayNode) v);
}
if (v.getClass().isArray() || v instanceof List<?>) {

View File

@ -33,14 +33,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions.strings;
import java.io.IOException;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONTokener;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.util.ParsingUtilities;
public class ParseJson implements Function {
@ -50,8 +49,8 @@ public class ParseJson implements Function {
Object o1 = args[0];
if (o1 != null) {
try {
return new JSONTokener(o1.toString()).nextValue();
} catch (JSONException e) {
return ParsingUtilities.mapper.readTree(o1.toString());
} catch (IOException e) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " failed: " + e.getMessage());
}
}

View File

@ -0,0 +1,37 @@
package com.google.refine.expr.util;
import com.fasterxml.jackson.databind.JsonNode;
/**
* Converts the a JSON value
* @author antonin
*
*/
public class JsonValueConverter {
public static Object convert(JsonNode value) {
if (value == null) {
return null;
}
if (value.isObject()) {
return value;
} else if (value.isArray()) {
return value;
} else if (value.isBigDecimal() || value.isDouble() || value.isFloat()) {
return value.asDouble();
} else if (value.isBigInteger()) {
return value.asLong();
} else if (value.isInt()) {
return value.asInt();
} else if (value.isBinary() || value.isTextual()) {
return value.asText();
} else if (value.isBoolean()) {
return value.asBoolean();
} else if (value.isNull()) {
return null;
} else {
return null;
}
}
}

View File

@ -41,6 +41,7 @@ import com.google.common.collect.Lists;
import com.google.refine.expr.Evaluable;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.HasFields;
import com.google.refine.expr.util.JsonValueConverter;
/**
* An abstract syntax tree node encapsulating a field accessor,
@ -67,21 +68,7 @@ public class FieldAccessorExpr implements Evaluable {
return ((HasFields) o).getField(_fieldName, bindings);
} else if (o instanceof ObjectNode) {
JsonNode value = ((ObjectNode) o).get(_fieldName);
if (value.isArray()) {
return Lists.newArrayList(value.elements());
} else if (value.isBigDecimal() || value.isDouble() || value.isFloat()) {
return value.asDouble();
} else if (value.isBigInteger() || value.isInt()) {
return value.asLong();
} else if (value.isBinary() || value.isTextual()) {
return value.asText();
} else if (value.isBoolean()) {
return value.asBoolean();
} else if (value.isNull()) {
return null;
} else {
return null;
}
return JsonValueConverter.convert(value);
} else {
return null;
}

View File

@ -38,12 +38,11 @@ import java.util.Collection;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.Evaluable;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.grel.Control;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.ast.VariableExpr;
@ -65,7 +64,7 @@ public class Filter implements Control {
Object o = args[0].evaluate(bindings);
if (ExpressionUtils.isError(o)) {
return o;
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) {
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) {
return new EvalError("First argument is not an array");
}
@ -91,14 +90,13 @@ public class Filter implements Control {
results.add(v);
}
}
} else if (o instanceof JSONArray) {
JSONArray a = (JSONArray) o;
int l = a.length();
} else if (o instanceof ArrayNode) {
ArrayNode a = (ArrayNode) o;
int l = a.size();
results = new ArrayList<Object>(l);
for (int i = 0; i < l; i++) {
try {
Object v = a.get(i);
Object v = JsonValueConverter.convert(a.get(i));
if (v != null) {
bindings.put(name, v);
@ -110,9 +108,6 @@ public class Filter implements Control {
if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
results.add(v);
}
} catch (JSONException e) {
results.add(new EvalError(e.getMessage()));
}
}
} else {
Collection<Object> collection = ExpressionUtils.toObjectCollection(o);

View File

@ -38,12 +38,11 @@ import java.util.Collection;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.Evaluable;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.grel.Control;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.ast.VariableExpr;
@ -65,7 +64,7 @@ public class ForEach implements Control {
Object o = args[0].evaluate(bindings);
if (ExpressionUtils.isError(o)) {
return o;
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) {
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) {
return new EvalError("First argument to forEach is not an array");
}
@ -90,14 +89,13 @@ public class ForEach implements Control {
results.add(r);
}
} else if (o instanceof JSONArray) {
JSONArray a = (JSONArray) o;
int l = a.length();
} else if (o instanceof ArrayNode) {
ArrayNode a = (ArrayNode) o;
int l = a.size();
results = new ArrayList<Object>(l);
for (int i = 0; i < l; i++) {
try {
Object v = a.get(i);
Object v = JsonValueConverter.convert(a.get(i));
if (v != null) {
bindings.put(name, v);
@ -108,9 +106,6 @@ public class ForEach implements Control {
Object r = args[2].evaluate(bindings);
results.add(r);
} catch (JSONException e) {
results.add(new EvalError(e.getMessage()));
}
}
} else {
Collection<Object> collection = ExpressionUtils.toObjectCollection(o);

View File

@ -37,12 +37,11 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.Evaluable;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.grel.Control;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.ast.VariableExpr;
@ -67,7 +66,7 @@ public class ForEachIndex implements Control {
Object o = args[0].evaluate(bindings);
if (ExpressionUtils.isError(o)) {
return o;
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof JSONArray)) {
} else if (!ExpressionUtils.isArrayOrCollection(o) && !(o instanceof ArrayNode)) {
return new EvalError("First argument to forEach is not an array");
}
@ -94,14 +93,13 @@ public class ForEachIndex implements Control {
results.add(r);
}
} else if (o instanceof JSONArray) {
JSONArray a = (JSONArray) o;
int l = a.length();
} else if (o instanceof ArrayNode) {
ArrayNode a = (ArrayNode) o;
int l = a.size();
results = new ArrayList<Object>(l);
for (int i = 0; i < l; i++) {
try {
Object v = a.get(i);
Object v = JsonValueConverter.convert(a.get(i));
bindings.put(indexName, i);
bindings.put(elementName, v);
@ -109,9 +107,6 @@ public class ForEachIndex implements Control {
Object r = args[3].evaluate(bindings);
results.add(r);
} catch (JSONException e) {
results.add(new EvalError(e.getMessage()));
}
}
} else {
List<Object> list = ExpressionUtils.toObjectList(o);

View File

@ -50,6 +50,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.refine.expr.util.JsonValueConverter;
public class JSONUtilities {
@ -477,4 +478,15 @@ public class JSONUtilities {
public static JSONObject objectNodeToJsonNode(ObjectNode fieldJsonObj) {
return new JSONObject(fieldJsonObj.toString());
}
public static Object[] toArray(ArrayNode v) {
if (v == null) {
return null;
}
Object[] result = new Object[v.size()];
for (int i = 0; i != v.size(); i++) {
result[i] = JsonValueConverter.convert(v.get(i));
}
return result;
}
}

View File

@ -0,0 +1,64 @@
package com.google.refine.tests.expr.util;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
import java.util.Arrays;
import org.testng.annotations.Test;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.expr.util.JsonValueConverter;
import com.google.refine.util.ParsingUtilities;
public class JsonValueConverterTests {
private void fieldEquals(String json, Object expectedValue) {
try {
ObjectNode n = (ObjectNode) ParsingUtilities.mapper.readTree(json);
assertEquals(expectedValue, JsonValueConverter.convert(n.get("foo")));
} catch (IOException e) {
e.printStackTrace();
}
}
@Test
public void testConvertJsonObject() throws IOException {
fieldEquals("{\"foo\":{\"ob\":\"ject\"}}", ParsingUtilities.mapper.readTree("{\"ob\":\"ject\"}"));
}
@Test
public void testConvertJsonArray() throws IOException {
fieldEquals("{\"foo\":[1,2]}", ParsingUtilities.mapper.readTree("[1,2]"));
}
@Test
public void testConvertInt() {
fieldEquals("{\"foo\":3}", 3);
}
@Test
public void testConvertFloat() {
fieldEquals("{\"foo\":3.14}", 3.14);
}
@Test
public void testConvertBool() {
fieldEquals("{\"foo\":true}", true);
}
@Test
public void testConvertNull() {
fieldEquals("{\"foo\":null}", null);
}
@Test
public void testConvertString() {
fieldEquals("{\"foo\":\"bar\"}", "bar");
}
@Test
public void testConvertNoField() {
fieldEquals("{}", null);
}
}