From fe6abedc525915c98f10f1034253bb1c6617befc Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Sun, 24 Mar 2019 19:50:25 +0000 Subject: [PATCH 1/2] Add inArray function to GREL --- .../refine/expr/functions/arrays/InArray.java | 89 ++++++++++++++ .../refine/grel/ControlFunctionRegistry.java | 2 + .../expr/functions/arrays/InArrayTests.java | 111 ++++++++++++++++++ 3 files changed, 202 insertions(+) create mode 100644 main/src/com/google/refine/expr/functions/arrays/InArray.java create mode 100644 main/tests/server/src/com/google/refine/tests/expr/functions/arrays/InArrayTests.java diff --git a/main/src/com/google/refine/expr/functions/arrays/InArray.java b/main/src/com/google/refine/expr/functions/arrays/InArray.java new file mode 100644 index 000000000..1fd47ceea --- /dev/null +++ b/main/src/com/google/refine/expr/functions/arrays/InArray.java @@ -0,0 +1,89 @@ +/* + +Copyright 2016, Knowledge Integration Ltd +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Knowledge Integration Ltd nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.expr.functions.arrays; + +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import com.fasterxml.jackson.databind.node.ArrayNode; + +import com.google.refine.expr.EvalError; +import com.google.refine.grel.ControlFunctionRegistry; +import com.google.refine.grel.Function; +import com.google.refine.util.JSONUtilities; + +public class InArray implements Function { + + @Override + @SuppressWarnings("unchecked") + public Object call(Properties bindings, Object[] args) { + if (args.length == 2) { + Object v = args[0]; + Object s = args[1]; + + if (v != null && s != null && s instanceof String) { + if (v.getClass().isArray()) { + Object[] a = (Object[]) v; + Object[] r = a.clone(); + return Arrays.asList(r).contains(s); + } else if (v instanceof ArrayNode) { + Object[] r = JSONUtilities.toArray((ArrayNode) v); + return Arrays.asList(r).contains(s); + } else if (v instanceof List) { + List> a = (List>) v; + return a.contains(s); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects an array"); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a string"); + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects two parameters: an array and a string"); + } + + @Override + public String getDescription() { + return "Checks if array a contains string s"; + } + + @Override + public String getParams() { + return "array a, string s"; + } + + @Override + public String getReturns() { + return "boolean"; + } +} diff --git a/main/src/com/google/refine/grel/ControlFunctionRegistry.java b/main/src/com/google/refine/grel/ControlFunctionRegistry.java index a51599747..ffc48ba53 100644 --- a/main/src/com/google/refine/grel/ControlFunctionRegistry.java +++ b/main/src/com/google/refine/grel/ControlFunctionRegistry.java @@ -51,6 +51,7 @@ import com.google.refine.expr.functions.ToDate; import com.google.refine.expr.functions.ToNumber; import com.google.refine.expr.functions.ToString; import com.google.refine.expr.functions.Type; +import com.google.refine.expr.functions.arrays.InArray; import com.google.refine.expr.functions.arrays.Join; import com.google.refine.expr.functions.arrays.Reverse; import com.google.refine.expr.functions.arrays.Sort; @@ -263,6 +264,7 @@ public class ControlFunctionRegistry { registerFunction("reverse", new Reverse()); registerFunction("sort", new Sort()); registerFunction("uniques", new Uniques()); + registerFunction("inArray", new InArray()); registerFunction("now", new Now()); registerFunction("inc", new Inc()); diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/arrays/InArrayTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/arrays/InArrayTests.java new file mode 100644 index 000000000..32f8314f8 --- /dev/null +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/arrays/InArrayTests.java @@ -0,0 +1,111 @@ +/******************************************************************************* + * Copyright (C) 2018, OpenRefine contributors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.google.refine.tests.expr.functions.arrays; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.fasterxml.jackson.databind.node.ArrayNode; + +import com.google.refine.expr.EvalError; +import com.google.refine.expr.functions.arrays.InArray; +import com.google.refine.grel.ControlFunctionRegistry; +import com.google.refine.grel.Function; +import com.google.refine.tests.RefineTest; +import com.google.refine.tests.util.TestUtils; + +public class InArrayTests extends RefineTest { + + static Properties bindings; + static final List listArray = Arrays.asList("v1", "v2", "v3"); + static final String stringArray[] = {"v1","v2","v3"}; + + + @BeforeMethod + public void SetUp() { + bindings = new Properties(); + } + + @AfterMethod + public void TearDown() { + bindings = null; + } + + @Test + public void serializeInArray() { + String json = "{\"description\":\"Checks if array a contains string s\",\"params\":\"array a, string s\",\"returns\":\"boolean\"}"; + TestUtils.isSerializedTo(new InArray(), json); + } + + @Test + public void testInArrayParameters() { + Assert.assertTrue(invoke("inArray") instanceof EvalError); + Assert.assertTrue(invoke("inArray", "string1") instanceof EvalError); + Assert.assertTrue(invoke("inArray", "string1","string2") instanceof EvalError); + Assert.assertTrue(invoke("inArray", "string1","string2","string3") instanceof EvalError); + } + + @Test + public void testInArray() { + Assert.assertTrue((boolean) invoke("inArray", listArray, "v1")); + Assert.assertFalse((boolean) invoke("inArray", listArray, "v4")); + Assert.assertTrue((boolean) invoke("inArray", stringArray, "v1")); + Assert.assertFalse((boolean) invoke("inArray", stringArray, "v4")); + } + + @Test + public void testInArrayWithArrayNode() { + ObjectMapper mapper = new ObjectMapper(); + ArrayNode arrayNode = mapper.createArrayNode(); + for (int i = 1; i < 4; i++) { + arrayNode.add("v" + i); + } + Assert.assertTrue((boolean) invoke("inArray", arrayNode, "v1")); + Assert.assertFalse((boolean) invoke("inArray", arrayNode, "v4")); + } + + private static Object invoke(String name,Object... args) { + // registry uses static initializer, so no need to set it up + Function function = ControlFunctionRegistry.getFunction(name); + if (function == null) { + throw new IllegalArgumentException("Unknown function "+name); + } + if (args == null) { + return function.call(bindings,new Object[0]); + } else { + return function.call(bindings,args); + } + } +} + From 7425418011a51429cb8e0622127215cabbe1f09f Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Sun, 24 Mar 2019 20:16:43 +0000 Subject: [PATCH 2/2] Add RandomNumber GREL function --- .../expr/functions/math/RandomNumber.java | 71 +++++++++++++++ .../refine/grel/ControlFunctionRegistry.java | 2 + .../functions/math/RandomNumberTests.java | 90 +++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100644 main/src/com/google/refine/expr/functions/math/RandomNumber.java create mode 100644 main/tests/server/src/com/google/refine/tests/expr/functions/math/RandomNumberTests.java diff --git a/main/src/com/google/refine/expr/functions/math/RandomNumber.java b/main/src/com/google/refine/expr/functions/math/RandomNumber.java new file mode 100644 index 000000000..f1f0a70c1 --- /dev/null +++ b/main/src/com/google/refine/expr/functions/math/RandomNumber.java @@ -0,0 +1,71 @@ +/* + +Copyright 2010, Knowledge Integration Ltd. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Knowledge Integration Ltd. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.expr.functions.math; + +import java.util.Properties; + +import java.util.concurrent.ThreadLocalRandom; + +import com.google.refine.expr.EvalError; +import com.google.refine.grel.ControlFunctionRegistry; +import com.google.refine.grel.Function; + +public class RandomNumber implements Function { + + @Override + public Object call(Properties bindings, Object[] args) { + if (args.length == 2 && args[0] != null && args[0] instanceof Number + && args[1] != null && args[1] instanceof Number && ((Number) args[0]).intValue()<((Number) args[1]).intValue()) { + int randomNum = ThreadLocalRandom.current().nextInt(((Number) args[0]).intValue(), ((Number) args[1]).intValue()+1); + return randomNum; + } + return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects two numbers, the first must be less than the second"); + } + + @Override + public String getDescription() { + return "Returns a pseudo-random integer between the lower and upper bound (inclusive)"; + } + + @Override + public String getParams() { + return "number lower bound, number upper bound"; + } + + @Override + public String getReturns() { + return "number"; + } + +} \ No newline at end of file diff --git a/main/src/com/google/refine/grel/ControlFunctionRegistry.java b/main/src/com/google/refine/grel/ControlFunctionRegistry.java index ffc48ba53..3a6502ba1 100644 --- a/main/src/com/google/refine/grel/ControlFunctionRegistry.java +++ b/main/src/com/google/refine/grel/ControlFunctionRegistry.java @@ -92,6 +92,7 @@ import com.google.refine.expr.functions.math.Odd; import com.google.refine.expr.functions.math.Pow; import com.google.refine.expr.functions.math.Quotient; import com.google.refine.expr.functions.math.Radians; +import com.google.refine.expr.functions.math.RandomNumber; import com.google.refine.expr.functions.math.Round; import com.google.refine.expr.functions.math.Sin; import com.google.refine.expr.functions.math.Sinh; @@ -299,6 +300,7 @@ public class ControlFunctionRegistry { registerFunction("combin", new Combin()); registerFunction("degrees", new Degrees()); registerFunction("radians", new Radians()); + registerFunction("randomNumber", new RandomNumber()); registerFunction("gcd", new GreatestCommonDenominator()); registerFunction("lcm", new LeastCommonMultiple()); registerFunction("multinomial", new Multinomial()); diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/math/RandomNumberTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/math/RandomNumberTests.java new file mode 100644 index 000000000..8c67f0995 --- /dev/null +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/math/RandomNumberTests.java @@ -0,0 +1,90 @@ +/******************************************************************************* + * Copyright (C) 2018, OpenRefine contributors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ +package com.google.refine.tests.expr.functions.math; + +import java.util.Properties; + +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.google.refine.expr.functions.math.RandomNumber; +import com.google.refine.expr.EvalError; +import com.google.refine.expr.functions.arrays.InArray; +import com.google.refine.grel.ControlFunctionRegistry; +import com.google.refine.grel.Function; +import com.google.refine.tests.RefineTest; +import com.google.refine.tests.util.TestUtils; + +public class RandomNumberTests extends RefineTest { + static Properties bindings; + + @BeforeMethod + public void SetUp() { + bindings = new Properties(); + } + + @AfterMethod + public void TearDown() { + bindings = null; + } + + @Test + public void serializeRandomNumber() { + String json = "{\"description\":\"Returns a pseudo-random integer between the lower and upper bound (inclusive)\",\"params\":\"number lower bound, number upper bound\",\"returns\":\"number\"}"; + TestUtils.isSerializedTo(new RandomNumber(), json); + } + + @Test + public void testRandomNumberParameters() { + Assert.assertTrue(invoke("randomNumber") instanceof EvalError); + Assert.assertTrue(invoke("randomNumber", "string1") instanceof EvalError); + Assert.assertTrue(invoke("randomNumber", "string1","string2") instanceof EvalError); + Assert.assertTrue(invoke("randomNumber", 3, 2) instanceof EvalError); + } + + @Test + public void testRandomNumber() { + Object a = invoke("randomNumber", 1, 10); + Assert.assertTrue((int) a < 11 && (int) a > 0); + } + + private static Object invoke(String name,Object... args) { + // registry uses static initializer, so no need to set it up + Function function = ControlFunctionRegistry.getFunction(name); + if (function == null) { + throw new IllegalArgumentException("Unknown function "+name); + } + if (args == null) { + return function.call(bindings,new Object[0]); + } else { + return function.call(bindings,args); + } + } +} +