From 22072c498bd925f549a11a8f84e0f79226597538 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 28 Jun 2018 06:07:41 +0100 Subject: [PATCH 01/11] Add tests for grouper for list facet --- .../util/ExpressionNominalValueGrouper.java | 62 ++++++++++++++++--- .../ExpressionNominalValueGrouperTests.java | 37 +++++++---- 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java b/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java index a0039d7ce..7e399fa03 100644 --- a/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java +++ b/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java @@ -57,15 +57,6 @@ import com.google.refine.util.StringUtils; * from a given expression. */ public class ExpressionNominalValueGrouper implements RowVisitor, RecordVisitor { - static public class IndexedNominalFacetChoice extends NominalFacetChoice { - int _latestIndex; - - public IndexedNominalFacetChoice(DecoratedValue decoratedValue, int latestIndex) { - super(decoratedValue); - _latestIndex = latestIndex; - } - } - /* * Configuration */ @@ -77,14 +68,29 @@ public class ExpressionNominalValueGrouper implements RowVisitor, RecordVisitor * Computed results */ final public Map choices = new HashMap(); + public int numberCount = 0; + public int datetimeCount = 0; + public int booleanCount = 0; public int blankCount = 0; public int errorCount = 0; /* * Scratch pad variables */ + protected boolean hasNumber; + protected boolean hasDateTime; + protected boolean hasBoolean; protected boolean hasBlank; protected boolean hasError; + + static public class IndexedNominalFacetChoice extends NominalFacetChoice { + int _latestIndex; + + public IndexedNominalFacetChoice(DecoratedValue decoratedValue, int latestIndex) { + super(decoratedValue); + _latestIndex = latestIndex; + } + } public ExpressionNominalValueGrouper(Evaluable evaluable, String columnName, int cellIndex) { _evaluable = evaluable; @@ -104,13 +110,25 @@ public class ExpressionNominalValueGrouper implements RowVisitor, RecordVisitor @Override public boolean visit(Project project, int rowIndex, Row row) { + hasNumber = false; + hasDateTime = false; + hasBoolean = false; hasError = false; hasBlank = false; Properties bindings = ExpressionUtils.createBindings(project); visitRow(project, rowIndex, row, bindings, rowIndex); - + + if (hasNumber) { + numberCount++; + } + if (hasDateTime) { + datetimeCount++; + } + if (hasBoolean) { + booleanCount++; + } if (hasError) { errorCount++; } @@ -123,6 +141,9 @@ public class ExpressionNominalValueGrouper implements RowVisitor, RecordVisitor @Override public boolean visit(Project project, Record record) { + hasNumber = false; + hasDateTime = false; + hasBoolean = false; hasError = false; hasBlank = false; @@ -133,6 +154,15 @@ public class ExpressionNominalValueGrouper implements RowVisitor, RecordVisitor visitRow(project, r, row, bindings, record.recordIndex); } + if (hasNumber) { + numberCount++; + } + if (hasDateTime) { + datetimeCount++; + } + if (hasBoolean) { + booleanCount++; + } if (hasError) { errorCount++; } @@ -174,6 +204,12 @@ public class ExpressionNominalValueGrouper implements RowVisitor, RecordVisitor protected void processValue(Object value, int index) { if (ExpressionUtils.isError(value)) { hasError = true; + } else if (ExpressionUtils.isNumber(value)) { + hasNumber = true; + } else if (ExpressionUtils.isDateTime(value)) { + hasDateTime = true; + } else if (ExpressionUtils.isBoolean(value)) { + hasBoolean = true; } else if (ExpressionUtils.isNonBlankData(value)) { String valueString = StringUtils.toString(value); IndexedNominalFacetChoice facetChoice = choices.get(valueString); @@ -236,6 +272,12 @@ public class ExpressionNominalValueGrouper implements RowVisitor, RecordVisitor public Integer getChoiceValueCount(Object choiceValue) { if (ExpressionUtils.isError(choiceValue)) { return errorCount; + } else if (ExpressionUtils.isNumber(choiceValue)) { + return numberCount; + } else if (ExpressionUtils.isDateTime(choiceValue)) { + return datetimeCount; + } else if (ExpressionUtils.isBoolean(choiceValue)) { + return booleanCount; } else if (ExpressionUtils.isNonBlankData(choiceValue)) { IndexedNominalFacetChoice choice = choices.get(StringUtils.toString(choiceValue)); return choice != null ? choice.count : 0; diff --git a/main/tests/server/src/com/google/refine/tests/browsing/util/ExpressionNominalValueGrouperTests.java b/main/tests/server/src/com/google/refine/tests/browsing/util/ExpressionNominalValueGrouperTests.java index 99c5a3715..6b482856e 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/util/ExpressionNominalValueGrouperTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/util/ExpressionNominalValueGrouperTests.java @@ -62,10 +62,9 @@ public class ExpressionNominalValueGrouperTests extends RefineTest { private static Properties bindings; private static OffsetDateTime dateTimeValue = OffsetDateTime.parse("2017-05-12T05:45:00+00:00", DateTimeFormatter.ISO_OFFSET_DATE_TIME); - private static String dateTimeStringValue = "2017-05-12T05:45:00Z"; private static int integerValue = 1; - private static String integerStringValue = "1"; private static String stringStringValue = "a"; + private static Boolean booleanValue = true; private static ExpressionNominalValueGrouper grouper; private static Evaluable eval; @@ -143,11 +142,7 @@ public class ExpressionNominalValueGrouperTests extends RefineTest { grouper.end(project); } - Assert.assertEquals(grouper.choices.size(),1); - - Assert.assertTrue(grouper.choices.containsKey(integerStringValue)); - Assert.assertEquals(grouper.choices.get(integerStringValue).decoratedValue.label,integerStringValue); - Assert.assertEquals(grouper.choices.get(integerStringValue).decoratedValue.value.toString(),integerStringValue); + Assert.assertEquals(grouper.choices.size(),0); } @Test @@ -171,10 +166,30 @@ public class ExpressionNominalValueGrouperTests extends RefineTest { grouper.end(project); } - Assert.assertEquals(grouper.choices.size(),1); + Assert.assertEquals(grouper.choices.size(),0); + } + + @Test + public void expressionNominalValueGrouperBooleans() throws Exception { + //populate project + for (int i = 0; i < numberOfRows; i++) { + Row row = new Row(1); + row.setCell(0, new Cell(booleanValue, null)); + project.rows.add(row); + } + //create grouper + eval = MetaParser.parse("value"); + grouper = new ExpressionNominalValueGrouper(eval, columnName, cellIndex); + try { + grouper.start(project); + for (int rowIndex = 0; rowIndex < numberOfRows; rowIndex++) { + Row row = project.rows.get(rowIndex); + grouper.visit(project, rowIndex, row); + } + } finally { + grouper.end(project); + } - Assert.assertTrue(grouper.choices.containsKey(dateTimeStringValue)); - Assert.assertEquals(grouper.choices.get(dateTimeStringValue).decoratedValue.label,dateTimeStringValue); - Assert.assertEquals(grouper.choices.get(dateTimeStringValue).decoratedValue.value.toString(),dateTimeStringValue); + Assert.assertEquals(grouper.choices.size(),0); } } From 55e3914c8d54a05966b8c2638e66895622ff7162 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 28 Jun 2018 06:08:28 +0100 Subject: [PATCH 02/11] Add test for new List/Text facet behaviour --- .../browsing/facets/TextListFacetTests.java | 442 ++++++++++++++++++ 1 file changed, 442 insertions(+) create mode 100644 main/tests/server/src/com/google/refine/tests/browsing/facets/TextListFacetTests.java diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextListFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextListFacetTests.java new file mode 100644 index 000000000..041dc8892 --- /dev/null +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextListFacetTests.java @@ -0,0 +1,442 @@ +/* + +Copyright 2018, Owen Stephens +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.tests.browsing.facets; + +import com.google.refine.model.Cell; +import com.google.refine.model.Row; + +import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; + +import org.json.JSONException; +import org.json.JSONObject; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import com.google.refine.model.ModelException; +import com.google.refine.model.Project; +import com.google.refine.browsing.RowFilter; +import com.google.refine.browsing.facets.ListFacet; +import com.google.refine.tests.RefineTest; + + +public class TextListFacetTests extends RefineTest { + // dependencies + private Project project; + private ListFacet facet; + private RowFilter rowfilter; + private JSONObject listfacetconfig; + + // Variables + private static OffsetDateTime dateTimeValue = OffsetDateTime.parse("2017-05-12T05:45:00+00:00", DateTimeFormatter.ISO_OFFSET_DATE_TIME); + private static int integerValue = 1; + private static String stringValue = "a"; + private static String emptyStringValue = ""; + private static Boolean booleanValue = true; + + private static final String projectName = "TextListFacet"; + private static final String columnName = "Col1"; + private static final int numberOfRows = 5; + + @Override + @BeforeTest + public void init() { + logger = LoggerFactory.getLogger(this.getClass()); + } + + @BeforeMethod + public void setUp() throws JSONException, IOException, ModelException { + project = createProjectWithColumns(projectName, columnName); + for (int i = 0; i < numberOfRows; i++) { + Row row = new Row(1); + row.setCell(0, new Cell(stringValue, null)); + project.rows.add(row); + } + for (int i = 0; i < numberOfRows; i++) { + Row row = new Row(1); + row.setCell(0, new Cell(dateTimeValue, null)); + project.rows.add(row); + } + for (int i = 0; i < numberOfRows; i++) { + Row row = new Row(1); + row.setCell(0, new Cell(integerValue, null)); + project.rows.add(row); + } + for (int i = 0; i < numberOfRows; i++) { + Row row = new Row(1); + row.setCell(0, new Cell(booleanValue, null)); + project.rows.add(row); + } + for (int i = 0; i < numberOfRows; i++) { + Row row = new Row(1); + row.setCell(0, new Cell(null, null)); + project.rows.add(row); + } + for (int i = 0; i < numberOfRows; i++) { + Row row = new Row(1); + row.setCell(0, new Cell(emptyStringValue, null)); + project.rows.add(row); + } + } + + @Test + public void testTextSelection() throws Exception { + //Need to work out the correct facet config for these tests to work + //Also need all rows in all tests so can check that rows aren't being selected when they shouldn't be + String facetconfig = "{" + + "\"type\": \"list\"," + + "\"name\": \"Value\"," + + "\"columnName\": \"" + columnName + "\"," + + "\"expression\": \"value\"," + + "\"omitBlank\": false," + + "\"omitError\": false," + + "\"selection\": [" + + " {" + + "\"v\": {" + + "\"v\": \"a\"," + + "\"l\": \"a\"" + + "}" + + "}" + + "]," + + "\"selectNumber\": false," + + "\"selectDateTime\": false," + + "\"selectBoolean\": false," + + "\"selectBlank\": false," + + "\"selectError\": false," + + "\"invert\": false" + + "}"; + + //Add the facet to the project and create a row filter + facet = new ListFacet(); + listfacetconfig = new JSONObject(facetconfig); + facet.initializeFromJSON(project,listfacetconfig); + rowfilter = facet.getRowFilter(project); + + //Check each row in the project against the filter + //Rows 1-5 are strings + Assert.assertEquals(rowfilter.filterRow(project, 0, project.rows.get(0)),true); + Assert.assertEquals(rowfilter.filterRow(project, 1, project.rows.get(1)),true); + Assert.assertEquals(rowfilter.filterRow(project, 2, project.rows.get(2)),true); + Assert.assertEquals(rowfilter.filterRow(project, 3, project.rows.get(3)),true); + Assert.assertEquals(rowfilter.filterRow(project, 4, project.rows.get(4)),true); + //Rows 6-10 are DateTimes + Assert.assertEquals(rowfilter.filterRow(project, 5, project.rows.get(5)),false); + Assert.assertEquals(rowfilter.filterRow(project, 6, project.rows.get(6)),false); + Assert.assertEquals(rowfilter.filterRow(project, 7, project.rows.get(7)),false); + Assert.assertEquals(rowfilter.filterRow(project, 8, project.rows.get(8)),false); + Assert.assertEquals(rowfilter.filterRow(project, 9, project.rows.get(9)),false); + //Rows 11-15 are integers + Assert.assertEquals(rowfilter.filterRow(project, 10, project.rows.get(10)),false); + Assert.assertEquals(rowfilter.filterRow(project, 11, project.rows.get(11)),false); + Assert.assertEquals(rowfilter.filterRow(project, 12, project.rows.get(12)),false); + Assert.assertEquals(rowfilter.filterRow(project, 13, project.rows.get(13)),false); + Assert.assertEquals(rowfilter.filterRow(project, 14, project.rows.get(14)),false); + //Rows 16-20 are booleans + Assert.assertEquals(rowfilter.filterRow(project, 15, project.rows.get(15)),false); + Assert.assertEquals(rowfilter.filterRow(project, 16, project.rows.get(16)),false); + Assert.assertEquals(rowfilter.filterRow(project, 17, project.rows.get(17)),false); + Assert.assertEquals(rowfilter.filterRow(project, 18, project.rows.get(18)),false); + Assert.assertEquals(rowfilter.filterRow(project, 19, project.rows.get(19)),false); + //Rows 21-25 are nulls + Assert.assertEquals(rowfilter.filterRow(project, 20, project.rows.get(20)),false); + Assert.assertEquals(rowfilter.filterRow(project, 21, project.rows.get(21)),false); + Assert.assertEquals(rowfilter.filterRow(project, 22, project.rows.get(22)),false); + Assert.assertEquals(rowfilter.filterRow(project, 23, project.rows.get(23)),false); + Assert.assertEquals(rowfilter.filterRow(project, 24, project.rows.get(24)),false); + //Rows 26-30 are empty strings + Assert.assertEquals(rowfilter.filterRow(project, 25, project.rows.get(25)),false); + Assert.assertEquals(rowfilter.filterRow(project, 26, project.rows.get(26)),false); + Assert.assertEquals(rowfilter.filterRow(project, 27, project.rows.get(27)),false); + Assert.assertEquals(rowfilter.filterRow(project, 28, project.rows.get(28)),false); + Assert.assertEquals(rowfilter.filterRow(project, 29, project.rows.get(29)),false); + } + + @Test + public void testDateSelection() throws Exception { + String facetconfig = "{" + + "\"type\": \"list\"," + + "\"name\": \"Value\"," + + "\"columnName\": \"" + columnName + "\"," + + "\"expression\": \"value\"," + + "\"omitBlank\": false," + + "\"omitError\": false," + + "\"selection\": []," + + "\"selectNumber\": false," + + "\"selectDateTime\": true," + + "\"selectBoolean\": false," + + "\"selectBlank\": false," + + "\"selectError\": false," + + "\"invert\": false" + + "}"; + + //Add the facet to the project and create a row filter + facet = new ListFacet(); + listfacetconfig = new JSONObject(facetconfig); + facet.initializeFromJSON(project,listfacetconfig); + rowfilter = facet.getRowFilter(project); + + //Check each row in the project against the filter + //Rows 1-5 are strings + Assert.assertEquals(rowfilter.filterRow(project, 0, project.rows.get(0)),false); + Assert.assertEquals(rowfilter.filterRow(project, 1, project.rows.get(1)),false); + Assert.assertEquals(rowfilter.filterRow(project, 2, project.rows.get(2)),false); + Assert.assertEquals(rowfilter.filterRow(project, 3, project.rows.get(3)),false); + Assert.assertEquals(rowfilter.filterRow(project, 4, project.rows.get(4)),false); + //Rows 6-10 are DateTimes + Assert.assertEquals(rowfilter.filterRow(project, 5, project.rows.get(5)),true); + Assert.assertEquals(rowfilter.filterRow(project, 6, project.rows.get(6)),true); + Assert.assertEquals(rowfilter.filterRow(project, 7, project.rows.get(7)),true); + Assert.assertEquals(rowfilter.filterRow(project, 8, project.rows.get(8)),true); + Assert.assertEquals(rowfilter.filterRow(project, 9, project.rows.get(9)),true); + //Rows 11-15 are integers + Assert.assertEquals(rowfilter.filterRow(project, 10, project.rows.get(10)),false); + Assert.assertEquals(rowfilter.filterRow(project, 11, project.rows.get(11)),false); + Assert.assertEquals(rowfilter.filterRow(project, 12, project.rows.get(12)),false); + Assert.assertEquals(rowfilter.filterRow(project, 13, project.rows.get(13)),false); + Assert.assertEquals(rowfilter.filterRow(project, 14, project.rows.get(14)),false); + //Rows 16-20 are booleans + Assert.assertEquals(rowfilter.filterRow(project, 15, project.rows.get(15)),false); + Assert.assertEquals(rowfilter.filterRow(project, 16, project.rows.get(16)),false); + Assert.assertEquals(rowfilter.filterRow(project, 17, project.rows.get(17)),false); + Assert.assertEquals(rowfilter.filterRow(project, 18, project.rows.get(18)),false); + Assert.assertEquals(rowfilter.filterRow(project, 19, project.rows.get(19)),false); + //Rows 21-25 are nulls + Assert.assertEquals(rowfilter.filterRow(project, 20, project.rows.get(20)),false); + Assert.assertEquals(rowfilter.filterRow(project, 21, project.rows.get(21)),false); + Assert.assertEquals(rowfilter.filterRow(project, 22, project.rows.get(22)),false); + Assert.assertEquals(rowfilter.filterRow(project, 23, project.rows.get(23)),false); + Assert.assertEquals(rowfilter.filterRow(project, 24, project.rows.get(24)),false); + //Rows 26-30 are empty strings + Assert.assertEquals(rowfilter.filterRow(project, 25, project.rows.get(25)),false); + Assert.assertEquals(rowfilter.filterRow(project, 26, project.rows.get(26)),false); + Assert.assertEquals(rowfilter.filterRow(project, 27, project.rows.get(27)),false); + Assert.assertEquals(rowfilter.filterRow(project, 28, project.rows.get(28)),false); + Assert.assertEquals(rowfilter.filterRow(project, 29, project.rows.get(29)),false); + } + + @Test + public void testIntegerSelection() throws Exception { + String facetconfig = "{" + + "\"type\": \"list\"," + + "\"name\": \"Value\"," + + "\"columnName\": \"" + columnName + "\"," + + "\"expression\": \"value\"," + + "\"omitBlank\": false," + + "\"omitError\": false," + + "\"selection\": []," + + "\"selectNumber\": true," + + "\"selectDateTime\": false," + + "\"selectBoolean\": false," + + "\"selectBlank\": false," + + "\"selectError\": false," + + "\"invert\": false" + + "}"; + + //Add the facet to the project and create a row filter + facet = new ListFacet(); + listfacetconfig = new JSONObject(facetconfig); + facet.initializeFromJSON(project,listfacetconfig); + rowfilter = facet.getRowFilter(project); + + //Check each row in the project against the filter + //Rows 1-5 are strings + Assert.assertEquals(rowfilter.filterRow(project, 0, project.rows.get(0)),false); + Assert.assertEquals(rowfilter.filterRow(project, 1, project.rows.get(1)),false); + Assert.assertEquals(rowfilter.filterRow(project, 2, project.rows.get(2)),false); + Assert.assertEquals(rowfilter.filterRow(project, 3, project.rows.get(3)),false); + Assert.assertEquals(rowfilter.filterRow(project, 4, project.rows.get(4)),false); + //Rows 6-10 are DateTimes + Assert.assertEquals(rowfilter.filterRow(project, 5, project.rows.get(5)),false); + Assert.assertEquals(rowfilter.filterRow(project, 6, project.rows.get(6)),false); + Assert.assertEquals(rowfilter.filterRow(project, 7, project.rows.get(7)),false); + Assert.assertEquals(rowfilter.filterRow(project, 8, project.rows.get(8)),false); + Assert.assertEquals(rowfilter.filterRow(project, 9, project.rows.get(9)),false); + //Rows 11-15 are integers + Assert.assertEquals(rowfilter.filterRow(project, 10, project.rows.get(10)),true); + Assert.assertEquals(rowfilter.filterRow(project, 11, project.rows.get(11)),true); + Assert.assertEquals(rowfilter.filterRow(project, 12, project.rows.get(12)),true); + Assert.assertEquals(rowfilter.filterRow(project, 13, project.rows.get(13)),true); + Assert.assertEquals(rowfilter.filterRow(project, 14, project.rows.get(14)),true); + //Rows 16-20 are booleans + Assert.assertEquals(rowfilter.filterRow(project, 15, project.rows.get(15)),false); + Assert.assertEquals(rowfilter.filterRow(project, 16, project.rows.get(16)),false); + Assert.assertEquals(rowfilter.filterRow(project, 17, project.rows.get(17)),false); + Assert.assertEquals(rowfilter.filterRow(project, 18, project.rows.get(18)),false); + Assert.assertEquals(rowfilter.filterRow(project, 19, project.rows.get(19)),false); + //Rows 21-25 are nulls + Assert.assertEquals(rowfilter.filterRow(project, 20, project.rows.get(20)),false); + Assert.assertEquals(rowfilter.filterRow(project, 21, project.rows.get(21)),false); + Assert.assertEquals(rowfilter.filterRow(project, 22, project.rows.get(22)),false); + Assert.assertEquals(rowfilter.filterRow(project, 23, project.rows.get(23)),false); + Assert.assertEquals(rowfilter.filterRow(project, 24, project.rows.get(24)),false); + //Rows 26-30 are empty strings + Assert.assertEquals(rowfilter.filterRow(project, 25, project.rows.get(25)),false); + Assert.assertEquals(rowfilter.filterRow(project, 26, project.rows.get(26)),false); + Assert.assertEquals(rowfilter.filterRow(project, 27, project.rows.get(27)),false); + Assert.assertEquals(rowfilter.filterRow(project, 28, project.rows.get(28)),false); + Assert.assertEquals(rowfilter.filterRow(project, 29, project.rows.get(29)),false); + } + + @Test + public void testBooleanSelection() throws Exception { + String facetconfig = "{" + + "\"type\": \"list\"," + + "\"name\": \"Value\"," + + "\"columnName\": \"" + columnName + "\"," + + "\"expression\": \"value\"," + + "\"omitBlank\": false," + + "\"omitError\": false," + + "\"selection\": []," + + "\"selectNumber\": false," + + "\"selectDateTime\": false," + + "\"selectBoolean\": true," + + "\"selectBlank\": false," + + "\"selectError\": false," + + "\"invert\": false" + + "}"; + + //Add the facet to the project and create a row filter + facet = new ListFacet(); + listfacetconfig = new JSONObject(facetconfig); + facet.initializeFromJSON(project,listfacetconfig); + rowfilter = facet.getRowFilter(project); + + //Check each row in the project against the filter + //Rows 1-5 are strings + Assert.assertEquals(rowfilter.filterRow(project, 0, project.rows.get(0)),false); + Assert.assertEquals(rowfilter.filterRow(project, 1, project.rows.get(1)),false); + Assert.assertEquals(rowfilter.filterRow(project, 2, project.rows.get(2)),false); + Assert.assertEquals(rowfilter.filterRow(project, 3, project.rows.get(3)),false); + Assert.assertEquals(rowfilter.filterRow(project, 4, project.rows.get(4)),false); + //Rows 6-10 are DateTimes + Assert.assertEquals(rowfilter.filterRow(project, 5, project.rows.get(5)),false); + Assert.assertEquals(rowfilter.filterRow(project, 6, project.rows.get(6)),false); + Assert.assertEquals(rowfilter.filterRow(project, 7, project.rows.get(7)),false); + Assert.assertEquals(rowfilter.filterRow(project, 8, project.rows.get(8)),false); + Assert.assertEquals(rowfilter.filterRow(project, 9, project.rows.get(9)),false); + //Rows 11-15 are integers + Assert.assertEquals(rowfilter.filterRow(project, 10, project.rows.get(10)),false); + Assert.assertEquals(rowfilter.filterRow(project, 11, project.rows.get(11)),false); + Assert.assertEquals(rowfilter.filterRow(project, 12, project.rows.get(12)),false); + Assert.assertEquals(rowfilter.filterRow(project, 13, project.rows.get(13)),false); + Assert.assertEquals(rowfilter.filterRow(project, 14, project.rows.get(14)),false); + //Rows 16-20 are booleans + Assert.assertEquals(rowfilter.filterRow(project, 15, project.rows.get(15)),true); + Assert.assertEquals(rowfilter.filterRow(project, 16, project.rows.get(16)),true); + Assert.assertEquals(rowfilter.filterRow(project, 17, project.rows.get(17)),true); + Assert.assertEquals(rowfilter.filterRow(project, 18, project.rows.get(18)),true); + Assert.assertEquals(rowfilter.filterRow(project, 19, project.rows.get(19)),true); + //Rows 21-25 are nulls + Assert.assertEquals(rowfilter.filterRow(project, 20, project.rows.get(20)),false); + Assert.assertEquals(rowfilter.filterRow(project, 21, project.rows.get(21)),false); + Assert.assertEquals(rowfilter.filterRow(project, 22, project.rows.get(22)),false); + Assert.assertEquals(rowfilter.filterRow(project, 23, project.rows.get(23)),false); + Assert.assertEquals(rowfilter.filterRow(project, 24, project.rows.get(24)),false); + //Rows 26-30 are empty strings + Assert.assertEquals(rowfilter.filterRow(project, 25, project.rows.get(25)),false); + Assert.assertEquals(rowfilter.filterRow(project, 26, project.rows.get(26)),false); + Assert.assertEquals(rowfilter.filterRow(project, 27, project.rows.get(27)),false); + Assert.assertEquals(rowfilter.filterRow(project, 28, project.rows.get(28)),false); + Assert.assertEquals(rowfilter.filterRow(project, 29, project.rows.get(29)),false); + } + + @Test + public void testBlankSelection() throws Exception { + String facetconfig = "{" + + "\"type\": \"list\"," + + "\"name\": \"Value\"," + + "\"columnName\": \"" + columnName + "\"," + + "\"expression\": \"value\"," + + "\"omitBlank\": false," + + "\"omitError\": false," + + "\"selection\": []," + + "\"selectNumber\": false," + + "\"selectDateTime\": false," + + "\"selectBoolean\": false," + + "\"selectBlank\": true," + + "\"selectError\": false," + + "\"invert\": false" + + "}"; + + //Add the facet to the project and create a row filter + facet = new ListFacet(); + listfacetconfig = new JSONObject(facetconfig); + facet.initializeFromJSON(project,listfacetconfig); + rowfilter = facet.getRowFilter(project); + + //Check each row in the project against the filter + //Rows 1-5 are strings + Assert.assertEquals(rowfilter.filterRow(project, 0, project.rows.get(0)),false); + Assert.assertEquals(rowfilter.filterRow(project, 1, project.rows.get(1)),false); + Assert.assertEquals(rowfilter.filterRow(project, 2, project.rows.get(2)),false); + Assert.assertEquals(rowfilter.filterRow(project, 3, project.rows.get(3)),false); + Assert.assertEquals(rowfilter.filterRow(project, 4, project.rows.get(4)),false); + //Rows 6-10 are DateTimes + Assert.assertEquals(rowfilter.filterRow(project, 5, project.rows.get(5)),false); + Assert.assertEquals(rowfilter.filterRow(project, 6, project.rows.get(6)),false); + Assert.assertEquals(rowfilter.filterRow(project, 7, project.rows.get(7)),false); + Assert.assertEquals(rowfilter.filterRow(project, 8, project.rows.get(8)),false); + Assert.assertEquals(rowfilter.filterRow(project, 9, project.rows.get(9)),false); + //Rows 11-15 are integers + Assert.assertEquals(rowfilter.filterRow(project, 10, project.rows.get(10)),false); + Assert.assertEquals(rowfilter.filterRow(project, 11, project.rows.get(11)),false); + Assert.assertEquals(rowfilter.filterRow(project, 12, project.rows.get(12)),false); + Assert.assertEquals(rowfilter.filterRow(project, 13, project.rows.get(13)),false); + Assert.assertEquals(rowfilter.filterRow(project, 14, project.rows.get(14)),false); + //Rows 16-20 are booleans + Assert.assertEquals(rowfilter.filterRow(project, 15, project.rows.get(15)),false); + Assert.assertEquals(rowfilter.filterRow(project, 16, project.rows.get(16)),false); + Assert.assertEquals(rowfilter.filterRow(project, 17, project.rows.get(17)),false); + Assert.assertEquals(rowfilter.filterRow(project, 18, project.rows.get(18)),false); + Assert.assertEquals(rowfilter.filterRow(project, 19, project.rows.get(19)),false); + //Rows 21-25 are nulls + Assert.assertEquals(rowfilter.filterRow(project, 20, project.rows.get(20)),true); + Assert.assertEquals(rowfilter.filterRow(project, 21, project.rows.get(21)),true); + Assert.assertEquals(rowfilter.filterRow(project, 22, project.rows.get(22)),true); + Assert.assertEquals(rowfilter.filterRow(project, 23, project.rows.get(23)),true); + Assert.assertEquals(rowfilter.filterRow(project, 24, project.rows.get(24)),true); + //Rows 26-30 are empty strings + Assert.assertEquals(rowfilter.filterRow(project, 25, project.rows.get(25)),true); + Assert.assertEquals(rowfilter.filterRow(project, 26, project.rows.get(26)),true); + Assert.assertEquals(rowfilter.filterRow(project, 27, project.rows.get(27)),true); + Assert.assertEquals(rowfilter.filterRow(project, 28, project.rows.get(28)),true); + Assert.assertEquals(rowfilter.filterRow(project, 29, project.rows.get(29)),true); + } + + // should add tests for errors as well +} From 2a034b65e93499922bfa23af9a13c2fadfaaeaa4 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 28 Jun 2018 06:21:27 +0100 Subject: [PATCH 03/11] Add number, date and boolean counts to List facet --- .../refine/browsing/facets/ListFacet.java | 42 ++++++++++++++++++- .../filters/ExpressionEqualRowFilter.java | 18 +++++++- .../util/ExpressionNominalValueGrouper.java | 1 + .../google/refine/expr/ExpressionUtils.java | 16 +++++-- 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/main/src/com/google/refine/browsing/facets/ListFacet.java b/main/src/com/google/refine/browsing/facets/ListFacet.java index 5314820a7..f33d5fa78 100644 --- a/main/src/com/google/refine/browsing/facets/ListFacet.java +++ b/main/src/com/google/refine/browsing/facets/ListFacet.java @@ -74,6 +74,9 @@ public class ListFacet implements Facet { public boolean omitError; public List selection = new LinkedList<>(); + public boolean selectNumber; + public boolean selectDateTime; + public boolean selectBoolean; public boolean selectBlank; public boolean selectError; @@ -94,6 +97,9 @@ public class ListFacet implements Facet { writer.endObject(); } writer.endArray(); + writer.key("selectNumber"); writer.value(selectNumber); + writer.key("selectDateTime"); writer.value(selectDateTime); + writer.key("selectBoolean"); writer.value(selectBoolean); writer.key("omitBlank"); writer.value(omitBlank); writer.key("selectBlank"); writer.value(selectBlank); writer.key("omitError"); writer.value(omitError); @@ -124,6 +130,9 @@ public class ListFacet implements Facet { omitBlank = JSONUtilities.getBoolean(o, "omitBlank", false); omitError = JSONUtilities.getBoolean(o, "omitError", false); + selectNumber = JSONUtilities.getBoolean(o, "selectNumber", false); + selectDateTime = JSONUtilities.getBoolean(o, "selectDateTime", false); + selectBoolean = JSONUtilities.getBoolean(o, "selectBoolean", false); selectBlank = JSONUtilities.getBoolean(o, "selectBlank", false); selectError = JSONUtilities.getBoolean(o, "selectError", false); } @@ -149,6 +158,9 @@ public class ListFacet implements Facet { * Computed results */ protected List _choices = new LinkedList(); + protected int _numberCount; + protected int _datetimeCount; + protected int _booleanCount; protected int _blankCount; protected int _errorCount; @@ -176,7 +188,27 @@ public class ListFacet implements Facet { choice.write(writer, options); } writer.endArray(); - + if (_config.selectNumber || _numberCount > 0) { + writer.key("numberChoice"); + writer.object(); + writer.key("s"); writer.value(_config.selectNumber); + writer.key("c"); writer.value(_numberCount); + writer.endObject(); + } + if (_config.selectDateTime || _datetimeCount > 0) { + writer.key("datetimeChoice"); + writer.object(); + writer.key("s"); writer.value(_config.selectDateTime); + writer.key("c"); writer.value(_datetimeCount); + writer.endObject(); + } + if (_config.selectBoolean || _booleanCount > 0) { + writer.key("booleanChoice"); + writer.object(); + writer.key("s"); writer.value(_configselectBoolean); + writer.key("c"); writer.value(_booleanCount); + writer.endObject(); + } if (!_config.omitBlank && (_config.selectBlank || _blankCount > 0)) { writer.key("blankChoice"); writer.object(); @@ -245,6 +277,9 @@ public class ListFacet implements Facet { _config.columnName, _cellIndex, createMatches(), + _config.selectNumber, + _config.selectDateTime, + _config.selectBoolean, _config.selectBlank, _config.selectError, _config.invert); @@ -310,6 +345,9 @@ public class ListFacet implements Facet { } } + _numberCount = grouper.numberCount; + _datetimeCount = grouper.datetimeCount; + _booleanCount = grouper.booleanCount; _blankCount = grouper.blankCount; _errorCount = grouper.errorCount; } @@ -321,4 +359,4 @@ public class ListFacet implements Facet { } return a; } -} +} \ No newline at end of file diff --git a/main/src/com/google/refine/browsing/filters/ExpressionEqualRowFilter.java b/main/src/com/google/refine/browsing/filters/ExpressionEqualRowFilter.java index b5380a90f..30ec6b7e0 100644 --- a/main/src/com/google/refine/browsing/filters/ExpressionEqualRowFilter.java +++ b/main/src/com/google/refine/browsing/filters/ExpressionEqualRowFilter.java @@ -33,6 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.browsing.filters; +import java.time.OffsetDateTime; + import java.util.Collection; import java.util.Properties; @@ -58,8 +60,10 @@ public class ExpressionEqualRowFilter implements RowFilter { final protected int _cellIndex; // the expression is based on this column; // -1 if based on no column in particular, // for expression such as "row.starred". - final protected Object[] _matches; + final protected boolean _selectNumber; + final protected boolean _selectDateTime; + final protected boolean _selectBoolean; final protected boolean _selectBlank; final protected boolean _selectError; final protected boolean _invert; @@ -69,6 +73,9 @@ public class ExpressionEqualRowFilter implements RowFilter { String columnName, int cellIndex, Object[] matches, + boolean selectNumber, + boolean selectDateTime, + boolean selectBoolean, boolean selectBlank, boolean selectError, boolean invert @@ -77,6 +84,9 @@ public class ExpressionEqualRowFilter implements RowFilter { _columnName = columnName; _cellIndex = cellIndex; _matches = matches; + _selectNumber = selectNumber; + _selectDateTime = selectDateTime; + _selectBoolean = selectBoolean; _selectBlank = selectBlank; _selectError = selectError; _invert = invert; @@ -178,6 +188,12 @@ public class ExpressionEqualRowFilter implements RowFilter { protected boolean testValue(Object v) { if (ExpressionUtils.isError(v)) { return _selectError; + } else if (v instanceof Number) { + return _selectNumber; + } else if (v instanceof OffsetDateTime) { + return _selectDateTime; + } else if (v instanceof Boolean) { + return _selectBoolean; } else if (ExpressionUtils.isNonBlankData(v)) { for (Object match : _matches) { if (testValue(v, match)) { diff --git a/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java b/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java index 7e399fa03..df5e925dd 100644 --- a/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java +++ b/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java @@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.browsing.util; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; diff --git a/main/src/com/google/refine/expr/ExpressionUtils.java b/main/src/com/google/refine/expr/ExpressionUtils.java index 293ce3c86..f44b0cd52 100644 --- a/main/src/com/google/refine/expr/ExpressionUtils.java +++ b/main/src/com/google/refine/expr/ExpressionUtils.java @@ -103,11 +103,19 @@ public class ExpressionUtils { static public boolean isError(Object o) { return o instanceof EvalError; } - /* - static public boolean isBlank(Object o) { - return o == null || (o instanceof String && ((String) o).length() == 0); + + static public boolean isNumber(Object v) { + return v != null && (v instanceof Number); } - */ + + static public boolean isBoolean(Object v) { + return v != null && v instanceof Boolean; + } + + static public boolean isDateTime(Object v) { + return v != null && v instanceof OffsetDateTime; + } + static public boolean isNonBlankData(Object o) { return o != null && From 477d9acf9b982f5062625a11e72ed95a758289f9 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 28 Jun 2018 06:22:41 +0100 Subject: [PATCH 04/11] Add frontend support for number, date, boolean groups in list facet --- .../modules/core/scripts/facets/list-facet.js | 85 +++++++++++++++++-- 1 file changed, 76 insertions(+), 9 deletions(-) diff --git a/main/webapp/modules/core/scripts/facets/list-facet.js b/main/webapp/modules/core/scripts/facets/list-facet.js index 33b1b3094..ee104a913 100644 --- a/main/webapp/modules/core/scripts/facets/list-facet.js +++ b/main/webapp/modules/core/scripts/facets/list-facet.js @@ -44,6 +44,9 @@ function ListFacet(div, config, options, selection) { } this._selection = selection || []; + this._numberChoice = (config.selectNumber) ? { s : true, c : 0 } : null; + this._datetimeChoice = (config.selectDateTime) ? { s : true, c : 0 } : null; + this._booleanChoice = (config.selecBoolean) ? { s : true, c : 0 } : null; this._blankChoice = (config.selectBlank) ? { s : true, c : 0 } : null; this._errorChoice = (config.selectError) ? { s : true, c : 0 } : null; @@ -62,6 +65,9 @@ ListFacet.prototype.dispose = function() { ListFacet.prototype.reset = function() { this._selection = []; + this._numberChoice = null; + this._datetimeChoice = null; + this._booleanChoice = null; this._blankChoice = null; this._errorChoice = null; }; @@ -85,11 +91,14 @@ ListFacet.prototype.getJSON = function() { columnName: this._config.columnName, expression: this._config.expression, omitBlank: "omitBlank" in this._config ? this._config.omitBlank : false, - omitError: "omitError" in this._config ? this._config.omitError : false, - selection: [], - selectBlank: this._blankChoice !== null && this._blankChoice.s, - selectError: this._errorChoice !== null && this._errorChoice.s, - invert: this._config.invert + omitError: "omitError" in this._config ? this._config.omitError : false, + selection: [], + selectNumber: this._numberChoice !== null && this._numberChoice.s, + selectDateTime: this._datetimeChoice !== null && this._datetimeChoice.s, + selectBoolean: this._booleanChoice !== null && this._booleanChoice.s, + selectBlank: this._blankChoice !== null && this._blankChoice.s, + selectError: this._errorChoice !== null && this._errorChoice.s, + invert: this._config.invert }; for (var i = 0; i < this._selection.length; i++) { var choice = { @@ -102,8 +111,11 @@ ListFacet.prototype.getJSON = function() { ListFacet.prototype.hasSelection = function() { return this._selection.length > 0 || - (this._blankChoice !== null && this._blankChoice.s) || - (this._errorChoice !== null && this._errorChoice.s); + ( this._numberChoice !== null && this._numberChoice.s ) || + ( this._datetimeChoice !== null && this._datetimeChoice.s ) || + ( this._booleanChoice !== null && this._booleanChoice.s ) || + ( this._blankChoice !== null && this._blankChoice.s ) || + ( this._errorChoice !== null && this._errorChoice.s ); }; ListFacet.prototype.updateState = function(data) { @@ -121,6 +133,9 @@ ListFacet.prototype.updateState = function(data) { this._selection = selection; this._reSortChoices(); + this._numberChoice = data.numberChoice || null; + this._datetimeChoice = data.datetimeChoice || null; + this._booleanChoice = data.booleanChoice || null; this._blankChoice = data.blankChoice || null; this._errorChoice = data.errorChoice || null; } @@ -239,6 +254,15 @@ ListFacet.prototype._copyChoices = function() { var choice = this._data.choices[i]; lines.push(choice.v.l + "\t" + choice.c); } + if (this._numberChoice) { + lines.push("(number)\t" + this._numberChoice.c); + } + if (this._datetimeChoice) { + lines.push("(date)\t" + this._datetimeChoice.c); + } + if (this._booleanChoice) { + lines.push("(boolean)\t" + this._booleanChoice.c); + } if (this._blankChoice) { lines.push("(blank)\t" + this._blankChoice.c); } @@ -335,6 +359,9 @@ ListFacet.prototype._update = function(resetScroll) { var choices = this._data.choices; var selectionCount = this._selection.length + + (this._numberChoice !== null && this._numberChoice.s ? 1 : 0) + + (this._datetimeChoice !== null && this._datetimeChoice.s ? 1 : 0) + + (this._booleanChoice !== null && this._booleanChoice.s ? 1 : 0) + (this._blankChoice !== null && this._blankChoice.s ? 1 : 0) + (this._errorChoice !== null && this._errorChoice.s ? 1 : 0); @@ -389,6 +416,15 @@ ListFacet.prototype._update = function(resetScroll) { for (var i = 0; i < choices.length; i++) { renderChoice(i, choices[i]); } + if (this._numberChoice !== null) { + renderChoice(-5, this._numberChoice, "(number)"); + } + if (this._datetimeChoice !== null) { + renderChoice(-4, this._datetimeChoice, "(date)"); + } + if (this._booleanChoice !== null) { + renderChoice(-3, this._booleanChoice, "(boolean)"); + } if (this._blankChoice !== null) { renderChoice(-1, this._blankChoice, "(blank)"); } @@ -406,6 +442,12 @@ ListFacet.prototype._update = function(resetScroll) { return self._blankChoice; } else if (index === -2) { return self._errorChoice; + } else if (index === -3) { + return self._booleanChoice; + } else if (index === -4) { + return self._datetimeChoice; + } else if (index === -5) { + return self._numberChoice; } else { return choices[index]; } @@ -542,6 +584,12 @@ ListFacet.prototype._editChoice = function(choice, choiceDiv) { originalContent = "(blank)"; } else if (choice === this._errorChoice) { originalContent = "(error)"; + } else if (choice === this._booleanChoice) { + originalContent = "(boolean)"; + } else if (choice === this._datetimeChoice) { + originalContent = "(date)"; + } else if (choice === this._numberChoice) { + originalContent = "(number)"; } else { originalContent = choice.v.v; } @@ -617,6 +665,15 @@ ListFacet.prototype._editChoice = function(choice, choiceDiv) { ListFacet.prototype._select = function(choice, only) { if (only) { this._selection = []; + if (this._numberChoice !== null) { + this._numberChoice.s = false; + } + if (this._datetimeChoice !== null) { + this._datetimeChoice.s = false; + } + if (this._booleanChoice !== null) { + this._booleanChoice.s = false; + } if (this._blankChoice !== null) { this._blankChoice.s = false; } @@ -626,7 +683,11 @@ ListFacet.prototype._select = function(choice, only) { } choice.s = true; - if (choice !== this._errorChoice && choice !== this._blankChoice) { + if (choice !== this._errorChoice && + choice !== this._blankChoice && + choice !== this._numberChoice && + choice !== this._datetimeChoice && + choice !== this._booleanChoice ) { this._selection.push(choice); } @@ -634,7 +695,7 @@ ListFacet.prototype._select = function(choice, only) { }; ListFacet.prototype._deselect = function(choice) { - if (choice === this._errorChoice || choice === this._blankChoice) { + if (choice === this._errorChoice || choice === this._blankChoice || choice === this._numberChoice || choice === this._datetimeChoice || choice === this._booleanChoice) { choice.s = false; } else { for (var i = this._selection.length - 1; i >= 0; i--) { @@ -649,6 +710,9 @@ ListFacet.prototype._deselect = function(choice) { ListFacet.prototype._reset = function() { this._selection = []; + this._numberChoice = null; + this._datetimeChoice = null; + this._booleanChoice = null; this._blankChoice = null; this._errorChoice = null; this._config.invert = false; @@ -669,6 +733,9 @@ ListFacet.prototype._remove = function() { this._config = null; this._selection = null; + this._numberChoice = null; + this._datetimeChoice = null; + this._booleanChoice = null; this._blankChoice = null; this._errorChoice = null; this._data = null; From acea8e5671d06ec6284dd3eaa9c383297cd11a16 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 28 Jun 2018 06:24:35 +0100 Subject: [PATCH 05/11] Update cache test to ensure correct facet created --- .../server/src/com/google/refine/tests/model/CacheTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java index b9a4bb0b9..8a501ad8e 100644 --- a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java @@ -62,7 +62,7 @@ import com.google.refine.tests.RefineTest; public class CacheTests extends RefineTest { // Equivalent to duplicate facet on Column A with true selected - static final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"facetCount(value, 'value', 'Column A') > 1\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":true,\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}"; + static final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"(facetCount(value, 'value', 'Column A') > 1).toString()\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":\"true\",\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}"; @Override @BeforeTest From 62971b380477ee7ab8848e177b201d8c22577531 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 28 Jun 2018 07:21:17 +0100 Subject: [PATCH 06/11] Don't display 'edit' option for number/date/boolean groups in list facet --- main/webapp/modules/core/scripts/facets/list-facet.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main/webapp/modules/core/scripts/facets/list-facet.js b/main/webapp/modules/core/scripts/facets/list-facet.js index ee104a913..77a81cc41 100644 --- a/main/webapp/modules/core/scripts/facets/list-facet.js +++ b/main/webapp/modules/core/scripts/facets/list-facet.js @@ -417,18 +417,23 @@ ListFacet.prototype._update = function(resetScroll) { renderChoice(i, choices[i]); } if (this._numberChoice !== null) { + renderEdit = false; renderChoice(-5, this._numberChoice, "(number)"); } if (this._datetimeChoice !== null) { + renderEdit = false; renderChoice(-4, this._datetimeChoice, "(date)"); } if (this._booleanChoice !== null) { + renderEdit = false; renderChoice(-3, this._booleanChoice, "(boolean)"); } if (this._blankChoice !== null) { + renderEdit = false; renderChoice(-1, this._blankChoice, "(blank)"); } if (this._errorChoice !== null) { + renderEdit = false; renderChoice(-2, this._errorChoice, "(error)"); } From fb8697ef9c2db1cb668405ec0fe221acc524fa87 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 13 Sep 2018 12:45:42 +0100 Subject: [PATCH 07/11] Minor code tidy --- .../refine/browsing/util/ExpressionNominalValueGrouper.java | 1 - .../server/src/com/google/refine/tests/model/CacheTests.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java b/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java index df5e925dd..7e399fa03 100644 --- a/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java +++ b/main/src/com/google/refine/browsing/util/ExpressionNominalValueGrouper.java @@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.browsing.util; -import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; diff --git a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java index 8a501ad8e..4cf52dc52 100644 --- a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java @@ -62,7 +62,7 @@ import com.google.refine.tests.RefineTest; public class CacheTests extends RefineTest { // Equivalent to duplicate facet on Column A with true selected - static final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"(facetCount(value, 'value', 'Column A') > 1).toString()\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":\"true\",\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}"; + static private final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"(facetCount(value, 'value', 'Column A') > 1).toString()\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":\"true\",\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}"; @Override @BeforeTest From 795d7232aaf9d88f4e9a2eb7b101404683db6da4 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Sat, 15 Sep 2018 09:17:52 +0100 Subject: [PATCH 08/11] Update Custom facets that output a boolean to convert to string --- .../core/scripts/views/data-table/menu-facets.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/main/webapp/modules/core/scripts/views/data-table/menu-facets.js b/main/webapp/modules/core/scripts/views/data-table/menu-facets.js index 53d1ecc16..ce3124db2 100644 --- a/main/webapp/modules/core/scripts/views/data-table/menu-facets.js +++ b/main/webapp/modules/core/scripts/views/data-table/menu-facets.js @@ -147,8 +147,8 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { { "name": column.name, "columnName": column.name, - "expression": "facetCount(value, 'value', '" + - column.name + "') > 1" + "expression": "(facetCount(value, 'value', '" + + column.name + "') > 1).toString()" } ); } @@ -240,7 +240,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { { "name": column.name, "columnName": column.name, - "expression": "isError(value)" + "expression": "isError(value).toString()" } ); } @@ -254,7 +254,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { { "name": column.name, "columnName": column.name, - "expression": "isNull(value)" + "expression": "isNull(value).toString()" } ); } @@ -268,7 +268,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { { "name": column.name, "columnName": column.name, - "expression": "isEmptyString(value)" + "expression": "isEmptyString(value).toString()" } ); } @@ -282,7 +282,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { { "name": column.name, "columnName": column.name, - "expression": "isBlank(value)" + "expression": "isBlank(value).toString()" } ); } From e8f454afc7fa012e97b494fae2872eb57f189ead Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 22 Nov 2018 13:42:46 +0000 Subject: [PATCH 09/11] Fixing changes missed during rebase --- main/src/com/google/refine/browsing/facets/ListFacet.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/src/com/google/refine/browsing/facets/ListFacet.java b/main/src/com/google/refine/browsing/facets/ListFacet.java index f33d5fa78..d5cdfa45f 100644 --- a/main/src/com/google/refine/browsing/facets/ListFacet.java +++ b/main/src/com/google/refine/browsing/facets/ListFacet.java @@ -205,7 +205,7 @@ public class ListFacet implements Facet { if (_config.selectBoolean || _booleanCount > 0) { writer.key("booleanChoice"); writer.object(); - writer.key("s"); writer.value(_configselectBoolean); + writer.key("s"); writer.value(_config.selectBoolean); writer.key("c"); writer.value(_booleanCount); writer.endObject(); } @@ -270,7 +270,7 @@ public class ListFacet implements Facet { return _eval == null || _errorMessage != null || - (_config.selection.size() == 0 && !_config.selectBlank && !_config.selectError) ? + (_config.selection.size() == 0 && !_config.selectBlank && !_config.selectError && !_config.selectNumber && !_config.selectDateTime && !_config.selectBoolean) ? null : new ExpressionEqualRowFilter( _eval, From 2ad0e76aca074db734a31d130be6d09bb7b4ff91 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 22 Nov 2018 13:43:02 +0000 Subject: [PATCH 10/11] Updating tests for List Facets --- .../tests/browsing/facets/ListFacetTests.java | 5 ++- .../browsing/facets/TextListFacetTests.java | 45 +++++++++---------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/ListFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/ListFacetTests.java index 580f0971d..6a18c1782 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/ListFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/ListFacetTests.java @@ -15,11 +15,14 @@ public class ListFacetTests extends RefineTest { private static String jsonConfig = "{" + "\"type\":\"list\"," + "\"name\":\"facet A\"," - + "\"columnName\":\"Column A\"," + "\"expression\":\"value+\\\"bar\\\"\"," + + "\"columnName\":\"Column A\"," + "\"omitBlank\":false," + "\"omitError\":false," + "\"selection\":[{\"v\":{\"v\":\"foobar\",\"l\":\"true\"}}]," + + "\"selectNumber\":false," + + "\"selectDateTime\":false," + + "\"selectBoolean\":false," + "\"selectBlank\":false," + "\"selectError\":false," + "\"invert\":false" diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextListFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextListFacetTests.java index 041dc8892..14800d7f8 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextListFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextListFacetTests.java @@ -51,16 +51,15 @@ import org.testng.annotations.Test; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.browsing.RowFilter; -import com.google.refine.browsing.facets.ListFacet; +import com.google.refine.browsing.facets.Facet; +import com.google.refine.browsing.facets.ListFacet.ListFacetConfig; import com.google.refine.tests.RefineTest; public class TextListFacetTests extends RefineTest { // dependencies private Project project; - private ListFacet facet; private RowFilter rowfilter; - private JSONObject listfacetconfig; // Variables private static OffsetDateTime dateTimeValue = OffsetDateTime.parse("2017-05-12T05:45:00+00:00", DateTimeFormatter.ISO_OFFSET_DATE_TIME); @@ -118,7 +117,7 @@ public class TextListFacetTests extends RefineTest { public void testTextSelection() throws Exception { //Need to work out the correct facet config for these tests to work //Also need all rows in all tests so can check that rows aren't being selected when they shouldn't be - String facetconfig = "{" + String jsonConfig = "{" + "\"type\": \"list\"," + "\"name\": \"Value\"," + "\"columnName\": \"" + columnName + "\"," @@ -142,9 +141,9 @@ public class TextListFacetTests extends RefineTest { + "}"; //Add the facet to the project and create a row filter - facet = new ListFacet(); - listfacetconfig = new JSONObject(facetconfig); - facet.initializeFromJSON(project,listfacetconfig); + ListFacetConfig facetConfig = new ListFacetConfig(); + facetConfig.initializeFromJSON(new JSONObject(jsonConfig)); + Facet facet = facetConfig.apply(project); rowfilter = facet.getRowFilter(project); //Check each row in the project against the filter @@ -188,7 +187,7 @@ public class TextListFacetTests extends RefineTest { @Test public void testDateSelection() throws Exception { - String facetconfig = "{" + String jsonConfig = "{" + "\"type\": \"list\"," + "\"name\": \"Value\"," + "\"columnName\": \"" + columnName + "\"," @@ -205,9 +204,9 @@ public class TextListFacetTests extends RefineTest { + "}"; //Add the facet to the project and create a row filter - facet = new ListFacet(); - listfacetconfig = new JSONObject(facetconfig); - facet.initializeFromJSON(project,listfacetconfig); + ListFacetConfig facetConfig = new ListFacetConfig(); + facetConfig.initializeFromJSON(new JSONObject(jsonConfig)); + Facet facet = facetConfig.apply(project); rowfilter = facet.getRowFilter(project); //Check each row in the project against the filter @@ -251,7 +250,7 @@ public class TextListFacetTests extends RefineTest { @Test public void testIntegerSelection() throws Exception { - String facetconfig = "{" + String jsonConfig = "{" + "\"type\": \"list\"," + "\"name\": \"Value\"," + "\"columnName\": \"" + columnName + "\"," @@ -268,9 +267,9 @@ public class TextListFacetTests extends RefineTest { + "}"; //Add the facet to the project and create a row filter - facet = new ListFacet(); - listfacetconfig = new JSONObject(facetconfig); - facet.initializeFromJSON(project,listfacetconfig); + ListFacetConfig facetConfig = new ListFacetConfig(); + facetConfig.initializeFromJSON(new JSONObject(jsonConfig)); + Facet facet = facetConfig.apply(project); rowfilter = facet.getRowFilter(project); //Check each row in the project against the filter @@ -314,7 +313,7 @@ public class TextListFacetTests extends RefineTest { @Test public void testBooleanSelection() throws Exception { - String facetconfig = "{" + String jsonConfig = "{" + "\"type\": \"list\"," + "\"name\": \"Value\"," + "\"columnName\": \"" + columnName + "\"," @@ -331,9 +330,9 @@ public class TextListFacetTests extends RefineTest { + "}"; //Add the facet to the project and create a row filter - facet = new ListFacet(); - listfacetconfig = new JSONObject(facetconfig); - facet.initializeFromJSON(project,listfacetconfig); + ListFacetConfig facetConfig = new ListFacetConfig(); + facetConfig.initializeFromJSON(new JSONObject(jsonConfig)); + Facet facet = facetConfig.apply(project); rowfilter = facet.getRowFilter(project); //Check each row in the project against the filter @@ -377,7 +376,7 @@ public class TextListFacetTests extends RefineTest { @Test public void testBlankSelection() throws Exception { - String facetconfig = "{" + String jsonConfig = "{" + "\"type\": \"list\"," + "\"name\": \"Value\"," + "\"columnName\": \"" + columnName + "\"," @@ -394,9 +393,9 @@ public class TextListFacetTests extends RefineTest { + "}"; //Add the facet to the project and create a row filter - facet = new ListFacet(); - listfacetconfig = new JSONObject(facetconfig); - facet.initializeFromJSON(project,listfacetconfig); + ListFacetConfig facetConfig = new ListFacetConfig(); + facetConfig.initializeFromJSON(new JSONObject(jsonConfig)); + Facet facet = facetConfig.apply(project); rowfilter = facet.getRowFilter(project); //Check each row in the project against the filter From 90cd3dadd02f403b6066ee815c08ffa31b3fe1a7 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Thu, 22 Nov 2018 15:07:41 +0000 Subject: [PATCH 11/11] Ensure Facet by Star/Flag still work as expected --- .../modules/core/scripts/views/data-table/data-table-view.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/webapp/modules/core/scripts/views/data-table/data-table-view.js b/main/webapp/modules/core/scripts/views/data-table/data-table-view.js index 9c8c20d38..86adebdeb 100644 --- a/main/webapp/modules/core/scripts/views/data-table/data-table-view.js +++ b/main/webapp/modules/core/scripts/views/data-table/data-table-view.js @@ -636,7 +636,7 @@ DataTableView.prototype._createMenuForAllColumns = function(elmt) { { "name" : $.i18n('core-views/starred-rows'), "columnName" : "", - "expression" : "row.starred" + "expression" : "row.starred.toString()" }, { "scroll" : false @@ -653,7 +653,7 @@ DataTableView.prototype._createMenuForAllColumns = function(elmt) { { "name" : $.i18n('core-views/flagged-rows'), "columnName" : "", - "expression" : "row.flagged" + "expression" : "row.flagged.toString()" }, { "scroll" : false