From 6daa1b4f5c979d8c7bec629d3761cfe045f7f99d Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 25 Sep 2018 12:28:39 +0100 Subject: [PATCH] Jackson serialization for RowReorderOperation --- .../refine/commands/row/GetRowsCommand.java | 16 +-- .../commands/row/ReorderRowsCommand.java | 6 +- .../refine/exporters/TemplatingExporter.java | 7 +- .../operations/row/RowFlagOperation.java | 7 ++ .../operations/row/RowReorderOperation.java | 30 ++++-- .../operations/row/RowStarOperation.java | 7 ++ .../com/google/refine/sorting/BaseSorter.java | 102 +++++++++++------- .../refine/sorting/BooleanCriterion.java | 10 ++ .../com/google/refine/sorting/Criterion.java | 87 +++++++++++++-- .../google/refine/sorting/DateCriterion.java | 9 ++ .../refine/sorting/NumberCriterion.java | 5 + .../refine/sorting/StringCriterion.java | 23 ++-- .../row/RowReorderOperationTests.java | 7 +- 13 files changed, 236 insertions(+), 80 deletions(-) diff --git a/main/src/com/google/refine/commands/row/GetRowsCommand.java b/main/src/com/google/refine/commands/row/GetRowsCommand.java index ca5cdf192..95e0fe266 100644 --- a/main/src/com/google/refine/commands/row/GetRowsCommand.java +++ b/main/src/com/google/refine/commands/row/GetRowsCommand.java @@ -57,6 +57,7 @@ import com.google.refine.importing.ImportingManager; import com.google.refine.model.Project; import com.google.refine.model.Record; import com.google.refine.model.Row; +import com.google.refine.sorting.BaseSorter.SortingConfig; import com.google.refine.sorting.SortingRecordVisitor; import com.google.refine.sorting.SortingRowVisitor; import com.google.refine.util.ParsingUtilities; @@ -120,11 +121,14 @@ public class GetRowsCommand extends Command { RowWritingVisitor rwv = new RowWritingVisitor(start, limit, jsonWriter, options); - JSONObject sortingJson = null; + SortingConfig sortingConfig = null; try{ String json = request.getParameter("sorting"); - sortingJson = (json == null) ? null : + JSONObject sortingJson = (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json); + if (sortingJson != null) { + sortingConfig = SortingConfig.reconstruct(project, sortingJson); + } } catch (JSONException e) { } @@ -132,10 +136,10 @@ public class GetRowsCommand extends Command { FilteredRows filteredRows = engine.getAllFilteredRows(); RowVisitor visitor = rwv; - if (sortingJson != null) { + if (sortingConfig != null) { SortingRowVisitor srv = new SortingRowVisitor(visitor); - srv.initializeFromJSON(project, sortingJson); + srv.initializeFromConfig(project, sortingConfig); if (srv.hasCriteria()) { visitor = srv; } @@ -151,10 +155,10 @@ public class GetRowsCommand extends Command { FilteredRecords filteredRecords = engine.getFilteredRecords(); RecordVisitor visitor = rwv; - if (sortingJson != null) { + if (sortingConfig != null) { SortingRecordVisitor srv = new SortingRecordVisitor(visitor); - srv.initializeFromJSON(project, sortingJson); + srv.initializeFromConfig(project, sortingConfig); if (srv.hasCriteria()) { visitor = srv; } diff --git a/main/src/com/google/refine/commands/row/ReorderRowsCommand.java b/main/src/com/google/refine/commands/row/ReorderRowsCommand.java index 49ed991c5..c9825c56e 100644 --- a/main/src/com/google/refine/commands/row/ReorderRowsCommand.java +++ b/main/src/com/google/refine/commands/row/ReorderRowsCommand.java @@ -44,6 +44,7 @@ import com.google.refine.commands.EngineDependentCommand; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; import com.google.refine.operations.row.RowReorderOperation; +import com.google.refine.sorting.BaseSorter.SortingConfig; import com.google.refine.util.ParsingUtilities; public class ReorderRowsCommand extends EngineDependentCommand { @@ -53,12 +54,13 @@ public class ReorderRowsCommand extends EngineDependentCommand { HttpServletRequest request, EngineConfig engineConfig) throws Exception { String mode = request.getParameter("mode"); - JSONObject sorting = null; + SortingConfig sorting = null; try{ String json = request.getParameter("sorting"); - sorting = (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json); + JSONObject sortingJson = (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json); + sorting = (sortingJson == null) ? null : SortingConfig.reconstruct(project, sortingJson); } catch (JSONException e) { // ignore } diff --git a/main/src/com/google/refine/exporters/TemplatingExporter.java b/main/src/com/google/refine/exporters/TemplatingExporter.java index 4691ccf7a..43f8d4809 100644 --- a/main/src/com/google/refine/exporters/TemplatingExporter.java +++ b/main/src/com/google/refine/exporters/TemplatingExporter.java @@ -52,6 +52,7 @@ import com.google.refine.expr.ParsingException; import com.google.refine.model.Project; import com.google.refine.sorting.SortingRecordVisitor; import com.google.refine.sorting.SortingRowVisitor; +import com.google.refine.sorting.BaseSorter.SortingConfig; import com.google.refine.templating.Parser; import com.google.refine.templating.Template; import com.google.refine.util.ParsingUtilities; @@ -115,8 +116,9 @@ public class TemplatingExporter implements WriterExporter { if (sortingJson != null) { try { + SortingConfig sorting = SortingConfig.reconstruct(project, sortingJson); SortingRowVisitor srv = new SortingRowVisitor(visitor); - srv.initializeFromJSON(project, sortingJson); + srv.initializeFromConfig(project, sorting); if (srv.hasCriteria()) { visitor = srv; @@ -133,8 +135,9 @@ public class TemplatingExporter implements WriterExporter { if (sortingJson != null) { try { + SortingConfig sorting = SortingConfig.reconstruct(project, sortingJson); SortingRecordVisitor srv = new SortingRecordVisitor(visitor); - srv.initializeFromJSON(project, sortingJson); + srv.initializeFromConfig(project, sorting); if (srv.hasCriteria()) { visitor = srv; diff --git a/main/src/com/google/refine/operations/row/RowFlagOperation.java b/main/src/com/google/refine/operations/row/RowFlagOperation.java index 5dc07a7a2..fe8a1dbfc 100644 --- a/main/src/com/google/refine/operations/row/RowFlagOperation.java +++ b/main/src/com/google/refine/operations/row/RowFlagOperation.java @@ -41,6 +41,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; @@ -84,6 +86,11 @@ public class RowFlagOperation extends EngineDependentOperation { writer.key("flagged"); writer.value(_flagged); writer.endObject(); } + + @JsonProperty("flagged") + public boolean getFlagged() { + return _flagged; + } @Override protected String getBriefDescription(Project project) { diff --git a/main/src/com/google/refine/operations/row/RowReorderOperation.java b/main/src/com/google/refine/operations/row/RowReorderOperation.java index 6df761943..ace36d498 100644 --- a/main/src/com/google/refine/operations/row/RowReorderOperation.java +++ b/main/src/com/google/refine/operations/row/RowReorderOperation.java @@ -41,6 +41,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine.Mode; import com.google.refine.browsing.RecordVisitor; @@ -52,6 +54,8 @@ import com.google.refine.model.Record; import com.google.refine.model.Row; import com.google.refine.model.changes.RowReorderChange; import com.google.refine.operations.OperationRegistry; +import com.google.refine.sorting.BaseSorter; +import com.google.refine.sorting.BaseSorter.SortingConfig; import com.google.refine.sorting.SortingRecordVisitor; import com.google.refine.sorting.SortingRowVisitor; @@ -60,14 +64,14 @@ public class RowReorderOperation extends AbstractOperation { String mode = obj.getString("mode"); JSONObject sorting = obj.has("sorting") && !obj.isNull("sorting") ? obj.getJSONObject("sorting") : null; - - return new RowReorderOperation(Engine.stringToMode(mode), sorting); + BaseSorter.SortingConfig config = BaseSorter.SortingConfig.reconstruct(project, sorting); + return new RowReorderOperation(Engine.stringToMode(mode), config); } - + final protected Mode _mode; - final protected JSONObject _sorting; + final protected BaseSorter.SortingConfig _sorting; - public RowReorderOperation(Mode mode, JSONObject sorting) { + public RowReorderOperation(Mode mode, BaseSorter.SortingConfig sorting) { _mode = mode; _sorting = sorting; } @@ -80,9 +84,19 @@ public class RowReorderOperation extends AbstractOperation { writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); writer.key("description"); writer.value(getBriefDescription(null)); writer.key("mode"); writer.value(Engine.modeToString(_mode)); - writer.key("sorting"); writer.value(_sorting); + writer.key("sorting"); _sorting.write(writer, options); writer.endObject(); } + + @JsonProperty("mode") + public Mode getMode() { + return _mode; + } + + @JsonProperty("sorting") + public SortingConfig getSortingConfig() { + return _sorting; + } @Override protected String getBriefDescription(Project project) { @@ -100,7 +114,7 @@ public class RowReorderOperation extends AbstractOperation { if (_sorting != null) { SortingRowVisitor srv = new SortingRowVisitor(visitor); - srv.initializeFromJSON(project, _sorting); + srv.initializeFromConfig(project, _sorting); if (srv.hasCriteria()) { visitor = srv; } @@ -112,7 +126,7 @@ public class RowReorderOperation extends AbstractOperation { if (_sorting != null) { SortingRecordVisitor srv = new SortingRecordVisitor(visitor); - srv.initializeFromJSON(project, _sorting); + srv.initializeFromConfig(project, _sorting); if (srv.hasCriteria()) { visitor = srv; } diff --git a/main/src/com/google/refine/operations/row/RowStarOperation.java b/main/src/com/google/refine/operations/row/RowStarOperation.java index db337d06f..5508449b1 100644 --- a/main/src/com/google/refine/operations/row/RowStarOperation.java +++ b/main/src/com/google/refine/operations/row/RowStarOperation.java @@ -41,6 +41,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; @@ -84,6 +86,11 @@ public class RowStarOperation extends EngineDependentOperation { writer.key("starred"); writer.value(_starred); writer.endObject(); } + + @JsonProperty("starred") + public boolean getStarred() { + return _starred; + } @Override protected String getBriefDescription(Project project) { diff --git a/main/src/com/google/refine/sorting/BaseSorter.java b/main/src/com/google/refine/sorting/BaseSorter.java index 5ace4db75..6b807f261 100644 --- a/main/src/com/google/refine/sorting/BaseSorter.java +++ b/main/src/com/google/refine/sorting/BaseSorter.java @@ -34,11 +34,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.sorting; import java.util.List; +import java.util.Properties; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import com.google.refine.Jsonizable; import com.google.refine.expr.EvalError; import com.google.refine.model.Project; import com.google.refine.sorting.Criterion.KeyMaker; @@ -104,26 +110,63 @@ abstract public class BaseSorter { } } - public void initializeFromJSON(Project project, JSONObject obj) throws JSONException { - if (obj.has("criteria") && !obj.isNull("criteria")) { - JSONArray a = obj.getJSONArray("criteria"); - int count = a.length(); - - _criteria = new Criterion[count]; - _keyMakers = new KeyMaker[count]; - _comparatorWrappers = new ComparatorWrapper[count]; - - for (int i = 0; i < count; i++) { - JSONObject obj2 = a.getJSONObject(i); - - _criteria[i] = createCriterionFromJSON(project, obj2); - _keyMakers[i] = _criteria[i].createKeyMaker(); - _comparatorWrappers[i] = new ComparatorWrapper(i); + public final static class SortingConfig implements Jsonizable { + + protected Criterion[] _criteria; + + @JsonCreator + public SortingConfig( + @JsonProperty("criteria") + Criterion[] criteria) { + _criteria = criteria; + } + + @JsonProperty("criteria") + public Criterion[] getCriteria() { + return _criteria; + } + + @Override + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("criteria"); + for (int i = 0; i != _criteria.length; i++) { + _criteria[i].write(writer, options); } - } else { - _criteria = new Criterion[0]; - _keyMakers = new KeyMaker[0]; - _comparatorWrappers = new ComparatorWrapper[0]; + writer.endObject(); + } + + public static SortingConfig reconstruct(Project project, JSONObject obj) { + Criterion[] criteria; + if (obj != null && obj.has("criteria") && !obj.isNull("criteria")) { + JSONArray a = obj.getJSONArray("criteria"); + int count = a.length(); + + criteria = new Criterion[count]; + + + for (int i = 0; i < count; i++) { + JSONObject obj2 = a.getJSONObject(i); + + criteria[i] = Criterion.reconstruct(obj2); + } + } else { + criteria = new Criterion[0]; + } + return new SortingConfig(criteria); + } + } + + public void initializeFromConfig(Project project, BaseSorter.SortingConfig config) throws JSONException { + _criteria = config.getCriteria(); + int count = _criteria.length; + _keyMakers = new KeyMaker[count]; + _comparatorWrappers = new ComparatorWrapper[count]; + + for (int i = 0; i < count; i++) { + _keyMakers[i] = _criteria[i].createKeyMaker(); + _comparatorWrappers[i] = new ComparatorWrapper(i); } } @@ -131,27 +174,6 @@ abstract public class BaseSorter { return _criteria != null && _criteria.length > 0; } - protected Criterion createCriterionFromJSON(Project project, JSONObject obj) throws JSONException { - String valueType = "string"; - if (obj.has("valueType") && !obj.isNull("valueType")) { - valueType = obj.getString("valueType"); - } - - Criterion c = null; - if ("boolean".equals(valueType)) { - c = new BooleanCriterion(); - } else if ("date".equals(valueType)) { - c = new DateCriterion(); - } else if ("number".equals(valueType)) { - c = new NumberCriterion(); - } else { - c = new StringCriterion(); - } - - c.initializeFromJSON(project, obj); - return c; - } - abstract protected Object makeKey( Project project, KeyMaker keyMaker, Criterion c, Object o, int index); diff --git a/main/src/com/google/refine/sorting/BooleanCriterion.java b/main/src/com/google/refine/sorting/BooleanCriterion.java index 1f56c98de..07746e038 100644 --- a/main/src/com/google/refine/sorting/BooleanCriterion.java +++ b/main/src/com/google/refine/sorting/BooleanCriterion.java @@ -33,6 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.sorting; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + import com.google.refine.expr.EvalError; import com.google.refine.expr.ExpressionUtils; @@ -62,4 +67,9 @@ public class BooleanCriterion extends Criterion { } }; } + + @Override + public String getValueType() { + return "boolean"; + } } diff --git a/main/src/com/google/refine/sorting/Criterion.java b/main/src/com/google/refine/sorting/Criterion.java index a41374fb0..e19b69dd6 100644 --- a/main/src/com/google/refine/sorting/Criterion.java +++ b/main/src/com/google/refine/sorting/Criterion.java @@ -33,18 +33,24 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.sorting; +import java.util.Properties; + import org.json.JSONException; import org.json.JSONObject; +import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + +import com.google.refine.Jsonizable; import com.google.refine.expr.ExpressionUtils; import com.google.refine.model.Column; import com.google.refine.model.Project; import com.google.refine.model.Record; import com.google.refine.model.Row; -abstract public class Criterion { +abstract public class Criterion implements Jsonizable { public String columnName; - protected int cellIndex; + protected int cellIndex = -2; // These take on positive and negative values to indicate where blanks and errors // go relative to non-blank values. They are also relative to each another. @@ -54,13 +60,10 @@ abstract public class Criterion { public boolean reverse; - public void initializeFromJSON(Project project, JSONObject obj) + public void initializeFromJSON(JSONObject obj) throws JSONException { if (obj.has("column") && !obj.isNull("column")) { columnName = obj.getString("column"); - - Column column = project.columnModel.getColumnByName(columnName); - cellIndex = column != null ? column.getCellIndex() : -1; } if (obj.has("blankPosition") && !obj.isNull("blankPosition")) { @@ -74,7 +77,75 @@ abstract public class Criterion { reverse = obj.getBoolean("reverse"); } } + + public static Criterion reconstruct(JSONObject obj) throws JSONException { + String valueType = "string"; + if (obj.has("valueType") && !obj.isNull("valueType")) { + valueType = obj.getString("valueType"); + } + Criterion c = null; + if ("boolean".equals(valueType)) { + c = new BooleanCriterion(); + } else if ("date".equals(valueType)) { + c = new DateCriterion(); + } else if ("number".equals(valueType)) { + c = new NumberCriterion(); + } else { + c = new StringCriterion(obj.getBoolean("caseSensitive")); + } + + c.initializeFromJSON(obj); + return c; + } + + + @Override + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("valueType"); writer.value(getValueType()); + writer.key("reverse"); writer.value(getReverse()); + writer.key("column"); writer.value(getColumnName()); + writer.key("blankPosition"); writer.value(getBlankPosition()); + writer.key("errorPosition"); writer.value(getErrorPosition()); + writer.endObject(); + } + + @JsonProperty("valueType") + public abstract String getValueType(); + + @JsonProperty("reverse") + public boolean getReverse() { + return reverse; + } + + @JsonProperty("column") + public String getColumnName() { + return columnName; + } + + @JsonProperty("blankPosition") + public int getBlankPosition() { + return blankPosition; + } + + @JsonProperty("errorPosition") + public int getErrorPosition() { + return errorPosition; + } + + // Returns a cached cell index + // We delay this fetching because the column might not exist + // at deserialization (for instance if the column is created by an operation + // that has not been applied yet). + protected int getCellIndex(Project project) { + if (cellIndex == -2) { + Column column = project.columnModel.getColumnByName(columnName); + cellIndex = column != null ? column.getCellIndex() : -1; + } + return cellIndex; + } // TODO: We'd like things to be more strongly typed a la the following, but // it's too involved to change right now @@ -119,10 +190,10 @@ abstract public class Criterion { } public Object makeKey(Project project, Row row, int rowIndex) { - if (cellIndex < 0) { + if (getCellIndex(project) < 0) { return null; } else { - Object value = row.getCellValue(cellIndex); + Object value = row.getCellValue(getCellIndex(project)); return makeKey(value); } } diff --git a/main/src/com/google/refine/sorting/DateCriterion.java b/main/src/com/google/refine/sorting/DateCriterion.java index a3add8187..161abb2f0 100644 --- a/main/src/com/google/refine/sorting/DateCriterion.java +++ b/main/src/com/google/refine/sorting/DateCriterion.java @@ -35,6 +35,10 @@ package com.google.refine.sorting; import java.time.Instant; import java.time.OffsetDateTime; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; import com.google.refine.expr.EvalError; import com.google.refine.expr.ExpressionUtils; @@ -63,4 +67,9 @@ public class DateCriterion extends Criterion { } }; } + + @Override + public String getValueType() { + return "date"; + } } diff --git a/main/src/com/google/refine/sorting/NumberCriterion.java b/main/src/com/google/refine/sorting/NumberCriterion.java index 287618c35..ad92f4093 100644 --- a/main/src/com/google/refine/sorting/NumberCriterion.java +++ b/main/src/com/google/refine/sorting/NumberCriterion.java @@ -77,4 +77,9 @@ public class NumberCriterion extends Criterion { } }; } + + @Override + public String getValueType() { + return "number"; + } } diff --git a/main/src/com/google/refine/sorting/StringCriterion.java b/main/src/com/google/refine/sorting/StringCriterion.java index 4807c2507..e3d937fc2 100644 --- a/main/src/com/google/refine/sorting/StringCriterion.java +++ b/main/src/com/google/refine/sorting/StringCriterion.java @@ -36,11 +36,9 @@ package com.google.refine.sorting; import java.text.CollationKey; import java.text.Collator; -import org.json.JSONException; -import org.json.JSONObject; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.refine.expr.ExpressionUtils; -import com.google.refine.model.Project; public class StringCriterion extends Criterion { public boolean caseSensitive; @@ -49,21 +47,17 @@ public class StringCriterion extends Criterion { /** * */ - public StringCriterion() { + public StringCriterion(boolean caseSensitive) { super(); collator = Collator.getInstance(); collator.setDecomposition(Collator.FULL_DECOMPOSITION); collator.setStrength(Collator.SECONDARY); + this.caseSensitive = caseSensitive; } - @Override - public void initializeFromJSON(Project project, JSONObject obj) throws JSONException { - super.initializeFromJSON(project, obj); - - if (obj.has("caseSensitive") && !obj.isNull("caseSensitive")) { - caseSensitive = obj.getBoolean("caseSensitive"); - collator.setStrength(Collator.IDENTICAL); - } + @JsonProperty("caseSensitive") + public boolean isCaseSensitive() { + return caseSensitive; } @Override @@ -81,4 +75,9 @@ public class StringCriterion extends Criterion { } }; } + + @Override + public String getValueType() { + return "string"; + } } diff --git a/main/tests/server/src/com/google/refine/tests/operations/row/RowReorderOperationTests.java b/main/tests/server/src/com/google/refine/tests/operations/row/RowReorderOperationTests.java index 0bc492d24..bdcfb80a1 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/row/RowReorderOperationTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/row/RowReorderOperationTests.java @@ -20,6 +20,7 @@ import com.google.refine.model.Project; import com.google.refine.operations.OperationRegistry; import com.google.refine.operations.row.RowReorderOperation; import com.google.refine.process.Process; +import com.google.refine.sorting.BaseSorter.SortingConfig; import com.google.refine.tests.RefineTest; import com.google.refine.tests.util.TestUtils; @@ -49,10 +50,12 @@ public class RowReorderOperationTests extends RefineTest { @Test public void testSortEmptyString() throws Exception { + String sortingJson = "{\"criteria\":[{\"column\":\"key\",\"valueType\":\"number\",\"reverse\":false,\"blankPosition\":2,\"errorPosition\":1}]}"; + SortingConfig sortingConfig = SortingConfig.reconstruct(project, new JSONObject(sortingJson)); project.rows.get(1).cells.set(0, new Cell("", null)); AbstractOperation op = new RowReorderOperation( - Mode.RowBased, - new JSONObject("{\"criteria\":[{\"column\":\"key\",\"valueType\":\"number\",\"reverse\":false,\"blankPosition\":2,\"errorPosition\":1}]}")); + Mode.RowBased, sortingConfig + ); Process process = op.createProcess(project, new Properties()); process.performImmediate();