s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Jackson serialization for RowReorderOperation

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-09-25 12:28:39 +01:00
parent a3bba83c63
commit 6daa1b4f5c
13 changed files with 236 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
main/src/com/google/refine/commands/row/GetRowsCommand.java
View File

@ -57,6 +57,7 @@ import com.google.refine.importing.ImportingManager;
import com.google.refine.model.Project;
import com.google.refine.model.Record;
import com.google.refine.model.Row;
import com.google.refine.sorting.BaseSorter.SortingConfig;
import com.google.refine.sorting.SortingRecordVisitor;
import com.google.refine.sorting.SortingRowVisitor;
import com.google.refine.util.ParsingUtilities;
@ -120,11 +121,14 @@ public class GetRowsCommand extends Command {
RowWritingVisitor rwv = new RowWritingVisitor(start, limit, jsonWriter, options);
JSONObject sortingJson = null;
SortingConfig sortingConfig = null;
try{
String json = request.getParameter("sorting");
sortingJson = (json == null) ? null :
JSONObject sortingJson = (json == null) ? null :
ParsingUtilities.evaluateJsonStringToObject(json);
if (sortingJson != null) {
sortingConfig = SortingConfig.reconstruct(project, sortingJson);
}
} catch (JSONException e) {
}
@ -132,10 +136,10 @@ public class GetRowsCommand extends Command {
FilteredRows filteredRows = engine.getAllFilteredRows();
RowVisitor visitor = rwv;
if (sortingJson != null) {
if (sortingConfig != null) {
SortingRowVisitor srv = new SortingRowVisitor(visitor);
srv.initializeFromJSON(project, sortingJson);
srv.initializeFromConfig(project, sortingConfig);
if (srv.hasCriteria()) {
visitor = srv;
}
@ -151,10 +155,10 @@ public class GetRowsCommand extends Command {
FilteredRecords filteredRecords = engine.getFilteredRecords();
RecordVisitor visitor = rwv;
if (sortingJson != null) {
if (sortingConfig != null) {
SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
srv.initializeFromJSON(project, sortingJson);
srv.initializeFromConfig(project, sortingConfig);
if (srv.hasCriteria()) {
visitor = srv;
}

6
main/src/com/google/refine/commands/row/ReorderRowsCommand.java
View File

@ -44,6 +44,7 @@ import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.row.RowReorderOperation;
import com.google.refine.sorting.BaseSorter.SortingConfig;
import com.google.refine.util.ParsingUtilities;
public class ReorderRowsCommand extends EngineDependentCommand {
@ -53,12 +54,13 @@ public class ReorderRowsCommand extends EngineDependentCommand {
HttpServletRequest request, EngineConfig engineConfig) throws Exception {
String mode = request.getParameter("mode");
JSONObject sorting = null;
SortingConfig sorting = null;
try{
String json = request.getParameter("sorting");
sorting = (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json);
JSONObject sortingJson = (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json);
sorting = (sortingJson == null) ? null : SortingConfig.reconstruct(project, sortingJson);
} catch (JSONException e) {
// ignore
}

7
main/src/com/google/refine/exporters/TemplatingExporter.java
View File

@ -52,6 +52,7 @@ import com.google.refine.expr.ParsingException;
import com.google.refine.model.Project;
import com.google.refine.sorting.SortingRecordVisitor;
import com.google.refine.sorting.SortingRowVisitor;
import com.google.refine.sorting.BaseSorter.SortingConfig;
import com.google.refine.templating.Parser;
import com.google.refine.templating.Template;
import com.google.refine.util.ParsingUtilities;
@ -115,8 +116,9 @@ public class TemplatingExporter implements WriterExporter {
if (sortingJson != null) {
try {
SortingConfig sorting = SortingConfig.reconstruct(project, sortingJson);
SortingRowVisitor srv = new SortingRowVisitor(visitor);
srv.initializeFromJSON(project, sortingJson);
srv.initializeFromConfig(project, sorting);
if (srv.hasCriteria()) {
visitor = srv;
@ -133,8 +135,9 @@ public class TemplatingExporter implements WriterExporter {
if (sortingJson != null) {
try {
SortingConfig sorting = SortingConfig.reconstruct(project, sortingJson);
SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
srv.initializeFromJSON(project, sortingJson);
srv.initializeFromConfig(project, sorting);
if (srv.hasCriteria()) {
visitor = srv;

7
main/src/com/google/refine/operations/row/RowFlagOperation.java
View File

@ -41,6 +41,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -84,6 +86,11 @@ public class RowFlagOperation extends EngineDependentOperation {
writer.key("flagged"); writer.value(_flagged);
writer.endObject();
}
@JsonProperty("flagged")
public boolean getFlagged() {
return _flagged;
}
@Override
protected String getBriefDescription(Project project) {

30
main/src/com/google/refine/operations/row/RowReorderOperation.java
View File

@ -41,6 +41,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.Engine.Mode;
import com.google.refine.browsing.RecordVisitor;
@ -52,6 +54,8 @@ import com.google.refine.model.Record;
import com.google.refine.model.Row;
import com.google.refine.model.changes.RowReorderChange;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.sorting.BaseSorter;
import com.google.refine.sorting.BaseSorter.SortingConfig;
import com.google.refine.sorting.SortingRecordVisitor;
import com.google.refine.sorting.SortingRowVisitor;
@ -60,14 +64,14 @@ public class RowReorderOperation extends AbstractOperation {
String mode = obj.getString("mode");
JSONObject sorting = obj.has("sorting") && !obj.isNull("sorting") ?
obj.getJSONObject("sorting") : null;
return new RowReorderOperation(Engine.stringToMode(mode), sorting);
BaseSorter.SortingConfig config = BaseSorter.SortingConfig.reconstruct(project, sorting);
return new RowReorderOperation(Engine.stringToMode(mode), config);
}
final protected Mode _mode;
final protected JSONObject _sorting;
final protected BaseSorter.SortingConfig _sorting;
public RowReorderOperation(Mode mode, JSONObject sorting) {
public RowReorderOperation(Mode mode, BaseSorter.SortingConfig sorting) {
_mode = mode;
_sorting = sorting;
}
@ -80,9 +84,19 @@ public class RowReorderOperation extends AbstractOperation {
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("mode"); writer.value(Engine.modeToString(_mode));
writer.key("sorting"); writer.value(_sorting);
writer.key("sorting"); _sorting.write(writer, options);
writer.endObject();
}
@JsonProperty("mode")
public Mode getMode() {
return _mode;
}
@JsonProperty("sorting")
public SortingConfig getSortingConfig() {
return _sorting;
}
@Override
protected String getBriefDescription(Project project) {
@ -100,7 +114,7 @@ public class RowReorderOperation extends AbstractOperation {
if (_sorting != null) {
SortingRowVisitor srv = new SortingRowVisitor(visitor);
srv.initializeFromJSON(project, _sorting);
srv.initializeFromConfig(project, _sorting);
if (srv.hasCriteria()) {
visitor = srv;
}
@ -112,7 +126,7 @@ public class RowReorderOperation extends AbstractOperation {
if (_sorting != null) {
SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
srv.initializeFromJSON(project, _sorting);
srv.initializeFromConfig(project, _sorting);
if (srv.hasCriteria()) {
visitor = srv;
}

7
main/src/com/google/refine/operations/row/RowStarOperation.java
View File

@ -41,6 +41,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -84,6 +86,11 @@ public class RowStarOperation extends EngineDependentOperation {
writer.key("starred"); writer.value(_starred);
writer.endObject();
}
@JsonProperty("starred")
public boolean getStarred() {
return _starred;
}
@Override
protected String getBriefDescription(Project project) {

102
main/src/com/google/refine/sorting/BaseSorter.java
View File

@ -34,11 +34,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.sorting;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.expr.EvalError;
import com.google.refine.model.Project;
import com.google.refine.sorting.Criterion.KeyMaker;
@ -104,26 +110,63 @@ abstract public class BaseSorter {
}
}
public void initializeFromJSON(Project project, JSONObject obj) throws JSONException {
if (obj.has("criteria") && !obj.isNull("criteria")) {
JSONArray a = obj.getJSONArray("criteria");
int count = a.length();
_criteria = new Criterion[count];
_keyMakers = new KeyMaker[count];
_comparatorWrappers = new ComparatorWrapper[count];
for (int i = 0; i < count; i++) {
JSONObject obj2 = a.getJSONObject(i);
_criteria[i] = createCriterionFromJSON(project, obj2);
_keyMakers[i] = _criteria[i].createKeyMaker();
_comparatorWrappers[i] = new ComparatorWrapper(i);
public final static class SortingConfig implements Jsonizable {
protected Criterion[] _criteria;
@JsonCreator
public SortingConfig(
@JsonProperty("criteria")
Criterion[] criteria) {
_criteria = criteria;
}
@JsonProperty("criteria")
public Criterion[] getCriteria() {
return _criteria;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("criteria");
for (int i = 0; i != _criteria.length; i++) {
_criteria[i].write(writer, options);
}
} else {
_criteria = new Criterion[0];
_keyMakers = new KeyMaker[0];
_comparatorWrappers = new ComparatorWrapper[0];
writer.endObject();
}
public static SortingConfig reconstruct(Project project, JSONObject obj) {
Criterion[] criteria;
if (obj != null && obj.has("criteria") && !obj.isNull("criteria")) {
JSONArray a = obj.getJSONArray("criteria");
int count = a.length();
criteria = new Criterion[count];
for (int i = 0; i < count; i++) {
JSONObject obj2 = a.getJSONObject(i);
criteria[i] = Criterion.reconstruct(obj2);
}
} else {
criteria = new Criterion[0];
}
return new SortingConfig(criteria);
}
}
public void initializeFromConfig(Project project, BaseSorter.SortingConfig config) throws JSONException {
_criteria = config.getCriteria();
int count = _criteria.length;
_keyMakers = new KeyMaker[count];
_comparatorWrappers = new ComparatorWrapper[count];
for (int i = 0; i < count; i++) {
_keyMakers[i] = _criteria[i].createKeyMaker();
_comparatorWrappers[i] = new ComparatorWrapper(i);
}
}
@ -131,27 +174,6 @@ abstract public class BaseSorter {
return _criteria != null && _criteria.length > 0;
}
protected Criterion createCriterionFromJSON(Project project, JSONObject obj) throws JSONException {
String valueType = "string";
if (obj.has("valueType") && !obj.isNull("valueType")) {
valueType = obj.getString("valueType");
}
Criterion c = null;
if ("boolean".equals(valueType)) {
c = new BooleanCriterion();
} else if ("date".equals(valueType)) {
c = new DateCriterion();
} else if ("number".equals(valueType)) {
c = new NumberCriterion();
} else {
c = new StringCriterion();
}
c.initializeFromJSON(project, obj);
return c;
}
abstract protected Object makeKey(
Project project, KeyMaker keyMaker, Criterion c, Object o, int index);

10
main/src/com/google/refine/sorting/BooleanCriterion.java
View File

@ -33,6 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.sorting;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils;
@ -62,4 +67,9 @@ public class BooleanCriterion extends Criterion {
}
};
}
@Override
public String getValueType() {
return "boolean";
}
}

87
main/src/com/google/refine/sorting/Criterion.java
View File

@ -33,18 +33,24 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.sorting;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.Record;
import com.google.refine.model.Row;
abstract public class Criterion {
abstract public class Criterion implements Jsonizable {
public String columnName;
protected int cellIndex;
protected int cellIndex = -2;
// These take on positive and negative values to indicate where blanks and errors
// go relative to non-blank values. They are also relative to each another.
@ -54,13 +60,10 @@ abstract public class Criterion {
public boolean reverse;
public void initializeFromJSON(Project project, JSONObject obj)
public void initializeFromJSON(JSONObject obj)
throws JSONException {
if (obj.has("column") && !obj.isNull("column")) {
columnName = obj.getString("column");
Column column = project.columnModel.getColumnByName(columnName);
cellIndex = column != null ? column.getCellIndex() : -1;
}
if (obj.has("blankPosition") && !obj.isNull("blankPosition")) {
@ -74,7 +77,75 @@ abstract public class Criterion {
reverse = obj.getBoolean("reverse");
}
}
public static Criterion reconstruct(JSONObject obj) throws JSONException {
String valueType = "string";
if (obj.has("valueType") && !obj.isNull("valueType")) {
valueType = obj.getString("valueType");
}
Criterion c = null;
if ("boolean".equals(valueType)) {
c = new BooleanCriterion();
} else if ("date".equals(valueType)) {
c = new DateCriterion();
} else if ("number".equals(valueType)) {
c = new NumberCriterion();
} else {
c = new StringCriterion(obj.getBoolean("caseSensitive"));
}
c.initializeFromJSON(obj);
return c;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("valueType"); writer.value(getValueType());
writer.key("reverse"); writer.value(getReverse());
writer.key("column"); writer.value(getColumnName());
writer.key("blankPosition"); writer.value(getBlankPosition());
writer.key("errorPosition"); writer.value(getErrorPosition());
writer.endObject();
}
@JsonProperty("valueType")
public abstract String getValueType();
@JsonProperty("reverse")
public boolean getReverse() {
return reverse;
}
@JsonProperty("column")
public String getColumnName() {
return columnName;
}
@JsonProperty("blankPosition")
public int getBlankPosition() {
return blankPosition;
}
@JsonProperty("errorPosition")
public int getErrorPosition() {
return errorPosition;
}
// Returns a cached cell index
// We delay this fetching because the column might not exist
// at deserialization (for instance if the column is created by an operation
// that has not been applied yet).
protected int getCellIndex(Project project) {
if (cellIndex == -2) {
Column column = project.columnModel.getColumnByName(columnName);
cellIndex = column != null ? column.getCellIndex() : -1;
}
return cellIndex;
}
// TODO: We'd like things to be more strongly typed a la the following, but
// it's too involved to change right now
@ -119,10 +190,10 @@ abstract public class Criterion {
}
public Object makeKey(Project project, Row row, int rowIndex) {
if (cellIndex < 0) {
if (getCellIndex(project) < 0) {
return null;
} else {
Object value = row.getCellValue(cellIndex);
Object value = row.getCellValue(getCellIndex(project));
return makeKey(value);
}
}

9
main/src/com/google/refine/sorting/DateCriterion.java
View File

@ -35,6 +35,10 @@ package com.google.refine.sorting;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils;
@ -63,4 +67,9 @@ public class DateCriterion extends Criterion {
}
};
}
@Override
public String getValueType() {
return "date";
}
}

5
main/src/com/google/refine/sorting/NumberCriterion.java
View File

@ -77,4 +77,9 @@ public class NumberCriterion extends Criterion {
}
};
}
@Override
public String getValueType() {
return "number";
}
}

23
main/src/com/google/refine/sorting/StringCriterion.java
View File

@ -36,11 +36,9 @@ package com.google.refine.sorting;
import java.text.CollationKey;
import java.text.Collator;
import org.json.JSONException;
import org.json.JSONObject;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Project;
public class StringCriterion extends Criterion {
public boolean caseSensitive;
@ -49,21 +47,17 @@ public class StringCriterion extends Criterion {
/**
*
*/
public StringCriterion() {
public StringCriterion(boolean caseSensitive) {
super();
collator = Collator.getInstance();
collator.setDecomposition(Collator.FULL_DECOMPOSITION);
collator.setStrength(Collator.SECONDARY);
this.caseSensitive = caseSensitive;
}
@Override
public void initializeFromJSON(Project project, JSONObject obj) throws JSONException {
super.initializeFromJSON(project, obj);
if (obj.has("caseSensitive") && !obj.isNull("caseSensitive")) {
caseSensitive = obj.getBoolean("caseSensitive");
collator.setStrength(Collator.IDENTICAL);
}
@JsonProperty("caseSensitive")
public boolean isCaseSensitive() {
return caseSensitive;
}
@Override
@ -81,4 +75,9 @@ public class StringCriterion extends Criterion {
}
};
}
@Override
public String getValueType() {
return "string";
}
}

7
main/tests/server/src/com/google/refine/tests/operations/row/RowReorderOperationTests.java
View File

@ -20,6 +20,7 @@ import com.google.refine.model.Project;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.operations.row.RowReorderOperation;
import com.google.refine.process.Process;
import com.google.refine.sorting.BaseSorter.SortingConfig;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
@ -49,10 +50,12 @@ public class RowReorderOperationTests extends RefineTest {
@Test
public void testSortEmptyString() throws Exception {
String sortingJson = "{\"criteria\":[{\"column\":\"key\",\"valueType\":\"number\",\"reverse\":false,\"blankPosition\":2,\"errorPosition\":1}]}";
SortingConfig sortingConfig = SortingConfig.reconstruct(project, new JSONObject(sortingJson));
project.rows.get(1).cells.set(0, new Cell("", null));
AbstractOperation op = new RowReorderOperation(
Mode.RowBased,
new JSONObject("{\"criteria\":[{\"column\":\"key\",\"valueType\":\"number\",\"reverse\":false,\"blankPosition\":2,\"errorPosition\":1}]}"));
Mode.RowBased, sortingConfig
);
Process process = op.createProcess(project, new Properties());
process.performImmediate();