From 1fa101c3346b6e2f9f4b13843f6eb93309de8a6a Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Thu, 27 Sep 2018 11:56:42 +0100 Subject: [PATCH] Jackson serialization for the model classes --- main/src/com/google/refine/model/Cell.java | 98 ++++++++++---- main/src/com/google/refine/model/Column.java | 21 ++- .../com/google/refine/model/ColumnGroup.java | 28 ++++ .../com/google/refine/model/ColumnModel.java | 28 ++++ main/src/com/google/refine/model/Recon.java | 98 +++++++++++++- .../google/refine/model/ReconCandidate.java | 15 +++ .../com/google/refine/model/RecordModel.java | 14 +- main/src/com/google/refine/model/Row.java | 25 ++++ .../refine/model/recon/ReconConfig.java | 9 ++ .../model/recon/StandardReconConfig.java | 48 ++++++- .../src/com/google/refine/util/JsonViews.java | 18 +++ .../google/refine/util/ParsingUtilities.java | 15 +++ .../refine/util/SerializationFilters.java | 55 ++++++++ .../refine/tests/model/ColumnGroupTests.java | 8 +- .../refine/tests/model/ColumnModelTests.java | 45 ++++++- .../tests/model/ReconCandidateTests.java | 12 +- .../google/refine/tests/model/ReconTests.java | 79 +++++++----- .../google/refine/tests/model/RowTests.java | 4 +- .../google/refine/tests/util/TestUtils.java | 120 +++++++++++++++++- 19 files changed, 655 insertions(+), 85 deletions(-) create mode 100644 main/src/com/google/refine/util/JsonViews.java create mode 100644 main/src/com/google/refine/util/SerializationFilters.java diff --git a/main/src/com/google/refine/model/Cell.java b/main/src/com/google/refine/model/Cell.java index 8a0da6054..fcaa86b5e 100644 --- a/main/src/com/google/refine/model/Cell.java +++ b/main/src/com/google/refine/model/Cell.java @@ -44,6 +44,10 @@ import java.util.Properties; import org.json.JSONException; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; @@ -57,7 +61,9 @@ import com.google.refine.util.Pool; import com.google.refine.util.StringUtils; public class Cell implements HasFields, Jsonizable { + @JsonIgnore final public Serializable value; + @JsonIgnore final public Recon recon; public Cell(Serializable value, Recon recon) { @@ -88,30 +94,10 @@ public class Cell implements HasFields, Jsonizable { writer.value(((EvalError) value).message); } else { writer.key("v"); - if (value != null) { - Instant instant = null; - if (value instanceof OffsetDateTime) { - instant = ((OffsetDateTime)value).toInstant(); - } else if (value instanceof LocalDateTime) { - instant = ((LocalDateTime)value).toInstant(ZoneOffset.of("Z")); - } - - if (instant != null) { - writer.value(ParsingUtilities.instantToString(instant)); - writer.key("t"); writer.value("date"); - } else if (value instanceof Double - && (((Double)value).isNaN() || ((Double)value).isInfinite())) { - // write as a string - writer.value(((Double)value).toString()); - } else if (value instanceof Float - && (((Float)value).isNaN() || ((Float)value).isInfinite())) { - // TODO: Skip? Write as string? - writer.value(((Float)value).toString()); - } else { - writer.value(value); - } - } else { - writer.value(null); + writer.value(getValueAsString()); + String jsonType = getTypeString(); + if(jsonType != null) { + writer.key("t"); writer.value(jsonType); } } @@ -125,6 +111,70 @@ public class Cell implements HasFields, Jsonizable { writer.endObject(); } + @JsonProperty("e") + @JsonInclude(Include.NON_NULL) + public String getErrorMessage() { + if (ExpressionUtils.isError(value)) { + return ((EvalError) value).message; + } + return null; + } + + @JsonProperty("t") + @JsonInclude(Include.NON_NULL) + public String getTypeString() { + if (value instanceof OffsetDateTime || value instanceof LocalDateTime) { + return "date"; + } + return null; + } + + @JsonProperty("v") + @JsonInclude(Include.NON_NULL) + public String getValueAsString() { + if (value != null && !ExpressionUtils.isError(value)) { + Instant instant = null; + if (value instanceof OffsetDateTime) { + instant = ((OffsetDateTime)value).toInstant(); + } else if (value instanceof LocalDateTime) { + instant = ((LocalDateTime)value).toInstant(ZoneOffset.of("Z")); + } + + if (instant != null) { + return ParsingUtilities.instantToString(instant); + } else if (value instanceof Double + && (((Double)value).isNaN() || ((Double)value).isInfinite())) { + // write as a string + return ((Double)value).toString(); + } else if (value instanceof Float + && (((Float)value).isNaN() || ((Float)value).isInfinite())) { + // TODO: Skip? Write as string? + return ((Float)value).toString(); + } else { + return value.toString(); + } + } else { + return null; + } + } + + /** + * TODO + * - use JsonIdentityInfo on recon + * - implement custom resolver to tie it to a pool + * - figure it all out + * @return + */ + @JsonProperty("r") + @JsonInclude(Include.NON_NULL) + public String getReconIdString() { + if (recon != null) { + return Long.toString(recon.id); + } + // TODO pool the recon?? + return null; + } + public void save(Writer writer, Properties options) { JSONWriter jsonWriter = new JSONWriter(writer); try { diff --git a/main/src/com/google/refine/model/Column.java b/main/src/com/google/refine/model/Column.java index 3eafa45db..e3cdf4538 100644 --- a/main/src/com/google/refine/model/Column.java +++ b/main/src/com/google/refine/model/Column.java @@ -44,6 +44,10 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.InterProjectModel; import com.google.refine.Jsonizable; import com.google.refine.model.recon.ReconConfig; @@ -76,10 +80,12 @@ public class Column implements Jsonizable { _originalName = _name = originalName; } + @JsonProperty("cellIndex") public int getCellIndex() { return _cellIndex; } + @JsonProperty("originalName") public String getOriginalHeaderLabel() { return _originalName; } @@ -88,6 +94,7 @@ public class Column implements Jsonizable { this._name = name; } + @JsonProperty("name") public String getName() { return _name; } @@ -96,6 +103,8 @@ public class Column implements Jsonizable { this._reconConfig = config; } + @JsonProperty("reconConfig") + @JsonInclude(Include.NON_NULL) public ReconConfig getReconConfig() { return _reconConfig; } @@ -104,6 +113,8 @@ public class Column implements Jsonizable { this._reconStats = stats; } + @JsonProperty("reconStats") + @JsonInclude(Include.NON_NULL) public ReconStats getReconStats() { return _reconStats; } @@ -131,6 +142,7 @@ public class Column implements Jsonizable { } writer.endObject(); } + /** * Clear all cached precomputed values. @@ -160,7 +172,7 @@ public class Column implements Jsonizable { _precomputes.put(key, value); } - + @JsonProperty("type") public String getType() { return type; } @@ -171,6 +183,7 @@ public class Column implements Jsonizable { } + @JsonProperty("format") public String getFormat() { return format; } @@ -181,6 +194,7 @@ public class Column implements Jsonizable { } + @JsonProperty("title") public String getTitle() { return title; } @@ -191,6 +205,7 @@ public class Column implements Jsonizable { } + @JsonProperty("description") public String getDescription() { return description; } @@ -200,6 +215,10 @@ public class Column implements Jsonizable { this.description = description; } + @JsonProperty("constraints") + public String getConstraintsString() { + return (new JSONObject(constraints)).toString(); + } public Map getConstraints() { return constraints; diff --git a/main/src/com/google/refine/model/ColumnGroup.java b/main/src/com/google/refine/model/ColumnGroup.java index 9df67f2dc..1ba26d1ad 100644 --- a/main/src/com/google/refine/model/ColumnGroup.java +++ b/main/src/com/google/refine/model/ColumnGroup.java @@ -42,7 +42,13 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonView; + import com.google.refine.Jsonizable; +import com.google.refine.util.JsonViews; import com.google.refine.util.ParsingUtilities; public class ColumnGroup implements Jsonizable { @@ -80,6 +86,28 @@ public class ColumnGroup implements Jsonizable { writer.endObject(); } + @JsonProperty("startColumnIndex") + public int getStartColumnIndex() { + return startColumnIndex; + } + + @JsonProperty("columnSpan") + public int getColumnSpan() { + return columnSpan; + } + + @JsonProperty("keyColumnIndex") + public int getKeyColumnIndex() { + return keyColumnIndex; + } + + @JsonProperty("subgroups") + @JsonView(JsonViews.NonSaveMode.class) + @JsonInclude(Include.NON_EMPTY) + public List getSubGroups() { + return subgroups; + } + public boolean contains(ColumnGroup g) { return (g.startColumnIndex >= startColumnIndex && g.startColumnIndex < startColumnIndex + columnSpan); diff --git a/main/src/com/google/refine/model/ColumnModel.java b/main/src/com/google/refine/model/ColumnModel.java index f310d3807..a4109c1b3 100644 --- a/main/src/com/google/refine/model/ColumnModel.java +++ b/main/src/com/google/refine/model/ColumnModel.java @@ -48,10 +48,17 @@ import java.util.Properties; import org.json.JSONException; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.Jsonizable; public class ColumnModel implements Jsonizable { + @JsonProperty("columns") final public List columns = new LinkedList(); + @JsonProperty("columnGroups") final public List columnGroups = new LinkedList(); private int _maxCellIndex = -1; @@ -70,6 +77,7 @@ public class ColumnModel implements Jsonizable { this._maxCellIndex = Math.max(this._maxCellIndex, maxCellIndex); } + @JsonIgnore synchronized public int getMaxCellIndex() { return _maxCellIndex; } @@ -86,6 +94,7 @@ public class ColumnModel implements Jsonizable { this._keyColumnIndex = keyColumnIndex; } + @JsonIgnore synchronized public int getKeyColumnIndex() { return _keyColumnIndex; } @@ -166,6 +175,7 @@ public class ColumnModel implements Jsonizable { return _cellIndexToColumn.get(cellIndex); } + @JsonIgnore synchronized public List getColumnNames() { return _columnNames; } @@ -198,6 +208,24 @@ public class ColumnModel implements Jsonizable { writer.endObject(); } + @JsonProperty("keyCellIndex") + @JsonInclude(Include.NON_NULL) + public Integer getJsonKeyCellIndex() { + if(columns.size() > 0) { + return getKeyColumnIndex(); + } + return null; + } + + @JsonProperty("keyColumnName") + @JsonInclude(Include.NON_NULL) + public String getKeyColumnName() { + if(columns.size() > 0) { + return columns.get(_keyColumnIndex).getName(); + } + return null; + } + synchronized public void save(Writer writer, Properties options) throws IOException { writer.write("maxCellIndex="); writer.write(Integer.toString(_maxCellIndex)); writer.write('\n'); writer.write("keyColumnIndex="); writer.write(Integer.toString(_keyColumnIndex)); writer.write('\n'); diff --git a/main/src/com/google/refine/model/Recon.java b/main/src/com/google/refine/model/Recon.java index d131b3f7d..a4b726708 100644 --- a/main/src/com/google/refine/model/Recon.java +++ b/main/src/com/google/refine/model/Recon.java @@ -34,21 +34,31 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.model; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import org.json.JSONException; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonFilter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonView; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + import com.google.refine.Jsonizable; import com.google.refine.expr.HasFields; +import com.google.refine.util.JsonViews; import com.google.refine.util.Pool; +@JsonFilter("reconCandidateFilter") public class Recon implements HasFields, Jsonizable { /** @@ -61,11 +71,15 @@ public class Recon implements HasFields, Jsonizable { private static final String WIKIDATA_IDENTIFIER_SPACE = "http://www.wikidata.org/entity/"; static public enum Judgment { + @JsonProperty("none") None, + @JsonProperty("matched") Matched, + @JsonProperty("new") New } + @Deprecated static public String judgmentToString(Judgment judgment) { if (judgment == Judgment.Matched) { return "matched"; @@ -76,6 +90,7 @@ public class Recon implements HasFields, Jsonizable { } } + @Deprecated static public Judgment stringToJudgment(String s) { if ("matched".equals(s)) { return Judgment.Matched; @@ -186,6 +201,7 @@ public class Recon implements HasFields, Jsonizable { candidates.add(candidate); } + @JsonIgnore public ReconCandidate getBestCandidate() { if (candidates != null && candidates.size() > 0) { return candidates.get(0); @@ -257,6 +273,7 @@ public class Recon implements HasFields, Jsonizable { return "match".equals(name) || "best".equals(name); } + @Deprecated protected String judgmentToString() { return judgmentToString(judgment); } @@ -273,6 +290,81 @@ public class Recon implements HasFields, Jsonizable { return false; } } + + @JsonProperty("id") + public long getId() { + return id; + } + + @JsonProperty("judgmentHistoryEntry") + @JsonView(JsonViews.SaveMode.class) + public long getJudgmentHistoryEntry() { + return judgmentHistoryEntry; + } + + @JsonProperty("service") + public String getServiceURI() { + return service; + } + + @JsonProperty("identifierSpace") + public String getIdentifierSpace() { + return identifierSpace; + } + + @JsonProperty("schemaSpace") + public String getSchemaSpace() { + return schemaSpace; + } + + @JsonProperty("j") + public Judgment getJudgment() { + return judgment; + } + + @JsonProperty("m") + @JsonInclude(Include.NON_NULL) + public ReconCandidate getMatch() { + return match; + } + + @JsonProperty("c") + //@JsonView(JsonViews.SaveMode.class) + public List getCandidates() { + if (candidates != null) { + return candidates; + } + return Collections.emptyList(); + } + + + @JsonProperty("f") + @JsonView(JsonViews.SaveMode.class) + public Object[] getfeatures() { + return features; + } + + @JsonProperty("judgmentAction") + @JsonView(JsonViews.SaveMode.class) + public String getJudgmentAction() { + return judgmentAction; + } + + @JsonProperty("judgmentBatchSize") + @JsonView(JsonViews.SaveMode.class) + public int getJudgmentBatchSize() { + return judgmentBatchSize; + } + + @JsonProperty("matchRank") + @JsonView(JsonViews.SaveMode.class) + @JsonInclude(Include.NON_NULL) + public Integer getMatchRank() { + if (match != null) { + return matchRank; + } + return null; + } @Override public void write(JSONWriter writer, Properties options) diff --git a/main/src/com/google/refine/model/ReconCandidate.java b/main/src/com/google/refine/model/ReconCandidate.java index 76a54d82f..b25b0354c 100644 --- a/main/src/com/google/refine/model/ReconCandidate.java +++ b/main/src/com/google/refine/model/ReconCandidate.java @@ -37,6 +37,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; @@ -47,9 +49,13 @@ import com.google.refine.Jsonizable; import com.google.refine.expr.HasFields; public class ReconCandidate implements HasFields, Jsonizable { + @JsonProperty("id") final public String id; + @JsonProperty("name") final public String name; + @JsonProperty("types") final public String[] types; + @JsonIgnore final public double score; public ReconCandidate(String topicID, String topicName, String[] typeIDs, double score) { @@ -59,6 +65,15 @@ public class ReconCandidate implements HasFields, Jsonizable { this.score = score; } + // Serialize doubles that are ints without trailing ".0" for consistency with previous serialization. + @JsonProperty("score") + public Object getJsonScore() { + if ((double)(int)score == score) { + return (int)score; + } + return score; + } + @Override public Object getField(String name, Properties bindings) { if ("id".equals(name)) { diff --git a/main/src/com/google/refine/model/RecordModel.java b/main/src/com/google/refine/model/RecordModel.java index 47a6e24a7..2d1bf0569 100644 --- a/main/src/com/google/refine/model/RecordModel.java +++ b/main/src/com/google/refine/model/RecordModel.java @@ -45,6 +45,9 @@ import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.Jsonizable; import com.google.refine.expr.ExpressionUtils; @@ -85,6 +88,7 @@ public class RecordModel implements Jsonizable { _rowDependencies.get(rowIndex) : null; } + @JsonIgnore public int getRecordCount() { return _records.size(); } @@ -111,11 +115,15 @@ public class RecordModel implements Jsonizable { writer.object(); writer.key("hasRecords"); - writer.value( - _records != null && _rowDependencies != null && - _records.size() < _rowDependencies.size()); + writer.value(hasRecords()); writer.endObject(); } + + @JsonProperty("hasRecords") + public boolean hasRecords() { + return _records != null && _rowDependencies != null && + _records.size() < _rowDependencies.size(); + } static protected class KeyedGroup { int[] cellIndices; diff --git a/main/src/com/google/refine/model/Row.java b/main/src/com/google/refine/model/Row.java index 422fd76ec..780f1fe92 100644 --- a/main/src/com/google/refine/model/Row.java +++ b/main/src/com/google/refine/model/Row.java @@ -39,6 +39,9 @@ import java.util.List; import java.util.Map.Entry; import java.util.Properties; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonView; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; @@ -48,6 +51,7 @@ import org.json.JSONWriter; import com.google.refine.Jsonizable; import com.google.refine.expr.CellTuple; import com.google.refine.expr.HasFields; +import com.google.refine.util.JsonViews; import com.google.refine.util.Pool; /** @@ -106,6 +110,7 @@ public class Row implements HasFields, Jsonizable { return "cells".equals(name) || "record".equals(name); } + @JsonIgnore public boolean isEmpty() { for (Cell cell : cells) { if (cell != null && cell.value != null && !isValueBlank(cell.value)) { @@ -204,6 +209,26 @@ public class Row implements HasFields, Jsonizable { writer.endObject(); } + @JsonProperty(FLAGGED) + public boolean isFlagged() { + return flagged; + } + + @JsonProperty(STARRED) + public boolean isStarred() { + return starred; + } + + @JsonProperty("cells") + public List getCells() { + return cells; + } + + /* + @JsonView(JsonViews.SaveMode.class) + public + */ + public void save(Writer writer, Properties options) { JSONWriter jsonWriter = new JSONWriter(writer); try { diff --git a/main/src/com/google/refine/model/recon/ReconConfig.java b/main/src/com/google/refine/model/recon/ReconConfig.java index a85c70cef..7f11af0f8 100644 --- a/main/src/com/google/refine/model/recon/ReconConfig.java +++ b/main/src/com/google/refine/model/recon/ReconConfig.java @@ -47,6 +47,8 @@ import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.Jsonizable; import com.google.refine.model.Cell; import com.google.refine.model.Project; @@ -130,4 +132,11 @@ abstract public class ReconConfig implements Jsonizable { LOGGER.error("Save failed",e); } } + + /** + * Returns the identifier for the reconciliation mode, as serialized in JSON. + * This is the same identifier that was used to register the registration mode. + */ + @JsonProperty("mode") + abstract public String getMode(); } diff --git a/main/src/com/google/refine/model/recon/StandardReconConfig.java b/main/src/com/google/refine/model/recon/StandardReconConfig.java index 84adf5273..43e3754b8 100644 --- a/main/src/com/google/refine/model/recon/StandardReconConfig.java +++ b/main/src/com/google/refine/model/recon/StandardReconConfig.java @@ -53,12 +53,17 @@ import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +import com.google.refine.Jsonizable; import com.google.refine.expr.ExpressionUtils; import com.google.refine.model.Cell; import com.google.refine.model.Project; import com.google.refine.model.Recon; import com.google.refine.model.Recon.Judgment; import com.google.refine.model.ReconCandidate; +import com.google.refine.model.ReconType; import com.google.refine.model.RecordModel.RowDependency; import com.google.refine.model.Row; import com.google.refine.util.ParsingUtilities; @@ -66,9 +71,12 @@ import com.google.refine.util.ParsingUtilities; public class StandardReconConfig extends ReconConfig { final static Logger logger = LoggerFactory.getLogger("refine-standard-recon"); - static public class ColumnDetail { + static public class ColumnDetail implements Jsonizable { + @JsonProperty("column") final public String columnName; + @JsonProperty("propertyName") final public String propertyName; + @JsonProperty("propertyID") final public String propertyID; public ColumnDetail(String columnName, String propertyName, String propertyID) { @@ -76,6 +84,17 @@ public class StandardReconConfig extends ReconConfig { this.propertyName = propertyName; this.propertyID = propertyID; } + + @Override + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("column"); writer.value(columnName); + writer.key("propertyName"); writer.value(propertyName); + writer.key("propertyID"); writer.value(propertyID); + writer.endObject(); + + } } static public ReconConfig reconstruct(JSONObject obj) throws Exception { @@ -133,14 +152,22 @@ public class StandardReconConfig extends ReconConfig { } } + @JsonProperty("service") final public String service; + @JsonProperty("identifierSpace") final public String identifierSpace; + @JsonProperty("schemaSpace") final public String schemaSpace; + @JsonIgnore final public String typeID; + @JsonIgnore final public String typeName; + @JsonProperty("autoMatch") final public boolean autoMatch; + @JsonProperty("columnDetails") final public List columnDetails; + @JsonProperty("limit") final private int limit; public StandardReconConfig( @@ -210,18 +237,21 @@ public class StandardReconConfig extends ReconConfig { writer.key("columnDetails"); writer.array(); for (ColumnDetail c : columnDetails) { - writer.object(); - writer.key("column"); writer.value(c.columnName); - writer.key("propertyName"); writer.value(c.propertyName); - writer.key("propertyID"); writer.value(c.propertyID); - writer.endObject(); + c.write(writer, options); } writer.endArray(); writer.key("limit"); writer.value(limit); writer.endObject(); } + + @JsonProperty("type") + public ReconType getReconType() { + ReconType t = new ReconType(typeID, typeName); + return t; + } @Override + @JsonIgnore public int getBatchSize() { return 10; } @@ -541,4 +571,10 @@ public class StandardReconConfig extends ReconConfig { } return set; } + + + @Override + public String getMode() { + return "standard-service"; + } } diff --git a/main/src/com/google/refine/util/JsonViews.java b/main/src/com/google/refine/util/JsonViews.java new file mode 100644 index 000000000..45074a41f --- /dev/null +++ b/main/src/com/google/refine/util/JsonViews.java @@ -0,0 +1,18 @@ +package com.google.refine.util; + + +/** + * Set of classes which define JSON visibility of certain fields. + * @author Antonin Delpeuch + * + */ +public class JsonViews { + + public static class SaveMode { + ; + } + + public static class NonSaveMode { + ; + } +} diff --git a/main/src/com/google/refine/util/ParsingUtilities.java b/main/src/com/google/refine/util/ParsingUtilities.java index 26adc6487..1cb62bc47 100644 --- a/main/src/com/google/refine/util/ParsingUtilities.java +++ b/main/src/com/google/refine/util/ParsingUtilities.java @@ -59,7 +59,22 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONTokener; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.ser.FilterProvider; +import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; + public class ParsingUtilities { + + public static final ObjectMapper mapper = new ObjectMapper(); + public static final FilterProvider defaultFilters = new SimpleFilterProvider() + .addFilter("reconCandidateFilter", SerializationFilters.reconCandidateFilter); + public static final FilterProvider saveFilters = new SimpleFilterProvider() + .addFilter("reconCandidateFilter", SerializationFilters.noFilter); + + public static final ObjectWriter saveWriter = mapper.writerWithView(JsonViews.SaveMode.class).with(saveFilters); + public static final ObjectWriter defaultWriter = mapper.writerWithView(JsonViews.NonSaveMode.class).with(defaultFilters); + public static final DateTimeFormatter ISO8601 = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); static public Properties parseUrlParameters(HttpServletRequest request) { diff --git a/main/src/com/google/refine/util/SerializationFilters.java b/main/src/com/google/refine/util/SerializationFilters.java new file mode 100644 index 000000000..606d463d6 --- /dev/null +++ b/main/src/com/google/refine/util/SerializationFilters.java @@ -0,0 +1,55 @@ +package com.google.refine.util; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.BeanPropertyWriter; +import com.fasterxml.jackson.databind.ser.PropertyFilter; +import com.fasterxml.jackson.databind.ser.PropertyWriter; +import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; + +import com.google.refine.model.Recon; +import com.google.refine.model.Recon.Judgment; + +public class SerializationFilters { + static class BaseFilter extends SimpleBeanPropertyFilter { + @Override + public void serializeAsField(Object obj, JsonGenerator jgen, SerializerProvider provider, PropertyWriter writer) + throws Exception { + if (include(writer)) { + writer.serializeAsField(obj, jgen, provider); + } else if (!jgen.canOmitFields()) { + writer.serializeAsOmittedField(obj, jgen, provider); + } + } + + @Override + protected boolean include(BeanPropertyWriter writer) { + return true; + } + + @Override + protected boolean include(PropertyWriter writer) { + return true; + } + } + + public static PropertyFilter noFilter = new BaseFilter(); + public static PropertyFilter reconCandidateFilter = new BaseFilter() { + @Override + public void serializeAsField(Object obj, JsonGenerator jgen, SerializerProvider provider, PropertyWriter writer) + throws Exception { + if (include(writer)) { + if (!writer.getName().equals("c") || ! (obj instanceof Recon)) { + writer.serializeAsField(obj, jgen, provider); + return; + } + Recon recon = (Recon)obj; + if (recon.judgment == Judgment.None) { + writer.serializeAsField(obj, jgen, provider); + } + } else if (!jgen.canOmitFields()) { + writer.serializeAsOmittedField(obj, jgen, provider); + } + } + }; +} diff --git a/main/tests/server/src/com/google/refine/tests/model/ColumnGroupTests.java b/main/tests/server/src/com/google/refine/tests/model/ColumnGroupTests.java index 83b6d8761..a964b07da 100644 --- a/main/tests/server/src/com/google/refine/tests/model/ColumnGroupTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/ColumnGroupTests.java @@ -1,7 +1,5 @@ package com.google.refine.tests.model; -import java.util.Properties; - import org.testng.annotations.Test; import com.google.refine.model.ColumnGroup; @@ -34,9 +32,7 @@ public class ColumnGroupTests { + " \"keyColumnIndex\":1" + "}]" + "}"; - Properties options = new Properties(); - options.setProperty("mode", "save"); - TestUtils.isSerializedTo(cg, json, options); - TestUtils.isSerializedTo(cg, fullJson); + TestUtils.isSerializedTo(cg, json, true); + TestUtils.isSerializedTo(cg, fullJson, false); } } diff --git a/main/tests/server/src/com/google/refine/tests/model/ColumnModelTests.java b/main/tests/server/src/com/google/refine/tests/model/ColumnModelTests.java index 89f986165..56c17abfd 100644 --- a/main/tests/server/src/com/google/refine/tests/model/ColumnModelTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/ColumnModelTests.java @@ -2,9 +2,50 @@ package com.google.refine.tests.model; import org.testng.annotations.Test; -public class ColumnModelTests { +import com.google.refine.model.ColumnModel; +import com.google.refine.model.Project; +import com.google.refine.tests.RefineTest; +import com.google.refine.tests.util.TestUtils; + +public class ColumnModelTests extends RefineTest { @Test public void serializeColumnModel() { - String json = ""; + Project project = createCSVProject("a,b\n"+ + "e,e"); + String json = "{\n" + + " \"columnGroups\" : [ ],\n" + + " \"columns\" : [ {\n" + + " \"cellIndex\" : 0,\n" + + " \"constraints\" : \"{}\",\n" + + " \"description\" : \"\",\n" + + " \"format\" : \"default\",\n" + + " \"name\" : \"a\",\n" + + " \"originalName\" : \"a\",\n" + + " \"title\" : \"\",\n" + + " \"type\" : \"\"\n" + + " }, {\n" + + " \"cellIndex\" : 1,\n" + + " \"constraints\" : \"{}\",\n" + + " \"description\" : \"\",\n" + + " \"format\" : \"default\",\n" + + " \"name\" : \"b\",\n" + + " \"originalName\" : \"b\",\n" + + " \"title\" : \"\",\n" + + " \"type\" : \"\"\n" + + " } ],\n" + + " \"keyCellIndex\" : 0,\n" + + " \"keyColumnName\" : \"a\"\n" + + " }"; + TestUtils.isSerializedTo(project.columnModel, json); + } + + @Test + public void serializeColumnModelEmpty() { + String json = "{" + + "\"columns\":[]," + + "\"columnGroups\":[]" + + "}"; + ColumnModel m = new ColumnModel(); + TestUtils.isSerializedTo(m, json); } } diff --git a/main/tests/server/src/com/google/refine/tests/model/ReconCandidateTests.java b/main/tests/server/src/com/google/refine/tests/model/ReconCandidateTests.java index 88fed65e4..3a24cd00d 100644 --- a/main/tests/server/src/com/google/refine/tests/model/ReconCandidateTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/ReconCandidateTests.java @@ -7,7 +7,7 @@ import com.google.refine.tests.util.TestUtils; public class ReconCandidateTests { @Test - public void serializeReconCandidate() throws Exception { + public void serializeReconCandidateInt() throws Exception { String json = "{\"id\":\"Q49213\"," + "\"name\":\"University of Texas at Austin\"," + "\"score\":100," @@ -15,4 +15,14 @@ public class ReconCandidateTests { ReconCandidate rc = ReconCandidate.loadStreaming(json); TestUtils.isSerializedTo(rc, json); } + + @Test + public void serializeReconCandidateDouble() throws Exception { + String json = "{\"id\":\"Q49213\"," + + "\"name\":\"University of Texas at Austin\"," + + "\"score\":0.5," + + "\"types\":[\"Q875538\",\"Q15936437\",\"Q20971972\",\"Q23002039\"]}"; + ReconCandidate rc = ReconCandidate.loadStreaming(json); + TestUtils.isSerializedTo(rc, json); + } } diff --git a/main/tests/server/src/com/google/refine/tests/model/ReconTests.java b/main/tests/server/src/com/google/refine/tests/model/ReconTests.java index a4ad792e8..fbfcfbfa0 100644 --- a/main/tests/server/src/com/google/refine/tests/model/ReconTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/ReconTests.java @@ -1,42 +1,44 @@ package com.google.refine.tests.model; -import java.util.Properties; - import org.testng.annotations.Test; import com.google.refine.model.Recon; +import com.google.refine.model.Recon.Judgment; import com.google.refine.tests.util.TestUtils; public class ReconTests { + + String fullJson = "{\"id\":1533651559492945033," + + "\"judgmentHistoryEntry\":1533651616890," + + "\"service\":\"https://tools.wmflabs.org/openrefine-wikidata/en/api\"," + + "\"identifierSpace\":\"http://www.wikidata.org/entity/\"," + + "\"schemaSpace\":\"http://www.wikidata.org/prop/direct/\"," + + "\"j\":\"matched\"," + + "\"m\":{" + + " \"id\":\"Q2892284\"," + + " \"name\":\"Baylor College of Medicine\"," + + " \"score\":98.57142857142858," + + " \"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]" + + "}," + + "\"c\":[" + + " {\"id\":\"Q2892284\",\"name\":\"Baylor College of Medicine\",\"score\":98.57142857142858,\"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]}," + + " {\"id\":\"Q16165943\",\"name\":\"Baylor College of Medicine Academy at Ryan\",\"score\":82.14285714285715,\"types\":[\"Q149566\"]}," + + " {\"id\":\"Q30284245\",\"name\":\"Baylor College of Medicine Children\\u2019s Foundation\",\"score\":48.57142857142858,\"types\":[\"Q163740\"]}" + + "]," + + "\"f\":[false,false,1,0.6666666666666666]," + + "\"judgmentAction\":\"mass\"," + + "\"judgmentBatchSize\":1," + + "\"matchRank\":0}"; + @Test - public void serializeRecon() throws Exception { - Properties options = new Properties(); - options.put("mode", "save"); - - String fullJson = "{\"id\":1533651559492945033," - + "\"judgmentHistoryEntry\":1533651616890," - + "\"service\":\"https://tools.wmflabs.org/openrefine-wikidata/en/api\"," - + "\"identifierSpace\":\"http://www.wikidata.org/entity/\"," - + "\"schemaSpace\":\"http://www.wikidata.org/prop/direct/\"," - + "\"j\":\"matched\"," - + "\"m\":{" - + " \"id\":\"Q2892284\"," - + " \"name\":\"Baylor College of Medicine\"," - + " \"score\":98.57142857142858," - + " \"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]" - + "}," - + "\"c\":[" - + " {\"id\":\"Q2892284\",\"name\":\"Baylor College of Medicine\",\"score\":98.57142857142858,\"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]}," - + " {\"id\":\"Q16165943\",\"name\":\"Baylor College of Medicine Academy at Ryan\",\"score\":82.14285714285715,\"types\":[\"Q149566\"]}," - + " {\"id\":\"Q30284245\",\"name\":\"Baylor College of Medicine Children\\u2019s Foundation\",\"score\":48.57142857142858,\"types\":[\"Q163740\"]}" - + "]," - + "\"f\":[false,false,1,0.6666666666666666]," - + "\"judgmentAction\":\"mass\"," - + "\"judgmentBatchSize\":1," - + "\"matchRank\":0}"; + public void serializeReconSaveMode() throws Exception { Recon r = Recon.loadStreaming(fullJson, null); - TestUtils.isSerializedTo(r, fullJson, options); + TestUtils.isSerializedTo(r, fullJson, true); + } + @Test + public void serializeReconViewMode() throws Exception { + Recon r = Recon.loadStreaming(fullJson, null); String shortJson = "{\"id\":1533651559492945033," + "\"service\":\"https://tools.wmflabs.org/openrefine-wikidata/en/api\"," + "\"identifierSpace\":\"http://www.wikidata.org/entity/\"," @@ -48,9 +50,26 @@ public class ReconTests { + " \"score\":98.57142857142858," + " \"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]" + "}}"; - options.put("mode", "normal"); - TestUtils.isSerializedTo(r, shortJson, options); + TestUtils.isSerializedTo(r, shortJson, false); } + @Test + public void serializeReconSaveModeNoMatch() throws Exception { + String json = "{\"id\":1533651559492945033," + + "\"service\":\"https://tools.wmflabs.org/openrefine-wikidata/en/api\"," + + "\"identifierSpace\":\"http://www.wikidata.org/entity/\"," + + "\"schemaSpace\":\"http://www.wikidata.org/prop/direct/\"," + + "\"j\":\"none\"," + + "\"c\":[" + + " {\"id\":\"Q2892284\",\"name\":\"Baylor College of Medicine\",\"score\":98.57142857142858,\"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]}," + + " {\"id\":\"Q16165943\",\"name\":\"Baylor College of Medicine Academy at Ryan\",\"score\":82.14285714285715,\"types\":[\"Q149566\"]}," + + " {\"id\":\"Q30284245\",\"name\":\"Baylor College of Medicine Children\\u2019s Foundation\",\"score\":48.57142857142858,\"types\":[\"Q163740\"]}" + + "]" + + "}"; + Recon r = Recon.loadStreaming(fullJson, null); + r.match = null; + r.judgment = Judgment.None; + TestUtils.isSerializedTo(r, json); + } } diff --git a/main/tests/server/src/com/google/refine/tests/model/RowTests.java b/main/tests/server/src/com/google/refine/tests/model/RowTests.java index ef0a765e0..6d36f4838 100644 --- a/main/tests/server/src/com/google/refine/tests/model/RowTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/RowTests.java @@ -107,7 +107,7 @@ public class RowTests extends RefineTest { Row row = new Row(5); row.setCell(0, new Cell("I'm not empty", null)); row.save(writer, options); - TestUtils.equalAsJson(writer.getBuffer().toString(), + TestUtils.assertEqualAsJson(writer.getBuffer().toString(), "{\"flagged\":false,\"starred\":false,\"cells\":[{\"v\":\"I'm not empty\"}]}"); } @@ -120,7 +120,7 @@ public class RowTests extends RefineTest { when(options.containsKey("recordIndex")).thenReturn(true); when(options.get("recordIndex")).thenReturn(1); row.save(writer, options); - TestUtils.equalAsJson( + TestUtils.assertEqualAsJson( writer.getBuffer().toString(), "{\"flagged\":false,\"starred\":false,\"cells\":[{\"v\":\"I'm not empty\"}],\"i\":0,\"j\":1}"); } diff --git a/main/tests/server/src/com/google/refine/tests/util/TestUtils.java b/main/tests/server/src/com/google/refine/tests/util/TestUtils.java index b0984c645..0e00b7e0c 100644 --- a/main/tests/server/src/com/google/refine/tests/util/TestUtils.java +++ b/main/tests/server/src/com/google/refine/tests/util/TestUtils.java @@ -5,19 +5,32 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.io.StringWriter; import java.util.Properties; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; import com.google.refine.Jsonizable; import com.google.refine.util.JSONUtilities; +import com.google.refine.util.JsonViews; +import com.google.refine.util.ParsingUtilities; public class TestUtils { static ObjectMapper mapper = new ObjectMapper(); + static { + mapper = mapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + } /** * Create a temporary directory. NOTE: This is a quick and dirty @@ -38,7 +51,7 @@ public class TestUtils { /** * Compare two JSON strings for equality. */ - public static void equalAsJson(String expected, String actual) { + public static void assertEqualAsJson(String expected, String actual) { try { JsonNode jsonA = mapper.readValue(expected, JsonNode.class); JsonNode jsonB = mapper.readValue(actual, JsonNode.class); @@ -48,19 +61,52 @@ public class TestUtils { } } + public static boolean equalAsJson(String expected, String actual) { + try { + JsonNode jsonA = mapper.readValue(expected, JsonNode.class); + JsonNode jsonB = mapper.readValue(actual, JsonNode.class); + return (jsonA == null && jsonB == null) || jsonA.equals(jsonB); + } catch(Exception e) { + return false; + } + } + /** * Checks that a serializable object is serialized to the target JSON string. + * @throws IOException */ public static void isSerializedTo(Jsonizable o, String targetJson, Properties options) { - equalAsJson(targetJson, JSONUtilities.serialize(o, options)); + String orgJson = JSONUtilities.serialize(o, options); + if(!equalAsJson(targetJson, orgJson)) { + System.out.println("org.json, "+o.getClass().getName()); + try { + jsonDiff(targetJson, orgJson); + } catch (JsonParseException | JsonMappingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + assertEqualAsJson(targetJson, orgJson); // also check Jackson serialization try { - equalAsJson(targetJson, mapper.writeValueAsString(o)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - fail("jackson serialization failed"); - } + String saveMode = options.getProperty("mode"); + ObjectWriter writer = null; + if("save".equals(saveMode)) { + writer = ParsingUtilities.saveWriter; + } else { + writer = ParsingUtilities.defaultWriter; + } + String jacksonJson = writer.writeValueAsString(o); + if(!equalAsJson(targetJson, jacksonJson)) { + System.out.println("jackson, "+o.getClass().getName()); + jsonDiff(targetJson, jacksonJson); + } + assertEqualAsJson(targetJson, jacksonJson); + } catch (JsonProcessingException e) { + e.printStackTrace(); + fail("jackson serialization failed"); + } } /** @@ -69,4 +115,64 @@ public class TestUtils { public static void isSerializedTo(Jsonizable o, String targetJson) { isSerializedTo(o, targetJson, new Properties()); } + + /** + * Checks that a serializable object is serialized to the target JSON string. + * This specifies the "save mode" for objects that are stored differently depending on + * whether they are written to disk or sent over the network. + */ + public static void isSerializedTo(Jsonizable o, String targetJson, boolean saveMode) { + Properties options = new Properties(); + if(saveMode) { + options.setProperty("mode", "save"); + options.put("mode", "save"); + } + isSerializedTo(o, targetJson, options); + } + + public static void jsonDiff(String a, String b) throws JsonParseException, JsonMappingException { + ObjectMapper myMapper = mapper.copy().configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) + .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + .configure(SerializationFeature.INDENT_OUTPUT, true); + try { + JsonNode nodeA = myMapper.readValue(a, JsonNode.class); + JsonNode nodeB = myMapper.readValue(b, JsonNode.class); + String prettyA = myMapper.writeValueAsString(myMapper.treeToValue(nodeA, Object.class)); + String prettyB = myMapper.writeValueAsString(myMapper.treeToValue(nodeB, Object.class)); + + // Compute the max line length of A + LineNumberReader readerA = new LineNumberReader(new StringReader(prettyA)); + int maxLength = 0; + String line = readerA.readLine(); + while (line != null) { + if(line.length() > maxLength) { + maxLength = line.length(); + } + line = readerA.readLine(); + } + + // Pad all lines + readerA = new LineNumberReader(new StringReader(prettyA)); + LineNumberReader readerB = new LineNumberReader(new StringReader(prettyB)); + StringWriter writer = new StringWriter(); + String lineA = readerA.readLine(); + String lineB = readerB.readLine(); + while(lineA != null || lineB != null) { + if (lineA == null) { + lineA = ""; + } + if (lineB == null) { + lineB = ""; + } + String paddedLineA = lineA + new String(new char[maxLength + 2 - lineA.length()]).replace("\0", " "); + writer.write(paddedLineA); + writer.write(lineB + "\n"); + lineA = readerA.readLine(); + lineB = readerB.readLine(); + } + System.out.print(writer.toString()); + } catch(IOException e) { + e.printStackTrace(); + } + } }