s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Jackson serialization for the model classes

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-09-27 11:56:42 +01:00
parent 1f40393028
commit 1fa101c334
19 changed files with 655 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
main/src/com/google/refine/model/Cell.java
View File

@ -44,6 +44,10 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
@ -57,7 +61,9 @@ import com.google.refine.util.Pool;
import com.google.refine.util.StringUtils;
public class Cell implements HasFields, Jsonizable {
@JsonIgnore
final public Serializable value;
@JsonIgnore
final public Recon recon;
public Cell(Serializable value, Recon recon) {
@ -88,30 +94,10 @@ public class Cell implements HasFields, Jsonizable {
writer.value(((EvalError) value).message);
} else {
writer.key("v");
if (value != null) {
Instant instant = null;
if (value instanceof OffsetDateTime) {
instant = ((OffsetDateTime)value).toInstant();
} else if (value instanceof LocalDateTime) {
instant = ((LocalDateTime)value).toInstant(ZoneOffset.of("Z"));
}
if (instant != null) {
writer.value(ParsingUtilities.instantToString(instant));
writer.key("t"); writer.value("date");
} else if (value instanceof Double
&& (((Double)value).isNaN() || ((Double)value).isInfinite())) {
// write as a string
writer.value(((Double)value).toString());
} else if (value instanceof Float
&& (((Float)value).isNaN() || ((Float)value).isInfinite())) {
// TODO: Skip? Write as string?
writer.value(((Float)value).toString());
} else {
writer.value(value);
}
} else {
writer.value(null);
writer.value(getValueAsString());
String jsonType = getTypeString();
if(jsonType != null) {
writer.key("t"); writer.value(jsonType);
}
}
@ -125,6 +111,70 @@ public class Cell implements HasFields, Jsonizable {
writer.endObject();
}
@JsonProperty("e")
@JsonInclude(Include.NON_NULL)
public String getErrorMessage() {
if (ExpressionUtils.isError(value)) {
return ((EvalError) value).message;
}
return null;
}
@JsonProperty("t")
@JsonInclude(Include.NON_NULL)
public String getTypeString() {
if (value instanceof OffsetDateTime || value instanceof LocalDateTime) {
return "date";
}
return null;
}
@JsonProperty("v")
@JsonInclude(Include.NON_NULL)
public String getValueAsString() {
if (value != null && !ExpressionUtils.isError(value)) {
Instant instant = null;
if (value instanceof OffsetDateTime) {
instant = ((OffsetDateTime)value).toInstant();
} else if (value instanceof LocalDateTime) {
instant = ((LocalDateTime)value).toInstant(ZoneOffset.of("Z"));
}
if (instant != null) {
return ParsingUtilities.instantToString(instant);
} else if (value instanceof Double
&& (((Double)value).isNaN() || ((Double)value).isInfinite())) {
// write as a string
return ((Double)value).toString();
} else if (value instanceof Float
&& (((Float)value).isNaN() || ((Float)value).isInfinite())) {
// TODO: Skip? Write as string?
return ((Float)value).toString();
} else {
return value.toString();
}
} else {
return null;
}
}
/**
* TODO
* - use JsonIdentityInfo on recon
* - implement custom resolver to tie it to a pool
* - figure it all out
* @return
*/
@JsonProperty("r")
@JsonInclude(Include.NON_NULL)
public String getReconIdString() {
if (recon != null) {
return Long.toString(recon.id);
}
// TODO pool the recon??
return null;
}
public void save(Writer writer, Properties options) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {

21
main/src/com/google/refine/model/Column.java
View File

@ -44,6 +44,10 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.InterProjectModel;
import com.google.refine.Jsonizable;
import com.google.refine.model.recon.ReconConfig;
@ -76,10 +80,12 @@ public class Column implements Jsonizable {
_originalName = _name = originalName;
}
@JsonProperty("cellIndex")
public int getCellIndex() {
return _cellIndex;
}
@JsonProperty("originalName")
public String getOriginalHeaderLabel() {
return _originalName;
}
@ -88,6 +94,7 @@ public class Column implements Jsonizable {
this._name = name;
}
@JsonProperty("name")
public String getName() {
return _name;
}
@ -96,6 +103,8 @@ public class Column implements Jsonizable {
this._reconConfig = config;
}
@JsonProperty("reconConfig")
@JsonInclude(Include.NON_NULL)
public ReconConfig getReconConfig() {
return _reconConfig;
}
@ -104,6 +113,8 @@ public class Column implements Jsonizable {
this._reconStats = stats;
}
@JsonProperty("reconStats")
@JsonInclude(Include.NON_NULL)
public ReconStats getReconStats() {
return _reconStats;
}
@ -131,6 +142,7 @@ public class Column implements Jsonizable {
}
writer.endObject();
}
/**
* Clear all cached precomputed values.
@ -160,7 +172,7 @@ public class Column implements Jsonizable {
_precomputes.put(key, value);
}
@JsonProperty("type")
public String getType() {
return type;
}
@ -171,6 +183,7 @@ public class Column implements Jsonizable {
}
@JsonProperty("format")
public String getFormat() {
return format;
}
@ -181,6 +194,7 @@ public class Column implements Jsonizable {
}
@JsonProperty("title")
public String getTitle() {
return title;
}
@ -191,6 +205,7 @@ public class Column implements Jsonizable {
}
@JsonProperty("description")
public String getDescription() {
return description;
}
@ -200,6 +215,10 @@ public class Column implements Jsonizable {
this.description = description;
}
@JsonProperty("constraints")
public String getConstraintsString() {
return (new JSONObject(constraints)).toString();
}
public Map<String, Object> getConstraints() {
return constraints;

28
main/src/com/google/refine/model/ColumnGroup.java
View File

@ -42,7 +42,13 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.google.refine.Jsonizable;
import com.google.refine.util.JsonViews;
import com.google.refine.util.ParsingUtilities;
public class ColumnGroup implements Jsonizable {
@ -80,6 +86,28 @@ public class ColumnGroup implements Jsonizable {
writer.endObject();
}
@JsonProperty("startColumnIndex")
public int getStartColumnIndex() {
return startColumnIndex;
}
@JsonProperty("columnSpan")
public int getColumnSpan() {
return columnSpan;
}
@JsonProperty("keyColumnIndex")
public int getKeyColumnIndex() {
return keyColumnIndex;
}
@JsonProperty("subgroups")
@JsonView(JsonViews.NonSaveMode.class)
@JsonInclude(Include.NON_EMPTY)
public List<ColumnGroup> getSubGroups() {
return subgroups;
}
public boolean contains(ColumnGroup g) {
return (g.startColumnIndex >= startColumnIndex &&
g.startColumnIndex < startColumnIndex + columnSpan);

28
main/src/com/google/refine/model/ColumnModel.java
View File

@ -48,10 +48,17 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
public class ColumnModel implements Jsonizable {
@JsonProperty("columns")
final public List<Column> columns = new LinkedList<Column>();
@JsonProperty("columnGroups")
final public List<ColumnGroup> columnGroups = new LinkedList<ColumnGroup>();
private int _maxCellIndex = -1;
@ -70,6 +77,7 @@ public class ColumnModel implements Jsonizable {
this._maxCellIndex = Math.max(this._maxCellIndex, maxCellIndex);
}
@JsonIgnore
synchronized public int getMaxCellIndex() {
return _maxCellIndex;
}
@ -86,6 +94,7 @@ public class ColumnModel implements Jsonizable {
this._keyColumnIndex = keyColumnIndex;
}
@JsonIgnore
synchronized public int getKeyColumnIndex() {
return _keyColumnIndex;
}
@ -166,6 +175,7 @@ public class ColumnModel implements Jsonizable {
return _cellIndexToColumn.get(cellIndex);
}
@JsonIgnore
synchronized public List<String> getColumnNames() {
return _columnNames;
}
@ -198,6 +208,24 @@ public class ColumnModel implements Jsonizable {
writer.endObject();
}
@JsonProperty("keyCellIndex")
@JsonInclude(Include.NON_NULL)
public Integer getJsonKeyCellIndex() {
if(columns.size() > 0) {
return getKeyColumnIndex();
}
return null;
}
@JsonProperty("keyColumnName")
@JsonInclude(Include.NON_NULL)
public String getKeyColumnName() {
if(columns.size() > 0) {
return columns.get(_keyColumnIndex).getName();
}
return null;
}
synchronized public void save(Writer writer, Properties options) throws IOException {
writer.write("maxCellIndex="); writer.write(Integer.toString(_maxCellIndex)); writer.write('\n');
writer.write("keyColumnIndex="); writer.write(Integer.toString(_keyColumnIndex)); writer.write('\n');

98
main/src/com/google/refine/model/Recon.java
View File

@ -34,21 +34,31 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import org.json.JSONException;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonFilter;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.refine.Jsonizable;
import com.google.refine.expr.HasFields;
import com.google.refine.util.JsonViews;
import com.google.refine.util.Pool;
@JsonFilter("reconCandidateFilter")
public class Recon implements HasFields, Jsonizable {
/**
@ -61,11 +71,15 @@ public class Recon implements HasFields, Jsonizable {
private static final String WIKIDATA_IDENTIFIER_SPACE = "http://www.wikidata.org/entity/";
static public enum Judgment {
@JsonProperty("none")
None,
@JsonProperty("matched")
Matched,
@JsonProperty("new")
New
}
@Deprecated
static public String judgmentToString(Judgment judgment) {
if (judgment == Judgment.Matched) {
return "matched";
@ -76,6 +90,7 @@ public class Recon implements HasFields, Jsonizable {
}
}
@Deprecated
static public Judgment stringToJudgment(String s) {
if ("matched".equals(s)) {
return Judgment.Matched;
@ -186,6 +201,7 @@ public class Recon implements HasFields, Jsonizable {
candidates.add(candidate);
}
@JsonIgnore
public ReconCandidate getBestCandidate() {
if (candidates != null && candidates.size() > 0) {
return candidates.get(0);
@ -257,6 +273,7 @@ public class Recon implements HasFields, Jsonizable {
return "match".equals(name) || "best".equals(name);
}
@Deprecated
protected String judgmentToString() {
return judgmentToString(judgment);
}
@ -273,6 +290,81 @@ public class Recon implements HasFields, Jsonizable {
return false;
}
}
@JsonProperty("id")
public long getId() {
return id;
}
@JsonProperty("judgmentHistoryEntry")
@JsonView(JsonViews.SaveMode.class)
public long getJudgmentHistoryEntry() {
return judgmentHistoryEntry;
}
@JsonProperty("service")
public String getServiceURI() {
return service;
}
@JsonProperty("identifierSpace")
public String getIdentifierSpace() {
return identifierSpace;
}
@JsonProperty("schemaSpace")
public String getSchemaSpace() {
return schemaSpace;
}
@JsonProperty("j")
public Judgment getJudgment() {
return judgment;
}
@JsonProperty("m")
@JsonInclude(Include.NON_NULL)
public ReconCandidate getMatch() {
return match;
}
@JsonProperty("c")
//@JsonView(JsonViews.SaveMode.class)
public List<ReconCandidate> getCandidates() {
if (candidates != null) {
return candidates;
}
return Collections.emptyList();
}
@JsonProperty("f")
@JsonView(JsonViews.SaveMode.class)
public Object[] getfeatures() {
return features;
}
@JsonProperty("judgmentAction")
@JsonView(JsonViews.SaveMode.class)
public String getJudgmentAction() {
return judgmentAction;
}
@JsonProperty("judgmentBatchSize")
@JsonView(JsonViews.SaveMode.class)
public int getJudgmentBatchSize() {
return judgmentBatchSize;
}
@JsonProperty("matchRank")
@JsonView(JsonViews.SaveMode.class)
@JsonInclude(Include.NON_NULL)
public Integer getMatchRank() {
if (match != null) {
return matchRank;
}
return null;
}
@Override
public void write(JSONWriter writer, Properties options)

15
main/src/com/google/refine/model/ReconCandidate.java
View File

@ -37,6 +37,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
@ -47,9 +49,13 @@ import com.google.refine.Jsonizable;
import com.google.refine.expr.HasFields;
public class ReconCandidate implements HasFields, Jsonizable {
@JsonProperty("id")
final public String id;
@JsonProperty("name")
final public String name;
@JsonProperty("types")
final public String[] types;
@JsonIgnore
final public double score;
public ReconCandidate(String topicID, String topicName, String[] typeIDs, double score) {
@ -59,6 +65,15 @@ public class ReconCandidate implements HasFields, Jsonizable {
this.score = score;
}
// Serialize doubles that are ints without trailing ".0" for consistency with previous serialization.
@JsonProperty("score")
public Object getJsonScore() {
if ((double)(int)score == score) {
return (int)score;
}
return score;
}
@Override
public Object getField(String name, Properties bindings) {
if ("id".equals(name)) {

14
main/src/com/google/refine/model/RecordModel.java
View File

@ -45,6 +45,9 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.expr.ExpressionUtils;
@ -85,6 +88,7 @@ public class RecordModel implements Jsonizable {
_rowDependencies.get(rowIndex) : null;
}
@JsonIgnore
public int getRecordCount() {
return _records.size();
}
@ -111,11 +115,15 @@ public class RecordModel implements Jsonizable {
writer.object();
writer.key("hasRecords");
writer.value(
_records != null && _rowDependencies != null &&
_records.size() < _rowDependencies.size());
writer.value(hasRecords());
writer.endObject();
}
@JsonProperty("hasRecords")
public boolean hasRecords() {
return _records != null && _rowDependencies != null &&
_records.size() < _rowDependencies.size();
}
static protected class KeyedGroup {
int[] cellIndices;

25
main/src/com/google/refine/model/Row.java
View File

@ -39,6 +39,9 @@ import java.util.List;
import java.util.Map.Entry;
import java.util.Properties;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
@ -48,6 +51,7 @@ import org.json.JSONWriter;
import com.google.refine.Jsonizable;
import com.google.refine.expr.CellTuple;
import com.google.refine.expr.HasFields;
import com.google.refine.util.JsonViews;
import com.google.refine.util.Pool;
/**
@ -106,6 +110,7 @@ public class Row implements HasFields, Jsonizable {
return "cells".equals(name) || "record".equals(name);
}
@JsonIgnore
public boolean isEmpty() {
for (Cell cell : cells) {
if (cell != null && cell.value != null && !isValueBlank(cell.value)) {
@ -204,6 +209,26 @@ public class Row implements HasFields, Jsonizable {
writer.endObject();
}
@JsonProperty(FLAGGED)
public boolean isFlagged() {
return flagged;
}
@JsonProperty(STARRED)
public boolean isStarred() {
return starred;
}
@JsonProperty("cells")
public List<Cell> getCells() {
return cells;
}
/*
@JsonView(JsonViews.SaveMode.class)
public
*/
public void save(Writer writer, Properties options) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {

9
main/src/com/google/refine/model/recon/ReconConfig.java
View File

@ -47,6 +47,8 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
@ -130,4 +132,11 @@ abstract public class ReconConfig implements Jsonizable {
LOGGER.error("Save failed",e);
}
}
/**
* Returns the identifier for the reconciliation mode, as serialized in JSON.
* This is the same identifier that was used to register the registration mode.
*/
@JsonProperty("mode")
abstract public String getMode();
}

48
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -53,12 +53,17 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.Jsonizable;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconType;
import com.google.refine.model.RecordModel.RowDependency;
import com.google.refine.model.Row;
import com.google.refine.util.ParsingUtilities;
@ -66,9 +71,12 @@ import com.google.refine.util.ParsingUtilities;
public class StandardReconConfig extends ReconConfig {
final static Logger logger = LoggerFactory.getLogger("refine-standard-recon");
static public class ColumnDetail {
static public class ColumnDetail implements Jsonizable {
@JsonProperty("column")
final public String columnName;
@JsonProperty("propertyName")
final public String propertyName;
@JsonProperty("propertyID")
final public String propertyID;
public ColumnDetail(String columnName, String propertyName, String propertyID) {
@ -76,6 +84,17 @@ public class StandardReconConfig extends ReconConfig {
this.propertyName = propertyName;
this.propertyID = propertyID;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("column"); writer.value(columnName);
writer.key("propertyName"); writer.value(propertyName);
writer.key("propertyID"); writer.value(propertyID);
writer.endObject();
}
}
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
@ -133,14 +152,22 @@ public class StandardReconConfig extends ReconConfig {
}
}
@JsonProperty("service")
final public String service;
@JsonProperty("identifierSpace")
final public String identifierSpace;
@JsonProperty("schemaSpace")
final public String schemaSpace;
@JsonIgnore
final public String typeID;
@JsonIgnore
final public String typeName;
@JsonProperty("autoMatch")
final public boolean autoMatch;
@JsonProperty("columnDetails")
final public List<ColumnDetail> columnDetails;
@JsonProperty("limit")
final private int limit;
public StandardReconConfig(
@ -210,18 +237,21 @@ public class StandardReconConfig extends ReconConfig {
writer.key("columnDetails");
writer.array();
for (ColumnDetail c : columnDetails) {
writer.object();
writer.key("column"); writer.value(c.columnName);
writer.key("propertyName"); writer.value(c.propertyName);
writer.key("propertyID"); writer.value(c.propertyID);
writer.endObject();
c.write(writer, options);
}
writer.endArray();
writer.key("limit"); writer.value(limit);
writer.endObject();
}
@JsonProperty("type")
public ReconType getReconType() {
ReconType t = new ReconType(typeID, typeName);
return t;
}
@Override
@JsonIgnore
public int getBatchSize() {
return 10;
}
@ -541,4 +571,10 @@ public class StandardReconConfig extends ReconConfig {
}
return set;
}
@Override
public String getMode() {
return "standard-service";
}
}

18
main/src/com/google/refine/util/JsonViews.java Normal file
View File

@ -0,0 +1,18 @@
package com.google.refine.util;
/**
* Set of classes which define JSON visibility of certain fields.
* @author Antonin Delpeuch
*
*/
public class JsonViews {
public static class SaveMode {
;
}
public static class NonSaveMode {
;
}
}

15
main/src/com/google/refine/util/ParsingUtilities.java
View File

@ -59,7 +59,22 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.ser.FilterProvider;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
public class ParsingUtilities {
public static final ObjectMapper mapper = new ObjectMapper();
public static final FilterProvider defaultFilters = new SimpleFilterProvider()
.addFilter("reconCandidateFilter", SerializationFilters.reconCandidateFilter);
public static final FilterProvider saveFilters = new SimpleFilterProvider()
.addFilter("reconCandidateFilter", SerializationFilters.noFilter);
public static final ObjectWriter saveWriter = mapper.writerWithView(JsonViews.SaveMode.class).with(saveFilters);
public static final ObjectWriter defaultWriter = mapper.writerWithView(JsonViews.NonSaveMode.class).with(defaultFilters);
public static final DateTimeFormatter ISO8601 = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'");
static public Properties parseUrlParameters(HttpServletRequest request) {

55
main/src/com/google/refine/util/SerializationFilters.java Normal file
View File

@ -0,0 +1,55 @@
package com.google.refine.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.BeanPropertyWriter;
import com.fasterxml.jackson.databind.ser.PropertyFilter;
import com.fasterxml.jackson.databind.ser.PropertyWriter;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
public class SerializationFilters {
static class BaseFilter extends SimpleBeanPropertyFilter {
@Override
public void serializeAsField(Object obj, JsonGenerator jgen, SerializerProvider provider, PropertyWriter writer)
throws Exception {
if (include(writer)) {
writer.serializeAsField(obj, jgen, provider);
} else if (!jgen.canOmitFields()) {
writer.serializeAsOmittedField(obj, jgen, provider);
}
}
@Override
protected boolean include(BeanPropertyWriter writer) {
return true;
}
@Override
protected boolean include(PropertyWriter writer) {
return true;
}
}
public static PropertyFilter noFilter = new BaseFilter();
public static PropertyFilter reconCandidateFilter = new BaseFilter() {
@Override
public void serializeAsField(Object obj, JsonGenerator jgen, SerializerProvider provider, PropertyWriter writer)
throws Exception {
if (include(writer)) {
if (!writer.getName().equals("c") || ! (obj instanceof Recon)) {
writer.serializeAsField(obj, jgen, provider);
return;
}
Recon recon = (Recon)obj;
if (recon.judgment == Judgment.None) {
writer.serializeAsField(obj, jgen, provider);
}
} else if (!jgen.canOmitFields()) {
writer.serializeAsOmittedField(obj, jgen, provider);
}
}
};
}

8
main/tests/server/src/com/google/refine/tests/model/ColumnGroupTests.java
View File

@ -1,7 +1,5 @@
package com.google.refine.tests.model;
import java.util.Properties;
import org.testng.annotations.Test;
import com.google.refine.model.ColumnGroup;
@ -34,9 +32,7 @@ public class ColumnGroupTests {
+ " \"keyColumnIndex\":1"
+ "}]"
+ "}";
Properties options = new Properties();
options.setProperty("mode", "save");
TestUtils.isSerializedTo(cg, json, options);
TestUtils.isSerializedTo(cg, fullJson);
TestUtils.isSerializedTo(cg, json, true);
TestUtils.isSerializedTo(cg, fullJson, false);
}
}

45
main/tests/server/src/com/google/refine/tests/model/ColumnModelTests.java
View File

@ -2,9 +2,50 @@ package com.google.refine.tests.model;
import org.testng.annotations.Test;
public class ColumnModelTests {
import com.google.refine.model.ColumnModel;
import com.google.refine.model.Project;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class ColumnModelTests extends RefineTest {
@Test
public void serializeColumnModel() {
String json = "";
Project project = createCSVProject("a,b\n"+
"e,e");
String json = "{\n" +
" \"columnGroups\" : [ ],\n" +
" \"columns\" : [ {\n" +
" \"cellIndex\" : 0,\n" +
" \"constraints\" : \"{}\",\n" +
" \"description\" : \"\",\n" +
" \"format\" : \"default\",\n" +
" \"name\" : \"a\",\n" +
" \"originalName\" : \"a\",\n" +
" \"title\" : \"\",\n" +
" \"type\" : \"\"\n" +
" }, {\n" +
" \"cellIndex\" : 1,\n" +
" \"constraints\" : \"{}\",\n" +
" \"description\" : \"\",\n" +
" \"format\" : \"default\",\n" +
" \"name\" : \"b\",\n" +
" \"originalName\" : \"b\",\n" +
" \"title\" : \"\",\n" +
" \"type\" : \"\"\n" +
" } ],\n" +
" \"keyCellIndex\" : 0,\n" +
" \"keyColumnName\" : \"a\"\n" +
" }";
TestUtils.isSerializedTo(project.columnModel, json);
}
@Test
public void serializeColumnModelEmpty() {
String json = "{"
+ "\"columns\":[],"
+ "\"columnGroups\":[]"
+ "}";
ColumnModel m = new ColumnModel();
TestUtils.isSerializedTo(m, json);
}
}

12
main/tests/server/src/com/google/refine/tests/model/ReconCandidateTests.java
View File

@ -7,7 +7,7 @@ import com.google.refine.tests.util.TestUtils;
public class ReconCandidateTests {
@Test
public void serializeReconCandidate() throws Exception {
public void serializeReconCandidateInt() throws Exception {
String json = "{\"id\":\"Q49213\","
+ "\"name\":\"University of Texas at Austin\","
+ "\"score\":100,"
@ -15,4 +15,14 @@ public class ReconCandidateTests {
ReconCandidate rc = ReconCandidate.loadStreaming(json);
TestUtils.isSerializedTo(rc, json);
}
@Test
public void serializeReconCandidateDouble() throws Exception {
String json = "{\"id\":\"Q49213\","
+ "\"name\":\"University of Texas at Austin\","
+ "\"score\":0.5,"
+ "\"types\":[\"Q875538\",\"Q15936437\",\"Q20971972\",\"Q23002039\"]}";
ReconCandidate rc = ReconCandidate.loadStreaming(json);
TestUtils.isSerializedTo(rc, json);
}
}

79
main/tests/server/src/com/google/refine/tests/model/ReconTests.java
View File

@ -1,42 +1,44 @@
package com.google.refine.tests.model;
import java.util.Properties;
import org.testng.annotations.Test;
import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.tests.util.TestUtils;
public class ReconTests {
String fullJson = "{\"id\":1533651559492945033,"
+ "\"judgmentHistoryEntry\":1533651616890,"
+ "\"service\":\"https://tools.wmflabs.org/openrefine-wikidata/en/api\","
+ "\"identifierSpace\":\"http://www.wikidata.org/entity/\","
+ "\"schemaSpace\":\"http://www.wikidata.org/prop/direct/\","
+ "\"j\":\"matched\","
+ "\"m\":{"
+ " \"id\":\"Q2892284\","
+ " \"name\":\"Baylor College of Medicine\","
+ " \"score\":98.57142857142858,"
+ " \"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]"
+ "},"
+ "\"c\":["
+ " {\"id\":\"Q2892284\",\"name\":\"Baylor College of Medicine\",\"score\":98.57142857142858,\"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]},"
+ " {\"id\":\"Q16165943\",\"name\":\"Baylor College of Medicine Academy at Ryan\",\"score\":82.14285714285715,\"types\":[\"Q149566\"]},"
+ " {\"id\":\"Q30284245\",\"name\":\"Baylor College of Medicine Children\\u2019s Foundation\",\"score\":48.57142857142858,\"types\":[\"Q163740\"]}"
+ "],"
+ "\"f\":[false,false,1,0.6666666666666666],"
+ "\"judgmentAction\":\"mass\","
+ "\"judgmentBatchSize\":1,"
+ "\"matchRank\":0}";
@Test
public void serializeRecon() throws Exception {
Properties options = new Properties();
options.put("mode", "save");
String fullJson = "{\"id\":1533651559492945033,"
+ "\"judgmentHistoryEntry\":1533651616890,"
+ "\"service\":\"https://tools.wmflabs.org/openrefine-wikidata/en/api\","
+ "\"identifierSpace\":\"http://www.wikidata.org/entity/\","
+ "\"schemaSpace\":\"http://www.wikidata.org/prop/direct/\","
+ "\"j\":\"matched\","
+ "\"m\":{"
+ " \"id\":\"Q2892284\","
+ " \"name\":\"Baylor College of Medicine\","
+ " \"score\":98.57142857142858,"
+ " \"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]"
+ "},"
+ "\"c\":["
+ " {\"id\":\"Q2892284\",\"name\":\"Baylor College of Medicine\",\"score\":98.57142857142858,\"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]},"
+ " {\"id\":\"Q16165943\",\"name\":\"Baylor College of Medicine Academy at Ryan\",\"score\":82.14285714285715,\"types\":[\"Q149566\"]},"
+ " {\"id\":\"Q30284245\",\"name\":\"Baylor College of Medicine Children\\u2019s Foundation\",\"score\":48.57142857142858,\"types\":[\"Q163740\"]}"
+ "],"
+ "\"f\":[false,false,1,0.6666666666666666],"
+ "\"judgmentAction\":\"mass\","
+ "\"judgmentBatchSize\":1,"
+ "\"matchRank\":0}";
public void serializeReconSaveMode() throws Exception {
Recon r = Recon.loadStreaming(fullJson, null);
TestUtils.isSerializedTo(r, fullJson, options);
TestUtils.isSerializedTo(r, fullJson, true);
}
@Test
public void serializeReconViewMode() throws Exception {
Recon r = Recon.loadStreaming(fullJson, null);
String shortJson = "{\"id\":1533651559492945033,"
+ "\"service\":\"https://tools.wmflabs.org/openrefine-wikidata/en/api\","
+ "\"identifierSpace\":\"http://www.wikidata.org/entity/\","
@ -48,9 +50,26 @@ public class ReconTests {
+ " \"score\":98.57142857142858,"
+ " \"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]"
+ "}}";
options.put("mode", "normal");
TestUtils.isSerializedTo(r, shortJson, options);
TestUtils.isSerializedTo(r, shortJson, false);
}
@Test
public void serializeReconSaveModeNoMatch() throws Exception {
String json = "{\"id\":1533651559492945033,"
+ "\"service\":\"https://tools.wmflabs.org/openrefine-wikidata/en/api\","
+ "\"identifierSpace\":\"http://www.wikidata.org/entity/\","
+ "\"schemaSpace\":\"http://www.wikidata.org/prop/direct/\","
+ "\"j\":\"none\","
+ "\"c\":["
+ " {\"id\":\"Q2892284\",\"name\":\"Baylor College of Medicine\",\"score\":98.57142857142858,\"types\":[\"Q16917\",\"Q23002054\",\"Q494230\"]},"
+ " {\"id\":\"Q16165943\",\"name\":\"Baylor College of Medicine Academy at Ryan\",\"score\":82.14285714285715,\"types\":[\"Q149566\"]},"
+ " {\"id\":\"Q30284245\",\"name\":\"Baylor College of Medicine Children\\u2019s Foundation\",\"score\":48.57142857142858,\"types\":[\"Q163740\"]}"
+ "]"
+ "}";
Recon r = Recon.loadStreaming(fullJson, null);
r.match = null;
r.judgment = Judgment.None;
TestUtils.isSerializedTo(r, json);
}
}

4
main/tests/server/src/com/google/refine/tests/model/RowTests.java
View File

@ -107,7 +107,7 @@ public class RowTests extends RefineTest {
Row row = new Row(5);
row.setCell(0, new Cell("I'm not empty", null));
row.save(writer, options);
TestUtils.equalAsJson(writer.getBuffer().toString(),
TestUtils.assertEqualAsJson(writer.getBuffer().toString(),
"{\"flagged\":false,\"starred\":false,\"cells\":[{\"v\":\"I'm not empty\"}]}");
}
@ -120,7 +120,7 @@ public class RowTests extends RefineTest {
when(options.containsKey("recordIndex")).thenReturn(true);
when(options.get("recordIndex")).thenReturn(1);
row.save(writer, options);
TestUtils.equalAsJson(
TestUtils.assertEqualAsJson(
writer.getBuffer().toString(),
"{\"flagged\":false,\"starred\":false,\"cells\":[{\"v\":\"I'm not empty\"}],\"i\":0,\"j\":1}");
}

120
main/tests/server/src/com/google/refine/tests/util/TestUtils.java
View File

@ -5,19 +5,32 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.Properties;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.refine.Jsonizable;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.JsonViews;
import com.google.refine.util.ParsingUtilities;
public class TestUtils {
static ObjectMapper mapper = new ObjectMapper();
static {
mapper = mapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true);
}
/**
* Create a temporary directory. NOTE: This is a quick and dirty
@ -38,7 +51,7 @@ public class TestUtils {
/**
* Compare two JSON strings for equality.
*/
public static void equalAsJson(String expected, String actual) {
public static void assertEqualAsJson(String expected, String actual) {
try {
JsonNode jsonA = mapper.readValue(expected, JsonNode.class);
JsonNode jsonB = mapper.readValue(actual, JsonNode.class);
@ -48,19 +61,52 @@ public class TestUtils {
}
}
public static boolean equalAsJson(String expected, String actual) {
try {
JsonNode jsonA = mapper.readValue(expected, JsonNode.class);
JsonNode jsonB = mapper.readValue(actual, JsonNode.class);
return (jsonA == null && jsonB == null) || jsonA.equals(jsonB);
} catch(Exception e) {
return false;
}
}
/**
* Checks that a serializable object is serialized to the target JSON string.
* @throws IOException
*/
public static void isSerializedTo(Jsonizable o, String targetJson, Properties options) {
equalAsJson(targetJson, JSONUtilities.serialize(o, options));
String orgJson = JSONUtilities.serialize(o, options);
if(!equalAsJson(targetJson, orgJson)) {
System.out.println("org.json, "+o.getClass().getName());
try {
jsonDiff(targetJson, orgJson);
} catch (JsonParseException | JsonMappingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
assertEqualAsJson(targetJson, orgJson);
// also check Jackson serialization
try {
equalAsJson(targetJson, mapper.writeValueAsString(o));
} catch (JsonProcessingException e) {
e.printStackTrace();
fail("jackson serialization failed");
}
String saveMode = options.getProperty("mode");
ObjectWriter writer = null;
if("save".equals(saveMode)) {
writer = ParsingUtilities.saveWriter;
} else {
writer = ParsingUtilities.defaultWriter;
}
String jacksonJson = writer.writeValueAsString(o);
if(!equalAsJson(targetJson, jacksonJson)) {
System.out.println("jackson, "+o.getClass().getName());
jsonDiff(targetJson, jacksonJson);
}
assertEqualAsJson(targetJson, jacksonJson);
} catch (JsonProcessingException e) {
e.printStackTrace();
fail("jackson serialization failed");
}
}
/**
@ -69,4 +115,64 @@ public class TestUtils {
public static void isSerializedTo(Jsonizable o, String targetJson) {
isSerializedTo(o, targetJson, new Properties());
}
/**
* Checks that a serializable object is serialized to the target JSON string.
* This specifies the "save mode" for objects that are stored differently depending on
* whether they are written to disk or sent over the network.
*/
public static void isSerializedTo(Jsonizable o, String targetJson, boolean saveMode) {
Properties options = new Properties();
if(saveMode) {
options.setProperty("mode", "save");
options.put("mode", "save");
}
isSerializedTo(o, targetJson, options);
}
public static void jsonDiff(String a, String b) throws JsonParseException, JsonMappingException {
ObjectMapper myMapper = mapper.copy().configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true)
.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
.configure(SerializationFeature.INDENT_OUTPUT, true);
try {
JsonNode nodeA = myMapper.readValue(a, JsonNode.class);
JsonNode nodeB = myMapper.readValue(b, JsonNode.class);
String prettyA = myMapper.writeValueAsString(myMapper.treeToValue(nodeA, Object.class));
String prettyB = myMapper.writeValueAsString(myMapper.treeToValue(nodeB, Object.class));
// Compute the max line length of A
LineNumberReader readerA = new LineNumberReader(new StringReader(prettyA));
int maxLength = 0;
String line = readerA.readLine();
while (line != null) {
if(line.length() > maxLength) {
maxLength = line.length();
}
line = readerA.readLine();
}
// Pad all lines
readerA = new LineNumberReader(new StringReader(prettyA));
LineNumberReader readerB = new LineNumberReader(new StringReader(prettyB));
StringWriter writer = new StringWriter();
String lineA = readerA.readLine();
String lineB = readerB.readLine();
while(lineA != null || lineB != null) {
if (lineA == null) {
lineA = "";
}
if (lineB == null) {
lineB = "";
}
String paddedLineA = lineA + new String(new char[maxLength + 2 - lineA.length()]).replace("\0", " ");
writer.write(paddedLineA);
writer.write(lineB + "\n");
lineA = readerA.readLine();
lineB = readerB.readLine();
}
System.out.print(writer.toString());
} catch(IOException e) {
e.printStackTrace();
}
}
}