s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Jackson serialization for reconciliation operations

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-09-28 09:18:52 +01:00
parent 8f1c7cc2ab
commit cf45f23e1d
17 changed files with 388 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
main/src/com/google/refine/commands/recon/ExtendDataCommand.java
View File

@ -35,13 +35,12 @@ package com.google.refine.commands.recon;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.operations.recon.ExtendDataOperation;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig;
import com.google.refine.operations.recon.ExtendDataOperation;
import com.google.refine.util.ParsingUtilities;
public class ExtendDataCommand extends EngineDependentCommand {
@ -56,7 +55,7 @@ public class ExtendDataCommand extends EngineDependentCommand {
String schemaSpace = request.getParameter("schemaSpace");
String jsonString = request.getParameter("extension");
JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString);
DataExtensionConfig extension = DataExtensionConfig.reconstruct(ParsingUtilities.evaluateJsonStringToObject(jsonString));
return new ExtendDataOperation(
engineConfig,

14
main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java
View File

@ -46,19 +46,19 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONArray;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.commands.Command;
import com.google.refine.model.recon.ReconciledDataExtensionJob;
import com.google.refine.model.recon.ReconciledDataExtensionJob.ColumnInfo;
import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.Project;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.model.Column;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.ReconciledDataExtensionJob;
import com.google.refine.model.recon.ReconciledDataExtensionJob.ColumnInfo;
import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension;
import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig;
import com.google.refine.model.recon.StandardReconConfig;
import com.google.refine.util.ParsingUtilities;
@ -79,7 +79,7 @@ public class PreviewExtendDataCommand extends Command {
}
String jsonString = request.getParameter("extension");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
DataExtensionConfig config = DataExtensionConfig.reconstruct(ParsingUtilities.evaluateJsonStringToObject(jsonString));
JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString);
int length = rowIndices.length();
@ -120,7 +120,7 @@ public class PreviewExtendDataCommand extends Command {
}
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(json, endpoint);
ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(config, endpoint);
Map<String, DataExtension> map = job.extend(ids, reconCandidateMap);
response.setCharacterEncoding("UTF-8");

6
main/src/com/google/refine/commands/recon/ReconMatchSpecificTopicCommand.java
View File

@ -39,7 +39,6 @@ import com.google.refine.browsing.EngineConfig;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.model.ReconCandidate;
import com.google.refine.operations.recon.ReconMatchSpecificTopicOperation;
public class ReconMatchSpecificTopicCommand extends EngineDependentCommand {
@ -49,11 +48,10 @@ public class ReconMatchSpecificTopicCommand extends EngineDependentCommand {
HttpServletRequest request, EngineConfig engineConfig) throws Exception {
String columnName = request.getParameter("columnName");
ReconCandidate match = new ReconCandidate(
ReconMatchSpecificTopicOperation.ReconItem match = new ReconMatchSpecificTopicOperation.ReconItem(
request.getParameter("topicID"),
request.getParameter("topicName"),
request.getParameter("types").split(","),
100
request.getParameter("types").split(",")
);
return new ReconMatchSpecificTopicOperation(

186
main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java
View File

@ -39,7 +39,6 @@ package com.google.refine.model.recon;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.io.StringWriter;
import java.io.Writer;
import java.net.URL;
@ -48,21 +47,155 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.model.ReconType;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.Jsonizable;
import com.google.refine.expr.functions.ToDate;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.recon.StandardReconConfig;
import com.google.refine.model.ReconType;
import com.google.refine.util.JSONUtilities;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.expr.functions.ToDate;
public class ReconciledDataExtensionJob {
static public class DataExtensionProperty implements Jsonizable {
@JsonProperty("id")
public final String id;
@JsonProperty("name")
public final String name;
@JsonProperty("settings")
@JsonInclude(Include.NON_NULL)
public final Map<String, Object> settings;
@JsonCreator
public DataExtensionProperty(
@JsonProperty("id")
String id,
@JsonProperty("name")
String name,
@JsonProperty("settings")
Map<String, Object> settings) {
this.id = id;
this.name = name;
this.settings = settings;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(id);
if(!"query".equals(options.getProperty("mode"))) {
writer.key("name"); writer.value(name);
}
if (settings != null) {
writer.key("settings");
writer.object();
for(Map.Entry<String, Object> entry : settings.entrySet()) {
writer.key(entry.getKey());
writer.value(entry.getValue());
}
writer.endObject();
}
writer.endObject();
}
}
static public class DataExtensionConfig implements Jsonizable {
@JsonProperty("properties")
public final List<DataExtensionProperty> properties;
@JsonCreator
public DataExtensionConfig(
@JsonProperty("properties")
List<DataExtensionProperty> properties) {
this.properties = properties;
}
public static DataExtensionConfig reconstruct(JSONObject obj) throws JSONException {
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(obj.toString(), DataExtensionConfig.class);
} catch(IOException e) {
throw new JSONException(e.toString());
}
}
@Override
public void write(JSONWriter jsonWriter, Properties options)
throws JSONException {
jsonWriter.object();
jsonWriter.key("properties");
jsonWriter.array();
for (DataExtensionProperty property : properties) {
property.write(jsonWriter, options);
}
jsonWriter.endArray();
jsonWriter.endObject();
}
}
static public class DataExtensionQuery extends DataExtensionConfig {
@JsonProperty("ids")
public final List<String> ids;
@JsonCreator
public DataExtensionQuery(
@JsonProperty("ids")
List<String> ids,
@JsonProperty("properties")
List<DataExtensionProperty> properties) {
super(properties);
this.ids = ids;
}
@Override
public void write(JSONWriter jsonWriter, Properties options)
throws JSONException {
jsonWriter.object();
if(ids != null) {
jsonWriter.key("ids");
jsonWriter.array();
for (String id : ids) {
if (id != null) {
jsonWriter.value(id);
}
}
jsonWriter.endArray();
}
jsonWriter.key("properties");
jsonWriter.array();
for (DataExtensionProperty property : properties) {
property.write(jsonWriter, options);
}
jsonWriter.endArray();
jsonWriter.endObject();
}
}
static public class DataExtension {
final public Object[][] data;
@ -83,11 +216,11 @@ public class ReconciledDataExtensionJob {
}
}
final public JSONObject extension;
final public String endpoint;
final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>();
final public DataExtensionConfig extension;
final public String endpoint;
final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>();
public ReconciledDataExtensionJob(JSONObject obj, String endpoint) throws JSONException {
public ReconciledDataExtensionJob(DataExtensionConfig obj, String endpoint) throws JSONException {
this.extension = obj;
this.endpoint = endpoint;
}
@ -249,39 +382,12 @@ public class ReconciledDataExtensionJob {
}
static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException {
static protected void formulateQuery(Set<String> ids, DataExtensionConfig node, Writer writer) throws JSONException {
JSONWriter jsonWriter = new JSONWriter(writer);
jsonWriter.object();
jsonWriter.key("ids");
jsonWriter.array();
for (String id : ids) {
if (id != null) {
jsonWriter.value(id);
}
}
jsonWriter.endArray();
jsonWriter.key("properties");
jsonWriter.array();
JSONArray properties = node.getJSONArray("properties");
int l = properties.length();
for (int i = 0; i < l; i++) {
JSONObject property = properties.getJSONObject(i);
jsonWriter.object();
jsonWriter.key("id");
jsonWriter.value(property.getString("id"));
if (property.has("settings")) {
JSONObject settings = property.getJSONObject("settings");
jsonWriter.key("settings");
jsonWriter.value(settings);
}
jsonWriter.endObject();
}
jsonWriter.endArray();
jsonWriter.endObject();
Properties options = new Properties();
DataExtensionQuery query = new DataExtensionQuery(ids.stream().collect(Collectors.toList()), node.properties);
options.setProperty("mode", "query");
query.write(jsonWriter, options);
}
static protected void gatherColumnInfo(JSONArray meta, List<ColumnInfo> columns) throws JSONException {

42
main/src/com/google/refine/operations/recon/ExtendDataOperation.java
View File

@ -45,6 +45,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -62,41 +64,51 @@ import com.google.refine.model.changes.DataExtensionChange;
import com.google.refine.model.recon.ReconciledDataExtensionJob;
import com.google.refine.model.recon.ReconciledDataExtensionJob.ColumnInfo;
import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension;
import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.process.LongRunningProcess;
import com.google.refine.process.Process;
public class ExtendDataOperation extends EngineDependentOperation {
final protected String _baseColumnName;
final protected String _endpoint;
final protected String _identifierSpace;
final protected String _schemaSpace;
final protected JSONObject _extension;
final protected int _columnInsertIndex;
@JsonProperty("baseColumnName")
final protected String _baseColumnName;
@JsonProperty("endpoint")
final protected String _endpoint;
@JsonProperty("identifierSpace")
final protected String _identifierSpace;
@JsonProperty("schemaSpace")
final protected String _schemaSpace;
@JsonProperty("extension")
final protected DataExtensionConfig _extension;
@JsonProperty("columnInsertIndex")
final protected int _columnInsertIndex;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
DataExtensionConfig dataExtensionConfig = DataExtensionConfig.reconstruct(obj.getJSONObject("extension"));
return new ExtendDataOperation(
EngineConfig.reconstruct(engineConfig),
obj.getString("baseColumnName"),
obj.getString("endpoint"),
obj.getString("identifierSpace"),
obj.getString("schemaSpace"),
obj.getJSONObject("extension"),
dataExtensionConfig,
obj.getInt("columnInsertIndex")
);
}
public ExtendDataOperation(
EngineConfig engineConfig,
String baseColumnName,
String endpoint,
String identifierSpace,
String schemaSpace,
JSONObject extension,
int columnInsertIndex
EngineConfig engineConfig,
String baseColumnName,
String endpoint,
String identifierSpace,
String schemaSpace,
DataExtensionConfig extension,
int columnInsertIndex
) {
super(engineConfig);
@ -121,7 +133,7 @@ public class ExtendDataOperation extends EngineDependentOperation {
writer.key("endpoint"); writer.value(_endpoint);
writer.key("identifierSpace"); writer.value(_identifierSpace);
writer.key("schemaSpace"); writer.value(_schemaSpace);
writer.key("extension"); writer.value(_extension);
writer.key("extension"); _extension.write(writer, options);
writer.endObject();
}

12
main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java
View File

@ -40,6 +40,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.history.Change;
@ -88,6 +90,16 @@ public class ReconClearSimilarCellsOperation extends EngineDependentMassCellOper
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;
}
@JsonProperty("similarValue")
public String getSimilarValue() {
return _similarValue;
}
@Override
protected String getBriefDescription(Project project) {
return "Clear recon data for cells containing \"" +

22
main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java
View File

@ -46,6 +46,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -118,6 +120,26 @@ public class ReconCopyAcrossColumnsOperation extends EngineDependentOperation {
writer.key("applyToJudgedCells"); writer.value(_applyToJudgedCells);
writer.endObject();
}
@JsonProperty("fromColumnName")
public String getFromColumnName() {
return _fromColumnName;
}
@JsonProperty("toColumnNames")
public String[] getToColumnNames() {
return _toColumnNames;
}
@JsonProperty("judgments")
public String[] getJudgments() {
return _judgments;
}
@JsonProperty("applyToJudgedCells")
public boolean getApplyToJudgedCells() {
return _applyToJudgedCells;
}
@Override
protected HistoryEntry createHistoryEntry(final Project project, final long historyEntryID) throws Exception {

12
main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java
View File

@ -42,6 +42,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.history.Change;
@ -86,6 +88,16 @@ public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOpera
writer.key("clearData"); writer.value(_clearData);
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;
}
@JsonProperty("clearData")
public boolean getClearData() {
return _clearData;
}
@Override
protected String getBriefDescription(Project project) {

32
main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java
View File

@ -43,6 +43,10 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.expr.ExpressionUtils;
@ -67,7 +71,7 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
final protected ReconCandidate _match;
final protected boolean _shareNewTopics;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
static public AbstractOperation reconstruct(JSONObject obj) throws JSONException {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
ReconCandidate match = null;
@ -137,6 +141,32 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;
}
@JsonProperty("similarValue")
public String getSimilarValue() {
return _similarValue;
}
@JsonProperty("judgment")
public Judgment getJudgment() {
return _judgment;
}
@JsonProperty("match")
@JsonInclude(Include.NON_NULL)
public ReconCandidate getMatch() {
return _match;
}
@JsonProperty("shareNewTopics")
public boolean getShareNewTopics() {
return _shareNewTopics;
}
@Override
protected String getBriefDescription(Project project) {
if (_judgment == Judgment.None) {

12
main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java
View File

@ -42,6 +42,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.history.Change;
@ -89,6 +91,16 @@ public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperatio
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;
}
@JsonProperty("shareNewTopics")
public boolean getShareNewTopics() {
return _shareNewTopics;
}
@Override
protected String getBriefDescription(Project project) {
return "Mark to create new items for cells in column " + _columnName +

7
main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java
View File

@ -42,6 +42,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.history.Change;
@ -84,6 +86,11 @@ public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOp
writer.key("columnName"); writer.value(_columnName);
writer.endObject();
}
@JsonProperty
public String getColumnName() {
return _columnName;
}
@Override
protected String getBriefDescription(Project project) {

43
main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java
View File

@ -43,6 +43,10 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.history.Change;
@ -60,8 +64,35 @@ import com.google.refine.operations.EngineDependentMassCellOperation;
import com.google.refine.operations.OperationRegistry;
public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOperation {
final protected ReconCandidate match;
public static class ReconItem {
@JsonProperty("id")
public final String id;
@JsonProperty("name")
public final String name;
@JsonProperty("types")
public final String[] types;
@JsonCreator
public ReconItem(
@JsonProperty("id") String id,
@JsonProperty("name") String name,
@JsonProperty("types") String[] types) {
this.id = id;
this.name = name;
this.types = types;
}
@JsonIgnore
public ReconCandidate getCandidate() {
return new ReconCandidate(id, name, types, 100);
}
}
@JsonProperty("match")
final protected ReconItem match;
@JsonProperty("identifierSpace")
final protected String identifierSpace;
@JsonProperty("schemaSpace")
final protected String schemaSpace;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
@ -78,11 +109,10 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
return new ReconMatchSpecificTopicOperation(
EngineConfig.reconstruct(engineConfig),
obj.getString("columnName"),
new ReconCandidate(
new ReconItem(
match.getString("id"),
match.getString("name"),
typeIDs,
100
typeIDs
),
obj.getString("identifierSpace"),
obj.getString("schemaSpace")
@ -92,7 +122,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
public ReconMatchSpecificTopicOperation(
EngineConfig engineConfig,
String columnName,
ReconCandidate match,
ReconItem match,
String identifierSpace,
String schemaSpace
) {
@ -146,6 +176,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
@Override
protected RowVisitor createRowVisitor(Project project, List<CellChange> cellChanges, long historyEntryID) throws Exception {
Column column = project.columnModel.getColumnByName(_columnName);
ReconCandidate candidate = match.getCandidate();
return new RowVisitor() {
int cellIndex;
@ -188,7 +219,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
identifierSpace,
schemaSpace);
newRecon.match = match;
newRecon.match = candidate;
newRecon.matchRank = -1;
newRecon.judgment = Judgment.Matched;
newRecon.judgmentAction = "mass";

12
main/src/com/google/refine/operations/recon/ReconOperation.java
View File

@ -45,6 +45,8 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -120,6 +122,16 @@ public class ReconOperation extends EngineDependentOperation {
writer.key("engineConfig"); getEngineConfig().write(writer, options);
writer.endObject();
}
@JsonProperty("config")
public ReconConfig getReconConfig() {
return _reconConfig;
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;
}
static protected class ReconEntry {
final public int rowIndex;

55
main/tests/server/src/com/google/refine/tests/operations/recon/ExtendDataOperationTests.java
View File

@ -36,9 +36,12 @@ package com.google.refine.tests.operations.recon;
import static org.mockito.Mockito.mock;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.json.JSONException;
import org.json.JSONObject;
@ -57,11 +60,13 @@ import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Row;
import com.google.refine.process.Process;
import com.google.refine.process.ProcessManager;
import com.google.refine.model.recon.ReconciledDataExtensionJob;
import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.operations.recon.ExtendDataOperation;
import com.google.refine.process.Process;
import com.google.refine.process.ProcessManager;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
@ -72,6 +77,26 @@ public class ExtendDataOperationTests extends RefineTest {
static final String RECON_SERVICE = "https://tools.wmflabs.org/openrefine-wikidata/en/api";
static final String RECON_IDENTIFIER_SPACE = "http://www.wikidata.org/entity/";
static final String RECON_SCHEMA_SPACE = "http://www.wikidata.org/prop/direct/";
private String dataExtensionConfigJson = "{"
+ " \"properties\":["
+ " {\"name\":\"inception\",\"id\":\"P571\"},"
+ " {\"name\":\"headquarters location\",\"id\":\"P159\"},"
+ " {\"name\":\"coordinate location\",\"id\":\"P625\"}"
+ " ]"
+ "}";
static public class ReconciledDataExtensionJobStub extends ReconciledDataExtensionJob {
public ReconciledDataExtensionJobStub(DataExtensionConfig obj, String endpoint) throws JSONException {
super(obj, endpoint);
}
public String formulateQueryStub(Set<String> ids, DataExtensionConfig node) throws JSONException {
StringWriter writer = new StringWriter();
super.formulateQuery(ids, node, writer);
return writer.toString();
}
}
@Override
@BeforeTest
@ -132,6 +157,21 @@ public class ExtendDataOperationTests extends RefineTest {
+ "}}";
TestUtils.isSerializedTo(ExtendDataOperation.reconstruct(project, new JSONObject(json)), json);
}
@Test
public void serializeDataExtensionConfig() {
TestUtils.isSerializedTo(DataExtensionConfig.reconstruct(new JSONObject(dataExtensionConfigJson)), dataExtensionConfigJson);
}
@Test
public void testFormulateQuery() {
DataExtensionConfig config = DataExtensionConfig.reconstruct(new JSONObject(dataExtensionConfigJson));
Set<String> ids = Collections.singleton("Q2");
String json = "{\"ids\":[\"Q2\"],\"properties\":[{\"id\":\"P571\"},{\"id\":\"P159\"},{\"id\":\"P625\"}]}";
ReconciledDataExtensionJobStub stub = new ReconciledDataExtensionJobStub(config, "http://endpoint");
TestUtils.assertEqualAsJson(json, stub.formulateQueryStub(ids, config));
}
@AfterMethod
public void TearDown() {
@ -157,7 +197,7 @@ public class ExtendDataOperationTests extends RefineTest {
@Test
public void testFetchStrings() throws Exception {
JSONObject extension = new JSONObject("{\"properties\":[{\"id\":\"P297\",\"name\":\"ISO 3166-1 alpha-2 code\"}]}");
DataExtensionConfig extension = DataExtensionConfig.reconstruct(new JSONObject("{\"properties\":[{\"id\":\"P297\",\"name\":\"ISO 3166-1 alpha-2 code\"}]}"));
EngineDependentOperation op = new ExtendDataOperation(engine_config,
"country",
@ -194,7 +234,8 @@ public class ExtendDataOperationTests extends RefineTest {
@Test
public void testFetchCounts() throws Exception {
JSONObject extension = new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}");
DataExtensionConfig extension = DataExtensionConfig.reconstruct(
new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}"));
EngineDependentOperation op = new ExtendDataOperation(engine_config,
"country",
@ -228,7 +269,8 @@ public class ExtendDataOperationTests extends RefineTest {
*/
@Test
public void testFetchCurrent() throws Exception {
JSONObject extension = new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"best\"}}]}");
DataExtensionConfig extension = DataExtensionConfig.reconstruct(
new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"best\"}}]}"));
EngineDependentOperation op = new ExtendDataOperation(engine_config,
"country",
@ -268,7 +310,8 @@ public class ExtendDataOperationTests extends RefineTest {
*/
@Test
public void testFetchRecord() throws Exception {
JSONObject extension = new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"any\"}}]}");
DataExtensionConfig extension = DataExtensionConfig.reconstruct(
new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"any\"}}]}"));
EngineDependentOperation op = new ExtendDataOperation(engine_config,
"country",

6
main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsOperationTests.java
View File

@ -1,11 +1,8 @@
package com.google.refine.tests.operations.recon;
import static org.mockito.Mockito.mock;
import org.json.JSONObject;
import org.testng.annotations.BeforeSuite;
import org.testng.annotations.Test;
import com.google.refine.model.Project;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.operations.recon.ReconJudgeSimilarCellsOperation;
import com.google.refine.tests.RefineTest;
@ -27,7 +24,6 @@ public class ReconJudgeSimilarCellsOperationTests extends RefineTest {
+ "\"judgment\":\"matched\","
+ "\"match\":{\"id\":\"Q7884717\",\"name\":\"Unicef Indonesia\",\"score\":71.42857142857143,\"types\":[\"Q43229\"]},"
+ "\"shareNewTopics\":false}";
Project project = mock(Project.class);
TestUtils.isSerializedTo(ReconJudgeSimilarCellsOperation.reconstruct(project, new JSONObject(json)), json);
TestUtils.isSerializedTo(ReconJudgeSimilarCellsOperation.reconstruct(new JSONObject(json)), json);
}
}

14
main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsTests.java
View File

@ -6,6 +6,7 @@ import static org.junit.Assert.assertNull;
import java.util.Collections;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
import org.testng.annotations.BeforeTest;
@ -37,20 +38,15 @@ public class ReconJudgeSimilarCellsTests extends RefineTest {
}
@Test
public void serializeReconJudgeSimilarCellsOperation() {
AbstractOperation op = new ReconJudgeSimilarCellsOperation(
ENGINE_CONFIG,
"A",
"foo",
Recon.Judgment.New,
null, true);
TestUtils.isSerializedTo(op, "{\"op\":\"core/recon-judge-similar-cells\","
public void serializeReconJudgeSimilarCellsOperation() throws JSONException {
String json = "{\"op\":\"core/recon-judge-similar-cells\","
+ "\"description\":\"Mark to create one single new item for all cells containing \\\"foo\\\" in column A\","
+ "\"engineConfig\":{\"mode\":\"row-based\",\"facets\":[]},"
+ "\"columnName\":\"A\","
+ "\"similarValue\":\"foo\","
+ "\"judgment\":\"new\","
+ "\"shareNewTopics\":true}");
+ "\"shareNewTopics\":true}";
TestUtils.isSerializedTo(ReconJudgeSimilarCellsOperation.reconstruct(new JSONObject(json)), json);
}
@Test

3
main/tests/server/src/com/google/refine/tests/operations/recon/ReconOperationTests.java
View File

@ -8,6 +8,8 @@ import org.testng.annotations.BeforeSuite;
import org.testng.annotations.Test;
import com.google.refine.model.Project;
import com.google.refine.model.recon.ReconConfig;
import com.google.refine.model.recon.StandardReconConfig;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.operations.recon.ReconOperation;
import com.google.refine.tests.RefineTest;
@ -18,6 +20,7 @@ public class ReconOperationTests extends RefineTest {
@BeforeSuite
public void registerOperation() {
OperationRegistry.registerOperation(getCoreModule(), "recon", ReconOperation.class);
ReconConfig.registerReconConfig(getCoreModule(), "standard-service", StandardReconConfig.class);
}
@Test