diff --git a/main/src/com/google/refine/commands/recon/ExtendDataCommand.java b/main/src/com/google/refine/commands/recon/ExtendDataCommand.java index 292ae79f2..30adcb526 100644 --- a/main/src/com/google/refine/commands/recon/ExtendDataCommand.java +++ b/main/src/com/google/refine/commands/recon/ExtendDataCommand.java @@ -35,13 +35,12 @@ package com.google.refine.commands.recon; import javax.servlet.http.HttpServletRequest; -import org.json.JSONObject; - import com.google.refine.browsing.EngineConfig; import com.google.refine.commands.EngineDependentCommand; -import com.google.refine.operations.recon.ExtendDataOperation; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; +import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig; +import com.google.refine.operations.recon.ExtendDataOperation; import com.google.refine.util.ParsingUtilities; public class ExtendDataCommand extends EngineDependentCommand { @@ -56,7 +55,7 @@ public class ExtendDataCommand extends EngineDependentCommand { String schemaSpace = request.getParameter("schemaSpace"); String jsonString = request.getParameter("extension"); - JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString); + DataExtensionConfig extension = DataExtensionConfig.reconstruct(ParsingUtilities.evaluateJsonStringToObject(jsonString)); return new ExtendDataOperation( engineConfig, diff --git a/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java b/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java index 037b1e87d..f0c6906a7 100644 --- a/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java +++ b/main/src/com/google/refine/commands/recon/PreviewExtendDataCommand.java @@ -46,19 +46,19 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.json.JSONArray; -import org.json.JSONObject; import org.json.JSONWriter; import com.google.refine.commands.Command; -import com.google.refine.model.recon.ReconciledDataExtensionJob; -import com.google.refine.model.recon.ReconciledDataExtensionJob.ColumnInfo; -import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension; import com.google.refine.model.Cell; +import com.google.refine.model.Column; import com.google.refine.model.Project; import com.google.refine.model.ReconCandidate; import com.google.refine.model.Row; -import com.google.refine.model.Column; import com.google.refine.model.recon.ReconConfig; +import com.google.refine.model.recon.ReconciledDataExtensionJob; +import com.google.refine.model.recon.ReconciledDataExtensionJob.ColumnInfo; +import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension; +import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig; import com.google.refine.model.recon.StandardReconConfig; import com.google.refine.util.ParsingUtilities; @@ -79,7 +79,7 @@ public class PreviewExtendDataCommand extends Command { } String jsonString = request.getParameter("extension"); - JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); + DataExtensionConfig config = DataExtensionConfig.reconstruct(ParsingUtilities.evaluateJsonStringToObject(jsonString)); JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString); int length = rowIndices.length(); @@ -120,7 +120,7 @@ public class PreviewExtendDataCommand extends Command { } Map reconCandidateMap = new HashMap(); - ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(json, endpoint); + ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(config, endpoint); Map map = job.extend(ids, reconCandidateMap); response.setCharacterEncoding("UTF-8"); diff --git a/main/src/com/google/refine/commands/recon/ReconMatchSpecificTopicCommand.java b/main/src/com/google/refine/commands/recon/ReconMatchSpecificTopicCommand.java index 18ca0b9f5..0951f1c27 100644 --- a/main/src/com/google/refine/commands/recon/ReconMatchSpecificTopicCommand.java +++ b/main/src/com/google/refine/commands/recon/ReconMatchSpecificTopicCommand.java @@ -39,7 +39,6 @@ import com.google.refine.browsing.EngineConfig; import com.google.refine.commands.EngineDependentCommand; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; -import com.google.refine.model.ReconCandidate; import com.google.refine.operations.recon.ReconMatchSpecificTopicOperation; public class ReconMatchSpecificTopicCommand extends EngineDependentCommand { @@ -49,11 +48,10 @@ public class ReconMatchSpecificTopicCommand extends EngineDependentCommand { HttpServletRequest request, EngineConfig engineConfig) throws Exception { String columnName = request.getParameter("columnName"); - ReconCandidate match = new ReconCandidate( + ReconMatchSpecificTopicOperation.ReconItem match = new ReconMatchSpecificTopicOperation.ReconItem( request.getParameter("topicID"), request.getParameter("topicName"), - request.getParameter("types").split(","), - 100 + request.getParameter("types").split(",") ); return new ReconMatchSpecificTopicOperation( diff --git a/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java b/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java index 07f994842..2d971a06d 100644 --- a/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java +++ b/main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java @@ -39,7 +39,6 @@ package com.google.refine.model.recon; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.Serializable; import java.io.StringWriter; import java.io.Writer; import java.net.URL; @@ -48,21 +47,155 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; +import java.util.stream.Collectors; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; -import com.google.refine.model.ReconType; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.google.refine.Jsonizable; +import com.google.refine.expr.functions.ToDate; import com.google.refine.model.ReconCandidate; -import com.google.refine.model.recon.StandardReconConfig; +import com.google.refine.model.ReconType; import com.google.refine.util.JSONUtilities; import com.google.refine.util.ParsingUtilities; -import com.google.refine.expr.functions.ToDate; public class ReconciledDataExtensionJob { + + + static public class DataExtensionProperty implements Jsonizable { + @JsonProperty("id") + public final String id; + @JsonProperty("name") + public final String name; + @JsonProperty("settings") + @JsonInclude(Include.NON_NULL) + public final Map settings; + + @JsonCreator + public DataExtensionProperty( + @JsonProperty("id") + String id, + @JsonProperty("name") + String name, + @JsonProperty("settings") + Map settings) { + this.id = id; + this.name = name; + this.settings = settings; + } + + @Override + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("id"); writer.value(id); + if(!"query".equals(options.getProperty("mode"))) { + writer.key("name"); writer.value(name); + } + if (settings != null) { + writer.key("settings"); + writer.object(); + for(Map.Entry entry : settings.entrySet()) { + writer.key(entry.getKey()); + writer.value(entry.getValue()); + } + writer.endObject(); + } + writer.endObject(); + } + + } + + static public class DataExtensionConfig implements Jsonizable { + + @JsonProperty("properties") + public final List properties; + + @JsonCreator + public DataExtensionConfig( + @JsonProperty("properties") + List properties) { + this.properties = properties; + } + + public static DataExtensionConfig reconstruct(JSONObject obj) throws JSONException { + ObjectMapper mapper = new ObjectMapper(); + try { + return mapper.readValue(obj.toString(), DataExtensionConfig.class); + } catch(IOException e) { + throw new JSONException(e.toString()); + } + } + + @Override + public void write(JSONWriter jsonWriter, Properties options) + throws JSONException { + jsonWriter.object(); + jsonWriter.key("properties"); + jsonWriter.array(); + + for (DataExtensionProperty property : properties) { + property.write(jsonWriter, options); + } + jsonWriter.endArray(); + jsonWriter.endObject(); + } + + } + + static public class DataExtensionQuery extends DataExtensionConfig { + + @JsonProperty("ids") + public final List ids; + + @JsonCreator + public DataExtensionQuery( + @JsonProperty("ids") + List ids, + @JsonProperty("properties") + List properties) { + super(properties); + this.ids = ids; + } + + @Override + public void write(JSONWriter jsonWriter, Properties options) + throws JSONException { + jsonWriter.object(); + + if(ids != null) { + jsonWriter.key("ids"); + jsonWriter.array(); + for (String id : ids) { + if (id != null) { + jsonWriter.value(id); + } + } + jsonWriter.endArray(); + } + + jsonWriter.key("properties"); + jsonWriter.array(); + + for (DataExtensionProperty property : properties) { + property.write(jsonWriter, options); + } + jsonWriter.endArray(); + jsonWriter.endObject(); + } + + } + static public class DataExtension { final public Object[][] data; @@ -83,11 +216,11 @@ public class ReconciledDataExtensionJob { } } - final public JSONObject extension; - final public String endpoint; - final public List columns = new ArrayList(); + final public DataExtensionConfig extension; + final public String endpoint; + final public List columns = new ArrayList(); - public ReconciledDataExtensionJob(JSONObject obj, String endpoint) throws JSONException { + public ReconciledDataExtensionJob(DataExtensionConfig obj, String endpoint) throws JSONException { this.extension = obj; this.endpoint = endpoint; } @@ -249,39 +382,12 @@ public class ReconciledDataExtensionJob { } - static protected void formulateQuery(Set ids, JSONObject node, Writer writer) throws JSONException { + static protected void formulateQuery(Set ids, DataExtensionConfig node, Writer writer) throws JSONException { JSONWriter jsonWriter = new JSONWriter(writer); - - jsonWriter.object(); - - jsonWriter.key("ids"); - jsonWriter.array(); - for (String id : ids) { - if (id != null) { - jsonWriter.value(id); - } - } - jsonWriter.endArray(); - - jsonWriter.key("properties"); - jsonWriter.array(); - JSONArray properties = node.getJSONArray("properties"); - int l = properties.length(); - - for (int i = 0; i < l; i++) { - JSONObject property = properties.getJSONObject(i); - jsonWriter.object(); - jsonWriter.key("id"); - jsonWriter.value(property.getString("id")); - if (property.has("settings")) { - JSONObject settings = property.getJSONObject("settings"); - jsonWriter.key("settings"); - jsonWriter.value(settings); - } - jsonWriter.endObject(); - } - jsonWriter.endArray(); - jsonWriter.endObject(); + Properties options = new Properties(); + DataExtensionQuery query = new DataExtensionQuery(ids.stream().collect(Collectors.toList()), node.properties); + options.setProperty("mode", "query"); + query.write(jsonWriter, options); } static protected void gatherColumnInfo(JSONArray meta, List columns) throws JSONException { diff --git a/main/src/com/google/refine/operations/recon/ExtendDataOperation.java b/main/src/com/google/refine/operations/recon/ExtendDataOperation.java index ae00cb5fd..4905bc682 100644 --- a/main/src/com/google/refine/operations/recon/ExtendDataOperation.java +++ b/main/src/com/google/refine/operations/recon/ExtendDataOperation.java @@ -45,6 +45,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; @@ -62,41 +64,51 @@ import com.google.refine.model.changes.DataExtensionChange; import com.google.refine.model.recon.ReconciledDataExtensionJob; import com.google.refine.model.recon.ReconciledDataExtensionJob.ColumnInfo; import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension; +import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.OperationRegistry; import com.google.refine.process.LongRunningProcess; import com.google.refine.process.Process; public class ExtendDataOperation extends EngineDependentOperation { - final protected String _baseColumnName; - final protected String _endpoint; - final protected String _identifierSpace; - final protected String _schemaSpace; - final protected JSONObject _extension; - final protected int _columnInsertIndex; + @JsonProperty("baseColumnName") + final protected String _baseColumnName; + @JsonProperty("endpoint") + final protected String _endpoint; + @JsonProperty("identifierSpace") + final protected String _identifierSpace; + @JsonProperty("schemaSpace") + final protected String _schemaSpace; + @JsonProperty("extension") + final protected DataExtensionConfig _extension; + @JsonProperty("columnInsertIndex") + final protected int _columnInsertIndex; + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { JSONObject engineConfig = obj.getJSONObject("engineConfig"); + DataExtensionConfig dataExtensionConfig = DataExtensionConfig.reconstruct(obj.getJSONObject("extension")); + return new ExtendDataOperation( EngineConfig.reconstruct(engineConfig), obj.getString("baseColumnName"), obj.getString("endpoint"), obj.getString("identifierSpace"), obj.getString("schemaSpace"), - obj.getJSONObject("extension"), + dataExtensionConfig, obj.getInt("columnInsertIndex") ); } public ExtendDataOperation( - EngineConfig engineConfig, - String baseColumnName, - String endpoint, - String identifierSpace, - String schemaSpace, - JSONObject extension, - int columnInsertIndex + EngineConfig engineConfig, + String baseColumnName, + String endpoint, + String identifierSpace, + String schemaSpace, + DataExtensionConfig extension, + int columnInsertIndex ) { super(engineConfig); @@ -121,7 +133,7 @@ public class ExtendDataOperation extends EngineDependentOperation { writer.key("endpoint"); writer.value(_endpoint); writer.key("identifierSpace"); writer.value(_identifierSpace); writer.key("schemaSpace"); writer.value(_schemaSpace); - writer.key("extension"); writer.value(_extension); + writer.key("extension"); _extension.write(writer, options); writer.endObject(); } diff --git a/main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java b/main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java index bebb80494..2b30ce3aa 100644 --- a/main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconClearSimilarCellsOperation.java @@ -40,6 +40,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; @@ -88,6 +90,16 @@ public class ReconClearSimilarCellsOperation extends EngineDependentMassCellOper writer.endObject(); } + @JsonProperty("columnName") + public String getColumnName() { + return _columnName; + } + + @JsonProperty("similarValue") + public String getSimilarValue() { + return _similarValue; + } + @Override protected String getBriefDescription(Project project) { return "Clear recon data for cells containing \"" + diff --git a/main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java b/main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java index 6141fa136..5d8c5c237 100644 --- a/main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconCopyAcrossColumnsOperation.java @@ -46,6 +46,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; @@ -118,6 +120,26 @@ public class ReconCopyAcrossColumnsOperation extends EngineDependentOperation { writer.key("applyToJudgedCells"); writer.value(_applyToJudgedCells); writer.endObject(); } + + @JsonProperty("fromColumnName") + public String getFromColumnName() { + return _fromColumnName; + } + + @JsonProperty("toColumnNames") + public String[] getToColumnNames() { + return _toColumnNames; + } + + @JsonProperty("judgments") + public String[] getJudgments() { + return _judgments; + } + + @JsonProperty("applyToJudgedCells") + public boolean getApplyToJudgedCells() { + return _applyToJudgedCells; + } @Override protected HistoryEntry createHistoryEntry(final Project project, final long historyEntryID) throws Exception { diff --git a/main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java b/main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java index 1873962a9..c95524805 100644 --- a/main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconDiscardJudgmentsOperation.java @@ -42,6 +42,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; @@ -86,6 +88,16 @@ public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOpera writer.key("clearData"); writer.value(_clearData); writer.endObject(); } + + @JsonProperty("columnName") + public String getColumnName() { + return _columnName; + } + + @JsonProperty("clearData") + public boolean getClearData() { + return _clearData; + } @Override protected String getBriefDescription(Project project) { diff --git a/main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java b/main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java index a3fc92e32..5afc6ce98 100644 --- a/main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconJudgeSimilarCellsOperation.java @@ -43,6 +43,10 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.expr.ExpressionUtils; @@ -67,7 +71,7 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper final protected ReconCandidate _match; final protected boolean _shareNewTopics; - static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + static public AbstractOperation reconstruct(JSONObject obj) throws JSONException { JSONObject engineConfig = obj.getJSONObject("engineConfig"); ReconCandidate match = null; @@ -137,6 +141,32 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper writer.endObject(); } + @JsonProperty("columnName") + public String getColumnName() { + return _columnName; + } + + @JsonProperty("similarValue") + public String getSimilarValue() { + return _similarValue; + } + + @JsonProperty("judgment") + public Judgment getJudgment() { + return _judgment; + } + + @JsonProperty("match") + @JsonInclude(Include.NON_NULL) + public ReconCandidate getMatch() { + return _match; + } + + @JsonProperty("shareNewTopics") + public boolean getShareNewTopics() { + return _shareNewTopics; + } + @Override protected String getBriefDescription(Project project) { if (_judgment == Judgment.None) { diff --git a/main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java b/main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java index cecba3833..363b52564 100644 --- a/main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconMarkNewTopicsOperation.java @@ -42,6 +42,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; @@ -89,6 +91,16 @@ public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperatio writer.endObject(); } + @JsonProperty("columnName") + public String getColumnName() { + return _columnName; + } + + @JsonProperty("shareNewTopics") + public boolean getShareNewTopics() { + return _shareNewTopics; + } + @Override protected String getBriefDescription(Project project) { return "Mark to create new items for cells in column " + _columnName + diff --git a/main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java b/main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java index 050eca7b3..c17bf4346 100644 --- a/main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconMatchBestCandidatesOperation.java @@ -42,6 +42,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; @@ -84,6 +86,11 @@ public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOp writer.key("columnName"); writer.value(_columnName); writer.endObject(); } + + @JsonProperty + public String getColumnName() { + return _columnName; + } @Override protected String getBriefDescription(Project project) { diff --git a/main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java b/main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java index 1bf466eb7..1dce15216 100644 --- a/main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconMatchSpecificTopicOperation.java @@ -43,6 +43,10 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.RowVisitor; import com.google.refine.history.Change; @@ -60,8 +64,35 @@ import com.google.refine.operations.EngineDependentMassCellOperation; import com.google.refine.operations.OperationRegistry; public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOperation { - final protected ReconCandidate match; + + public static class ReconItem { + @JsonProperty("id") + public final String id; + @JsonProperty("name") + public final String name; + @JsonProperty("types") + public final String[] types; + @JsonCreator + public ReconItem( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("types") String[] types) { + this.id = id; + this.name = name; + this.types = types; + } + + @JsonIgnore + public ReconCandidate getCandidate() { + return new ReconCandidate(id, name, types, 100); + } + } + + @JsonProperty("match") + final protected ReconItem match; + @JsonProperty("identifierSpace") final protected String identifierSpace; + @JsonProperty("schemaSpace") final protected String schemaSpace; static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { @@ -78,11 +109,10 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe return new ReconMatchSpecificTopicOperation( EngineConfig.reconstruct(engineConfig), obj.getString("columnName"), - new ReconCandidate( + new ReconItem( match.getString("id"), match.getString("name"), - typeIDs, - 100 + typeIDs ), obj.getString("identifierSpace"), obj.getString("schemaSpace") @@ -92,7 +122,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe public ReconMatchSpecificTopicOperation( EngineConfig engineConfig, String columnName, - ReconCandidate match, + ReconItem match, String identifierSpace, String schemaSpace ) { @@ -146,6 +176,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe @Override protected RowVisitor createRowVisitor(Project project, List cellChanges, long historyEntryID) throws Exception { Column column = project.columnModel.getColumnByName(_columnName); + ReconCandidate candidate = match.getCandidate(); return new RowVisitor() { int cellIndex; @@ -188,7 +219,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe identifierSpace, schemaSpace); - newRecon.match = match; + newRecon.match = candidate; newRecon.matchRank = -1; newRecon.judgment = Judgment.Matched; newRecon.judgmentAction = "mass"; diff --git a/main/src/com/google/refine/operations/recon/ReconOperation.java b/main/src/com/google/refine/operations/recon/ReconOperation.java index 87e9c4972..567fb6cef 100644 --- a/main/src/com/google/refine/operations/recon/ReconOperation.java +++ b/main/src/com/google/refine/operations/recon/ReconOperation.java @@ -45,6 +45,8 @@ import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; @@ -120,6 +122,16 @@ public class ReconOperation extends EngineDependentOperation { writer.key("engineConfig"); getEngineConfig().write(writer, options); writer.endObject(); } + + @JsonProperty("config") + public ReconConfig getReconConfig() { + return _reconConfig; + } + + @JsonProperty("columnName") + public String getColumnName() { + return _columnName; + } static protected class ReconEntry { final public int rowIndex; diff --git a/main/tests/server/src/com/google/refine/tests/operations/recon/ExtendDataOperationTests.java b/main/tests/server/src/com/google/refine/tests/operations/recon/ExtendDataOperationTests.java index d41618800..111da8f76 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/recon/ExtendDataOperationTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/recon/ExtendDataOperationTests.java @@ -36,9 +36,12 @@ package com.google.refine.tests.operations.recon; import static org.mockito.Mockito.mock; import java.io.IOException; +import java.io.StringWriter; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Properties; +import java.util.Set; import org.json.JSONException; import org.json.JSONObject; @@ -57,11 +60,13 @@ import com.google.refine.model.Project; import com.google.refine.model.Recon; import com.google.refine.model.ReconCandidate; import com.google.refine.model.Row; -import com.google.refine.process.Process; -import com.google.refine.process.ProcessManager; +import com.google.refine.model.recon.ReconciledDataExtensionJob; +import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.OperationRegistry; import com.google.refine.operations.recon.ExtendDataOperation; +import com.google.refine.process.Process; +import com.google.refine.process.ProcessManager; import com.google.refine.tests.RefineTest; import com.google.refine.tests.util.TestUtils; @@ -72,6 +77,26 @@ public class ExtendDataOperationTests extends RefineTest { static final String RECON_SERVICE = "https://tools.wmflabs.org/openrefine-wikidata/en/api"; static final String RECON_IDENTIFIER_SPACE = "http://www.wikidata.org/entity/"; static final String RECON_SCHEMA_SPACE = "http://www.wikidata.org/prop/direct/"; + + private String dataExtensionConfigJson = "{" + + " \"properties\":[" + + " {\"name\":\"inception\",\"id\":\"P571\"}," + + " {\"name\":\"headquarters location\",\"id\":\"P159\"}," + + " {\"name\":\"coordinate location\",\"id\":\"P625\"}" + + " ]" + + "}"; + + static public class ReconciledDataExtensionJobStub extends ReconciledDataExtensionJob { + public ReconciledDataExtensionJobStub(DataExtensionConfig obj, String endpoint) throws JSONException { + super(obj, endpoint); + } + + public String formulateQueryStub(Set ids, DataExtensionConfig node) throws JSONException { + StringWriter writer = new StringWriter(); + super.formulateQuery(ids, node, writer); + return writer.toString(); + } + } @Override @BeforeTest @@ -132,6 +157,21 @@ public class ExtendDataOperationTests extends RefineTest { + "}}"; TestUtils.isSerializedTo(ExtendDataOperation.reconstruct(project, new JSONObject(json)), json); } + + @Test + public void serializeDataExtensionConfig() { + TestUtils.isSerializedTo(DataExtensionConfig.reconstruct(new JSONObject(dataExtensionConfigJson)), dataExtensionConfigJson); + } + + @Test + public void testFormulateQuery() { + DataExtensionConfig config = DataExtensionConfig.reconstruct(new JSONObject(dataExtensionConfigJson)); + Set ids = Collections.singleton("Q2"); + String json = "{\"ids\":[\"Q2\"],\"properties\":[{\"id\":\"P571\"},{\"id\":\"P159\"},{\"id\":\"P625\"}]}"; + ReconciledDataExtensionJobStub stub = new ReconciledDataExtensionJobStub(config, "http://endpoint"); + TestUtils.assertEqualAsJson(json, stub.formulateQueryStub(ids, config)); + } + @AfterMethod public void TearDown() { @@ -157,7 +197,7 @@ public class ExtendDataOperationTests extends RefineTest { @Test public void testFetchStrings() throws Exception { - JSONObject extension = new JSONObject("{\"properties\":[{\"id\":\"P297\",\"name\":\"ISO 3166-1 alpha-2 code\"}]}"); + DataExtensionConfig extension = DataExtensionConfig.reconstruct(new JSONObject("{\"properties\":[{\"id\":\"P297\",\"name\":\"ISO 3166-1 alpha-2 code\"}]}")); EngineDependentOperation op = new ExtendDataOperation(engine_config, "country", @@ -194,7 +234,8 @@ public class ExtendDataOperationTests extends RefineTest { @Test public void testFetchCounts() throws Exception { - JSONObject extension = new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}"); + DataExtensionConfig extension = DataExtensionConfig.reconstruct( + new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}")); EngineDependentOperation op = new ExtendDataOperation(engine_config, "country", @@ -228,7 +269,8 @@ public class ExtendDataOperationTests extends RefineTest { */ @Test public void testFetchCurrent() throws Exception { - JSONObject extension = new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"best\"}}]}"); + DataExtensionConfig extension = DataExtensionConfig.reconstruct( + new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"best\"}}]}")); EngineDependentOperation op = new ExtendDataOperation(engine_config, "country", @@ -268,7 +310,8 @@ public class ExtendDataOperationTests extends RefineTest { */ @Test public void testFetchRecord() throws Exception { - JSONObject extension = new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"any\"}}]}"); + DataExtensionConfig extension = DataExtensionConfig.reconstruct( + new JSONObject("{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"any\"}}]}")); EngineDependentOperation op = new ExtendDataOperation(engine_config, "country", diff --git a/main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsOperationTests.java b/main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsOperationTests.java index 117193d58..4662cd016 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsOperationTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsOperationTests.java @@ -1,11 +1,8 @@ package com.google.refine.tests.operations.recon; -import static org.mockito.Mockito.mock; - import org.json.JSONObject; import org.testng.annotations.BeforeSuite; import org.testng.annotations.Test; -import com.google.refine.model.Project; import com.google.refine.operations.OperationRegistry; import com.google.refine.operations.recon.ReconJudgeSimilarCellsOperation; import com.google.refine.tests.RefineTest; @@ -27,7 +24,6 @@ public class ReconJudgeSimilarCellsOperationTests extends RefineTest { + "\"judgment\":\"matched\"," + "\"match\":{\"id\":\"Q7884717\",\"name\":\"Unicef Indonesia\",\"score\":71.42857142857143,\"types\":[\"Q43229\"]}," + "\"shareNewTopics\":false}"; - Project project = mock(Project.class); - TestUtils.isSerializedTo(ReconJudgeSimilarCellsOperation.reconstruct(project, new JSONObject(json)), json); + TestUtils.isSerializedTo(ReconJudgeSimilarCellsOperation.reconstruct(new JSONObject(json)), json); } } diff --git a/main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsTests.java b/main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsTests.java index 5d9f2e063..413797e7f 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/recon/ReconJudgeSimilarCellsTests.java @@ -6,6 +6,7 @@ import static org.junit.Assert.assertNull; import java.util.Collections; import java.util.Properties; +import org.json.JSONException; import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.annotations.BeforeTest; @@ -37,20 +38,15 @@ public class ReconJudgeSimilarCellsTests extends RefineTest { } @Test - public void serializeReconJudgeSimilarCellsOperation() { - AbstractOperation op = new ReconJudgeSimilarCellsOperation( - ENGINE_CONFIG, - "A", - "foo", - Recon.Judgment.New, - null, true); - TestUtils.isSerializedTo(op, "{\"op\":\"core/recon-judge-similar-cells\"," + public void serializeReconJudgeSimilarCellsOperation() throws JSONException { + String json = "{\"op\":\"core/recon-judge-similar-cells\"," + "\"description\":\"Mark to create one single new item for all cells containing \\\"foo\\\" in column A\"," + "\"engineConfig\":{\"mode\":\"row-based\",\"facets\":[]}," + "\"columnName\":\"A\"," + "\"similarValue\":\"foo\"," + "\"judgment\":\"new\"," - + "\"shareNewTopics\":true}"); + + "\"shareNewTopics\":true}"; + TestUtils.isSerializedTo(ReconJudgeSimilarCellsOperation.reconstruct(new JSONObject(json)), json); } @Test diff --git a/main/tests/server/src/com/google/refine/tests/operations/recon/ReconOperationTests.java b/main/tests/server/src/com/google/refine/tests/operations/recon/ReconOperationTests.java index d5f16b420..4da313b37 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/recon/ReconOperationTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/recon/ReconOperationTests.java @@ -8,6 +8,8 @@ import org.testng.annotations.BeforeSuite; import org.testng.annotations.Test; import com.google.refine.model.Project; +import com.google.refine.model.recon.ReconConfig; +import com.google.refine.model.recon.StandardReconConfig; import com.google.refine.operations.OperationRegistry; import com.google.refine.operations.recon.ReconOperation; import com.google.refine.tests.RefineTest; @@ -18,6 +20,7 @@ public class ReconOperationTests extends RefineTest { @BeforeSuite public void registerOperation() { OperationRegistry.registerOperation(getCoreModule(), "recon", ReconOperation.class); + ReconConfig.registerReconConfig(getCoreModule(), "standard-service", StandardReconConfig.class); } @Test