From a3bba83c630d9ad3c40f5b6188184afff31781ec Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sun, 23 Sep 2018 16:52:07 +0100 Subject: [PATCH] Jackson serialization for column operations --- .../AddColumnByFetchingURLsCommand.java | 11 +- .../com/google/refine/operations/OnError.java | 5 + ...ColumnAdditionByFetchingURLsOperation.java | 102 ++++++++++++++++-- .../column/ColumnAdditionOperation.java | 27 +++++ .../column/ColumnMoveOperation.java | 13 ++- .../column/ColumnRemovalOperation.java | 7 ++ .../column/ColumnRenameOperation.java | 11 ++ .../column/ColumnReorderOperation.java | 7 ++ .../column/ColumnSplitOperation.java | 56 +++++++++- ...nAdditionByFetchingURLsOperationTests.java | 15 +-- .../column/ColumnSplitOperationTests.java | 21 +++- 11 files changed, 248 insertions(+), 27 deletions(-) diff --git a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java index fa0112e25..13c7ea762 100644 --- a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java +++ b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java @@ -33,9 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.commands.column; +import java.util.Arrays; +import java.util.List; + import javax.servlet.http.HttpServletRequest; -import org.json.JSONArray; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.refine.browsing.EngineConfig; import com.google.refine.commands.EngineDependentCommand; @@ -43,6 +46,7 @@ import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; import com.google.refine.operations.cell.TextTransformOperation; import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation; +import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation.HttpHeader; public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { @Override @@ -56,7 +60,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { int delay = Integer.parseInt(request.getParameter("delay")); String onError = request.getParameter("onError"); boolean cacheResponses = Boolean.parseBoolean(request.getParameter("cacheResponses")); - JSONArray httpHeadersJson = new JSONArray(request.getParameter("httpHeaders")); + ObjectMapper mapper = new ObjectMapper(); + List headers = Arrays.asList(mapper.readValue(request.getParameter("httpHeaders"), HttpHeader[].class)); return new ColumnAdditionByFetchingURLsOperation( engineConfig, @@ -67,7 +72,7 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { columnInsertIndex, delay, cacheResponses, - httpHeadersJson + headers ); } diff --git a/main/src/com/google/refine/operations/OnError.java b/main/src/com/google/refine/operations/OnError.java index 3fc0c7695..a43b604dd 100644 --- a/main/src/com/google/refine/operations/OnError.java +++ b/main/src/com/google/refine/operations/OnError.java @@ -33,8 +33,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.operations; +import com.fasterxml.jackson.annotation.JsonProperty; + public enum OnError { + @JsonProperty("keep-original") KeepOriginal, + @JsonProperty("set-to-blank") SetToBlank, + @JsonProperty("store-error") StoreError } \ No newline at end of file diff --git a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java index fa6a6f12a..8af2c0dcf 100644 --- a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java @@ -50,10 +50,13 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.google.refine.Jsonizable; import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; @@ -81,6 +84,27 @@ import com.google.refine.util.ParsingUtilities; public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperation { + public static final class HttpHeader implements Jsonizable { + @JsonProperty("name") + final public String name; + @JsonProperty("value") + final public String value; + + public HttpHeader(String name, String value) { + this.name = name; + this.value = value; + } + + @Override + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("name"); writer.value(name); + writer.key("value"); writer.value(value); + writer.endObject(); + } + } + final protected String _baseColumnName; final protected String _urlExpression; final protected OnError _onError; @@ -89,10 +113,21 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat final protected int _columnInsertIndex; final protected int _delay; final protected boolean _cacheResponses; - final protected JSONArray _httpHeadersJson; + final protected List _httpHeadersJson; static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + List headers = null; + JSONArray headersJson = obj.optJSONArray("httpHeadersJson"); + if (headersJson != null) { + headers = new ArrayList<>(headersJson.length()); + for (int i = 0; i < headersJson.length(); i++) { + String headerLabel = headersJson.getJSONObject(i).getString("name"); + String headerValue = headersJson.getJSONObject(i).getString("value"); + headers.add(new HttpHeader(headerLabel, headerValue)); + } + } return new ColumnAdditionByFetchingURLsOperation( EngineConfig.reconstruct(engineConfig), @@ -103,7 +138,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat obj.getInt("columnInsertIndex"), obj.getInt("delay"), obj.optBoolean("cacheResponses", false), // false for retro-compatibility - obj.optJSONArray("httpHeadersJson") // will be null if it doesn't exist for retro-compatibility + headers // will be null if it doesn't exist for retro-compatibility ); } @@ -116,7 +151,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat int columnInsertIndex, int delay, boolean cacheResponses, - JSONArray httpHeadersJson + List httpHeadersJson ) { super(engineConfig); @@ -147,9 +182,56 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError)); writer.key("delay"); writer.value(_delay); writer.key("cacheResponses"); writer.value(_cacheResponses); - writer.key("httpHeadersJson"); writer.value(_httpHeadersJson); + if (_httpHeadersJson != null) { + writer.key("httpHeadersJson"); + writer.array(); + for(HttpHeader header : _httpHeadersJson) { + header.write(writer, options); + } + writer.endArray(); + } writer.endObject(); } + + @JsonProperty("newColumnName") + public String getNewColumnName() { + return _newColumnName; + } + + @JsonProperty("columnInsertIndex") + public int getColumnInsertIndex() { + return _columnInsertIndex; + } + + @JsonProperty("baseColumnName") + public String getBaseColumnName() { + return _baseColumnName; + } + + @JsonProperty("urlExpression") + public String getUrlExpression() { + return _urlExpression; + } + + @JsonProperty("onError") + public OnError getOnError() { + return _onError; + } + + @JsonProperty("delay") + public int getDelay() { + return _delay; + } + + @JsonProperty("httpHeadersJson") + public List getHttpHeadersJson() { + return _httpHeadersJson; + } + + @JsonProperty("cacheResponses") + public boolean getCacheResponses() { + return _cacheResponses; + } @Override protected String getBriefDescription(Project project) { @@ -179,8 +261,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat engine, eval, getBriefDescription(null), - _cacheResponses, - _httpHeadersJson + _cacheResponses ); } @@ -197,8 +278,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat Engine engine, Evaluable eval, String description, - boolean cacheResponses, - JSONArray httpHeadersJson + boolean cacheResponses ) throws JSONException { super(description); _project = project; @@ -335,9 +415,9 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat try { URLConnection urlConnection = url.openConnection(); if (_httpHeadersJson != null) { - for (int i = 0; i < _httpHeadersJson.length(); i++) { - String headerLabel = _httpHeadersJson.getJSONObject(i).getString("name"); - String headerValue = _httpHeadersJson.getJSONObject(i).getString("value"); + for (int i = 0; i < _httpHeadersJson.size(); i++) { + String headerLabel = _httpHeadersJson.get(i).name; + String headerValue = _httpHeadersJson.get(i).value; if (headerValue != null && !headerValue.isEmpty()) { urlConnection.setRequestProperty(headerLabel, headerValue); } diff --git a/main/src/com/google/refine/operations/column/ColumnAdditionOperation.java b/main/src/com/google/refine/operations/column/ColumnAdditionOperation.java index cf2d3b10b..0d4334a07 100644 --- a/main/src/com/google/refine/operations/column/ColumnAdditionOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnAdditionOperation.java @@ -42,6 +42,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; @@ -118,7 +120,32 @@ public class ColumnAdditionOperation extends EngineDependentOperation { writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError)); writer.endObject(); } + + @JsonProperty("newColumnName") + public String getNewColumnName() { + return _newColumnName; + } + + @JsonProperty("columnInsertIndex") + public int getColumnInsertIndex() { + return _columnInsertIndex; + } + + @JsonProperty("baseColumnName") + public String getBaseColumnName() { + return _baseColumnName; + } + + @JsonProperty("expression") + public String getExpression() { + return _expression; + } + @JsonProperty("onError") + public OnError getOnError() { + return _onError; + } + @Override protected String getBriefDescription(Project project) { return "Create column " + _newColumnName + diff --git a/main/src/com/google/refine/operations/column/ColumnMoveOperation.java b/main/src/com/google/refine/operations/column/ColumnMoveOperation.java index c4f18a3fc..a000859da 100644 --- a/main/src/com/google/refine/operations/column/ColumnMoveOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnMoveOperation.java @@ -39,6 +39,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.history.Change; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; @@ -76,7 +78,16 @@ public class ColumnMoveOperation extends AbstractOperation { writer.key("index"); writer.value(_index); writer.endObject(); } - + + @JsonProperty("columnName") + public String getColumnName() { + return _columnName; + } + + @JsonProperty("index") + public int getIndex() { + return _index; + } @Override protected String getBriefDescription(Project project) { diff --git a/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java b/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java index 54fb8116a..5de7a5895 100644 --- a/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnRemovalOperation.java @@ -39,6 +39,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.history.Change; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; @@ -72,6 +74,11 @@ public class ColumnRemovalOperation extends AbstractOperation { writer.key("columnName"); writer.value(_columnName); writer.endObject(); } + + @JsonProperty("columnName") + public String getColumnName() { + return _columnName; + } @Override diff --git a/main/src/com/google/refine/operations/column/ColumnRenameOperation.java b/main/src/com/google/refine/operations/column/ColumnRenameOperation.java index 846f66fbc..0d4b85c9a 100644 --- a/main/src/com/google/refine/operations/column/ColumnRenameOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnRenameOperation.java @@ -39,6 +39,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.history.Change; import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; @@ -76,7 +78,16 @@ public class ColumnRenameOperation extends AbstractOperation { writer.key("newColumnName"); writer.value(_newColumnName); writer.endObject(); } + + @JsonProperty("oldColumnName") + public String getOldColumnName() { + return _oldColumnName; + } + @JsonProperty("newColumnName") + public String getNewColumnName() { + return _newColumnName; + } @Override protected String getBriefDescription(Project project) { diff --git a/main/src/com/google/refine/operations/column/ColumnReorderOperation.java b/main/src/com/google/refine/operations/column/ColumnReorderOperation.java index 4920b2e5e..3b7f906a8 100644 --- a/main/src/com/google/refine/operations/column/ColumnReorderOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnReorderOperation.java @@ -41,6 +41,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.history.HistoryEntry; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; @@ -77,6 +79,11 @@ public class ColumnReorderOperation extends AbstractOperation { writer.endArray(); writer.endObject(); } + + @JsonProperty("columnNames") + public List getColumnNames() { + return _columnNames; + } @Override protected String getBriefDescription(Project project) { diff --git a/main/src/com/google/refine/operations/column/ColumnSplitOperation.java b/main/src/com/google/refine/operations/column/ColumnSplitOperation.java index f4c7aeeea..adc7d1860 100644 --- a/main/src/com/google/refine/operations/column/ColumnSplitOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnSplitOperation.java @@ -44,6 +44,10 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.FilteredRows; @@ -68,8 +72,8 @@ public class ColumnSplitOperation extends EngineDependentOperation { final protected String _mode; final protected String _separator; - final protected boolean _regex; - final protected int _maxColumns; + final protected Boolean _regex; + final protected Integer _maxColumns; final protected int[] _fieldLengths; @@ -136,8 +140,8 @@ public class ColumnSplitOperation extends EngineDependentOperation { _mode = "lengths"; _separator = null; - _regex = false; - _maxColumns = -1; + _regex = null; + _maxColumns = null; _fieldLengths = fieldLengths; } @@ -167,6 +171,50 @@ public class ColumnSplitOperation extends EngineDependentOperation { } writer.endObject(); } + + @JsonProperty("columnName") + public String getColumnName() { + return _columnName; + } + + @JsonProperty("guessCellType") + public boolean getGuessCellType() { + return _guessCellType; + } + + @JsonProperty("removeOriginalColumn") + public boolean getRemoveOriginalColumn() { + return _removeOriginalColumn; + } + + @JsonProperty("mode") + public String getMode() { + return _mode; + } + + @JsonProperty("separator") + @JsonInclude(Include.NON_NULL) + public String getSeparator() { + return _separator; + } + + @JsonProperty("regex") + @JsonInclude(Include.NON_NULL) + public Boolean getRegex() { + return _regex; + } + + @JsonProperty("maxColumns") + @JsonInclude(Include.NON_NULL) + public Integer getMaxColumns() { + return _maxColumns; + } + + @JsonProperty("fieldLengths") + @JsonInclude(Include.NON_NULL) + public int[] getFieldLengths() { + return _fieldLengths; + } @Override protected String getBriefDescription(Project project) { diff --git a/main/tests/server/src/com/google/refine/tests/operations/column/ColumnAdditionByFetchingURLsOperationTests.java b/main/tests/server/src/com/google/refine/tests/operations/column/ColumnAdditionByFetchingURLsOperationTests.java index cbef00f53..7d29fd608 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/column/ColumnAdditionByFetchingURLsOperationTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/column/ColumnAdditionByFetchingURLsOperationTests.java @@ -35,9 +35,10 @@ package com.google.refine.tests.operations.column; import java.io.IOException; import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; -import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.LoggerFactory; @@ -56,6 +57,7 @@ import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.OnError; import com.google.refine.operations.OperationRegistry; import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation; +import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation.HttpHeader; import com.google.refine.process.Process; import com.google.refine.process.ProcessManager; import com.google.refine.tests.RefineTest; @@ -233,11 +235,10 @@ public class ColumnAdditionByFetchingURLsOperationTests extends RefineTest { String userAgentValue = "OpenRefine"; String authorizationValue = "Basic"; String acceptValue = "*/*"; - String jsonString = "[{\"name\": \"authorization\",\"value\": \""+authorizationValue+ - "\"},{\"name\": \"user-agent\",\"value\": \""+userAgentValue+ - "\"},{\"name\": \"accept\",\"value\": \""+acceptValue+"\"}]"; - - JSONArray httpHeadersJson = new JSONArray(jsonString); + List headers = new ArrayList<>(); + headers.add(new HttpHeader("authorization", authorizationValue)); + headers.add(new HttpHeader("user-agent", userAgentValue)); + headers.add(new HttpHeader("accept", acceptValue)); EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config, "fruits", @@ -247,7 +248,7 @@ public class ColumnAdditionByFetchingURLsOperationTests extends RefineTest { 1, 50, true, - httpHeadersJson); + headers); ProcessManager pm = project.getProcessManager(); Process process = op.createProcess(project, options); process.startPerforming(pm); diff --git a/main/tests/server/src/com/google/refine/tests/operations/column/ColumnSplitOperationTests.java b/main/tests/server/src/com/google/refine/tests/operations/column/ColumnSplitOperationTests.java index 1aa3a997c..57949edd9 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/column/ColumnSplitOperationTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/column/ColumnSplitOperationTests.java @@ -20,7 +20,7 @@ public class ColumnSplitOperationTests extends RefineTest { } @Test - public void serializeColumnSplitOperation() throws JSONException, Exception { + public void serializeColumnSplitOperationBySeparator() throws JSONException, Exception { String json = "{\n" + " \"op\": \"core/column-split\",\n" + " \"description\": \"Split column ea by separator\",\n" + @@ -39,4 +39,23 @@ public class ColumnSplitOperationTests extends RefineTest { Project project = mock(Project.class); TestUtils.isSerializedTo(ColumnSplitOperation.reconstruct(project, new JSONObject(json)), json); } + + @Test + public void serializeColumnSplitOperationByLengths() throws JSONException, Exception { + String json = "{\n" + + " \"op\": \"core/column-split\",\n" + + " \"description\": \"Split column ea by field lengths\",\n" + + " \"engineConfig\": {\n" + + " \"mode\": \"row-based\",\n" + + " \"facets\": []\n" + + " },\n" + + " \"columnName\": \"ea\",\n" + + " \"guessCellType\": true,\n" + + " \"removeOriginalColumn\": true,\n" + + " \"mode\": \"lengths\",\n" + + " \"fieldLengths\": [1,1]\n" + + " }"; + Project project = mock(Project.class); + TestUtils.isSerializedTo(ColumnSplitOperation.reconstruct(project, new JSONObject(json)), json); + } }