s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Jackson serialization for column operations

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-09-23 16:52:07 +01:00
parent 539585fc5e
commit a3bba83c63
11 changed files with 248 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java
View File

@ -33,9 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.commands.column;
import java.util.Arrays;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONArray;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.commands.EngineDependentCommand;
@ -43,6 +46,7 @@ import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.cell.TextTransformOperation;
import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation;
import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation.HttpHeader;
public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
@Override
@ -56,7 +60,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
int delay = Integer.parseInt(request.getParameter("delay"));
String onError = request.getParameter("onError");
boolean cacheResponses = Boolean.parseBoolean(request.getParameter("cacheResponses"));
JSONArray httpHeadersJson = new JSONArray(request.getParameter("httpHeaders"));
ObjectMapper mapper = new ObjectMapper();
List<HttpHeader> headers = Arrays.asList(mapper.readValue(request.getParameter("httpHeaders"), HttpHeader[].class));
return new ColumnAdditionByFetchingURLsOperation(
engineConfig,
@ -67,7 +72,7 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
columnInsertIndex,
delay,
cacheResponses,
httpHeadersJson
headers
);
}

5
main/src/com/google/refine/operations/OnError.java
View File

@ -33,8 +33,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations;
import com.fasterxml.jackson.annotation.JsonProperty;
public enum OnError {
@JsonProperty("keep-original")
KeepOriginal,
@JsonProperty("set-to-blank")
SetToBlank,
@JsonProperty("store-error")
StoreError
}

102
main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java
View File

@ -50,10 +50,13 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.refine.Jsonizable;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -81,6 +84,27 @@ import com.google.refine.util.ParsingUtilities;
public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperation {
public static final class HttpHeader implements Jsonizable {
@JsonProperty("name")
final public String name;
@JsonProperty("value")
final public String value;
public HttpHeader(String name, String value) {
this.name = name;
this.value = value;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("name"); writer.value(name);
writer.key("value"); writer.value(value);
writer.endObject();
}
}
final protected String _baseColumnName;
final protected String _urlExpression;
final protected OnError _onError;
@ -89,10 +113,21 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
final protected int _columnInsertIndex;
final protected int _delay;
final protected boolean _cacheResponses;
final protected JSONArray _httpHeadersJson;
final protected List<HttpHeader> _httpHeadersJson;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
List<HttpHeader> headers = null;
JSONArray headersJson = obj.optJSONArray("httpHeadersJson");
if (headersJson != null) {
headers = new ArrayList<>(headersJson.length());
for (int i = 0; i < headersJson.length(); i++) {
String headerLabel = headersJson.getJSONObject(i).getString("name");
String headerValue = headersJson.getJSONObject(i).getString("value");
headers.add(new HttpHeader(headerLabel, headerValue));
}
}
return new ColumnAdditionByFetchingURLsOperation(
EngineConfig.reconstruct(engineConfig),
@ -103,7 +138,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
obj.getInt("columnInsertIndex"),
obj.getInt("delay"),
obj.optBoolean("cacheResponses", false), // false for retro-compatibility
obj.optJSONArray("httpHeadersJson") // will be null if it doesn't exist for retro-compatibility
headers // will be null if it doesn't exist for retro-compatibility
);
}
@ -116,7 +151,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
int columnInsertIndex,
int delay,
boolean cacheResponses,
JSONArray httpHeadersJson
List<HttpHeader> httpHeadersJson
) {
super(engineConfig);
@ -147,9 +182,56 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError));
writer.key("delay"); writer.value(_delay);
writer.key("cacheResponses"); writer.value(_cacheResponses);
writer.key("httpHeadersJson"); writer.value(_httpHeadersJson);
if (_httpHeadersJson != null) {
writer.key("httpHeadersJson");
writer.array();
for(HttpHeader header : _httpHeadersJson) {
header.write(writer, options);
}
writer.endArray();
}
writer.endObject();
}
@JsonProperty("newColumnName")
public String getNewColumnName() {
return _newColumnName;
}
@JsonProperty("columnInsertIndex")
public int getColumnInsertIndex() {
return _columnInsertIndex;
}
@JsonProperty("baseColumnName")
public String getBaseColumnName() {
return _baseColumnName;
}
@JsonProperty("urlExpression")
public String getUrlExpression() {
return _urlExpression;
}
@JsonProperty("onError")
public OnError getOnError() {
return _onError;
}
@JsonProperty("delay")
public int getDelay() {
return _delay;
}
@JsonProperty("httpHeadersJson")
public List<HttpHeader> getHttpHeadersJson() {
return _httpHeadersJson;
}
@JsonProperty("cacheResponses")
public boolean getCacheResponses() {
return _cacheResponses;
}
@Override
protected String getBriefDescription(Project project) {
@ -179,8 +261,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
engine,
eval,
getBriefDescription(null),
_cacheResponses,
_httpHeadersJson
_cacheResponses
);
}
@ -197,8 +278,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
Engine engine,
Evaluable eval,
String description,
boolean cacheResponses,
JSONArray httpHeadersJson
boolean cacheResponses
) throws JSONException {
super(description);
_project = project;
@ -335,9 +415,9 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
try {
URLConnection urlConnection = url.openConnection();
if (_httpHeadersJson != null) {
for (int i = 0; i < _httpHeadersJson.length(); i++) {
String headerLabel = _httpHeadersJson.getJSONObject(i).getString("name");
String headerValue = _httpHeadersJson.getJSONObject(i).getString("value");
for (int i = 0; i < _httpHeadersJson.size(); i++) {
String headerLabel = _httpHeadersJson.get(i).name;
String headerValue = _httpHeadersJson.get(i).value;
if (headerValue != null && !headerValue.isEmpty()) {
urlConnection.setRequestProperty(headerLabel, headerValue);
}

27
main/src/com/google/refine/operations/column/ColumnAdditionOperation.java
View File

@ -42,6 +42,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -118,7 +120,32 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError));
writer.endObject();
}
@JsonProperty("newColumnName")
public String getNewColumnName() {
return _newColumnName;
}
@JsonProperty("columnInsertIndex")
public int getColumnInsertIndex() {
return _columnInsertIndex;
}
@JsonProperty("baseColumnName")
public String getBaseColumnName() {
return _baseColumnName;
}
@JsonProperty("expression")
public String getExpression() {
return _expression;
}
@JsonProperty("onError")
public OnError getOnError() {
return _onError;
}
@Override
protected String getBriefDescription(Project project) {
return "Create column " + _newColumnName +

13
main/src/com/google/refine/operations/column/ColumnMoveOperation.java
View File

@ -39,6 +39,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
@ -76,7 +78,16 @@ public class ColumnMoveOperation extends AbstractOperation {
writer.key("index"); writer.value(_index);
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;
}
@JsonProperty("index")
public int getIndex() {
return _index;
}
@Override
protected String getBriefDescription(Project project) {

7
main/src/com/google/refine/operations/column/ColumnRemovalOperation.java
View File

@ -39,6 +39,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
@ -72,6 +74,11 @@ public class ColumnRemovalOperation extends AbstractOperation {
writer.key("columnName"); writer.value(_columnName);
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;
}
@Override

11
main/src/com/google/refine/operations/column/ColumnRenameOperation.java
View File

@ -39,6 +39,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
@ -76,7 +78,16 @@ public class ColumnRenameOperation extends AbstractOperation {
writer.key("newColumnName"); writer.value(_newColumnName);
writer.endObject();
}
@JsonProperty("oldColumnName")
public String getOldColumnName() {
return _oldColumnName;
}
@JsonProperty("newColumnName")
public String getNewColumnName() {
return _newColumnName;
}
@Override
protected String getBriefDescription(Project project) {

7
main/src/com/google/refine/operations/column/ColumnReorderOperation.java
View File

@ -41,6 +41,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
@ -77,6 +79,11 @@ public class ColumnReorderOperation extends AbstractOperation {
writer.endArray();
writer.endObject();
}
@JsonProperty("columnNames")
public List<String> getColumnNames() {
return _columnNames;
}
@Override
protected String getBriefDescription(Project project) {

56
main/src/com/google/refine/operations/column/ColumnSplitOperation.java
View File

@ -44,6 +44,10 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig;
import com.google.refine.browsing.FilteredRows;
@ -68,8 +72,8 @@ public class ColumnSplitOperation extends EngineDependentOperation {
final protected String _mode;
final protected String _separator;
final protected boolean _regex;
final protected int _maxColumns;
final protected Boolean _regex;
final protected Integer _maxColumns;
final protected int[] _fieldLengths;
@ -136,8 +140,8 @@ public class ColumnSplitOperation extends EngineDependentOperation {
_mode = "lengths";
_separator = null;
_regex = false;
_maxColumns = -1;
_regex = null;
_maxColumns = null;
_fieldLengths = fieldLengths;
}
@ -167,6 +171,50 @@ public class ColumnSplitOperation extends EngineDependentOperation {
}
writer.endObject();
}
@JsonProperty("columnName")
public String getColumnName() {
return _columnName;
}
@JsonProperty("guessCellType")
public boolean getGuessCellType() {
return _guessCellType;
}
@JsonProperty("removeOriginalColumn")
public boolean getRemoveOriginalColumn() {
return _removeOriginalColumn;
}
@JsonProperty("mode")
public String getMode() {
return _mode;
}
@JsonProperty("separator")
@JsonInclude(Include.NON_NULL)
public String getSeparator() {
return _separator;
}
@JsonProperty("regex")
@JsonInclude(Include.NON_NULL)
public Boolean getRegex() {
return _regex;
}
@JsonProperty("maxColumns")
@JsonInclude(Include.NON_NULL)
public Integer getMaxColumns() {
return _maxColumns;
}
@JsonProperty("fieldLengths")
@JsonInclude(Include.NON_NULL)
public int[] getFieldLengths() {
return _fieldLengths;
}
@Override
protected String getBriefDescription(Project project) {

15
main/tests/server/src/com/google/refine/tests/operations/column/ColumnAdditionByFetchingURLsOperationTests.java
View File

@ -35,9 +35,10 @@ package com.google.refine.tests.operations.column;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
@ -56,6 +57,7 @@ import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OnError;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation;
import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation.HttpHeader;
import com.google.refine.process.Process;
import com.google.refine.process.ProcessManager;
import com.google.refine.tests.RefineTest;
@ -233,11 +235,10 @@ public class ColumnAdditionByFetchingURLsOperationTests extends RefineTest {
String userAgentValue = "OpenRefine";
String authorizationValue = "Basic";
String acceptValue = "*/*";
String jsonString = "[{\"name\": \"authorization\",\"value\": \""+authorizationValue+
"\"},{\"name\": \"user-agent\",\"value\": \""+userAgentValue+
"\"},{\"name\": \"accept\",\"value\": \""+acceptValue+"\"}]";
JSONArray httpHeadersJson = new JSONArray(jsonString);
List<HttpHeader> headers = new ArrayList<>();
headers.add(new HttpHeader("authorization", authorizationValue));
headers.add(new HttpHeader("user-agent", userAgentValue));
headers.add(new HttpHeader("accept", acceptValue));
EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config,
"fruits",
@ -247,7 +248,7 @@ public class ColumnAdditionByFetchingURLsOperationTests extends RefineTest {
1,
50,
true,
httpHeadersJson);
headers);
ProcessManager pm = project.getProcessManager();
Process process = op.createProcess(project, options);
process.startPerforming(pm);

21
main/tests/server/src/com/google/refine/tests/operations/column/ColumnSplitOperationTests.java
View File

@ -20,7 +20,7 @@ public class ColumnSplitOperationTests extends RefineTest {
}
@Test
public void serializeColumnSplitOperation() throws JSONException, Exception {
public void serializeColumnSplitOperationBySeparator() throws JSONException, Exception {
String json = "{\n" +
" \"op\": \"core/column-split\",\n" +
" \"description\": \"Split column ea by separator\",\n" +
@ -39,4 +39,23 @@ public class ColumnSplitOperationTests extends RefineTest {
Project project = mock(Project.class);
TestUtils.isSerializedTo(ColumnSplitOperation.reconstruct(project, new JSONObject(json)), json);
}
@Test
public void serializeColumnSplitOperationByLengths() throws JSONException, Exception {
String json = "{\n" +
" \"op\": \"core/column-split\",\n" +
" \"description\": \"Split column ea by field lengths\",\n" +
" \"engineConfig\": {\n" +
" \"mode\": \"row-based\",\n" +
" \"facets\": []\n" +
" },\n" +
" \"columnName\": \"ea\",\n" +
" \"guessCellType\": true,\n" +
" \"removeOriginalColumn\": true,\n" +
" \"mode\": \"lengths\",\n" +
" \"fieldLengths\": [1,1]\n" +
" }";
Project project = mock(Project.class);
TestUtils.isSerializedTo(ColumnSplitOperation.reconstruct(project, new JSONObject(json)), json);
}
}