From 7464e83766f8cc3e18f2ed17bafff5d0b86c7301 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Fri, 12 Jan 2018 14:15:30 +0000 Subject: [PATCH] Get and set any http headers submitted when using Add Column by Fetching URLs --- .../AddColumnByFetchingURLsCommand.java | 10 +++-- ...ColumnAdditionByFetchingURLsOperation.java | 44 +++++++++++++++---- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java index 717fb100f..d6ae70d3d 100644 --- a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java +++ b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java @@ -36,6 +36,8 @@ package com.google.refine.commands.column; import javax.servlet.http.HttpServletRequest; import org.json.JSONObject; +import org.json.JSONArray; +import java.util.Enumeration; import com.google.refine.commands.EngineDependentCommand; import com.google.refine.model.AbstractOperation; @@ -46,7 +48,7 @@ import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { @Override protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { + HttpServletRequest request, JSONObject engineConfig) throws Exception { String baseColumnName = request.getParameter("baseColumnName"); String urlExpression = request.getParameter("urlExpression"); @@ -55,7 +57,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { int delay = Integer.parseInt(request.getParameter("delay")); String onError = request.getParameter("onError"); boolean cacheResponses = Boolean.parseBoolean(request.getParameter("cacheResponses")); - + JSONArray httpHeadersJson = new JSONArray(request.getParameter("httpHeaders")); + return new ColumnAdditionByFetchingURLsOperation( engineConfig, baseColumnName, @@ -64,7 +67,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { newColumnName, columnInsertIndex, delay, - cacheResponses + cacheResponses, + httpHeadersJson ); } diff --git a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java index f08f83887..6799b782f 100644 --- a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java @@ -42,12 +42,15 @@ import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.HashMap; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.ExecutionException; import org.json.JSONException; import org.json.JSONObject; +import org.json.JSONArray; import org.json.JSONWriter; import com.google.refine.browsing.Engine; @@ -66,6 +69,8 @@ import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.model.changes.CellAtRow; import com.google.refine.model.changes.ColumnAdditionChange; +import com.google.refine.commands.HttpHeadersSupport; +import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.OnError; import com.google.refine.operations.OperationRegistry; @@ -77,6 +82,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.LoadingCache; import com.google.common.cache.CacheLoader; + public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperation { final protected String _baseColumnName; final protected String _urlExpression; @@ -86,6 +92,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat final protected int _columnInsertIndex; final protected int _delay; final protected boolean _cacheResponses; + final protected JSONArray _httpHeadersJson; static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { JSONObject engineConfig = obj.getJSONObject("engineConfig"); @@ -98,7 +105,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat obj.getString("newColumnName"), obj.getInt("columnInsertIndex"), obj.getInt("delay"), - obj.optBoolean("cacheResponses", false) // false for retro-compatibility + obj.optBoolean("cacheResponses", false), // false for retro-compatibility + obj.optJSONArray("httpHeadersJson") // will be null if it doesn't exist for retro-compatibility ); } @@ -110,7 +118,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat String newColumnName, int columnInsertIndex, int delay, - boolean cacheResponses + boolean cacheResponses, + JSONArray httpHeadersJson ) { super(engineConfig); @@ -123,6 +132,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat _delay = delay; _cacheResponses = cacheResponses; + _httpHeadersJson = httpHeadersJson; } @Override @@ -140,6 +150,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError)); writer.key("delay"); writer.value(_delay); writer.key("cacheResponses"); writer.value(_cacheResponses); + writer.key("httpHeadersJson"); writer.value(_httpHeadersJson); writer.endObject(); } @@ -171,7 +182,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat engine, eval, getBriefDescription(null), - _cacheResponses + _cacheResponses, + _httpHeadersJson ); } @@ -188,7 +200,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat Engine engine, Evaluable eval, String description, - boolean cacheResponses + boolean cacheResponses, + JSONArray httpHeadersJson ) throws JSONException { super(description); _project = project; @@ -217,13 +230,13 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat result = null; } - if (result == null) { - // the load method should not return any null value - throw new Exception("null result returned by fetch"); - } + if (result == null) { + // the load method should not return any null value + throw new Exception("null result returned by fetch"); + } return result; } - }); + }); } } @@ -324,6 +337,19 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat try { URLConnection urlConnection = url.openConnection(); + if (_httpHeadersJson != null) { + Map httpHeaders = new HashMap<>(); + for (int i = 0; i < _httpHeadersJson.length(); i++) { + String headerLabel = _httpHeadersJson.getJSONObject(i).getString("name"); + String headerValue = _httpHeadersJson.getJSONObject(i).getString("value"); + httpHeaders.put(headerLabel, headerValue); + } + for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) { + HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel); + + urlConnection.setRequestProperty(info.header, httpHeaders.get(headerLabel)); + } + } // urlConnection.setRequestProperty(_headerKey, _headerValue); try {