From 34e83fa98c0254334940c92f6ddb1aef3aa256ff Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Sat, 2 Dec 2017 12:32:06 +0000 Subject: [PATCH 01/11] Initial work to add option to set http headers in Add column by fetching URLs --- .../webapp/modules/core/MOD-INF/controller.js | 3 +- .../modules/core/langs/translation-en.json | 1 + .../scripts/dialogs/http-headers-dialog.html | 6 ++ .../scripts/dialogs/http-headers-dialog.js | 68 +++++++++++++++++++ .../add-column-by-fetching-urls-dialog.html | 3 + .../views/data-table/menu-edit-column.js | 5 +- 6 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html create mode 100644 main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js diff --git a/main/webapp/modules/core/MOD-INF/controller.js b/main/webapp/modules/core/MOD-INF/controller.js index 0acb0d393..7ecdff3e4 100644 --- a/main/webapp/modules/core/MOD-INF/controller.js +++ b/main/webapp/modules/core/MOD-INF/controller.js @@ -459,7 +459,8 @@ function init() { "scripts/dialogs/templating-exporter-dialog.js", "scripts/dialogs/column-reordering-dialog.js", "scripts/dialogs/custom-tabular-exporter-dialog.js", - "scripts/dialogs/expression-column-dialog.js" + "scripts/dialogs/expression-column-dialog.js", + "scripts/dialogs/http-headers-dialog.js", ] ); diff --git a/main/webapp/modules/core/langs/translation-en.json b/main/webapp/modules/core/langs/translation-en.json index 15e0acec8..c9a571ce2 100644 --- a/main/webapp/modules/core/langs/translation-en.json +++ b/main/webapp/modules/core/langs/translation-en.json @@ -541,6 +541,7 @@ "throttle-delay": "Throttle delay", "milli": "milliseconds", "url-fetch": "Formulate the URLs to fetch:", + "http-headers": "Define any HTTP headers to be used when fetching URLs:", "enter-col-name": "Enter new column name", "split-col": "Split column", "several-col": "into several columns", diff --git a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html new file mode 100644 index 000000000..cebfa2407 --- /dev/null +++ b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html @@ -0,0 +1,6 @@ +
+ + + + +
KEY
VALUE
\ No newline at end of file diff --git a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js new file mode 100644 index 000000000..96a7a170a --- /dev/null +++ b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js @@ -0,0 +1,68 @@ +/* + +Copyright 2017, Owen Stephens +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +function HttpHeadersDialog(title, headers, onDone) { + this._onDone = onDone; + + var self = this; + var frame = DialogSystem.createDialog(); + frame.width("700px"); + + var header = $('
').addClass("dialog-header").text(title).appendTo(frame); + var body = $('
').addClass("dialog-body").appendTo(frame); + var footer = $('
').addClass("dialog-footer").appendTo(frame); + var html = $(HttpHeadersDialog.generateWidgetHtml()).appendTo(body); + + this._elmts = DOM.bind(html); + + DialogSystem.showDialog(frame); + this._previewWidget = new HttpHeadersDialog.Widget( + this._elmts, + headers + ); +} + +HttpHeadersDialog.generateWidgetHtml = function() { + var html = DOM.loadHTML("core", "scripts/dialogs/http-headers-dialog.html"); + return html; +}; + +HttpHeadersDialog.Widget = function( + elmts, + headers +) { + this._elmts = elmts; + this._headers = headers; + + var self = this; +}; \ No newline at end of file diff --git a/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html b/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html index e51bc2e52..31fa826cc 100644 --- a/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html +++ b/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html @@ -21,6 +21,9 @@ +

+ $HTTP_HEADERS_WIDGET$ +

$EXPRESSION_PREVIEW_WIDGET$ diff --git a/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js b/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js index e43491e8c..351fa3027 100644 --- a/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js +++ b/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js @@ -91,7 +91,9 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { var doAddColumnByFetchingURLs = function() { var frame = $( DOM.loadHTML("core", "scripts/views/data-table/add-column-by-fetching-urls-dialog.html") - .replace("$EXPRESSION_PREVIEW_WIDGET$", ExpressionPreviewDialog.generateWidgetHtml())); + .replace("$EXPRESSION_PREVIEW_WIDGET$", ExpressionPreviewDialog.generateWidgetHtml()) + .replace("$HTTP_HEADERS_WIDGET$", HttpHeadersDialog.generateWidgetHtml()) + ); var elmts = DOM.bind(frame); elmts.dialogHeader.text($.i18n._('core-views')["add-col-fetch"]+" " + column.name); @@ -103,6 +105,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { elmts.or_views_setBlank.text($.i18n._('core-views')["set-blank"]); elmts.or_views_storeErr.text($.i18n._('core-views')["store-err"]); elmts.or_views_cacheResponses.text($.i18n._('core-views')["cache-responses"]); + elmts.or_views_httpHeaders.text($.i18n._('core-views')["http-headers"]); elmts.or_views_urlFetch.text($.i18n._('core-views')["url-fetch"]); elmts.okButton.html($.i18n._('core-buttons')["ok"]); elmts.cancelButton.text($.i18n._('core-buttons')["cancel"]); From 4fb18679807e10b88571a4b044698fe397ad7268 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Fri, 12 Jan 2018 14:15:02 +0000 Subject: [PATCH 02/11] Option to add http headers in Add column by fetching URLs - support adding list of headers in backend, and exposing these in UI for user to edit and use --- .../refine/commands/HttpHeadersSupport.java | 79 +++++++++++++++++++ .../commands/project/GetModelsCommand.java | 15 ++++ .../scripts/dialogs/http-headers-dialog.html | 7 +- .../scripts/dialogs/http-headers-dialog.js | 75 +++++++++++++++--- .../views/data-table/menu-edit-column.js | 6 +- 5 files changed, 161 insertions(+), 21 deletions(-) create mode 100644 main/src/com/google/refine/commands/HttpHeadersSupport.java diff --git a/main/src/com/google/refine/commands/HttpHeadersSupport.java b/main/src/com/google/refine/commands/HttpHeadersSupport.java new file mode 100644 index 000000000..6b1690bbd --- /dev/null +++ b/main/src/com/google/refine/commands/HttpHeadersSupport.java @@ -0,0 +1,79 @@ +/* + +Copyright 2017, Owen Stephens +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.commands; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import com.google.refine.RefineServlet; + +abstract public class HttpHeadersSupport { + + static public class HttpHeaderInfo { + final public String name; + final public String header; + final public String defaultValue; + + HttpHeaderInfo(String header, String defaultValue) { + this.name = header.toLowerCase(); + this.header = header; + this.defaultValue = defaultValue; + } + } + + static final protected Map s_headers = new HashMap(); + + static { + registerHttpHeader("User-Agent", RefineServlet.FULLNAME); + registerHttpHeader("Accept", "*/*"); + registerHttpHeader("Authorization", ""); + } + + /** + * @param header + * @param defaultValue + */ + static public void registerHttpHeader(String header, String defaultValue) { + s_headers.put(header.toLowerCase(), new HttpHeaderInfo(header, defaultValue)); + } + + static public HttpHeaderInfo getHttpHeaderInfo(String header) { + return s_headers.get(header.toLowerCase()); + } + + static public Set getHttpHeaderLabels() { + return s_headers.keySet(); + } +} diff --git a/main/src/com/google/refine/commands/project/GetModelsCommand.java b/main/src/com/google/refine/commands/project/GetModelsCommand.java index e8a2a39f1..ea433fa9c 100644 --- a/main/src/com/google/refine/commands/project/GetModelsCommand.java +++ b/main/src/com/google/refine/commands/project/GetModelsCommand.java @@ -45,6 +45,9 @@ import org.json.JSONWriter; import com.google.refine.commands.Command; import com.google.refine.commands.HttpUtilities; +import com.google.refine.commands.HttpHeadersSupport; +import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo; + import com.google.refine.expr.MetaParser; import com.google.refine.expr.MetaParser.LanguageInfo; import com.google.refine.importing.ImportingJob; @@ -116,6 +119,18 @@ public class GetModelsCommand extends Command { writer.endObject(); } writer.endObject(); + + writer.key("httpHeaders"); + writer.object(); + for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) { + HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel); + writer.key(headerLabel); + writer.object(); + writer.key("header"); writer.value(info.header); + writer.key("defaultValue"); writer.value(info.defaultValue); + writer.endObject(); + } + writer.endObject(); writer.endObject(); } catch (JSONException e) { diff --git a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html index cebfa2407..eaa087d00 100644 --- a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html +++ b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html @@ -1,6 +1 @@ -
- - - - -
KEY
VALUE
\ No newline at end of file +
$HTTP_HEADER_OPTIONS$
\ No newline at end of file diff --git a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js index 96a7a170a..aad7f1122 100644 --- a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js +++ b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js @@ -33,20 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. function HttpHeadersDialog(title, headers, onDone) { this._onDone = onDone; - var self = this; - var frame = DialogSystem.createDialog(); - frame.width("700px"); var header = $('
').addClass("dialog-header").text(title).appendTo(frame); var body = $('
').addClass("dialog-body").appendTo(frame); var footer = $('
').addClass("dialog-footer").appendTo(frame); var html = $(HttpHeadersDialog.generateWidgetHtml()).appendTo(body); - this._elmts = DOM.bind(html); - DialogSystem.showDialog(frame); - this._previewWidget = new HttpHeadersDialog.Widget( + this._httpHeadersWidget = new HttpHeadersDialog.Widget( this._elmts, headers ); @@ -54,15 +49,69 @@ function HttpHeadersDialog(title, headers, onDone) { HttpHeadersDialog.generateWidgetHtml = function() { var html = DOM.loadHTML("core", "scripts/dialogs/http-headers-dialog.html"); - return html; + var httpheaderOptions = []; + + var httpHeaders = []; + for (var headerLabel in theProject.httpHeaders) { + if (theProject.httpHeaders.hasOwnProperty(headerLabel)) { + var info = theProject.httpHeaders[headerLabel]; + httpheaderOptions.push('
'); + } + } + + return html.replace("$HTTP_HEADER_OPTIONS$", httpheaderOptions.join("")); }; -HttpHeadersDialog.Widget = function( - elmts, - headers -) { +HttpHeadersDialog.Widget = function(elmts) { this._elmts = elmts; - this._headers = headers; var self = this; -}; \ No newline at end of file + self._getSupportedHeaders(); +}; + +HttpHeadersDialog.Widget.prototype._getSupportedHeaders = function() { + var self = this; + $.getJSON( + "command/core/get-http-headers", + null, + function(data) { + self._renderSetHttpHeaders(data); + }, + "json" + ); +}; + +HttpHeadersDialog.Widget.prototype._renderSetHttpHeaders = function(data) { + var self = this; + var elmt = this._elmts.setHttpHeadersContainer.empty(); + + var table = $( + '' + + '' + + '
'+$.i18n._('core-dialogs')["http-header-key"]+''+$.i18n._('core-dialogs')["http-header-value"]+'
' + ).appendTo($('
').addClass("set-httpheaders-table-wrapper").appendTo(elmt))[0]; + + var renderHeadersList = function(self,tr,header) { + $(tr).empty(); + $(""+header+"").appendTo(tr.insertCell(0)); + $("").appendTo(tr.insertCell(2)); + }; + + for (var i = 0; i < data["http-headers"].length; i++) { + var tr = table.insertRow(table.rows.length); + var header = data["http-headers"][i]; + renderHeadersList(self,tr,header); + } + +}; + diff --git a/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js b/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js index 351fa3027..424e20f41 100644 --- a/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js +++ b/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js @@ -121,7 +121,8 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { o.values, null ); - + + elmts.cancelButton.click(dismiss); elmts.okButton.click(function() { var columnName = $.trim(elmts.columnNameInput[0].value); @@ -129,7 +130,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { alert($.i18n._('core-views')["warning-col-name"]); return; } - + Refine.postCoreProcess( "add-column-by-fetching-urls", { @@ -140,6 +141,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { delay: elmts.throttleDelayInput[0].value, onError: $('input[name="dialog-onerror-choice"]:checked')[0].value, cacheResponses: $('input[name="dialog-cache-responses"]')[0].checked, + httpHeaders: JSON.stringify(elmts.setHttpHeadersContainer.find("input").serializeArray()) }, null, { modelsChanged: true } From 7464e83766f8cc3e18f2ed17bafff5d0b86c7301 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Fri, 12 Jan 2018 14:15:30 +0000 Subject: [PATCH 03/11] Get and set any http headers submitted when using Add Column by Fetching URLs --- .../AddColumnByFetchingURLsCommand.java | 10 +++-- ...ColumnAdditionByFetchingURLsOperation.java | 44 +++++++++++++++---- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java index 717fb100f..d6ae70d3d 100644 --- a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java +++ b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java @@ -36,6 +36,8 @@ package com.google.refine.commands.column; import javax.servlet.http.HttpServletRequest; import org.json.JSONObject; +import org.json.JSONArray; +import java.util.Enumeration; import com.google.refine.commands.EngineDependentCommand; import com.google.refine.model.AbstractOperation; @@ -46,7 +48,7 @@ import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { @Override protected AbstractOperation createOperation(Project project, - HttpServletRequest request, JSONObject engineConfig) throws Exception { + HttpServletRequest request, JSONObject engineConfig) throws Exception { String baseColumnName = request.getParameter("baseColumnName"); String urlExpression = request.getParameter("urlExpression"); @@ -55,7 +57,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { int delay = Integer.parseInt(request.getParameter("delay")); String onError = request.getParameter("onError"); boolean cacheResponses = Boolean.parseBoolean(request.getParameter("cacheResponses")); - + JSONArray httpHeadersJson = new JSONArray(request.getParameter("httpHeaders")); + return new ColumnAdditionByFetchingURLsOperation( engineConfig, baseColumnName, @@ -64,7 +67,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand { newColumnName, columnInsertIndex, delay, - cacheResponses + cacheResponses, + httpHeadersJson ); } diff --git a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java index f08f83887..6799b782f 100644 --- a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java @@ -42,12 +42,15 @@ import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.HashMap; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.ExecutionException; import org.json.JSONException; import org.json.JSONObject; +import org.json.JSONArray; import org.json.JSONWriter; import com.google.refine.browsing.Engine; @@ -66,6 +69,8 @@ import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.model.changes.CellAtRow; import com.google.refine.model.changes.ColumnAdditionChange; +import com.google.refine.commands.HttpHeadersSupport; +import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.OnError; import com.google.refine.operations.OperationRegistry; @@ -77,6 +82,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.LoadingCache; import com.google.common.cache.CacheLoader; + public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperation { final protected String _baseColumnName; final protected String _urlExpression; @@ -86,6 +92,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat final protected int _columnInsertIndex; final protected int _delay; final protected boolean _cacheResponses; + final protected JSONArray _httpHeadersJson; static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { JSONObject engineConfig = obj.getJSONObject("engineConfig"); @@ -98,7 +105,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat obj.getString("newColumnName"), obj.getInt("columnInsertIndex"), obj.getInt("delay"), - obj.optBoolean("cacheResponses", false) // false for retro-compatibility + obj.optBoolean("cacheResponses", false), // false for retro-compatibility + obj.optJSONArray("httpHeadersJson") // will be null if it doesn't exist for retro-compatibility ); } @@ -110,7 +118,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat String newColumnName, int columnInsertIndex, int delay, - boolean cacheResponses + boolean cacheResponses, + JSONArray httpHeadersJson ) { super(engineConfig); @@ -123,6 +132,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat _delay = delay; _cacheResponses = cacheResponses; + _httpHeadersJson = httpHeadersJson; } @Override @@ -140,6 +150,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError)); writer.key("delay"); writer.value(_delay); writer.key("cacheResponses"); writer.value(_cacheResponses); + writer.key("httpHeadersJson"); writer.value(_httpHeadersJson); writer.endObject(); } @@ -171,7 +182,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat engine, eval, getBriefDescription(null), - _cacheResponses + _cacheResponses, + _httpHeadersJson ); } @@ -188,7 +200,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat Engine engine, Evaluable eval, String description, - boolean cacheResponses + boolean cacheResponses, + JSONArray httpHeadersJson ) throws JSONException { super(description); _project = project; @@ -217,13 +230,13 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat result = null; } - if (result == null) { - // the load method should not return any null value - throw new Exception("null result returned by fetch"); - } + if (result == null) { + // the load method should not return any null value + throw new Exception("null result returned by fetch"); + } return result; } - }); + }); } } @@ -324,6 +337,19 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat try { URLConnection urlConnection = url.openConnection(); + if (_httpHeadersJson != null) { + Map httpHeaders = new HashMap<>(); + for (int i = 0; i < _httpHeadersJson.length(); i++) { + String headerLabel = _httpHeadersJson.getJSONObject(i).getString("name"); + String headerValue = _httpHeadersJson.getJSONObject(i).getString("value"); + httpHeaders.put(headerLabel, headerValue); + } + for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) { + HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel); + + urlConnection.setRequestProperty(info.header, httpHeaders.get(headerLabel)); + } + } // urlConnection.setRequestProperty(_headerKey, _headerValue); try { From 5acdb1429c41bb2fadf6d0c38f03d9d3f1667231 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Fri, 12 Jan 2018 14:32:18 +0000 Subject: [PATCH 04/11] Fix tests - Include new httpHeaders JSON array when creating new ColumnAdditionByFetchingURLsOperation --- .../com/google/refine/tests/model/UrlFetchingTests.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java b/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java index 66c4564e8..e548fbd38 100644 --- a/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java @@ -143,6 +143,7 @@ public class UrlFetchingTests extends RefineTest { row.setCell(0, new Cell(i < 5 ? "apple":"orange", null)); project.rows.add(row); } + EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config, "fruits", "\"https://www.random.org/integers/?num=1&min=1&max=100&col=1&base=10&format=plain&rnd=new&city=\"+value", @@ -150,7 +151,8 @@ public class UrlFetchingTests extends RefineTest { "rand", 1, 500, - true); + true, + null); ProcessManager pm = project.getProcessManager(); Process process = op.createProcess(project, options); process.startPerforming(pm); @@ -195,6 +197,7 @@ public class UrlFetchingTests extends RefineTest { Row row2 = new Row(2); row2.setCell(0, new Cell("http://anursiebcuiesldcresturce.detur/anusclbc", null)); // well-formed but invalid project.rows.add(row2); + EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config, "fruits", "value", @@ -202,7 +205,8 @@ public class UrlFetchingTests extends RefineTest { "junk", 1, 50, - true); + true, + null); ProcessManager pm = project.getProcessManager(); Process process = op.createProcess(project, options); process.startPerforming(pm); From 4320ef8c10518fa00377801e01cd11bf20835e0c Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Tue, 16 Jan 2018 17:41:56 +0000 Subject: [PATCH 05/11] Resolved merge conflict --- .../refine/tests/model/UrlFetchingTests.java | 61 ++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java b/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java index e548fbd38..52ff09328 100644 --- a/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java @@ -42,6 +42,7 @@ import java.util.Properties; import org.json.JSONException; import org.json.JSONObject; +import org.json.JSONArray; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -171,7 +172,7 @@ public class UrlFetchingTests extends RefineTest { // Inspect rows String ref_val = (String)project.rows.get(0).getCellValue(1).toString(); - if (ref_val.startsWith("HTTP error")) + if (ref_val.startsWith("HTTP error")) return; Assert.assertTrue(ref_val != "apple"); // just to make sure I picked the right column for (int i = 1; i < 4; i++) { @@ -179,7 +180,7 @@ public class UrlFetchingTests extends RefineTest { // all random values should be equal due to caching Assert.assertEquals(project.rows.get(i).getCellValue(1).toString(), ref_val); } - Assert.assertFalse(process.isRunning()); + Assert.assertFalse(process.isRunning()); } /** @@ -207,6 +208,7 @@ public class UrlFetchingTests extends RefineTest { 50, true, null); + ProcessManager pm = project.getProcessManager(); Process process = op.createProcess(project, options); process.startPerforming(pm); @@ -225,4 +227,59 @@ public class UrlFetchingTests extends RefineTest { Assert.assertTrue(ExpressionUtils.isError(project.rows.get(2).getCellValue(newCol))); } + @Test + public void testHttpHeaders() throws Exception { + Row row0 = new Row(2); + row0.setCell(0, new Cell("http://headers.jsontest.com", null)); + /* + http://headers.jsontest.com is a service which returns the HTTP request headers + as JSON. For example: + { + "X-Cloud-Trace-Context": "579a1a2ee5c778dfc0810a3bf131ba4e/11053223648711966807", + "Authorization": "Basic", + "Host": "headers.jsontest.com", + "User-Agent": "OpenRefine", + "Accept": "*" + } + */ + + project.rows.add(row0); + + String userAgentValue = "OpenRefine"; + String authorizationValue = "Basic"; + String acceptValue = "*/*"; + String jsonString = "[{\"name\": \"authorization\",\"value\": \""+authorizationValue+ + "\"},{\"name\": \"user-agent\",\"value\": \""+userAgentValue+ + "\"},{\"name\": \"accept\",\"value\": \""+acceptValue+"\"}]"; + + JSONArray httpHeadersJson = new JSONArray(jsonString); + + EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config, + "fruits", + "value", + OnError.StoreError, + "junk", + 1, + 50, + true, + httpHeadersJson); + ProcessManager pm = project.getProcessManager(); + Process process = op.createProcess(project, options); + process.startPerforming(pm); + Assert.assertTrue(process.isRunning()); + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + Assert.fail("Test interrupted"); + } + Assert.assertFalse(process.isRunning()); + + int newCol = project.columnModel.getColumnByName("junk").getCellIndex(); + JSONObject headersUsed = new JSONObject(project.rows.get(0).getCellValue(newCol).toString()); + // Inspect the results we got from remote service + Assert.assertEquals(headersUsed.getString("User-Agent"), userAgentValue); + Assert.assertEquals(headersUsed.getString("Authorization"), authorizationValue); + Assert.assertEquals(headersUsed.getString("Accept"), acceptValue); + } + } From f1d23f9e3b9d65dcaf28d8019e522746e9279bb9 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Tue, 16 Jan 2018 16:19:17 +0000 Subject: [PATCH 06/11] Add/update text strings --- main/webapp/modules/core/langs/translation-en.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main/webapp/modules/core/langs/translation-en.json b/main/webapp/modules/core/langs/translation-en.json index c9a571ce2..07ce3d8b7 100644 --- a/main/webapp/modules/core/langs/translation-en.json +++ b/main/webapp/modules/core/langs/translation-en.json @@ -541,7 +541,7 @@ "throttle-delay": "Throttle delay", "milli": "milliseconds", "url-fetch": "Formulate the URLs to fetch:", - "http-headers": "Define any HTTP headers to be used when fetching URLs:", + "http-headers": "HTTP headers to be used when fetching URLs:", "enter-col-name": "Enter new column name", "split-col": "Split column", "several-col": "into several columns", @@ -650,7 +650,8 @@ "ctrl-enter": "Ctrl-Enter", "rows": "rows", "records": "records", - "show": "Show" + "show": "Show", + "hide": "Hide" }, "core-buttons": { "cancel": "Cancel", From 91edc810a750442374bbeaa25c30aae251b55b9f Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Tue, 16 Jan 2018 16:19:52 +0000 Subject: [PATCH 07/11] Removed unused code --- .../scripts/dialogs/http-headers-dialog.js | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js index aad7f1122..b47a6b74e 100644 --- a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js +++ b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js @@ -51,7 +51,6 @@ HttpHeadersDialog.generateWidgetHtml = function() { var html = DOM.loadHTML("core", "scripts/dialogs/http-headers-dialog.html"); var httpheaderOptions = []; - var httpHeaders = []; for (var headerLabel in theProject.httpHeaders) { if (theProject.httpHeaders.hasOwnProperty(headerLabel)) { var info = theProject.httpHeaders[headerLabel]; @@ -72,46 +71,3 @@ HttpHeadersDialog.generateWidgetHtml = function() { return html.replace("$HTTP_HEADER_OPTIONS$", httpheaderOptions.join("")); }; -HttpHeadersDialog.Widget = function(elmts) { - this._elmts = elmts; - - var self = this; - self._getSupportedHeaders(); -}; - -HttpHeadersDialog.Widget.prototype._getSupportedHeaders = function() { - var self = this; - $.getJSON( - "command/core/get-http-headers", - null, - function(data) { - self._renderSetHttpHeaders(data); - }, - "json" - ); -}; - -HttpHeadersDialog.Widget.prototype._renderSetHttpHeaders = function(data) { - var self = this; - var elmt = this._elmts.setHttpHeadersContainer.empty(); - - var table = $( - '' + - '' + - '
'+$.i18n._('core-dialogs')["http-header-key"]+''+$.i18n._('core-dialogs')["http-header-value"]+'
' - ).appendTo($('
').addClass("set-httpheaders-table-wrapper").appendTo(elmt))[0]; - - var renderHeadersList = function(self,tr,header) { - $(tr).empty(); - $(""+header+"").appendTo(tr.insertCell(0)); - $("").appendTo(tr.insertCell(2)); - }; - - for (var i = 0; i < data["http-headers"].length; i++) { - var tr = table.insertRow(table.rows.length); - var header = data["http-headers"][i]; - renderHeadersList(self,tr,header); - } - -}; - From a9be18d9dfbf0b05f5e112da687e4a27ea6bc203 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Tue, 16 Jan 2018 16:24:52 +0000 Subject: [PATCH 08/11] Remove div no longer needed --- .../modules/core/scripts/dialogs/http-headers-dialog.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html index eaa087d00..3827490ae 100644 --- a/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html +++ b/main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html @@ -1 +1 @@ -
$HTTP_HEADER_OPTIONS$
\ No newline at end of file +
$HTTP_HEADER_OPTIONS$
\ No newline at end of file From 9cf379e625dd380eca4c2986cdf3fe4a92c43164 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Tue, 16 Jan 2018 16:26:15 +0000 Subject: [PATCH 09/11] Add show/hide toggle and some style adjustments --- .../add-column-by-fetching-urls-dialog.html | 6 ++++-- .../scripts/views/data-table/menu-edit-column.js | 10 ++++++++++ main/webapp/modules/core/styles/common.less | 10 ++++++++++ .../core/styles/views/data-table-view.less | 15 +++++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html b/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html index 31fa826cc..e1ee9073e 100644 --- a/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html +++ b/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html @@ -21,8 +21,10 @@ -

- $HTTP_HEADERS_WIDGET$ +

+ + $HTTP_HEADERS_WIDGET$ +

$EXPRESSION_PREVIEW_WIDGET$ diff --git a/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js b/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js index 424e20f41..3d6a25335 100644 --- a/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js +++ b/main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js @@ -106,6 +106,16 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { elmts.or_views_storeErr.text($.i18n._('core-views')["store-err"]); elmts.or_views_cacheResponses.text($.i18n._('core-views')["cache-responses"]); elmts.or_views_httpHeaders.text($.i18n._('core-views')["http-headers"]); + elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["show"]); + elmts.or_views_httpHeadersShowHide.click(function() { + $( ".set-httpheaders-container" ).toggle( "slow", function() { + if ($(this).is(':visible')) { + elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["hide"]); + } else { + elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["show"]); + } + }); + }); elmts.or_views_urlFetch.text($.i18n._('core-views')["url-fetch"]); elmts.okButton.html($.i18n._('core-buttons')["ok"]); elmts.cancelButton.text($.i18n._('core-buttons')["cancel"]); diff --git a/main/webapp/modules/core/styles/common.less b/main/webapp/modules/core/styles/common.less index 9beccc81f..7d45d2758 100644 --- a/main/webapp/modules/core/styles/common.less +++ b/main/webapp/modules/core/styles/common.less @@ -178,6 +178,10 @@ a img { font-weight: bold; } +.inline-header { + display: inline; + } + .list-table { width: 100%; } @@ -242,6 +246,12 @@ a img { opacity: 0.3; } +.toggle-text { + font-size: x-small; + border-bottom: 1px blue dotted; + left-margin: 1em +} + #header { height: 40px; margin: 10px; diff --git a/main/webapp/modules/core/styles/views/data-table-view.less b/main/webapp/modules/core/styles/views/data-table-view.less index 35eff8fc8..3ea49872a 100644 --- a/main/webapp/modules/core/styles/views/data-table-view.less +++ b/main/webapp/modules/core/styles/views/data-table-view.less @@ -316,6 +316,21 @@ a.data-table-flag-off { color: @light_grey; } +.set-httpheaders-container { + display: none; + } + +.set-httpheaders-container label { + display: inline-block; + width: 15%; + text-align: left; + } + +.set-httpheaders-container input { + display: inline-block; + width: 50%; + } + ul.sorting-dialog-blank-error-positions { margin: 0; padding: 5px; From af3aa44e6c3fca4fa0614d846418e3bcfeb262f8 Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Tue, 16 Jan 2018 18:27:13 +0000 Subject: [PATCH 10/11] Tidy up for codacy --- main/src/com/google/refine/commands/HttpHeadersSupport.java | 6 +++--- .../commands/column/AddColumnByFetchingURLsCommand.java | 1 - .../column/ColumnAdditionByFetchingURLsOperation.java | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/main/src/com/google/refine/commands/HttpHeadersSupport.java b/main/src/com/google/refine/commands/HttpHeadersSupport.java index 6b1690bbd..6bac9db5b 100644 --- a/main/src/com/google/refine/commands/HttpHeadersSupport.java +++ b/main/src/com/google/refine/commands/HttpHeadersSupport.java @@ -41,6 +41,8 @@ import com.google.refine.RefineServlet; abstract public class HttpHeadersSupport { + static final protected Map s_headers = new HashMap(); + static public class HttpHeaderInfo { final public String name; final public String header; @@ -52,9 +54,7 @@ abstract public class HttpHeadersSupport { this.defaultValue = defaultValue; } } - - static final protected Map s_headers = new HashMap(); - + static { registerHttpHeader("User-Agent", RefineServlet.FULLNAME); registerHttpHeader("Accept", "*/*"); diff --git a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java index d6ae70d3d..ebf6a89e0 100644 --- a/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java +++ b/main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java @@ -37,7 +37,6 @@ import javax.servlet.http.HttpServletRequest; import org.json.JSONObject; import org.json.JSONArray; -import java.util.Enumeration; import com.google.refine.commands.EngineDependentCommand; import com.google.refine.model.AbstractOperation; diff --git a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java index 6799b782f..d231fad04 100644 --- a/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java +++ b/main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java @@ -350,7 +350,6 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat urlConnection.setRequestProperty(info.header, httpHeaders.get(headerLabel)); } } -// urlConnection.setRequestProperty(_headerKey, _headerValue); try { InputStream is = urlConnection.getInputStream(); From c0fda7de8cd5824cf13bd33d5588f73188d9dd5e Mon Sep 17 00:00:00 2001 From: Owen Stephens Date: Wed, 17 Jan 2018 09:56:11 +0000 Subject: [PATCH 11/11] Removed h3 tag and related styling --- .../views/data-table/add-column-by-fetching-urls-dialog.html | 2 +- main/webapp/modules/core/styles/common.less | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html b/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html index e1ee9073e..5ee43fdaa 100644 --- a/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html +++ b/main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html @@ -21,7 +21,7 @@ -

+ $HTTP_HEADERS_WIDGET$ diff --git a/main/webapp/modules/core/styles/common.less b/main/webapp/modules/core/styles/common.less index 7d45d2758..d3a7ce8e4 100644 --- a/main/webapp/modules/core/styles/common.less +++ b/main/webapp/modules/core/styles/common.less @@ -178,10 +178,6 @@ a img { font-weight: bold; } -.inline-header { - display: inline; - } - .list-table { width: 100%; }