Merge pull request #1434 from ostephens/set-http-req-headers

Set http req headers

main/src/com/google/refine/commands/HttpHeadersSupport.java

@@ -0,0 +1,79 @@
+/*
+
+Copyright 2017, Owen Stephens
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+package com.google.refine.commands;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.refine.RefineServlet;
+
+abstract public class HttpHeadersSupport {
+
+    static final protected Map<String, HttpHeaderInfo> s_headers = new HashMap<String, HttpHeaderInfo>();
+
+    static public class HttpHeaderInfo {
+        final public String                 name;
+        final public String                 header;
+        final public String                 defaultValue;
+        
+        HttpHeaderInfo(String header, String defaultValue) {
+            this.name = header.toLowerCase();
+            this.header = header;
+            this.defaultValue = defaultValue;
+        }
+    }
+
+    static {
+        registerHttpHeader("User-Agent", RefineServlet.FULLNAME);
+        registerHttpHeader("Accept", "*/*");
+        registerHttpHeader("Authorization", "");
+    }
+    
+    /**
+     * @param header
+     * @param defaultValue
+     */
+    static public void registerHttpHeader(String header, String defaultValue) {
+        s_headers.put(header.toLowerCase(), new HttpHeaderInfo(header, defaultValue));
+    }
+    
+    static public HttpHeaderInfo getHttpHeaderInfo(String header) {
+        return s_headers.get(header.toLowerCase());
+    }
+    
+    static public Set<String> getHttpHeaderLabels() {
+        return s_headers.keySet();
+    }
+}

main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java

@@ -36,6 +36,7 @@ package com.google.refine.commands.column;
 import javax.servlet.http.HttpServletRequest;
 
 import org.json.JSONObject;
+import org.json.JSONArray;
 
 import com.google.refine.commands.EngineDependentCommand;
 import com.google.refine.model.AbstractOperation;
@@ -55,6 +56,7 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
         int delay = Integer.parseInt(request.getParameter("delay"));
         String onError = request.getParameter("onError");
         boolean cacheResponses = Boolean.parseBoolean(request.getParameter("cacheResponses"));
+        JSONArray httpHeadersJson = new JSONArray(request.getParameter("httpHeaders"));
 
         return new ColumnAdditionByFetchingURLsOperation(
             engineConfig, 
@@ -64,7 +66,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
             newColumnName,
             columnInsertIndex,
             delay,
-            cacheResponses
+            cacheResponses,
+            httpHeadersJson
         );
     }
 

main/src/com/google/refine/commands/project/GetModelsCommand.java

@@ -45,6 +45,9 @@ import org.json.JSONWriter;
 
 import com.google.refine.commands.Command;
 import com.google.refine.commands.HttpUtilities;
+import com.google.refine.commands.HttpHeadersSupport;
+import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo;
+
 import com.google.refine.expr.MetaParser;
 import com.google.refine.expr.MetaParser.LanguageInfo;
 import com.google.refine.importing.ImportingJob;
@@ -117,6 +120,18 @@ public class GetModelsCommand extends Command {
             }
             writer.endObject();
 
+            writer.key("httpHeaders");
+            writer.object();
+            for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) {
+                HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel);
+                writer.key(headerLabel);
+                writer.object();
+                    writer.key("header"); writer.value(info.header);
+                    writer.key("defaultValue"); writer.value(info.defaultValue);
+                writer.endObject();
+            }
+            writer.endObject();
+            
             writer.endObject();
         } catch (JSONException e) {
             HttpUtilities.respondException(response, e);

main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java

@@ -42,12 +42,15 @@ import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.ExecutionException;
 
 import org.json.JSONException;
 import org.json.JSONObject;
+import org.json.JSONArray;
 import org.json.JSONWriter;
 
 import com.google.refine.browsing.Engine;
@@ -66,6 +69,8 @@ import com.google.refine.model.Project;
 import com.google.refine.model.Row;
 import com.google.refine.model.changes.CellAtRow;
 import com.google.refine.model.changes.ColumnAdditionChange;
+import com.google.refine.commands.HttpHeadersSupport;
+import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo;
 import com.google.refine.operations.EngineDependentOperation;
 import com.google.refine.operations.OnError;
 import com.google.refine.operations.OperationRegistry;
@@ -77,6 +82,7 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.LoadingCache;
 import com.google.common.cache.CacheLoader;
 
+
 public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperation {
     final protected String     _baseColumnName;
     final protected String     _urlExpression;
@@ -86,6 +92,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
     final protected int        _columnInsertIndex;
     final protected int        _delay;
     final protected boolean    _cacheResponses;
+    final protected JSONArray  _httpHeadersJson;
 
     static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
         JSONObject engineConfig = obj.getJSONObject("engineConfig");
@@ -98,7 +105,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
             obj.getString("newColumnName"),
             obj.getInt("columnInsertIndex"),
             obj.getInt("delay"),
-            obj.optBoolean("cacheResponses", false) // false for retro-compatibility
+            obj.optBoolean("cacheResponses", false), // false for retro-compatibility
+            obj.optJSONArray("httpHeadersJson") // will be null if it doesn't exist for retro-compatibility
         );
     }
 
@@ -110,7 +118,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
         String         newColumnName,
         int            columnInsertIndex,
         int            delay,
-        boolean        cacheResponses
+        boolean        cacheResponses,
+        JSONArray      httpHeadersJson
     ) {
         super(engineConfig);
 
@@ -123,6 +132,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
 
         _delay = delay;
         _cacheResponses = cacheResponses;
+        _httpHeadersJson = httpHeadersJson;
     }
 
     @Override
@@ -140,6 +150,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
         writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError));
         writer.key("delay"); writer.value(_delay);
         writer.key("cacheResponses"); writer.value(_cacheResponses);
+        writer.key("httpHeadersJson"); writer.value(_httpHeadersJson);
         writer.endObject();
     }
 
@@ -171,7 +182,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
             engine,
             eval,
             getBriefDescription(null),
-            _cacheResponses
+            _cacheResponses,
+            _httpHeadersJson
         );
     }
 
@@ -188,7 +200,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
             Engine engine,
             Evaluable eval,
             String description,
-            boolean cacheResponses
+            boolean cacheResponses,
+            JSONArray httpHeadersJson
         ) throws JSONException {
             super(description);
             _project = project;
@@ -324,7 +337,19 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
 
             try {
                 URLConnection urlConnection = url.openConnection();
-//                    urlConnection.setRequestProperty(_headerKey, _headerValue);
+                if (_httpHeadersJson != null) {
+                    Map<String, String> httpHeaders = new HashMap<>();
+                    for (int i = 0; i < _httpHeadersJson.length(); i++) {            
+                        String headerLabel = _httpHeadersJson.getJSONObject(i).getString("name");
+                        String headerValue = _httpHeadersJson.getJSONObject(i).getString("value");
+                        httpHeaders.put(headerLabel, headerValue);
+                    }
+                    for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) {
+                        HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel);
+
+                        urlConnection.setRequestProperty(info.header, httpHeaders.get(headerLabel));
+                    }
+                }
 
                 try {
                     InputStream is = urlConnection.getInputStream();

main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java

@@ -42,6 +42,7 @@ import java.util.Properties;
 
 import org.json.JSONException;
 import org.json.JSONObject;
+import org.json.JSONArray;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
@@ -143,6 +144,7 @@ public class UrlFetchingTests extends RefineTest {
             row.setCell(0, new Cell(i < 5 ? "apple":"orange", null));
             project.rows.add(row);
         }
+
         EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config,
                 "fruits",
                 "\"https://www.random.org/integers/?num=1&min=1&max=100&col=1&base=10&format=plain&rnd=new&city=\"+value",
@@ -150,7 +152,8 @@ public class UrlFetchingTests extends RefineTest {
                 "rand",
                 1,
                 500,
-                true);
+                true,
+                null);
         ProcessManager pm = project.getProcessManager();
         Process process = op.createProcess(project, options);
         process.startPerforming(pm);
@@ -195,6 +198,7 @@ public class UrlFetchingTests extends RefineTest {
         Row row2 = new Row(2);
         row2.setCell(0, new Cell("http://anursiebcuiesldcresturce.detur/anusclbc", null)); // well-formed but invalid
         project.rows.add(row2);
+
         EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config,
                 "fruits",
                 "value",
@@ -202,7 +206,9 @@ public class UrlFetchingTests extends RefineTest {
                 "junk",
                 1,
                 50,
-                true);
+                true,
+                null);
+
         ProcessManager pm = project.getProcessManager();
         Process process = op.createProcess(project, options);
         process.startPerforming(pm);
@@ -221,4 +227,59 @@ public class UrlFetchingTests extends RefineTest {
         Assert.assertTrue(ExpressionUtils.isError(project.rows.get(2).getCellValue(newCol)));
     }
 
+    @Test
+    public void testHttpHeaders() throws Exception {
+        Row row0 = new Row(2);
+        row0.setCell(0, new Cell("http://headers.jsontest.com", null));
+        /* 
+        http://headers.jsontest.com is a service which returns the HTTP request headers
+        as JSON. For example:
+        {
+           "X-Cloud-Trace-Context": "579a1a2ee5c778dfc0810a3bf131ba4e/11053223648711966807",
+           "Authorization": "Basic",
+           "Host": "headers.jsontest.com",
+           "User-Agent": "OpenRefine",
+           "Accept": "*"
+        }
+        */
+
+        project.rows.add(row0);
+
+        String userAgentValue =  "OpenRefine";
+        String authorizationValue = "Basic";
+        String acceptValue = "*/*";
+        String jsonString = "[{\"name\": \"authorization\",\"value\": \""+authorizationValue+
+                             "\"},{\"name\": \"user-agent\",\"value\": \""+userAgentValue+
+                             "\"},{\"name\": \"accept\",\"value\": \""+acceptValue+"\"}]";
+
+        JSONArray httpHeadersJson = new JSONArray(jsonString);
+
+        EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config,
+            "fruits",
+            "value",
+            OnError.StoreError,
+            "junk",
+            1,
+            50,
+            true,
+            httpHeadersJson);
+        ProcessManager pm = project.getProcessManager();
+        Process process = op.createProcess(project, options);
+        process.startPerforming(pm);
+        Assert.assertTrue(process.isRunning());
+        try {
+                Thread.sleep(5000);
+            } catch (InterruptedException e) {
+                Assert.fail("Test interrupted");
+            }
+        Assert.assertFalse(process.isRunning());
+
+        int newCol = project.columnModel.getColumnByName("junk").getCellIndex();
+        JSONObject headersUsed = new JSONObject(project.rows.get(0).getCellValue(newCol).toString());
+        // Inspect the results we got from remote service
+        Assert.assertEquals(headersUsed.getString("User-Agent"), userAgentValue);
+        Assert.assertEquals(headersUsed.getString("Authorization"), authorizationValue);
+        Assert.assertEquals(headersUsed.getString("Accept"), acceptValue);
+    }
+
 }

main/webapp/modules/core/MOD-INF/controller.js

@@ -459,7 +459,8 @@ function init() {
       "scripts/dialogs/templating-exporter-dialog.js",
       "scripts/dialogs/column-reordering-dialog.js",
       "scripts/dialogs/custom-tabular-exporter-dialog.js",
-      "scripts/dialogs/expression-column-dialog.js"
+      "scripts/dialogs/expression-column-dialog.js",
+      "scripts/dialogs/http-headers-dialog.js",
     ]
   );
 

main/webapp/modules/core/langs/translation-en.json

@@ -541,6 +541,7 @@
         "throttle-delay": "Throttle delay",
         "milli": "milliseconds",
         "url-fetch": "Formulate the URLs to fetch:",
+        "http-headers": "HTTP headers to be used when fetching URLs:",
         "enter-col-name": "Enter new column name",
         "split-col": "Split column",
         "several-col": "into several columns",
@@ -649,7 +650,8 @@
         "ctrl-enter": "Ctrl-Enter",
         "rows": "rows",
         "records": "records",
-        "show": "Show"
+        "show": "Show",
+        "hide": "Hide"
     },
     "core-buttons": {
         "cancel": "Cancel",

main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html

@@ -0,0 +1 @@
+<div class="set-httpheaders-container" bind="setHttpHeadersContainer">$HTTP_HEADER_OPTIONS$</div>

main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js

@@ -0,0 +1,73 @@
+/*
+
+Copyright 2017, Owen Stephens
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+function HttpHeadersDialog(title, headers, onDone) {
+    this._onDone = onDone;
+    var self = this;
+    
+    var header = $('<div></div>').addClass("dialog-header").text(title).appendTo(frame);
+    var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
+    var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
+    var html = $(HttpHeadersDialog.generateWidgetHtml()).appendTo(body);
+    this._elmts = DOM.bind(html);
+        
+    this._httpHeadersWidget = new HttpHeadersDialog.Widget(
+        this._elmts, 
+        headers
+    );
+}
+
+HttpHeadersDialog.generateWidgetHtml = function() {
+    var html = DOM.loadHTML("core", "scripts/dialogs/http-headers-dialog.html");
+    var httpheaderOptions = [];
+
+    for (var headerLabel in theProject.httpHeaders) {
+        if (theProject.httpHeaders.hasOwnProperty(headerLabel)) {
+            var info = theProject.httpHeaders[headerLabel];
+            httpheaderOptions.push('<label for="' +
+                                    headerLabel +
+                                    '">' +
+                                    info.header +
+                                    ': </label><input type="text" id="' +
+                                    headerLabel +
+                                    '" name="' +
+                                    headerLabel +
+                                    '" value="' +
+                                    info.defaultValue +
+                                    '" /></option><br />');
+        }
+    }
+
+    return html.replace("$HTTP_HEADER_OPTIONS$", httpheaderOptions.join(""));
+};
+

main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html

@@ -21,6 +21,11 @@
 	     <input type="checkbox" name="dialog-cache-responses" id="$add-column-cache-responses" checked="checked" />
 		<label for="$add-column-cache-responses" bind="or_views_cacheResponses"></label></td>
         </tr>
+        <tr><td colspan="4"><span bind="or_views_httpHeaders"></span>
+          <span class="toggle-text" bind="or_views_httpHeadersShowHide"></span>
+          $HTTP_HEADERS_WIDGET$
+        </td></tr>
+        </tr>
         <tr><td colspan="4"><h3><span bind="or_views_urlFetch"></span></h3></td></tr>
         <tr><td colspan="4">$EXPRESSION_PREVIEW_WIDGET$</td></tr>
       </table></div>

main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js

@@ -91,7 +91,9 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
   var doAddColumnByFetchingURLs = function() {
     var frame = $(
         DOM.loadHTML("core", "scripts/views/data-table/add-column-by-fetching-urls-dialog.html")
-        .replace("$EXPRESSION_PREVIEW_WIDGET$", ExpressionPreviewDialog.generateWidgetHtml()));
+        .replace("$EXPRESSION_PREVIEW_WIDGET$", ExpressionPreviewDialog.generateWidgetHtml())
+        .replace("$HTTP_HEADERS_WIDGET$", HttpHeadersDialog.generateWidgetHtml())
+        );
 
     var elmts = DOM.bind(frame);
     elmts.dialogHeader.text($.i18n._('core-views')["add-col-fetch"]+" " + column.name);
@@ -103,6 +105,17 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
     elmts.or_views_setBlank.text($.i18n._('core-views')["set-blank"]);
     elmts.or_views_storeErr.text($.i18n._('core-views')["store-err"]);
     elmts.or_views_cacheResponses.text($.i18n._('core-views')["cache-responses"]);
+    elmts.or_views_httpHeaders.text($.i18n._('core-views')["http-headers"]);
+    elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["show"]);
+    elmts.or_views_httpHeadersShowHide.click(function() {
+                                                          $( ".set-httpheaders-container" ).toggle( "slow", function() {
+                                                            if ($(this).is(':visible')) {
+                                                              elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["hide"]);
+                                                            } else {
+                                                              elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["show"]);
+                                                            }
+                                                          });
+                                                        });
     elmts.or_views_urlFetch.text($.i18n._('core-views')["url-fetch"]);
     elmts.okButton.html($.i18n._('core-buttons')["ok"]);
     elmts.cancelButton.text($.i18n._('core-buttons')["cancel"]);
@@ -119,6 +132,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
       null
     );
 
+
     elmts.cancelButton.click(dismiss);
     elmts.okButton.click(function() {
       var columnName = $.trim(elmts.columnNameInput[0].value);
@@ -137,6 +151,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
           delay: elmts.throttleDelayInput[0].value,
           onError: $('input[name="dialog-onerror-choice"]:checked')[0].value,
           cacheResponses: $('input[name="dialog-cache-responses"]')[0].checked,
+          httpHeaders: JSON.stringify(elmts.setHttpHeadersContainer.find("input").serializeArray())
         },
         null,
         { modelsChanged: true }

main/webapp/modules/core/styles/common.less

@@ -242,6 +242,12 @@ a img {
   opacity: 0.3;
   }
 
+.toggle-text {
+  font-size: x-small;
+  border-bottom: 1px blue dotted;
+  left-margin: 1em
+}
+
 #header {
   height: 40px;
   margin: 10px;

main/webapp/modules/core/styles/views/data-table-view.less

@@ -316,6 +316,21 @@ a.data-table-flag-off {
   color: @light_grey;
   }
 
+.set-httpheaders-container {
+  display: none;
+  }
+
+.set-httpheaders-container label {
+  display: inline-block;
+  width: 15%;
+  text-align: left;
+  }
+
+.set-httpheaders-container input {
+  display: inline-block;
+  width: 50%;
+  }
+
 ul.sorting-dialog-blank-error-positions {
   margin: 0;
   padding: 5px;