Option to add http headers in Add column by fetching URLs - support adding list of headers in backend, and exposing these in UI for user to edit and use

main/src/com/google/refine/commands/HttpHeadersSupport.java

@@ -0,0 +1,79 @@
+/*
+
+Copyright 2017, Owen Stephens
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+package com.google.refine.commands;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.refine.RefineServlet;
+
+abstract public class HttpHeadersSupport {
+
+    static public class HttpHeaderInfo {
+        final public String                 name;
+        final public String                 header;
+        final public String                 defaultValue;
+        
+        HttpHeaderInfo(String header, String defaultValue) {
+            this.name = header.toLowerCase();
+            this.header = header;
+            this.defaultValue = defaultValue;
+        }
+    }
+    
+    static final protected Map<String, HttpHeaderInfo> s_headers = new HashMap<String, HttpHeaderInfo>();
+    
+    static {
+        registerHttpHeader("User-Agent", RefineServlet.FULLNAME);
+        registerHttpHeader("Accept", "*/*");
+        registerHttpHeader("Authorization", "");
+    }
+    
+    /**
+     * @param header
+     * @param defaultValue
+     */
+    static public void registerHttpHeader(String header, String defaultValue) {
+        s_headers.put(header.toLowerCase(), new HttpHeaderInfo(header, defaultValue));
+    }
+    
+    static public HttpHeaderInfo getHttpHeaderInfo(String header) {
+        return s_headers.get(header.toLowerCase());
+    }
+    
+    static public Set<String> getHttpHeaderLabels() {
+        return s_headers.keySet();
+    }
+}

main/src/com/google/refine/commands/project/GetModelsCommand.java

@@ -45,6 +45,9 @@ import org.json.JSONWriter;
 
 import com.google.refine.commands.Command;
 import com.google.refine.commands.HttpUtilities;
+import com.google.refine.commands.HttpHeadersSupport;
+import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo;
+
 import com.google.refine.expr.MetaParser;
 import com.google.refine.expr.MetaParser.LanguageInfo;
 import com.google.refine.importing.ImportingJob;
@@ -116,6 +119,18 @@ public class GetModelsCommand extends Command {
                 writer.endObject();
             }
             writer.endObject();
+
+            writer.key("httpHeaders");
+            writer.object();
+            for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) {
+                HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel);
+                writer.key(headerLabel);
+                writer.object();
+                    writer.key("header"); writer.value(info.header);
+                    writer.key("defaultValue"); writer.value(info.defaultValue);
+                writer.endObject();
+            }
+            writer.endObject();
             
             writer.endObject();
         } catch (JSONException e) {

main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html

@@ -1,6 +1 @@
-<div class="grid-layout layout-tight layout-full"><table rows="4" cols="4">
+<div class="grid-layout layout-tight layout-full"><div class="set-httpheaders-container" bind="setHttpHeadersContainer">$HTTP_HEADER_OPTIONS$</div></div>
-  <tr>
-    <td colspan="2"><div class="input-container">KEY</div></td>
-    <td colspan="2"><div class="input-container">VALUE</div></td>
-  </tr>
-</table></div>

main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js

@@ -33,20 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 function HttpHeadersDialog(title, headers, onDone) {
     this._onDone = onDone;
-
     var self = this;
-    var frame = DialogSystem.createDialog();
-    frame.width("700px");
     
     var header = $('<div></div>').addClass("dialog-header").text(title).appendTo(frame);
     var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
     var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
     var html = $(HttpHeadersDialog.generateWidgetHtml()).appendTo(body);
-    
     this._elmts = DOM.bind(html);
         
-    DialogSystem.showDialog(frame);
+    this._httpHeadersWidget = new HttpHeadersDialog.Widget(
-    this._previewWidget = new HttpHeadersDialog.Widget(
         this._elmts, 
         headers
     );
@@ -54,15 +49,69 @@ function HttpHeadersDialog(title, headers, onDone) {
 
 HttpHeadersDialog.generateWidgetHtml = function() {
     var html = DOM.loadHTML("core", "scripts/dialogs/http-headers-dialog.html");
-    return html;
+    var httpheaderOptions = [];
+
+    var httpHeaders = [];
+    for (var headerLabel in theProject.httpHeaders) {
+        if (theProject.httpHeaders.hasOwnProperty(headerLabel)) {
+            var info = theProject.httpHeaders[headerLabel];
+            httpheaderOptions.push('<label for="' +
+                                    headerLabel +
+                                    '">' +
+                                    info.header +
+                                    ': </label><input type="text" id="' +
+                                    headerLabel +
+                                    '" name="' +
+                                    headerLabel +
+                                    '" value="' +
+                                    info.defaultValue +
+                                    '" /></option><br />');
+        }
+    }
+
+    return html.replace("$HTTP_HEADER_OPTIONS$", httpheaderOptions.join(""));
 };
 
-HttpHeadersDialog.Widget = function(
+HttpHeadersDialog.Widget = function(elmts) {
-    elmts, 
-    headers
-) {
     this._elmts = elmts;
-    this._headers = headers;
             
     var self = this;
-};
+    self._getSupportedHeaders();
+};
+
+HttpHeadersDialog.Widget.prototype._getSupportedHeaders = function() {
+    var self = this;
+    $.getJSON(
+        "command/core/get-http-headers",
+        null,
+        function(data) {
+            self._renderSetHttpHeaders(data);
+        },
+        "json"
+    );
+};
+
+HttpHeadersDialog.Widget.prototype._renderSetHttpHeaders = function(data) {
+    var self = this;
+    var elmt = this._elmts.setHttpHeadersContainer.empty();
+    
+    var table = $(
+        '<table cols="4">' +
+            '<tr><th colspan="2">'+$.i18n._('core-dialogs')["http-header-key"]+'</th><th colspan="2">'+$.i18n._('core-dialogs')["http-header-value"]+'</th></tr>' +
+        '</table>'
+    ).appendTo($('<div>').addClass("set-httpheaders-table-wrapper").appendTo(elmt))[0];
+    
+    var renderHeadersList = function(self,tr,header) {
+        $(tr).empty();
+        $("<span>"+header+"</span>").appendTo(tr.insertCell(0));
+        $("<input bind=\""+header+"\" />").appendTo(tr.insertCell(2));
+    };
+    
+    for (var i = 0; i < data["http-headers"].length; i++) {
+        var tr = table.insertRow(table.rows.length);
+        var header = data["http-headers"][i];
+        renderHeadersList(self,tr,header);
+    }
+   
+};
+

main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js

@@ -121,7 +121,8 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
       o.values,
       null
     );
-    
+
+
     elmts.cancelButton.click(dismiss);
     elmts.okButton.click(function() {
       var columnName = $.trim(elmts.columnNameInput[0].value);
@@ -129,7 +130,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
         alert($.i18n._('core-views')["warning-col-name"]);
         return;
       }
-
+      
       Refine.postCoreProcess(
         "add-column-by-fetching-urls", 
         {
@@ -140,6 +141,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
           delay: elmts.throttleDelayInput[0].value,
           onError: $('input[name="dialog-onerror-choice"]:checked')[0].value,
           cacheResponses: $('input[name="dialog-cache-responses"]')[0].checked,
+          httpHeaders: JSON.stringify(elmts.setHttpHeadersContainer.find("input").serializeArray())
         },
         null,
         { modelsChanged: true }