s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1434 from ostephens/set-http-req-headers

Browse Source 
Set http req headers
This commit is contained in:
Antonin Delpeuch 2018-01-26 17:28:22 +00:00 committed by GitHub
commit ab25f22ea3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 323 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
main/src/com/google/refine/commands/HttpHeadersSupport.java Normal file
View File

@ -0,0 +1,79 @@
/*
Copyright 2017, Owen Stephens
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.commands;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import com.google.refine.RefineServlet;
abstract public class HttpHeadersSupport {
static final protected Map<String, HttpHeaderInfo> s_headers = new HashMap<String, HttpHeaderInfo>();
static public class HttpHeaderInfo {
final public String name;
final public String header;
final public String defaultValue;
HttpHeaderInfo(String header, String defaultValue) {
this.name = header.toLowerCase();
this.header = header;
this.defaultValue = defaultValue;
}
}
static {
registerHttpHeader("User-Agent", RefineServlet.FULLNAME);
registerHttpHeader("Accept", "*/*");
registerHttpHeader("Authorization", "");
}
/**
* @param header
* @param defaultValue
*/
static public void registerHttpHeader(String header, String defaultValue) {
s_headers.put(header.toLowerCase(), new HttpHeaderInfo(header, defaultValue));
}
static public HttpHeaderInfo getHttpHeaderInfo(String header) {
return s_headers.get(header.toLowerCase());
}
static public Set<String> getHttpHeaderLabels() {
return s_headers.keySet();
}
}

5
main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java
View File

@ -36,6 +36,7 @@ package com.google.refine.commands.column;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import org.json.JSONArray;
import com.google.refine.commands.EngineDependentCommand;
import com.google.refine.model.AbstractOperation;
@ -55,6 +56,7 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
int delay = Integer.parseInt(request.getParameter("delay"));
String onError = request.getParameter("onError");
boolean cacheResponses = Boolean.parseBoolean(request.getParameter("cacheResponses"));
JSONArray httpHeadersJson = new JSONArray(request.getParameter("httpHeaders"));
return new ColumnAdditionByFetchingURLsOperation(
engineConfig,
@ -64,7 +66,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
newColumnName,
columnInsertIndex,
delay,
cacheResponses
cacheResponses,
httpHeadersJson
);
}

15
main/src/com/google/refine/commands/project/GetModelsCommand.java
View File

@ -45,6 +45,9 @@ import org.json.JSONWriter;
import com.google.refine.commands.Command;
import com.google.refine.commands.HttpUtilities;
import com.google.refine.commands.HttpHeadersSupport;
import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo;
import com.google.refine.expr.MetaParser;
import com.google.refine.expr.MetaParser.LanguageInfo;
import com.google.refine.importing.ImportingJob;
@ -117,6 +120,18 @@ public class GetModelsCommand extends Command {
}
writer.endObject();
writer.key("httpHeaders");
writer.object();
for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) {
HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel);
writer.key(headerLabel);
writer.object();
writer.key("header"); writer.value(info.header);
writer.key("defaultValue"); writer.value(info.defaultValue);
writer.endObject();
}
writer.endObject();
writer.endObject();
} catch (JSONException e) {
HttpUtilities.respondException(response, e);

35
main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java
View File

@ -42,12 +42,15 @@ import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.ExecutionException;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONArray;
import org.json.JSONWriter;
import com.google.refine.browsing.Engine;
@ -66,6 +69,8 @@ import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.changes.CellAtRow;
import com.google.refine.model.changes.ColumnAdditionChange;
import com.google.refine.commands.HttpHeadersSupport;
import com.google.refine.commands.HttpHeadersSupport.HttpHeaderInfo;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OnError;
import com.google.refine.operations.OperationRegistry;
@ -77,6 +82,7 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.CacheLoader;
public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperation {
final protected String _baseColumnName;
final protected String _urlExpression;
@ -86,6 +92,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
final protected int _columnInsertIndex;
final protected int _delay;
final protected boolean _cacheResponses;
final protected JSONArray _httpHeadersJson;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
@ -98,7 +105,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
obj.getString("newColumnName"),
obj.getInt("columnInsertIndex"),
obj.getInt("delay"),
obj.optBoolean("cacheResponses", false) // false for retro-compatibility
obj.optBoolean("cacheResponses", false), // false for retro-compatibility
obj.optJSONArray("httpHeadersJson") // will be null if it doesn't exist for retro-compatibility
);
}
@ -110,7 +118,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
String newColumnName,
int columnInsertIndex,
int delay,
boolean cacheResponses
boolean cacheResponses,
JSONArray httpHeadersJson
) {
super(engineConfig);
@ -123,6 +132,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
_delay = delay;
_cacheResponses = cacheResponses;
_httpHeadersJson = httpHeadersJson;
}
@Override
@ -140,6 +150,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError));
writer.key("delay"); writer.value(_delay);
writer.key("cacheResponses"); writer.value(_cacheResponses);
writer.key("httpHeadersJson"); writer.value(_httpHeadersJson);
writer.endObject();
}
@ -171,7 +182,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
engine,
eval,
getBriefDescription(null),
_cacheResponses
_cacheResponses,
_httpHeadersJson
);
}
@ -188,7 +200,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
Engine engine,
Evaluable eval,
String description,
boolean cacheResponses
boolean cacheResponses,
JSONArray httpHeadersJson
) throws JSONException {
super(description);
_project = project;
@ -324,7 +337,19 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
try {
URLConnection urlConnection = url.openConnection();
// urlConnection.setRequestProperty(_headerKey, _headerValue);
if (_httpHeadersJson != null) {
Map<String, String> httpHeaders = new HashMap<>();
for (int i = 0; i < _httpHeadersJson.length(); i++) {
String headerLabel = _httpHeadersJson.getJSONObject(i).getString("name");
String headerValue = _httpHeadersJson.getJSONObject(i).getString("value");
httpHeaders.put(headerLabel, headerValue);
}
for (String headerLabel : HttpHeadersSupport.getHttpHeaderLabels()) {
HttpHeaderInfo info = HttpHeadersSupport.getHttpHeaderInfo(headerLabel);
urlConnection.setRequestProperty(info.header, httpHeaders.get(headerLabel));
}
}
try {
InputStream is = urlConnection.getInputStream();

65
main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java
View File

@ -42,6 +42,7 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONArray;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
@ -143,6 +144,7 @@ public class UrlFetchingTests extends RefineTest {
row.setCell(0, new Cell(i < 5 ? "apple":"orange", null));
project.rows.add(row);
}
EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config,
"fruits",
"\"https://www.random.org/integers/?num=1&min=1&max=100&col=1&base=10&format=plain&rnd=new&city=\"+value",
@ -150,7 +152,8 @@ public class UrlFetchingTests extends RefineTest {
"rand",
1,
500,
true);
true,
null);
ProcessManager pm = project.getProcessManager();
Process process = op.createProcess(project, options);
process.startPerforming(pm);
@ -195,6 +198,7 @@ public class UrlFetchingTests extends RefineTest {
Row row2 = new Row(2);
row2.setCell(0, new Cell("http://anursiebcuiesldcresturce.detur/anusclbc", null)); // well-formed but invalid
project.rows.add(row2);
EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config,
"fruits",
"value",
@ -202,7 +206,9 @@ public class UrlFetchingTests extends RefineTest {
"junk",
1,
50,
true);
true,
null);
ProcessManager pm = project.getProcessManager();
Process process = op.createProcess(project, options);
process.startPerforming(pm);
@ -221,4 +227,59 @@ public class UrlFetchingTests extends RefineTest {
Assert.assertTrue(ExpressionUtils.isError(project.rows.get(2).getCellValue(newCol)));
}
@Test
public void testHttpHeaders() throws Exception {
Row row0 = new Row(2);
row0.setCell(0, new Cell("http://headers.jsontest.com", null));
/*
http://headers.jsontest.com is a service which returns the HTTP request headers
as JSON. For example:
{
"X-Cloud-Trace-Context": "579a1a2ee5c778dfc0810a3bf131ba4e/11053223648711966807",
"Authorization": "Basic",
"Host": "headers.jsontest.com",
"User-Agent": "OpenRefine",
"Accept": "*"
}
*/
project.rows.add(row0);
String userAgentValue = "OpenRefine";
String authorizationValue = "Basic";
String acceptValue = "*/*";
String jsonString = "[{\"name\": \"authorization\",\"value\": \""+authorizationValue+
"\"},{\"name\": \"user-agent\",\"value\": \""+userAgentValue+
"\"},{\"name\": \"accept\",\"value\": \""+acceptValue+"\"}]";
JSONArray httpHeadersJson = new JSONArray(jsonString);
EngineDependentOperation op = new ColumnAdditionByFetchingURLsOperation(engine_config,
"fruits",
"value",
OnError.StoreError,
"junk",
1,
50,
true,
httpHeadersJson);
ProcessManager pm = project.getProcessManager();
Process process = op.createProcess(project, options);
process.startPerforming(pm);
Assert.assertTrue(process.isRunning());
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
Assert.fail("Test interrupted");
}
Assert.assertFalse(process.isRunning());
int newCol = project.columnModel.getColumnByName("junk").getCellIndex();
JSONObject headersUsed = new JSONObject(project.rows.get(0).getCellValue(newCol).toString());
// Inspect the results we got from remote service
Assert.assertEquals(headersUsed.getString("User-Agent"), userAgentValue);
Assert.assertEquals(headersUsed.getString("Authorization"), authorizationValue);
Assert.assertEquals(headersUsed.getString("Accept"), acceptValue);
}
}

3
main/webapp/modules/core/MOD-INF/controller.js
View File

@ -459,7 +459,8 @@ function init() {
"scripts/dialogs/templating-exporter-dialog.js",
"scripts/dialogs/column-reordering-dialog.js",
"scripts/dialogs/custom-tabular-exporter-dialog.js",
"scripts/dialogs/expression-column-dialog.js"
"scripts/dialogs/expression-column-dialog.js",
"scripts/dialogs/http-headers-dialog.js",
]
);

4
main/webapp/modules/core/langs/translation-en.json
View File

@ -541,6 +541,7 @@
"throttle-delay": "Throttle delay",
"milli": "milliseconds",
"url-fetch": "Formulate the URLs to fetch:",
"http-headers": "HTTP headers to be used when fetching URLs:",
"enter-col-name": "Enter new column name",
"split-col": "Split column",
"several-col": "into several columns",
@ -649,7 +650,8 @@
"ctrl-enter": "Ctrl-Enter",
"rows": "rows",
"records": "records",
"show": "Show"
"show": "Show",
"hide": "Hide"
},
"core-buttons": {
"cancel": "Cancel",

1
main/webapp/modules/core/scripts/dialogs/http-headers-dialog.html Normal file
View File

@ -0,0 +1 @@
<div class="set-httpheaders-container" bind="setHttpHeadersContainer">$HTTP_HEADER_OPTIONS$</div>

73
main/webapp/modules/core/scripts/dialogs/http-headers-dialog.js Normal file
View File

@ -0,0 +1,73 @@
/*
Copyright 2017, Owen Stephens
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
function HttpHeadersDialog(title, headers, onDone) {
this._onDone = onDone;
var self = this;
var header = $('<div></div>').addClass("dialog-header").text(title).appendTo(frame);
var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
var footer = $('<div></div>').addClass("dialog-footer").appendTo(frame);
var html = $(HttpHeadersDialog.generateWidgetHtml()).appendTo(body);
this._elmts = DOM.bind(html);
this._httpHeadersWidget = new HttpHeadersDialog.Widget(
this._elmts,
headers
);
}
HttpHeadersDialog.generateWidgetHtml = function() {
var html = DOM.loadHTML("core", "scripts/dialogs/http-headers-dialog.html");
var httpheaderOptions = [];
for (var headerLabel in theProject.httpHeaders) {
if (theProject.httpHeaders.hasOwnProperty(headerLabel)) {
var info = theProject.httpHeaders[headerLabel];
httpheaderOptions.push('<label for="' +
headerLabel +
'">' +
info.header +
': </label><input type="text" id="' +
headerLabel +
'" name="' +
headerLabel +
'" value="' +
info.defaultValue +
'" /></option><br />');
}
}
return html.replace("$HTTP_HEADER_OPTIONS$", httpheaderOptions.join(""));
};

5
main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html
View File

@ -21,6 +21,11 @@
<input type="checkbox" name="dialog-cache-responses" id="$add-column-cache-responses" checked="checked" />
<label for="$add-column-cache-responses" bind="or_views_cacheResponses"></label></td>
</tr>
<tr><td colspan="4"><span bind="or_views_httpHeaders"></span>
<span class="toggle-text" bind="or_views_httpHeadersShowHide"></span>
$HTTP_HEADERS_WIDGET$
</td></tr>
</tr>
<tr><td colspan="4"><h3><span bind="or_views_urlFetch"></span></h3></td></tr>
<tr><td colspan="4">$EXPRESSION_PREVIEW_WIDGET$</td></tr>
</table></div>

17
main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js
View File

@ -91,7 +91,9 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
var doAddColumnByFetchingURLs = function() {
var frame = $(
DOM.loadHTML("core", "scripts/views/data-table/add-column-by-fetching-urls-dialog.html")
.replace("$EXPRESSION_PREVIEW_WIDGET$", ExpressionPreviewDialog.generateWidgetHtml()));
.replace("$EXPRESSION_PREVIEW_WIDGET$", ExpressionPreviewDialog.generateWidgetHtml())
.replace("$HTTP_HEADERS_WIDGET$", HttpHeadersDialog.generateWidgetHtml())
);
var elmts = DOM.bind(frame);
elmts.dialogHeader.text($.i18n._('core-views')["add-col-fetch"]+" " + column.name);
@ -103,6 +105,17 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
elmts.or_views_setBlank.text($.i18n._('core-views')["set-blank"]);
elmts.or_views_storeErr.text($.i18n._('core-views')["store-err"]);
elmts.or_views_cacheResponses.text($.i18n._('core-views')["cache-responses"]);
elmts.or_views_httpHeaders.text($.i18n._('core-views')["http-headers"]);
elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["show"]);
elmts.or_views_httpHeadersShowHide.click(function() {
$( ".set-httpheaders-container" ).toggle( "slow", function() {
if ($(this).is(':visible')) {
elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["hide"]);
} else {
elmts.or_views_httpHeadersShowHide.text($.i18n._('core-views')["show"]);
}
});
});
elmts.or_views_urlFetch.text($.i18n._('core-views')["url-fetch"]);
elmts.okButton.html($.i18n._('core-buttons')["ok"]);
elmts.cancelButton.text($.i18n._('core-buttons')["cancel"]);
@ -119,6 +132,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
null
);
elmts.cancelButton.click(dismiss);
elmts.okButton.click(function() {
var columnName = $.trim(elmts.columnNameInput[0].value);
@ -137,6 +151,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
delay: elmts.throttleDelayInput[0].value,
onError: $('input[name="dialog-onerror-choice"]:checked')[0].value,
cacheResponses: $('input[name="dialog-cache-responses"]')[0].checked,
httpHeaders: JSON.stringify(elmts.setHttpHeadersContainer.find("input").serializeArray())
},
null,
{ modelsChanged: true }

6
main/webapp/modules/core/styles/common.less
View File

@ -242,6 +242,12 @@ a img {
opacity: 0.3;
}
.toggle-text {
font-size: x-small;
border-bottom: 1px blue dotted;
left-margin: 1em
}
#header {
height: 40px;
margin: 10px;

15
main/webapp/modules/core/styles/views/data-table-view.less
View File

@ -316,6 +316,21 @@ a.data-table-flag-off {
color: @light_grey;
}
.set-httpheaders-container {
display: none;
}
.set-httpheaders-container label {
display: inline-block;
width: 15%;
text-align: left;
}
.set-httpheaders-container input {
display: inline-block;
width: 50%;
}
ul.sorting-dialog-blank-error-positions {
margin: 0;
padding: 5px;