s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate reconciliation calls to Apache HTTP client (#2906)

Browse Source 
* Migrate reconciliation calls to OkHTTP, for #2903

* Migrate to Apache HTTP Commons

* Migrate data extension to Apache HTTP client

* Deprecate HttpURLConnection in RefineServlet

* Use LaxRedirectStrategy, clean up imports

* Remove read and pool timeouts, only keep the connection timeout

* Adapt mocking of HTTP calls after migration
This commit is contained in:
Antonin Delpeuch 2020-08-23 14:04:59 +02:00 committed by GitHub
parent 259705ad5f
commit 9ac54edbba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 421 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
main/pom.xml
View File

@ -23,6 +23,7 @@
Jena 3.15.0 doesn't work. Versions through 3.14.0 appear to, but we'll be conservative Jena 3.15.0 doesn't work. Versions through 3.14.0 appear to, but we'll be conservative
--> -->
<jena.version>3.9.0</jena.version> <jena.version>3.9.0</jena.version>
<okhttp.version>4.7.2</okhttp.version>
</properties> </properties>
<scm> <scm>
@ -378,7 +379,7 @@
<dependency> <dependency>
<groupId>com.squareup.okhttp3</groupId> <groupId>com.squareup.okhttp3</groupId>
<artifactId>mockwebserver</artifactId> <artifactId>mockwebserver</artifactId>
<version>4.8.1</version> <version>${okhttp.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>

12
main/src/com/google/refine/RefineServlet.java
View File

@ -371,12 +371,24 @@ public class RefineServlet extends Butterfly {
return klass; return klass;
} }
/**
* @deprecated extensions relying on HttpURLConnection should rather
* migrate to a more high-level and mature HTTP client.
* Use {@link RefineServlet.getUserAgent()} instead.
*/
@Deprecated
static public void setUserAgent(URLConnection urlConnection) { static public void setUserAgent(URLConnection urlConnection) {
if (urlConnection instanceof HttpURLConnection) { if (urlConnection instanceof HttpURLConnection) {
setUserAgent((HttpURLConnection) urlConnection); setUserAgent((HttpURLConnection) urlConnection);
} }
} }
/**
* @deprecated extensions relying on HttpURLConnection should rather
* migrate to a more high-level and mature HTTP client.
* Use {@link RefineServlet.getUserAgent()} instead.
*/
@Deprecated
static public void setUserAgent(HttpURLConnection httpConnection) { static public void setUserAgent(HttpURLConnection httpConnection) {
httpConnection.addRequestProperty("User-Agent", getUserAgent()); httpConnection.addRequestProperty("User-Agent", getUserAgent());
} }

134
main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java
View File

@ -33,11 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.commands.recon; package com.google.refine.commands.recon;
import java.io.DataOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
@ -52,6 +48,19 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.http.Consts;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.message.BasicNameValuePair;
import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
@ -59,6 +68,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.RefineServlet;
import com.google.refine.commands.Command; import com.google.refine.commands.Command;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Column; import com.google.refine.model.Column;
@ -69,6 +79,9 @@ import com.google.refine.model.recon.StandardReconConfig.ReconResult;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
public class GuessTypesOfColumnCommand extends Command { public class GuessTypesOfColumnCommand extends Command {
final static int DEFAULT_SAMPLE_SIZE = 10;
private int sampleSize = DEFAULT_SAMPLE_SIZE;
protected static class TypesResponse { protected static class TypesResponse {
@JsonProperty("code") @JsonProperty("code")
@ -116,8 +129,6 @@ public class GuessTypesOfColumnCommand extends Command {
} }
} }
final static int SAMPLE_SIZE = 10;
protected static class IndividualQuery { protected static class IndividualQuery {
@JsonProperty("query") @JsonProperty("query")
protected String query; protected String query;
@ -146,7 +157,7 @@ public class GuessTypesOfColumnCommand extends Command {
int cellIndex = column.getCellIndex(); int cellIndex = column.getCellIndex();
List<String> samples = new ArrayList<String>(SAMPLE_SIZE); List<String> samples = new ArrayList<String>(sampleSize);
Set<String> sampleSet = new HashSet<String>(); Set<String> sampleSet = new HashSet<String>();
for (Row row : project.rows) { for (Row row : project.rows) {
@ -156,7 +167,7 @@ public class GuessTypesOfColumnCommand extends Command {
if (!sampleSet.contains(s)) { if (!sampleSet.contains(s)) {
samples.add(s); samples.add(s);
sampleSet.add(s); sampleSet.add(s);
if (samples.size() >= SAMPLE_SIZE) { if (samples.size() >= sampleSize) {
break; break;
} }
} }
@ -170,70 +181,62 @@ public class GuessTypesOfColumnCommand extends Command {
String queriesString = ParsingUtilities.defaultWriter.writeValueAsString(queryMap); String queriesString = ParsingUtilities.defaultWriter.writeValueAsString(queryMap);
try { try {
URL url = new URL(serviceUrl); RequestConfig defaultRequestConfig = RequestConfig.custom()
HttpURLConnection connection = (HttpURLConnection) url.openConnection(); .setConnectTimeout(30 * 1000)
{ .build();
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
connection.setConnectTimeout(30000); HttpClientBuilder httpClientBuilder = HttpClients.custom()
connection.setDoOutput(true); .setUserAgent(RefineServlet.getUserAgent())
.setRedirectStrategy(new LaxRedirectStrategy())
DataOutputStream dos = new DataOutputStream(connection.getOutputStream()); .setDefaultRequestConfig(defaultRequestConfig);
try {
String body = "queries=" + ParsingUtilities.encode(queriesString); CloseableHttpClient httpClient = httpClientBuilder.build();
HttpPost request = new HttpPost(serviceUrl);
dos.writeBytes(body); List<NameValuePair> body = Collections.singletonList(
} finally { new BasicNameValuePair("queries", queriesString));
dos.flush(); request.setEntity(new UrlEncodedFormEntity(body, Consts.UTF_8));
dos.close();
try (CloseableHttpResponse response = httpClient.execute(request)) {
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() >= 400) {
throw new IOException("Failed - code:"
+ Integer.toString(statusLine.getStatusCode())
+ " message: " + statusLine.getReasonPhrase());
} }
connection.connect(); String s = ParsingUtilities.inputStreamToString(response.getEntity().getContent());
} ObjectNode o = ParsingUtilities.evaluateJsonStringToObjectNode(s);
if (connection.getResponseCode() >= 400) { Iterator<JsonNode> iterator = o.iterator();
InputStream is = connection.getErrorStream(); while (iterator.hasNext()) {
throw new IOException("Failed - code:" JsonNode o2 = iterator.next();
+ Integer.toString(connection.getResponseCode()) if (!(o2.has("result") && o2.get("result") instanceof ArrayNode)) {
+ " message: " + is == null ? "" : ParsingUtilities.inputStreamToString(is)); continue;
} else { }
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
ObjectNode o = ParsingUtilities.evaluateJsonStringToObjectNode(s);
Iterator<JsonNode> iterator = o.iterator(); ArrayNode results = (ArrayNode) o2.get("result");
while (iterator.hasNext()) { List<ReconResult> reconResults = ParsingUtilities.mapper.convertValue(results, new TypeReference<List<ReconResult>>() {});
JsonNode o2 = iterator.next(); int count = reconResults.size();
if (!(o2.has("result") && o2.get("result") instanceof ArrayNode)) {
continue;
}
ArrayNode results = (ArrayNode) o2.get("result"); for (int j = 0; j < count; j++) {
List<ReconResult> reconResults = ParsingUtilities.mapper.convertValue(results, new TypeReference<List<ReconResult>>() {}); ReconResult result = reconResults.get(j);
int count = reconResults.size(); double score = 1.0 / (1 + j); // score by each result's rank
for (int j = 0; j < count; j++) { List<ReconType> types = result.types;
ReconResult result = reconResults.get(j); int typeCount = types.size();
double score = 1.0 / (1 + j); // score by each result's rank
List<ReconType> types = result.types; for (int t = 0; t < typeCount; t++) {
int typeCount = types.size(); ReconType type = types.get(t);
double score2 = score * (typeCount - t) / typeCount;
for (int t = 0; t < typeCount; t++) { if (map.containsKey(type.id)) {
ReconType type = types.get(t); TypeGroup tg = map.get(type.id);
double score2 = score * (typeCount - t) / typeCount; tg.score += score2;
if (map.containsKey(type.id)) { tg.count++;
TypeGroup tg = map.get(type.id); } else {
tg.score += score2; map.put(type.id, new TypeGroup(type.id, type.name, score2));
tg.count++;
} else {
map.put(type.id, new TypeGroup(type.id, type.name, score2));
}
} }
} }
} }
} finally {
is.close();
} }
} }
} catch (IOException e) { } catch (IOException e) {
@ -245,7 +248,7 @@ public class GuessTypesOfColumnCommand extends Command {
Collections.sort(types, new Comparator<TypeGroup>() { Collections.sort(types, new Comparator<TypeGroup>() {
@Override @Override
public int compare(TypeGroup o1, TypeGroup o2) { public int compare(TypeGroup o1, TypeGroup o2) {
int c = Math.min(SAMPLE_SIZE, o2.count) - Math.min(SAMPLE_SIZE, o1.count); int c = Math.min(sampleSize, o2.count) - Math.min(sampleSize, o1.count);
if (c != 0) { if (c != 0) {
return c; return c;
} }
@ -273,4 +276,9 @@ public class GuessTypesOfColumnCommand extends Command {
this.count = 1; this.count = 1;
} }
} }
// for testability
protected void setSampleSize(int sampleSize) {
this.sampleSize = sampleSize;
}
} }

130
main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java
View File

@ -36,20 +36,31 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package com.google.refine.model.recon; package com.google.refine.model.recon;
import java.io.DataOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.io.Writer; import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.http.Consts;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.message.BasicNameValuePair;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonInclude.Include;
@ -58,6 +69,7 @@ import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.RefineServlet;
import com.google.refine.expr.functions.ToDate; import com.google.refine.expr.functions.ToDate;
import com.google.refine.model.ReconCandidate; import com.google.refine.model.ReconCandidate;
import com.google.refine.model.ReconType; import com.google.refine.model.ReconType;
@ -159,6 +171,9 @@ public class ReconciledDataExtensionJob {
final public String endpoint; final public String endpoint;
final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>(); final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>();
// not final: initialized lazily
private static CloseableHttpClient httpClient = null;
public ReconciledDataExtensionJob(DataExtensionConfig obj, String endpoint) { public ReconciledDataExtensionJob(DataExtensionConfig obj, String endpoint) {
this.extension = obj; this.extension = obj;
this.endpoint = endpoint; this.endpoint = endpoint;
@ -172,63 +187,76 @@ public class ReconciledDataExtensionJob {
formulateQuery(ids, extension, writer); formulateQuery(ids, extension, writer);
String query = writer.toString(); String query = writer.toString();
InputStream is = performQuery(this.endpoint, query); String response = performQuery(this.endpoint, query);
try {
ObjectNode o = ParsingUtilities.mapper.readValue(is, ObjectNode.class); ObjectNode o = ParsingUtilities.mapper.readValue(response, ObjectNode.class);
if(columns.size() == 0) { if(columns.size() == 0) {
// Extract the column metadata // Extract the column metadata
List<ColumnInfo> newColumns = ParsingUtilities.mapper.convertValue(o.get("meta"), new TypeReference<List<ColumnInfo>>() {}); List<ColumnInfo> newColumns = ParsingUtilities.mapper.convertValue(o.get("meta"), new TypeReference<List<ColumnInfo>>() {});
columns.addAll(newColumns); columns.addAll(newColumns);
} }
Map<String, ReconciledDataExtensionJob.DataExtension> map = new HashMap<String, ReconciledDataExtensionJob.DataExtension>(); Map<String, ReconciledDataExtensionJob.DataExtension> map = new HashMap<String, ReconciledDataExtensionJob.DataExtension>();
if (o.has("rows") && o.get("rows") instanceof ObjectNode){ if (o.has("rows") && o.get("rows") instanceof ObjectNode){
ObjectNode records = (ObjectNode) o.get("rows"); ObjectNode records = (ObjectNode) o.get("rows");
// for each identifier // for each identifier
for (String id : ids) { for (String id : ids) {
if (records.has(id) && records.get(id) instanceof ObjectNode) { if (records.has(id) && records.get(id) instanceof ObjectNode) {
ObjectNode record = (ObjectNode) records.get(id); ObjectNode record = (ObjectNode) records.get(id);
ReconciledDataExtensionJob.DataExtension ext = collectResult(record, reconCandidateMap); ReconciledDataExtensionJob.DataExtension ext = collectResult(record, reconCandidateMap);
if (ext != null) { if (ext != null) {
map.put(id, ext); map.put(id, ext);
}
} }
} }
} }
}
return map;
} finally { return map;
is.close(); }
/**
* @todo this should be refactored to be unified with the HTTP querying code
* from StandardReconConfig. We should ideally extract a library to query
* reconciliation services and expose it as such for others to reuse.
*/
static protected String performQuery(String endpoint, String query) throws IOException {
HttpPost request = new HttpPost(endpoint);
List<NameValuePair> body = Collections.singletonList(
new BasicNameValuePair("extend", query));
request.setEntity(new UrlEncodedFormEntity(body, Consts.UTF_8));
try (CloseableHttpResponse response = getHttpClient().execute(request)) {
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() >= 400) {
throw new IOException("Data extension query failed - code: "
+ Integer.toString(statusLine.getStatusCode())
+ " message: " + statusLine.getReasonPhrase());
} else {
return ParsingUtilities.inputStreamToString(response.getEntity().getContent());
}
} }
} }
static protected InputStream performQuery(String endpoint, String query) throws IOException { private static CloseableHttpClient getHttpClient() {
URL url = new URL(endpoint); if (httpClient != null) {
return httpClient;
URLConnection connection = url.openConnection();
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.setConnectTimeout(5000);
connection.setDoOutput(true);
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
try {
String body = "extend=" + ParsingUtilities.encode(query);
dos.writeBytes(body);
} finally {
dos.flush();
dos.close();
} }
RequestConfig defaultRequestConfig = RequestConfig.custom()
.setConnectTimeout(30 * 1000)
.build();
connection.connect(); HttpClientBuilder httpClientBuilder = HttpClients.custom()
.setUserAgent(RefineServlet.getUserAgent())
return connection.getInputStream(); .setRedirectStrategy(new LaxRedirectStrategy())
.setDefaultRequestConfig(defaultRequestConfig);
httpClient = httpClientBuilder.build();
return httpClient;
} }
protected ReconciledDataExtensionJob.DataExtension collectResult( protected ReconciledDataExtensionJob.DataExtension collectResult(
ObjectNode record, ObjectNode record,

125
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -33,12 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model.recon; package com.google.refine.model.recon;
import java.io.DataOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
@ -49,6 +45,18 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.http.Consts;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.message.BasicNameValuePair;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -61,6 +69,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.RefineServlet;
import com.google.refine.expr.ExpressionUtils; import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Project; import com.google.refine.model.Project;
@ -154,6 +163,9 @@ public class StandardReconConfig extends ReconConfig {
@JsonProperty("limit") @JsonProperty("limit")
final private int limit; final private int limit;
// initialized lazily
private CloseableHttpClient httpClient = null;
@JsonCreator @JsonCreator
public StandardReconConfig( public StandardReconConfig(
@JsonProperty("service") @JsonProperty("service")
@ -428,6 +440,22 @@ public class StandardReconConfig extends ReconConfig {
return job; return job;
} }
private CloseableHttpClient getHttpClient() {
if (httpClient != null) {
return httpClient;
}
RequestConfig defaultRequestConfig = RequestConfig.custom()
.setConnectTimeout(30 * 1000)
.build();
HttpClientBuilder httpClientBuilder = HttpClients.custom()
.setUserAgent(RefineServlet.getUserAgent())
.setRedirectStrategy(new LaxRedirectStrategy())
.setDefaultRequestConfig(defaultRequestConfig);
httpClient = httpClientBuilder.build();
return httpClient;
}
@Override @Override
public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) { public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
List<Recon> recons = new ArrayList<Recon>(jobs.size()); List<Recon> recons = new ArrayList<Recon>(jobs.size());
@ -446,69 +474,48 @@ public class StandardReconConfig extends ReconConfig {
stringWriter.write("}"); stringWriter.write("}");
String queriesString = stringWriter.toString(); String queriesString = stringWriter.toString();
try { HttpPost request = new HttpPost(service);
URL url = new URL(service); List<NameValuePair> body = Collections.singletonList(
HttpURLConnection connection = (HttpURLConnection) url.openConnection(); new BasicNameValuePair("queries", queriesString));
{ request.setEntity(new UrlEncodedFormEntity(body, Consts.UTF_8));
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
connection.setConnectTimeout(30000); // TODO parameterize try (CloseableHttpResponse response = getHttpClient().execute(request)) {
connection.setDoOutput(true); StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() >= 400) {
DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
try {
String body = "queries=" + ParsingUtilities.encode(queriesString);
dos.writeBytes(body);
} finally {
dos.flush();
dos.close();
}
connection.connect();
}
if (connection.getResponseCode() >= 400) {
InputStream is = connection.getErrorStream();
String msg = is == null ? "" : ParsingUtilities.inputStreamToString(is);
logger.error("Failed - code: " logger.error("Failed - code: "
+ Integer.toString(connection.getResponseCode()) + Integer.toString(statusLine.getStatusCode())
+ " message: " + msg); + " message: " + statusLine.getReasonPhrase());
} else { } else {
InputStream is = connection.getInputStream(); String s = ParsingUtilities.inputStreamToString(response.getEntity().getContent());
try { ObjectNode o = ParsingUtilities.evaluateJsonStringToObjectNode(s);
String s = ParsingUtilities.inputStreamToString(is); if (o == null) { // utility method returns null instead of throwing
ObjectNode o = ParsingUtilities.evaluateJsonStringToObjectNode(s); logger.error("Failed to parse string as JSON: " + s);
if (o == null) { // utility method returns null instead of throwing } else {
logger.error("Failed to parse string as JSON: " + s); for (int i = 0; i < jobs.size(); i++) {
} else { StandardReconJob job = (StandardReconJob) jobs.get(i);
for (int i = 0; i < jobs.size(); i++) { Recon recon = null;
StandardReconJob job = (StandardReconJob) jobs.get(i);
Recon recon = null;
String text = job.text; String text = job.text;
String key = "q" + i; String key = "q" + i;
if (o.has(key) && o.get(key) instanceof ObjectNode) { if (o.has(key) && o.get(key) instanceof ObjectNode) {
ObjectNode o2 = (ObjectNode) o.get(key); ObjectNode o2 = (ObjectNode) o.get(key);
if (o2.has("result") && o2.get("result") instanceof ArrayNode) { if (o2.has("result") && o2.get("result") instanceof ArrayNode) {
ArrayNode results = (ArrayNode) o2.get("result"); ArrayNode results = (ArrayNode) o2.get("result");
recon = createReconServiceResults(text, results, historyEntryID); recon = createReconServiceResults(text, results, historyEntryID);
} else {
logger.warn("Service error for text: " + text + "\n Job code: " + job.code + "\n Response: " + o2.toString());
}
} else { } else {
// TODO: better error reporting logger.warn("Service error for text: " + text + "\n Job code: " + job.code + "\n Response: " + o2.toString());
logger.warn("Service error for text: " + text + "\n Job code: " + job.code);
} }
} else {
if (recon != null) { // TODO: better error reporting
recon.service = service; logger.warn("Service error for text: " + text + "\n Job code: " + job.code);
}
recons.add(recon);
} }
if (recon != null) {
recon.service = service;
}
recons.add(recon);
} }
} finally {
is.close();
} }
} }
} catch (Exception e) { } catch (Exception e) {

137
main/tests/server/src/com/google/refine/commands/recon/GuessTypesOfColumnCommandTests.java
View File

@ -1,23 +1,154 @@
package com.google.refine.commands.recon; package com.google.refine.commands.recon;
import com.google.refine.commands.CommandTestBase; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
public class GuessTypesOfColumnCommandTests extends CommandTestBase { import com.google.refine.RefineTest;
import com.google.refine.commands.Command;
import com.google.refine.model.Project;
import com.google.refine.util.TestUtils;
import okhttp3.HttpUrl;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import okhttp3.mockwebserver.RecordedRequest;
public class GuessTypesOfColumnCommandTests extends RefineTest {
HttpServletRequest request = null;
HttpServletResponse response = null;
GuessTypesOfColumnCommand command = null;
StringWriter writer = null;
Project project = null;
@BeforeMethod @BeforeMethod
public void setUpCommand() { public void setUpCommand() {
command = new GuessTypesOfColumnCommand(); command = new GuessTypesOfColumnCommand();
command.setSampleSize(2);
request = mock(HttpServletRequest.class);
response = mock(HttpServletResponse.class);
writer = new StringWriter();
try {
when(response.getWriter()).thenReturn(new PrintWriter(writer));
} catch (IOException e) {
e.printStackTrace();
}
project = createCSVProject(
"foo,bar\n"
+ "France,b\n"
+ "Japan,d\n"
+ "Paraguay,x");
} }
@Test @Test
public void testCSRFProtection() throws ServletException, IOException { public void testCSRFProtection() throws ServletException, IOException {
command.doPost(request, response); command.doPost(request, response);
assertCSRFCheckFailed(); TestUtils.assertEqualAsJson("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", writer.toString());
}
@Test
public void testGuessTypes() throws IOException, ServletException, InterruptedException {
when(request.getParameter("project")).thenReturn(Long.toString(project.id));
when(request.getParameter("columnName")).thenReturn("foo");
when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
String expectedQuery = "queries=%7B%22q1%22%3A%7B%22query%22%3A%22Japan%22%2C%22limit%22"+
"%3A3%7D%2C%22q0%22%3A%7B%22query%22%3A%22France%22%2C%22limit%22%3A3%7D%7D";
String serviceResponse = "{\n" +
" \"q0\": {\n" +
" \"result\": [\n" +
" {\n" +
" \"id\": \"Q17\",\n" +
" \"name\": \"Japan\",\n" +
" \"type\": [\n" +
" {\n" +
" \"id\": \"Q3624078\",\n" +
" \"name\": \"sovereign state\"\n" +
" },\n" +
" {\n" +
" \"id\": \"Q112099\",\n" +
" \"name\": \"island nation\"\n" +
" },\n" +
" {\n" +
" \"id\": \"Q6256\",\n" +
" \"name\": \"country\"\n" +
" }\n" +
" ]\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"q1\": {\n" +
" \"result\": [\n" +
" {\n" +
" \"id\": \"Q142\",\n" +
" \"name\": \"France\",\n" +
" \"type\": [\n" +
" {\n" +
" \"id\": \"Q3624078\",\n" +
" \"name\": \"sovereign state\"\n" +
" },\n" +
" {\n" +
" \"id\": \"Q20181813\",\n" +
" \"name\": \"colonial power\"\n" +
" }\n" +
" ]\n" +
" }\n" +
" ]\n" +
" }\n" +
"}";
String guessedTypes = "{\n" +
" \"code\" : \"ok\",\n" +
" \"types\" : [ {\n" +
" \"count\" : 2,\n" +
" \"id\" : \"Q3624078\",\n" +
" \"name\" : \"sovereign state\",\n" +
" \"score\" : 2\n" +
" }, {\n" +
" \"count\" : 1,\n" +
" \"id\" : \"Q112099\",\n" +
" \"name\" : \"island nation\",\n" +
" \"score\" : 0.6666666666666666\n" +
" }, {\n" +
" \"count\" : 1,\n" +
" \"id\" : \"Q20181813\",\n" +
" \"name\" : \"colonial power\",\n" +
" \"score\" : 0.5\n" +
" }, {\n" +
" \"count\" : 1,\n" +
" \"id\" : \"Q6256\",\n" +
" \"name\" : \"country\",\n" +
" \"score\" : 0.3333333333333333\n" +
" } ]\n" +
" }";
try (MockWebServer server = new MockWebServer()) {
server.start();
HttpUrl url = server.url("/api");
server.enqueue(new MockResponse().setBody(serviceResponse));
when(request.getParameter("service")).thenReturn(url.toString());
command.doPost(request, response);
TestUtils.assertEqualAsJson(guessedTypes, writer.toString());
RecordedRequest request = server.takeRequest();
Assert.assertEquals(request.getBody().readUtf8(), expectedQuery);
}
} }
} }

125
main/tests/server/src/com/google/refine/importers/WikitextImporterTests.java
View File

@ -34,14 +34,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.importers; package com.google.refine.importers;
import java.io.ByteArrayInputStream;
import java.io.OutputStream;
import java.io.StringReader; import java.io.StringReader;
import java.net.HttpURLConnection; import java.util.ArrayList;
import java.net.URL; import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.google.refine.model.recon.StandardReconConfig;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.powermock.api.mockito.PowerMockito; import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.core.classloader.annotations.PrepareForTest;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -51,12 +53,16 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest; import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.refine.importers.WikitextImporter; import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.recon.ReconJob;
import com.google.refine.model.recon.StandardReconConfig;
@PrepareForTest(StandardReconConfig.class) @PrepareForTest(WikitextImporter.class)
public class WikitextImporterTests extends ImporterTest { public class WikitextImporterTests extends ImporterTest {
private WikitextImporter importer = null; private WikitextImporter importer = null;
private Map<String, Recon> mockedRecons = null;
@Override @Override
@BeforeTest @BeforeTest
@ -69,6 +75,7 @@ public class WikitextImporterTests extends ImporterTest {
public void setUp() { public void setUp() {
super.setUp(); super.setUp();
importer = new WikitextImporter(); importer = new WikitextImporter();
mockedRecons = new HashMap<>();
} }
@Override @Override
@ -131,79 +138,43 @@ public class WikitextImporterTests extends ImporterTest {
Assert.assertEquals(project.rows.get(1).cells.get(2).value, "f"); Assert.assertEquals(project.rows.get(1).cells.get(2).value, "f");
} }
@BeforeMethod
public void mockReconCalls() throws Exception {
StandardReconConfig cfg = Mockito.spy(new StandardReconConfig(
"http://endpoint.com", "http://schemaspace", "http://schemaspace.com", null, true, Collections.emptyList(), 0));
PowerMockito.whenNew(StandardReconConfig.class).withAnyArguments().thenReturn(cfg);
Answer<List<Recon>> mockedResponse = new Answer<List<Recon>>() {
@Override
public List<Recon> answer(InvocationOnMock invocation) throws Throwable {
return fakeReconCall(invocation.getArgument(0));
}
};
PowerMockito.doAnswer(mockedResponse).when(cfg, "batchRecon", Mockito.any(), Mockito.anyLong());
}
private List<Recon> fakeReconCall(List<ReconJob> jobs) {
List<Recon> result = new ArrayList<>();
for(ReconJob job : jobs) {
result.add(mockedRecons.get(job.toString()));
}
return result;
}
@Test @Test
public void readTableWithLinks() throws Exception { public void readTableWithLinks() throws Exception {
String result = "{\n" +
" \"q0\": {\n" +
" \"result\": [\n" +
" {\n" +
" \"all_labels\": {\n" +
" \"score\": 100,\n" +
" \"weighted\": 100\n" +
" },\n" +
" \"score\": 100,\n" +
" \"id\": \"Q116214\",\n" +
" \"name\": \"European Centre for the Development of Vocational Training\",\n" +
" \"type\": [\n" +
" {\n" +
" \"id\": \"Q392918\",\n" +
" \"name\": \"agency of the European Union\"\n" +
" }\n" +
" ],\n" +
" \"match\": true\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"q1\": {\n" +
" \"result\": [\n" +
" {\n" +
" \"all_labels\": {\n" +
" \"score\": 100,\n" +
" \"weighted\": 100\n" +
" },\n" +
" \"score\": 100,\n" +
" \"id\": \"Q1377549\",\n" +
" \"name\": \"European Foundation for the Improvement of Living and Working Conditions\",\n" +
" \"type\": [\n" +
" {\n" +
" \"id\": \"Q392918\",\n" +
" \"name\": \"agency of the European Union\"\n" +
" }\n" +
" ],\n" +
" \"match\": true\n" +
" }\n" +
" ]\n" +
" },\n" +
" \"q2\": {\n" +
" \"result\": [\n" +
" {\n" +
" \"all_labels\": {\n" +
" \"score\": 100,\n" +
" \"weighted\": 100\n" +
" },\n" +
" \"score\": 100,\n" +
" \"id\": \"Q1377256\",\n" +
" \"name\": \"European Monitoring Centre for Drugs and Drug Addiction\",\n" +
" \"type\": [\n" +
" {\n" +
" \"id\": \"Q392918\",\n" +
" \"name\": \"agency of the European Union\"\n" +
" }\n" +
" ],\n" +
" \"match\": true\n" +
" }\n" +
" ]\n" +
" }\n" +
"}";
// This mock is used to avoid real network connection during test // This mock is used to avoid real network connection during test
URL url = PowerMockito.mock(URL.class); Recon ecdvt = Mockito.mock(Recon.class);
HttpURLConnection connection = Mockito.mock(HttpURLConnection.class); Mockito.when(ecdvt.getBestCandidate()).thenReturn(
Mockito.when(url.openConnection()).thenReturn(connection); new ReconCandidate("Q116214", "European Centre for the Development of Vocational Training", new String[] {"Q392918"}, 100));
OutputStream out = Mockito.mock(OutputStream.class); mockedRecons.put("{\"query\":\"https://de.wikipedia.org/wiki/Europäisches Zentrum für die Förderung der Berufsbildung\"}", ecdvt);
Mockito.when(connection.getOutputStream()).thenReturn(out); // avoid NullPointerException Recon efilwc = Mockito.mock(Recon.class);
Mockito.when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(result.getBytes())); Mockito.when(efilwc.getBestCandidate()).thenReturn(
PowerMockito.whenNew(URL.class).withAnyArguments().thenReturn(url); new ReconCandidate("Q1377549", "European Foundation for the Improvement of Living and Working Conditions", new String[] {"Q392918"}, 100));
mockedRecons.put("{\"query\":\"https://de.wikipedia.org/wiki/Europäische Stiftung zur Verbesserung der Lebens- und Arbeitsbedingungen\"}", efilwc);
Recon emcdda = Mockito.mock(Recon.class);
Mockito.when(emcdda.getBestCandidate()).thenReturn(
new ReconCandidate("Q1377256", "European Monitoring Centre for Drugs and Drug Addiction", new String[] {"Q392918"}, 100));
mockedRecons.put("{\"query\":\"https://de.wikipedia.org/wiki/Europäische Beobachtungsstelle für Drogen und Drogensucht\"}", emcdda);
// Data credits: Wikipedia contributors, https://de.wikipedia.org/w/index.php?title=Agenturen_der_Europäischen_Union&action=edit // Data credits: Wikipedia contributors, https://de.wikipedia.org/w/index.php?title=Agenturen_der_Europäischen_Union&action=edit
String input = "\n" String input = "\n"

18
main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java
View File

@ -230,9 +230,9 @@ public class ExtendDataOperationTests extends RefineTest {
public void mockHttpCalls() throws Exception { public void mockHttpCalls() throws Exception {
mockStatic(ReconciledDataExtensionJob.class); mockStatic(ReconciledDataExtensionJob.class);
PowerMockito.spy(ReconciledDataExtensionJob.class); PowerMockito.spy(ReconciledDataExtensionJob.class);
Answer<InputStream> mockedResponse = new Answer<InputStream>() { Answer<String> mockedResponse = new Answer<String>() {
@Override @Override
public InputStream answer(InvocationOnMock invocation) throws Throwable { public String answer(InvocationOnMock invocation) throws Throwable {
return fakeHttpCall(invocation.getArgument(0), invocation.getArgument(1)); return fakeHttpCall(invocation.getArgument(0), invocation.getArgument(1));
} }
}; };
@ -410,12 +410,12 @@ public class ExtendDataOperationTests extends RefineTest {
mockedResponses.put(ParsingUtilities.mapper.readTree(query), response); mockedResponses.put(ParsingUtilities.mapper.readTree(query), response);
} }
InputStream fakeHttpCall(String endpoint, String query) throws IOException { String fakeHttpCall(String endpoint, String query) throws IOException {
JsonNode parsedQuery = ParsingUtilities.mapper.readTree(query); JsonNode parsedQuery = ParsingUtilities.mapper.readTree(query);
if (mockedResponses.containsKey(parsedQuery)) { if (mockedResponses.containsKey(parsedQuery)) {
return IOUtils.toInputStream(mockedResponses.get(parsedQuery), StandardCharsets.UTF_8); return mockedResponses.get(parsedQuery);
} else { } else {
throw new IllegalArgumentException("HTTP call not mocked for query: "+query); throw new IllegalArgumentException("HTTP call not mocked for query: "+query);
} }
} }
} }