Migrate reconciliation calls to Apache HTTP client (#2906)

* Migrate reconciliation calls to OkHTTP, for #2903

* Migrate to Apache HTTP Commons

* Migrate data extension to Apache HTTP client

* Deprecate HttpURLConnection in RefineServlet

* Use LaxRedirectStrategy, clean up imports

* Remove read and pool timeouts, only keep the connection timeout

* Adapt mocking of HTTP calls after migration

main/pom.xml

@@ -23,6 +23,7 @@
          Jena 3.15.0 doesn't work. Versions through 3.14.0 appear to, but we'll be conservative
     -->
     <jena.version>3.9.0</jena.version>
+    <okhttp.version>4.7.2</okhttp.version>
   </properties>
 
   <scm>
@@ -378,7 +379,7 @@
     <dependency>
       <groupId>com.squareup.okhttp3</groupId>
       <artifactId>mockwebserver</artifactId>
-      <version>4.8.1</version>
+      <version>${okhttp.version}</version>
       <scope>test</scope>
      </dependency>
     <dependency>

main/src/com/google/refine/RefineServlet.java

@@ -371,12 +371,24 @@ public class RefineServlet extends Butterfly {
         return klass;
     }
     
+    /**
+     * @deprecated extensions relying on HttpURLConnection should rather
+     * migrate to a more high-level and mature HTTP client.
+     * Use {@link RefineServlet.getUserAgent()} instead.
+     */
+    @Deprecated
     static public void setUserAgent(URLConnection urlConnection) {
         if (urlConnection instanceof HttpURLConnection) {
             setUserAgent((HttpURLConnection) urlConnection);
         }
     }
     
+    /**
+     * @deprecated extensions relying on HttpURLConnection should rather
+     * migrate to a more high-level and mature HTTP client.
+     * Use {@link RefineServlet.getUserAgent()} instead.
+     */
+    @Deprecated
     static public void setUserAgent(HttpURLConnection httpConnection) {
         httpConnection.addRequestProperty("User-Agent", getUserAgent());
     }

main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java

@@ -33,11 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.commands.recon;
 
-import java.io.DataOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -52,6 +48,19 @@ import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.apache.http.Consts;
+import org.apache.http.NameValuePair;
+import org.apache.http.StatusLine;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
+import org.apache.http.message.BasicNameValuePair;
+
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -59,6 +68,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.refine.RefineServlet;
 import com.google.refine.commands.Command;
 import com.google.refine.expr.ExpressionUtils;
 import com.google.refine.model.Column;
@@ -70,6 +80,9 @@ import com.google.refine.util.ParsingUtilities;
 
 public class GuessTypesOfColumnCommand extends Command {
     
+    final static int DEFAULT_SAMPLE_SIZE = 10;
+    private int sampleSize = DEFAULT_SAMPLE_SIZE;
+     
     protected static class TypesResponse {
         @JsonProperty("code")
         protected String code;
@@ -116,8 +129,6 @@ public class GuessTypesOfColumnCommand extends Command {
         }
     }
     
-    final static int SAMPLE_SIZE = 10;
-    
     protected static class IndividualQuery {
         @JsonProperty("query")
         protected String query;
@@ -146,7 +157,7 @@ public class GuessTypesOfColumnCommand extends Command {
         
         int cellIndex = column.getCellIndex();
         
-        List<String> samples = new ArrayList<String>(SAMPLE_SIZE);
+        List<String> samples = new ArrayList<String>(sampleSize);
         Set<String> sampleSet = new HashSet<String>();
         
         for (Row row : project.rows) {
@@ -156,7 +167,7 @@ public class GuessTypesOfColumnCommand extends Command {
                 if (!sampleSet.contains(s)) {
                     samples.add(s);
                     sampleSet.add(s);
-                    if (samples.size() >= SAMPLE_SIZE) {
+                    if (samples.size() >= sampleSize) {
                         break;
                     }
                 }
@@ -170,35 +181,30 @@ public class GuessTypesOfColumnCommand extends Command {
         
         String queriesString = ParsingUtilities.defaultWriter.writeValueAsString(queryMap);
         try {
-            URL url = new URL(serviceUrl);
+            RequestConfig defaultRequestConfig = RequestConfig.custom()
-            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+                    .setConnectTimeout(30 * 1000)
-            {
+                    .build();
-                connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
-                connection.setConnectTimeout(30000);
-                connection.setDoOutput(true);
 
-                DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
+            HttpClientBuilder httpClientBuilder = HttpClients.custom()
-                try {
+                    .setUserAgent(RefineServlet.getUserAgent())
-                    String body = "queries=" + ParsingUtilities.encode(queriesString);
+                    .setRedirectStrategy(new LaxRedirectStrategy())
+                    .setDefaultRequestConfig(defaultRequestConfig);
             
-                    dos.writeBytes(body);
+            CloseableHttpClient httpClient = httpClientBuilder.build();
-                } finally {
+            HttpPost request = new HttpPost(serviceUrl);
-                    dos.flush();
+            List<NameValuePair> body = Collections.singletonList(
-                    dos.close();
+                    new BasicNameValuePair("queries", queriesString));
-                }
+            request.setEntity(new UrlEncodedFormEntity(body, Consts.UTF_8));
             
-                connection.connect();
+            try (CloseableHttpResponse response = httpClient.execute(request)) {
-            }
+                StatusLine statusLine = response.getStatusLine();
-
+                if (statusLine.getStatusCode() >= 400) {
-            if (connection.getResponseCode() >= 400) {
-                InputStream is = connection.getErrorStream();
                     throw new IOException("Failed  - code:" 
-                        + Integer.toString(connection.getResponseCode()) 
+                            + Integer.toString(statusLine.getStatusCode()) 
-                        + " message: " + is == null ? "" : ParsingUtilities.inputStreamToString(is));
+                            + " message: " + statusLine.getReasonPhrase());
-            } else {
+                }
-                InputStream is = connection.getInputStream();
+                
-                try {
+                String s = ParsingUtilities.inputStreamToString(response.getEntity().getContent());
-                    String s = ParsingUtilities.inputStreamToString(is);
                 ObjectNode o = ParsingUtilities.evaluateJsonStringToObjectNode(s);
 
                 Iterator<JsonNode> iterator = o.iterator();
@@ -232,9 +238,6 @@ public class GuessTypesOfColumnCommand extends Command {
                         }
                     }
                 }
-                } finally {
-                    is.close();
-                }
             }
         } catch (IOException e) {
             logger.error("Failed to guess cell types for load\n" + queriesString, e);
@@ -245,7 +248,7 @@ public class GuessTypesOfColumnCommand extends Command {
         Collections.sort(types, new Comparator<TypeGroup>() {
             @Override
             public int compare(TypeGroup o1, TypeGroup o2) {
-                int c = Math.min(SAMPLE_SIZE, o2.count) - Math.min(SAMPLE_SIZE, o1.count);
+                int c = Math.min(sampleSize, o2.count) - Math.min(sampleSize, o1.count);
                 if (c != 0) {
                     return c;
                 }
@@ -273,4 +276,9 @@ public class GuessTypesOfColumnCommand extends Command {
             this.count = 1;
         }
     }
+    
+    // for testability
+    protected void setSampleSize(int sampleSize) {
+        this.sampleSize = sampleSize;
+    }
 }

main/src/com/google/refine/model/recon/ReconciledDataExtensionJob.java

@@ -36,20 +36,31 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 package com.google.refine.model.recon;
 
-import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringWriter;
 import java.io.Writer;
-import java.net.URL;
-import java.net.URLConnection;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.apache.http.Consts;
+import org.apache.http.NameValuePair;
+import org.apache.http.StatusLine;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
+import org.apache.http.message.BasicNameValuePair;
+
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
@@ -58,6 +69,7 @@ import com.fasterxml.jackson.annotation.JsonView;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.refine.RefineServlet;
 import com.google.refine.expr.functions.ToDate;
 import com.google.refine.model.ReconCandidate;
 import com.google.refine.model.ReconType;
@@ -159,6 +171,9 @@ public class ReconciledDataExtensionJob {
     final public String              endpoint;
     final public List<ColumnInfo>    columns = new ArrayList<ColumnInfo>();
     
+    // not final: initialized lazily
+    private static CloseableHttpClient httpClient = null;
+    
     public ReconciledDataExtensionJob(DataExtensionConfig obj, String endpoint) {
         this.extension = obj;
         this.endpoint = endpoint;
@@ -172,9 +187,9 @@ public class ReconciledDataExtensionJob {
         formulateQuery(ids, extension, writer);
 
         String query = writer.toString();
-        InputStream is = performQuery(this.endpoint, query);
+        String response = performQuery(this.endpoint, query);
-        try {
+
-        	ObjectNode o = ParsingUtilities.mapper.readValue(is, ObjectNode.class);
+        ObjectNode o = ParsingUtilities.mapper.readValue(response, ObjectNode.class);
 
         if(columns.size() == 0) {
             // Extract the column metadata
@@ -201,35 +216,48 @@ public class ReconciledDataExtensionJob {
         }
         
         return map;
-        } finally {
+    }
-            is.close();
+
+    /**
+     * @todo this should be refactored to be unified with the HTTP querying code
+     * from StandardReconConfig. We should ideally extract a library to query
+     * reconciliation services and expose it as such for others to reuse.
+     */
+    
+    static protected String performQuery(String endpoint, String query) throws IOException {
+        HttpPost request = new HttpPost(endpoint);
+        List<NameValuePair> body = Collections.singletonList(
+                new BasicNameValuePair("extend", query));
+        request.setEntity(new UrlEncodedFormEntity(body, Consts.UTF_8));
+        
+        try (CloseableHttpResponse response = getHttpClient().execute(request)) {
+            StatusLine statusLine = response.getStatusLine();
+            if (statusLine.getStatusCode() >= 400) {
+                throw new IOException("Data extension query failed - code: "
+                        + Integer.toString(statusLine.getStatusCode())
+                        + " message: " + statusLine.getReasonPhrase());
+            } else {
+                return ParsingUtilities.inputStreamToString(response.getEntity().getContent());
+            }
         }
     }
 
-    static protected InputStream performQuery(String endpoint, String query) throws IOException {
+    private static CloseableHttpClient getHttpClient() {
-        URL url = new URL(endpoint);
+        if (httpClient != null) {
-
+            return httpClient;
-        URLConnection connection = url.openConnection();
-        connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
-        connection.setConnectTimeout(5000);
-        connection.setDoOutput(true);
-
-        DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
-        try {
-            String body = "extend=" + ParsingUtilities.encode(query);
-
-            dos.writeBytes(body);
-        } finally {
-            dos.flush();
-            dos.close();
         }
+        RequestConfig defaultRequestConfig = RequestConfig.custom()
+                .setConnectTimeout(30 * 1000)
+                .build();
 
-        connection.connect();
+        HttpClientBuilder httpClientBuilder = HttpClients.custom()
-
+                .setUserAgent(RefineServlet.getUserAgent())
-        return connection.getInputStream();
+                .setRedirectStrategy(new LaxRedirectStrategy())
+                .setDefaultRequestConfig(defaultRequestConfig);
+        httpClient = httpClientBuilder.build();
+        return httpClient;
     }
     
-    
     protected ReconciledDataExtensionJob.DataExtension collectResult(
         ObjectNode record,
         Map<String, ReconCandidate> reconCandidateMap

main/src/com/google/refine/model/recon/StandardReconConfig.java

@@ -33,12 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.model.recon;
 
-import java.io.DataOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.StringWriter;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -49,6 +45,18 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.Consts;
+import org.apache.http.NameValuePair;
+import org.apache.http.StatusLine;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.LaxRedirectStrategy;
+import org.apache.http.message.BasicNameValuePair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,6 +69,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.refine.RefineServlet;
 import com.google.refine.expr.ExpressionUtils;
 import com.google.refine.model.Cell;
 import com.google.refine.model.Project;
@@ -154,6 +163,9 @@ public class StandardReconConfig extends ReconConfig {
     @JsonProperty("limit")
     final private int limit;
 
+    // initialized lazily
+    private CloseableHttpClient httpClient = null;
+
     @JsonCreator
     public StandardReconConfig(
             @JsonProperty("service")
@@ -428,6 +440,22 @@ public class StandardReconConfig extends ReconConfig {
         return job;
     }
     
+    private CloseableHttpClient getHttpClient() {
+        if (httpClient != null) {
+            return httpClient;
+        }
+        RequestConfig defaultRequestConfig = RequestConfig.custom()
+                .setConnectTimeout(30 * 1000)
+                .build();
+
+        HttpClientBuilder httpClientBuilder = HttpClients.custom()
+                .setUserAgent(RefineServlet.getUserAgent())
+                .setRedirectStrategy(new LaxRedirectStrategy())
+                .setDefaultRequestConfig(defaultRequestConfig);
+        httpClient = httpClientBuilder.build();
+        return httpClient;
+    }
+    
     @Override
     public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID) {
         List<Recon> recons = new ArrayList<Recon>(jobs.size());
@@ -446,37 +474,19 @@ public class StandardReconConfig extends ReconConfig {
         stringWriter.write("}");
         String queriesString = stringWriter.toString();
         
-        try {
+        HttpPost request = new HttpPost(service);
-            URL url = new URL(service);
+        List<NameValuePair> body = Collections.singletonList(
-            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+                new BasicNameValuePair("queries", queriesString));
-            {
+        request.setEntity(new UrlEncodedFormEntity(body, Consts.UTF_8));
-                connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
-                connection.setConnectTimeout(30000); // TODO parameterize
-                connection.setDoOutput(true);
         
-                DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
+        try (CloseableHttpResponse response = getHttpClient().execute(request)) {
-                try {
+            StatusLine statusLine = response.getStatusLine();
-                    String body = "queries=" + ParsingUtilities.encode(queriesString);
+            if (statusLine.getStatusCode() >= 400) {
-                    
-                    dos.writeBytes(body);
-                } finally {
-                    dos.flush();
-                    dos.close();
-                }
-                
-                connection.connect();
-            }
-            
-            if (connection.getResponseCode() >= 400) {
-                InputStream is = connection.getErrorStream();
-                String msg = is == null ? "" : ParsingUtilities.inputStreamToString(is);
                 logger.error("Failed  - code: "
-                        + Integer.toString(connection.getResponseCode())
+                        + Integer.toString(statusLine.getStatusCode())
-                        + " message: " + msg);
+                        + " message: " + statusLine.getReasonPhrase());
             } else {
-                InputStream is = connection.getInputStream();
+                String s = ParsingUtilities.inputStreamToString(response.getEntity().getContent());
-                try {
-                    String s = ParsingUtilities.inputStreamToString(is);
                 ObjectNode o = ParsingUtilities.evaluateJsonStringToObjectNode(s);
                 if (o == null) { // utility method returns null instead of throwing
                     logger.error("Failed to parse string as JSON: " + s);
@@ -507,9 +517,6 @@ public class StandardReconConfig extends ReconConfig {
                         recons.add(recon);
                     }
                 }
-                } finally {
-                    is.close();
-                }
             }
         } catch (Exception e) {
             logger.error("Failed to batch recon with load:\n" + queriesString, e);

main/tests/server/src/com/google/refine/commands/recon/GuessTypesOfColumnCommandTests.java

@@ -1,23 +1,154 @@
 package com.google.refine.commands.recon;
 
-import com.google.refine.commands.CommandTestBase;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 
 import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
+import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-public class GuessTypesOfColumnCommandTests extends CommandTestBase {
+import com.google.refine.RefineTest;
+import com.google.refine.commands.Command;
+import com.google.refine.model.Project;
+import com.google.refine.util.TestUtils;
+
+import okhttp3.HttpUrl;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import okhttp3.mockwebserver.RecordedRequest;
+
+public class GuessTypesOfColumnCommandTests extends RefineTest {
+    
+    HttpServletRequest request = null;
+    HttpServletResponse response = null;
+    GuessTypesOfColumnCommand command = null;
+    StringWriter writer = null;
+    Project project = null;
 	
 	@BeforeMethod
 	public void setUpCommand() {
 		command = new GuessTypesOfColumnCommand();
+		command.setSampleSize(2);
+		request = mock(HttpServletRequest.class);
+        response = mock(HttpServletResponse.class);
+        writer = new StringWriter();
+        try {
+            when(response.getWriter()).thenReturn(new PrintWriter(writer));
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+		project = createCSVProject(
+		        "foo,bar\n"
+		        + "France,b\n"
+		        + "Japan,d\n"
+		        + "Paraguay,x");
+		
 	}
 	
 	@Test
 	public void testCSRFProtection() throws ServletException, IOException {
 		command.doPost(request, response);
-		assertCSRFCheckFailed();
+		TestUtils.assertEqualAsJson("{\"code\":\"error\",\"message\":\"Missing or invalid csrf_token parameter\"}", writer.toString());
+	}
+	
+	@Test
+	public void testGuessTypes() throws IOException, ServletException, InterruptedException {
+	    when(request.getParameter("project")).thenReturn(Long.toString(project.id));
+	    when(request.getParameter("columnName")).thenReturn("foo");
+	    when(request.getParameter("csrf_token")).thenReturn(Command.csrfFactory.getFreshToken());
+	    
+	    String expectedQuery = "queries=%7B%22q1%22%3A%7B%22query%22%3A%22Japan%22%2C%22limit%22"+
+	            "%3A3%7D%2C%22q0%22%3A%7B%22query%22%3A%22France%22%2C%22limit%22%3A3%7D%7D";
+	    
+	    String serviceResponse = "{\n" + 
+	            "  \"q0\": {\n" + 
+	            "    \"result\": [\n" + 
+	            "      {\n" + 
+	            "        \"id\": \"Q17\",\n" + 
+	            "        \"name\": \"Japan\",\n" + 
+	            "        \"type\": [\n" + 
+	            "          {\n" + 
+	            "            \"id\": \"Q3624078\",\n" + 
+	            "            \"name\": \"sovereign state\"\n" + 
+	            "          },\n" + 
+	            "          {\n" + 
+	            "            \"id\": \"Q112099\",\n" + 
+	            "            \"name\": \"island nation\"\n" + 
+	            "          },\n" + 
+	            "          {\n" + 
+	            "            \"id\": \"Q6256\",\n" + 
+	            "            \"name\": \"country\"\n" + 
+	            "          }\n" + 
+	            "        ]\n" + 
+	            "      }\n" + 
+	            "    ]\n" + 
+	            "  },\n" + 
+	            "  \"q1\": {\n" + 
+	            "    \"result\": [\n" + 
+	            "      {\n" + 
+	            "        \"id\": \"Q142\",\n" + 
+	            "        \"name\": \"France\",\n" + 
+	            "        \"type\": [\n" + 
+	            "          {\n" + 
+	            "            \"id\": \"Q3624078\",\n" + 
+	            "            \"name\": \"sovereign state\"\n" + 
+	            "          },\n" + 
+	            "          {\n" + 
+	            "            \"id\": \"Q20181813\",\n" + 
+	            "            \"name\": \"colonial power\"\n" + 
+	            "          }\n" + 
+	            "        ]\n" + 
+	            "      }\n" + 
+	            "    ]\n" + 
+	            "  }\n" + 
+	            "}";
+	    
+	    String guessedTypes = "{\n" + 
+	            "       \"code\" : \"ok\",\n" + 
+	            "       \"types\" : [ {\n" + 
+	            "         \"count\" : 2,\n" + 
+	            "         \"id\" : \"Q3624078\",\n" + 
+	            "         \"name\" : \"sovereign state\",\n" + 
+	            "         \"score\" : 2\n" + 
+	            "       }, {\n" + 
+	            "         \"count\" : 1,\n" + 
+	            "         \"id\" : \"Q112099\",\n" + 
+	            "         \"name\" : \"island nation\",\n" + 
+	            "         \"score\" : 0.6666666666666666\n" + 
+	            "       }, {\n" + 
+	            "         \"count\" : 1,\n" + 
+	            "         \"id\" : \"Q20181813\",\n" + 
+	            "         \"name\" : \"colonial power\",\n" + 
+	            "         \"score\" : 0.5\n" + 
+	            "       }, {\n" + 
+	            "         \"count\" : 1,\n" + 
+	            "         \"id\" : \"Q6256\",\n" + 
+	            "         \"name\" : \"country\",\n" + 
+	            "         \"score\" : 0.3333333333333333\n" + 
+	            "       } ]\n" + 
+	            "     }";
+	    
+	    try (MockWebServer server = new MockWebServer()) {
+            server.start();
+            HttpUrl url = server.url("/api");
+            server.enqueue(new MockResponse().setBody(serviceResponse));
+            
+            when(request.getParameter("service")).thenReturn(url.toString());
+            
+            command.doPost(request, response);
+            
+            TestUtils.assertEqualAsJson(guessedTypes, writer.toString());
+            
+            RecordedRequest request = server.takeRequest();
+            Assert.assertEquals(request.getBody().readUtf8(), expectedQuery);
+	    }
 	}
 }

main/tests/server/src/com/google/refine/importers/WikitextImporterTests.java

@@ -34,14 +34,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.importers;
 
 
-import java.io.ByteArrayInputStream;
-import java.io.OutputStream;
 import java.io.StringReader;
-import java.net.HttpURLConnection;
+import java.util.ArrayList;
-import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
-import com.google.refine.model.recon.StandardReconConfig;
 import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 import org.powermock.api.mockito.PowerMockito;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.slf4j.LoggerFactory;
@@ -51,12 +53,16 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.refine.importers.WikitextImporter;
+import com.google.refine.model.Recon;
+import com.google.refine.model.ReconCandidate;
+import com.google.refine.model.recon.ReconJob;
+import com.google.refine.model.recon.StandardReconConfig;
 
-@PrepareForTest(StandardReconConfig.class)
+@PrepareForTest(WikitextImporter.class)
 public class WikitextImporterTests extends ImporterTest {
 
     private WikitextImporter importer = null;
+    private Map<String, Recon> mockedRecons = null;
     
     @Override
     @BeforeTest
@@ -69,6 +75,7 @@ public class WikitextImporterTests extends ImporterTest {
     public void setUp() {
         super.setUp();
         importer = new WikitextImporter();
+        mockedRecons = new HashMap<>();
     }
 
     @Override
@@ -131,79 +138,43 @@ public class WikitextImporterTests extends ImporterTest {
         Assert.assertEquals(project.rows.get(1).cells.get(2).value, "f");
     }
     
+    @BeforeMethod
+    public void mockReconCalls() throws Exception {
+        StandardReconConfig cfg = Mockito.spy(new StandardReconConfig(
+                "http://endpoint.com", "http://schemaspace", "http://schemaspace.com", null, true, Collections.emptyList(), 0));
+        PowerMockito.whenNew(StandardReconConfig.class).withAnyArguments().thenReturn(cfg);
+        Answer<List<Recon>> mockedResponse = new Answer<List<Recon>>() {
+            @Override
+            public List<Recon> answer(InvocationOnMock invocation) throws Throwable {
+                return fakeReconCall(invocation.getArgument(0));
+            }
+        };
+        PowerMockito.doAnswer(mockedResponse).when(cfg, "batchRecon", Mockito.any(), Mockito.anyLong());
+    }
+    
+    private List<Recon> fakeReconCall(List<ReconJob> jobs) {
+        List<Recon> result = new ArrayList<>();
+        for(ReconJob job : jobs) {
+            result.add(mockedRecons.get(job.toString()));
+        }
+        return result;
+    }
+    
     @Test
     public void readTableWithLinks() throws Exception {
-        String result = "{\n" +
-                "    \"q0\": {\n" +
-                "        \"result\": [\n" +
-                "            {\n" +
-                "                \"all_labels\": {\n" +
-                "                    \"score\": 100,\n" +
-                "                    \"weighted\": 100\n" +
-                "                },\n" +
-                "                \"score\": 100,\n" +
-                "                \"id\": \"Q116214\",\n" +
-                "                \"name\": \"European Centre for the Development of Vocational Training\",\n" +
-                "                \"type\": [\n" +
-                "                    {\n" +
-                "                        \"id\": \"Q392918\",\n" +
-                "                        \"name\": \"agency of the European Union\"\n" +
-                "                    }\n" +
-                "                ],\n" +
-                "                \"match\": true\n" +
-                "            }\n" +
-                "        ]\n" +
-                "    },\n" +
-                "    \"q1\": {\n" +
-                "        \"result\": [\n" +
-                "            {\n" +
-                "                \"all_labels\": {\n" +
-                "                    \"score\": 100,\n" +
-                "                    \"weighted\": 100\n" +
-                "                },\n" +
-                "                \"score\": 100,\n" +
-                "                \"id\": \"Q1377549\",\n" +
-                "                \"name\": \"European Foundation for the Improvement of Living and Working Conditions\",\n" +
-                "                \"type\": [\n" +
-                "                    {\n" +
-                "                        \"id\": \"Q392918\",\n" +
-                "                        \"name\": \"agency of the European Union\"\n" +
-                "                    }\n" +
-                "                ],\n" +
-                "                \"match\": true\n" +
-                "            }\n" +
-                "        ]\n" +
-                "    },\n" +
-                "    \"q2\": {\n" +
-                "        \"result\": [\n" +
-                "            {\n" +
-                "                \"all_labels\": {\n" +
-                "                    \"score\": 100,\n" +
-                "                    \"weighted\": 100\n" +
-                "                },\n" +
-                "                \"score\": 100,\n" +
-                "                \"id\": \"Q1377256\",\n" +
-                "                \"name\": \"European Monitoring Centre for Drugs and Drug Addiction\",\n" +
-                "                \"type\": [\n" +
-                "                    {\n" +
-                "                        \"id\": \"Q392918\",\n" +
-                "                        \"name\": \"agency of the European Union\"\n" +
-                "                    }\n" +
-                "                ],\n" +
-                "                \"match\": true\n" +
-                "            }\n" +
-                "        ]\n" +
-                "    }\n" +
-                "}";
-
         // This mock is used to avoid real network connection during test
-        URL url = PowerMockito.mock(URL.class);
+        Recon ecdvt = Mockito.mock(Recon.class);
-        HttpURLConnection connection = Mockito.mock(HttpURLConnection.class);
+        Mockito.when(ecdvt.getBestCandidate()).thenReturn(
-        Mockito.when(url.openConnection()).thenReturn(connection);
+                new ReconCandidate("Q116214", "European Centre for the Development of Vocational Training", new String[] {"Q392918"}, 100));
-        OutputStream out = Mockito.mock(OutputStream.class);
+        mockedRecons.put("{\"query\":\"https://de.wikipedia.org/wiki/Europäisches Zentrum für die Förderung der Berufsbildung\"}", ecdvt);
-        Mockito.when(connection.getOutputStream()).thenReturn(out); // avoid NullPointerException
+        Recon efilwc = Mockito.mock(Recon.class);
-        Mockito.when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(result.getBytes()));
+        Mockito.when(efilwc.getBestCandidate()).thenReturn(
-        PowerMockito.whenNew(URL.class).withAnyArguments().thenReturn(url);
+                new ReconCandidate("Q1377549", "European Foundation for the Improvement of Living and Working Conditions", new String[] {"Q392918"}, 100));
+        mockedRecons.put("{\"query\":\"https://de.wikipedia.org/wiki/Europäische Stiftung zur Verbesserung der Lebens- und Arbeitsbedingungen\"}", efilwc);
+        Recon emcdda = Mockito.mock(Recon.class);
+        Mockito.when(emcdda.getBestCandidate()).thenReturn(
+                new ReconCandidate("Q1377256", "European Monitoring Centre for Drugs and Drug Addiction", new String[] {"Q392918"}, 100));
+        mockedRecons.put("{\"query\":\"https://de.wikipedia.org/wiki/Europäische Beobachtungsstelle für Drogen und Drogensucht\"}", emcdda);
 
         // Data credits: Wikipedia contributors, https://de.wikipedia.org/w/index.php?title=Agenturen_der_Europäischen_Union&action=edit
         String input = "\n"

main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java

@@ -230,9 +230,9 @@ public class ExtendDataOperationTests extends RefineTest {
     public void mockHttpCalls() throws Exception {
     	mockStatic(ReconciledDataExtensionJob.class);
     	PowerMockito.spy(ReconciledDataExtensionJob.class);
-    	Answer<InputStream> mockedResponse = new Answer<InputStream>() {
+    	Answer<String> mockedResponse = new Answer<String>() {
 			@Override
-			public InputStream answer(InvocationOnMock invocation) throws Throwable {
+			public String answer(InvocationOnMock invocation) throws Throwable {
 				return fakeHttpCall(invocation.getArgument(0), invocation.getArgument(1));
 			}
     	};
@@ -410,10 +410,10 @@ public class ExtendDataOperationTests extends RefineTest {
         mockedResponses.put(ParsingUtilities.mapper.readTree(query), response);
     }
 
-    InputStream fakeHttpCall(String endpoint, String query) throws IOException {
+    String fakeHttpCall(String endpoint, String query) throws IOException {
     	  JsonNode parsedQuery = ParsingUtilities.mapper.readTree(query);
     	  if (mockedResponses.containsKey(parsedQuery)) {
-            return IOUtils.toInputStream(mockedResponses.get(parsedQuery), StandardCharsets.UTF_8);
+    		    return mockedResponses.get(parsedQuery);
     	  } else {
     		    throw new IllegalArgumentException("HTTP call not mocked for query: "+query);
     	  }