Merge pull request #2141 from OpenRefine/issue-1904-mock-data-extension-tests

Mock HTTP calls in data extension tests.

appveyor.yml

@@ -51,6 +51,10 @@ test_script:
 
     refine test
 
+on_failure:
+- cmd: |-
+    dir
+
 cache:
   - C:\Users\appveyor\.m2
 

extensions/database/pom.xml

@@ -64,7 +64,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.0</version>
+        <version>2.22.2</version>
         <configuration>
           <suiteXmlFiles>
             <suiteXmlFile>tests/conf/tests.xml</suiteXmlFile>

extensions/gdata/pom.xml

@@ -64,7 +64,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.0</version>
+        <version>${surefire.version}</version>
         <configuration>
            <skip>true</skip>
         </configuration>

extensions/jython/pom.xml

@@ -66,7 +66,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.0</version>
+        <version>${surefire.version}</version>
         <configuration>
           <suiteXmlFiles>
             <suiteXmlFile>tests/conf/tests.xml</suiteXmlFile>

extensions/pc-axis/pom.xml

@@ -64,7 +64,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.0</version>
+        <version>${surefire.version}</version>
         <configuration>
           <skip>true</skip>
         </configuration>

extensions/phonetic/pom.xml

@@ -66,7 +66,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.0</version>
+        <version>${surefire.version}</version>
         <configuration>
           <suiteXmlFiles>
             <suiteXmlFile>tests/conf/tests.xml</suiteXmlFile>

extensions/wikidata/pom.xml

@@ -14,6 +14,10 @@
     <version>3.3-SNAPSHOT</version>
   </parent>
 
+  <properties>
+    <powermock.version>2.0.2</powermock.version>
+  </properties>
+
   <build>
     <finalName>openrefine-wikidata</finalName>
     <resources>
@@ -69,7 +73,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.0</version>
+        <version>${surefire.version}</version>
         <configuration>
           <suiteXmlFiles>
             <suiteXmlFile>tests/conf/tests.xml</suiteXmlFile>
@@ -154,20 +158,24 @@
       <version>6.9.10</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <version>4.12</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
       <version>2.23.4</version>
       <scope>test</scope>
     </dependency>
-
+    <dependency>
-
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-module-testng</artifactId>
+      <version>${powermock.version}</version>
+      <scope>test</scope>
+     </dependency>
+     <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-api-mockito2</artifactId>
+      <version>${powermock.version}</version>
+      <scope>test</scope>
+     </dependency> 
   </dependencies>
 
 </project>

extensions/wikidata/tests/src/org/openrefine/wikidata/commands/CommandTest.java

@@ -34,14 +34,14 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.openrefine.wikidata.testing.TestingData;
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
 
-import com.google.refine.RefineTest;
 import com.google.refine.commands.Command;
 import com.google.refine.model.Project;
 
-public abstract class CommandTest extends RefineTest {
+public abstract class CommandTest extends WikidataRefineTest {
 
     protected Project project = null;
     protected HttpServletRequest request = null;
@@ -49,6 +49,7 @@ public abstract class CommandTest extends RefineTest {
     protected StringWriter writer = null;
 
     protected Command command = null;
+   
 
     @BeforeMethod(alwaysRun = true)
     public void setUpProject() {

extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java

@@ -23,11 +23,11 @@
  ******************************************************************************/
 package org.openrefine.wikidata.editing;
 
-import static org.testng.Assert.assertEquals;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -37,6 +37,7 @@ import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.openrefine.wikidata.testing.TestingData;
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.openrefine.wikidata.updates.ItemUpdate;
 import org.openrefine.wikidata.updates.ItemUpdateBuilder;
 import org.testng.annotations.BeforeMethod;
@@ -51,9 +52,7 @@ import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
 import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
 import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
 
-import com.google.refine.RefineTest;
+public class EditBatchProcessorTest extends WikidataRefineTest {
-
-public class EditBatchProcessorTest extends RefineTest {
 
     private WikibaseDataFetcher fetcher = null;
     private WikibaseDataEditor editor = null;

extensions/wikidata/tests/src/org/openrefine/wikidata/editing/NewItemLibraryTest.java

@@ -30,6 +30,7 @@ import java.util.Collections;
 
 import org.openrefine.wikidata.testing.JacksonSerializationTest;
 import org.openrefine.wikidata.testing.TestingData;
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
@@ -39,7 +40,7 @@ import com.google.refine.model.Project;
 import com.google.refine.model.Recon;
 import com.google.refine.model.recon.StandardReconConfig;
 
-public class NewItemLibraryTest extends RefineTest {
+public class NewItemLibraryTest extends WikidataRefineTest {
 
     private NewItemLibrary library;
 

extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java

@@ -33,6 +33,7 @@ import java.util.Properties;
 
 import org.openrefine.wikidata.schema.WikibaseSchema;
 import org.openrefine.wikidata.testing.TestingData;
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.openrefine.wikidata.updates.ItemUpdate;
 import org.openrefine.wikidata.updates.ItemUpdateBuilder;
 import org.testng.annotations.Test;
@@ -48,7 +49,7 @@ import com.google.refine.RefineTest;
 import com.google.refine.browsing.Engine;
 import com.google.refine.model.Project;
 
-public class QuickStatementsExporterTest extends RefineTest {
+public class QuickStatementsExporterTest extends WikidataRefineTest {
 
     private QuickStatementsExporter exporter = new QuickStatementsExporter();
     private ItemIdValue newIdA = TestingData.newIdA;

extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/SchemaExporterTest.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.io.StringWriter;
 import java.util.Properties;
 
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.testng.annotations.Test;
 
 import com.google.refine.RefineTest;
@@ -11,7 +12,7 @@ import com.google.refine.browsing.Engine;
 import com.google.refine.model.Project;
 import com.google.refine.util.TestUtils;
 
-public class SchemaExporterTest extends RefineTest {
+public class SchemaExporterTest extends WikidataRefineTest {
 	
 	private SchemaExporter exporter = new SchemaExporter();
 	

extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java

@@ -32,10 +32,10 @@ import java.io.StringReader;
 import java.io.StringWriter;
 import java.util.Properties;
 
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import com.google.refine.RefineTest;
 import com.google.refine.history.Change;
 import com.google.refine.model.AbstractOperation;
 import com.google.refine.model.Project;
@@ -46,7 +46,7 @@ import com.google.refine.util.TestUtils;
 
 import edu.mit.simile.butterfly.ButterflyModule;
 
-public abstract class OperationTest extends RefineTest {
+public abstract class OperationTest extends WikidataRefineTest {
 
     protected Project project = null;
     protected ButterflyModule module = null;

extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java

@@ -26,13 +26,13 @@ package org.openrefine.wikidata.schema;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNull;
 
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import com.google.refine.RefineTest;
 import com.google.refine.model.Project;
 
-public class ExpressionContextTest extends RefineTest {
+public class ExpressionContextTest extends WikidataRefineTest {
 
     Project project = null;
 

extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java

@@ -29,17 +29,17 @@ import java.io.Serializable;
 import org.openrefine.wikidata.qa.QAWarningStore;
 import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
 import org.openrefine.wikidata.testing.TestingData;
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
 
-import com.google.refine.RefineTest;
 import com.google.refine.model.Cell;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.model.Recon;
 import com.google.refine.model.Row;
 
-public class WbExpressionTest<T> extends RefineTest {
+public class WbExpressionTest<T> extends WikidataRefineTest {
 
     protected Project project;
     protected Row row;

extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java

@@ -32,6 +32,7 @@ import java.util.Collections;
 import java.util.List;
 
 import org.openrefine.wikidata.testing.TestingData;
+import org.openrefine.wikidata.testing.WikidataRefineTest;
 import org.openrefine.wikidata.updates.ItemUpdate;
 import org.openrefine.wikidata.updates.ItemUpdateBuilder;
 import org.testng.annotations.BeforeMethod;
@@ -47,13 +48,12 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
 import org.wikidata.wdtk.datamodel.interfaces.StringValue;
 import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
 
-import com.google.refine.RefineTest;
 import com.google.refine.browsing.Engine;
 import com.google.refine.browsing.EngineConfig;
 import com.google.refine.model.Project;
 import com.google.refine.util.TestUtils;
 
-public class WikibaseSchemaTest extends RefineTest {
+public class WikibaseSchemaTest extends WikidataRefineTest {
 
     private ItemIdValue qid1 = Datamodel.makeWikidataItemIdValue("Q1377");
     private ItemIdValue qid2 = Datamodel.makeWikidataItemIdValue("Q865528");

extensions/wikidata/tests/src/org/openrefine/wikidata/testing/WikidataRefineTest.java

@@ -0,0 +1,63 @@
+package org.openrefine.wikidata.testing;
+
+import static org.mockito.Mockito.mock;
+
+import java.io.File;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.testng.annotations.BeforeMethod;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.refine.ProjectManager;
+import com.google.refine.ProjectManagerStub;
+import com.google.refine.ProjectMetadata;
+import com.google.refine.RefineServlet;
+import com.google.refine.RefineServletStub;
+import com.google.refine.RefineTest;
+import com.google.refine.importers.SeparatorBasedImporter;
+import com.google.refine.importing.ImportingJob;
+import com.google.refine.importing.ImportingManager;
+import com.google.refine.model.Project;
+
+public class WikidataRefineTest {
+	protected File workspaceDir;
+    protected RefineServlet servlet;
+    private List<Project> projects = new ArrayList<Project>();
+    private List<ImportingJob> importingJobs = new ArrayList<ImportingJob>();
+    
+    public Project createCSVProject(String input) {
+    	return createCSVProject("test project", input);
+    }
+    
+    protected Project createCSVProject(String projectName, String input) {
+        Project project = new Project();
+        
+        ProjectMetadata metadata = new ProjectMetadata();
+        metadata.setName(projectName);
+        
+        ObjectNode options = mock(ObjectNode.class);
+        RefineTest.prepareImportOptions(options, ",", -1, 0, 0, 1, false, false);
+        
+        ImportingJob job = ImportingManager.createJob();
+        
+        SeparatorBasedImporter importer = new SeparatorBasedImporter();
+        
+        List<Exception> exceptions = new ArrayList<Exception>();
+        importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions);
+        project.update();
+        ProjectManager.singleton.registerProject(project, metadata);
+        
+        projects.add(project);
+        importingJobs.add(job);
+        return project;
+    }
+    
+    @BeforeMethod(alwaysRun = true)
+    public void initServlet() {
+    	servlet = new RefineServletStub();
+        ProjectManager.singleton = new ProjectManagerStub();
+        ImportingManager.initialize(servlet);
+    }
+}

main/pom.xml

@@ -18,6 +18,7 @@
     <jee.path>/</jee.path>
     <jee.port>3333</jee.port>
     <refine.data>/tmp/refine</refine.data>
+    <powermock.version>2.0.2</powermock.version>
   </properties>
 
   <scm>
@@ -79,7 +80,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.0</version>
+        <version>${surefire.version}</version>
         <configuration>
           <suiteXmlFiles>
             <suiteXmlFile>tests/server/conf/tests.xml</suiteXmlFile>
@@ -463,7 +464,18 @@
       <artifactId>jaxb-api</artifactId>
       <version>2.3.0</version>
     </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-module-testng</artifactId>
+      <version>${powermock.version}</version>
+      <scope>test</scope>
+     </dependency>
+     <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-api-mockito2</artifactId>
+      <version>${powermock.version}</version>
+      <scope>test</scope>
+     </dependency> 
   </dependencies>
-
 </project>
 

main/tests/server/src/com/google/refine/RefineTest.java

@@ -45,6 +45,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.io.FileUtils;
+import org.powermock.modules.testng.PowerMockTestCase;
 import org.slf4j.Logger;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
@@ -75,7 +76,7 @@ import edu.mit.simile.butterfly.ButterflyModule;
 /**
  * A base class containing various utilities to help testing Refine.
  */
-public class RefineTest {
+public class RefineTest extends PowerMockTestCase {
 
     protected Logger logger;
     
@@ -194,7 +195,7 @@ public class RefineTest {
      * @param guessValueType
      * @param ignoreQuotes
      */
-    private void prepareImportOptions(ObjectNode options,
+    public static void prepareImportOptions(ObjectNode options,
             String sep, int limit, int skip, int ignoreLines,
             int headerLines, boolean guessValueType, boolean ignoreQuotes) {
             

main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java

@@ -33,16 +33,26 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.operations.recon;
 
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 
+import org.apache.commons.io.IOUtils;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
@@ -50,6 +60,7 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.refine.RefineTest;
 import com.google.refine.browsing.Engine;
 import com.google.refine.browsing.EngineConfig;
@@ -63,13 +74,12 @@ import com.google.refine.model.recon.ReconciledDataExtensionJob;
 import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig;
 import com.google.refine.operations.EngineDependentOperation;
 import com.google.refine.operations.OperationRegistry;
-import com.google.refine.operations.recon.ExtendDataOperation;
+import com.google.refine.process.LongRunningProcessStub;
 import com.google.refine.process.Process;
-import com.google.refine.process.ProcessManager;
 import com.google.refine.util.ParsingUtilities;
 import com.google.refine.util.TestUtils;
 
-
+@PrepareForTest(ReconciledDataExtensionJob.class)
 public class ExtendDataOperationTests extends RefineTest {
 
     static final String ENGINE_JSON_URLS = "{\"mode\":\"row-based\"}}";
@@ -113,6 +123,8 @@ public class ExtendDataOperationTests extends RefineTest {
             "       \"status\" : \"pending\"\n" + 
             "     }";
     
+    private Map<JsonNode, String> mockedResponses = new HashMap<>();
+    
     static public class ReconciledDataExtensionJobStub extends ReconciledDataExtensionJob {
         public ReconciledDataExtensionJobStub(DataExtensionConfig obj, String endpoint) {
             super(obj, endpoint);
@@ -137,6 +149,7 @@ public class ExtendDataOperationTests extends RefineTest {
     EngineConfig engine_config;
     Engine engine;
 
+
     @BeforeMethod
     public void SetUp() throws IOException, ModelException {
         OperationRegistry.registerOperation(getCoreModule(), "extend-reconciled-data", ExtendDataOperation.class);
@@ -209,12 +222,44 @@ public class ExtendDataOperationTests extends RefineTest {
 
     /**
      * Test to fetch simple strings
+     * @throws Exception 
      */
+    
+    @BeforeMethod
+    public void mockHttpCalls() throws Exception {
+    	mockStatic(ReconciledDataExtensionJob.class);
+    	PowerMockito.spy(ReconciledDataExtensionJob.class);
+    	Answer<InputStream> mockedResponse = new Answer<InputStream>() {
+			@Override
+			public InputStream answer(InvocationOnMock invocation) throws Throwable {
+				return fakeHttpCall(invocation.getArgument(0), invocation.getArgument(1));
+			}
+    	};
+    	PowerMockito.doAnswer(mockedResponse).when(ReconciledDataExtensionJob.class, "performQuery", anyString(), anyString());
+    }
+    
+    @AfterMethod
+    public void cleanupHttpMocks() {
+    	mockedResponses.clear();
+    }
 
     @Test
     public void testFetchStrings() throws Exception {
+  
         DataExtensionConfig extension = DataExtensionConfig.reconstruct("{\"properties\":[{\"id\":\"P297\",\"name\":\"ISO 3166-1 alpha-2 code\"}]}");
         
+        mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P297\"}]}",
+      		  "{"
+  			+ "\"rows\": {"
+  			+ "    \"Q794\": {\"P297\": [{\"str\": \"IR\"}]},"
+  			+ "    \"Q863\": {\"P297\": [{\"str\": \"TJ\"}]},"
+  			+ "    \"Q30\": {\"P297\": [{\"str\": \"US\"}]},"
+  			+ "    \"Q17\": {\"P297\": [{\"str\": \"JP\"}]}"
+  			+ "},"
+  			+ "\"meta\": ["
+  			+ "   {\"name\": \"ISO 3166-1 alpha-2 code\", \"id\": \"P297\"}"
+  			+ "]}");
+        
         EngineDependentOperation op = new ExtendDataOperation(engine_config,
                 "country",
                 RECON_SERVICE,
@@ -222,17 +267,8 @@ public class ExtendDataOperationTests extends RefineTest {
                 RECON_SCHEMA_SPACE,
                 extension,
                 1);
-        ProcessManager pm = project.getProcessManager();
+        LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options));
-        Process process = op.createProcess(project, options);
+        process.run();
-        process.startPerforming(pm);
-        Assert.assertTrue(process.isRunning());
-        try {
-            // This is 10 seconds because for some reason running this test on Travis takes longer.
-            Thread.sleep(10000);
-        } catch (InterruptedException e) {
-            Assert.fail("Test interrupted");
-        }
-        Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected.");
 
         // Inspect rows
         Assert.assertTrue("IR".equals(project.rows.get(0).getCellValue(1)), "Bad country code for Iran.");
@@ -253,6 +289,18 @@ public class ExtendDataOperationTests extends RefineTest {
         DataExtensionConfig extension = DataExtensionConfig.reconstruct(
                 "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}");
         
+        mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}",
+        		"{"
+        		+ "\"rows\": {"
+        		+ "    \"Q794\": {\"P38\": [{\"float\": 1}]},"
+        		+ "    \"Q863\": {\"P38\": [{\"float\": 2}]},"
+        		+ "    \"Q30\": {\"P38\": [{\"float\": 1}]},"
+        		+ "    \"Q17\": {\"P38\": [{\"float\": 1}]}"
+        		+ "},"
+        		+ "\"meta\": ["
+        		+ "    {\"settings\": {\"count\": \"on\", \"rank\": \"any\"}, \"name\": \"currency\", \"id\": \"P38\"}"
+        		+ "]}");
+        
         EngineDependentOperation op = new ExtendDataOperation(engine_config,
                 "country",
                 RECON_SERVICE,
@@ -260,17 +308,9 @@ public class ExtendDataOperationTests extends RefineTest {
                 RECON_SCHEMA_SPACE,
                 extension,
                 1);
-        ProcessManager pm = project.getProcessManager();
+        
-        Process process = op.createProcess(project, options);
+        LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options));
-        process.startPerforming(pm);
+        process.run();
-        Assert.assertTrue(process.isRunning());
-        try {
-            // This is 10 seconds because for some reason running this test on Travis takes longer.
-            Thread.sleep(10000);
-        } catch (InterruptedException e) {
-            Assert.fail("Test interrupted");
-        }
-        Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected.");
 
         // Test to be updated as countries change currencies!
         Assert.assertTrue(Math.round((double)project.rows.get(2).getCellValue(1)) == 2, "Incorrect number of currencies returned for Tajikistan.");
@@ -288,6 +328,16 @@ public class ExtendDataOperationTests extends RefineTest {
         DataExtensionConfig extension = DataExtensionConfig.reconstruct(
                 "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"best\"}}]}");
         
+        mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"rank\":\"best\"}}]}",
+        		"{\"rows\":{"
+        		+ "   \"Q794\": {\"P38\": [{\"name\": \"Iranian rial\", \"id\": \"Q188608\"}]},"
+        		+ "   \"Q863\": {\"P38\": [{\"name\": \"Tajikistani somoni\", \"id\": \"Q199886\"}]},"
+        		+ "   \"Q30\": {\"P38\": [{\"name\": \"United States dollar\", \"id\": \"Q4917\"}]},"
+        		+ "   \"Q17\": {\"P38\": [{\"name\": \"Japanese yen\", \"id\": \"Q8146\"}]}"
+        		+ "}, \"meta\": ["
+        		+ "     {\"settings\": {\"rank\": \"best\"}, \"name\": \"currency\", \"id\": \"P38\"}"
+        		+ "]}");
+        
         EngineDependentOperation op = new ExtendDataOperation(engine_config,
                 "country",
                 RECON_SERVICE,
@@ -295,17 +345,8 @@ public class ExtendDataOperationTests extends RefineTest {
                 RECON_SCHEMA_SPACE,
                 extension,
                 1);
-        ProcessManager pm = project.getProcessManager();
+        LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options));
-        Process process = op.createProcess(project, options);
+        process.run();
-        process.startPerforming(pm);
-        Assert.assertTrue(process.isRunning());
-        try {
-            // This is 10 seconds because for some reason running this test on Travis takes longer.
-            Thread.sleep(10000);
-        } catch (InterruptedException e) {
-            Assert.fail("Test interrupted");
-        }
-        Assert.assertFalse(process.isRunning());
 
         /*
           * Tajikistan has one "preferred" currency and one "normal" one
@@ -329,6 +370,17 @@ public class ExtendDataOperationTests extends RefineTest {
         DataExtensionConfig extension = DataExtensionConfig.reconstruct(
                 "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"any\"}}]}");
         
+        mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"rank\":\"any\"}}]}",
+    			"{\"rows\": {"
+    			+ "   \"Q794\": {\"P38\": [{\"name\": \"Iranian rial\", \"id\": \"Q188608\"}]},"
+    			+ "   \"Q863\": {\"P38\": [{\"name\": \"Tajikistani somoni\", \"id\": \"Q199886\"}, {\"name\": \"Tajikistani ruble\", \"id\": \"Q2423956\"}]},"
+    			+ "   \"Q30\": {\"P38\": [{\"name\": \"United States dollar\", \"id\": \"Q4917\"}]},"
+    			+ "   \"Q17\": {\"P38\": [{\"name\": \"Japanese yen\", \"id\": \"Q8146\"}]}"
+    			+ "},"
+    			+ "\"meta\": ["
+    			+ "    {\"settings\": {\"rank\": \"any\"}, \"name\": \"currency\", \"id\": \"P38\"}"
+    			+ "]}");
+        
         EngineDependentOperation op = new ExtendDataOperation(engine_config,
                 "country",
                 RECON_SERVICE,
@@ -336,17 +388,8 @@ public class ExtendDataOperationTests extends RefineTest {
                 RECON_SCHEMA_SPACE,
                 extension,
                 1);
-        ProcessManager pm = project.getProcessManager();
+        LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options));
-        Process process = op.createProcess(project, options);
+        process.run();
-        process.startPerforming(pm);
-        Assert.assertTrue(process.isRunning());
-        try {
-            // This is 10 seconds because for some reason running this test on Travis takes longer.
-            Thread.sleep(10000);
-        } catch (InterruptedException e) {
-            Assert.fail("Test interrupted");
-        }
-        Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected.");
 
         /*
           * Tajikistan has one "preferred" currency and one "normal" one
@@ -361,5 +404,17 @@ public class ExtendDataOperationTests extends RefineTest {
         // Make sure all the values are reconciled
         Assert.assertTrue(project.columnModel.getColumnByName("currency").getReconStats().matchedTopics == 5);
     }
+    
+    private void mockHttpCall(String query, String response) throws IOException {
+    	mockedResponses.put(ParsingUtilities.mapper.readTree(query), response);
+    }
      
+    InputStream fakeHttpCall(String endpoint, String query) throws IOException {
+    	JsonNode parsedQuery = ParsingUtilities.mapper.readTree(query);
+    	if (mockedResponses.containsKey(parsedQuery)) {
+    		return IOUtils.toInputStream(mockedResponses.get(parsedQuery));
+    	} else {
+    		throw new IllegalArgumentException("HTTP call not mocked for query: "+query);
+    	}
+    }
 }

main/tests/server/src/com/google/refine/process/LongRunningProcessStub.java

@@ -0,0 +1,31 @@
+package com.google.refine.process;
+
+/**
+ * A long running process that we can actually run in the main
+ * thread of the test runner, because during tests it is actually
+ * expected to be quick.
+ * 
+ * It wraps an existing LongRunningProcess
+ * 
+ * @author Antonin Delpeuch
+ *
+ */
+public class LongRunningProcessStub extends LongRunningProcess {
+	
+	protected LongRunningProcess wrapped;
+
+	public LongRunningProcessStub(Process process) {
+		super("some description");
+		this.wrapped = (LongRunningProcess)process;
+	}
+	
+	public void run() {
+		wrapped.getRunnable().run();
+	}
+
+	@Override
+	protected Runnable getRunnable() {
+		return wrapped.getRunnable();
+	}
+
+}

pom.xml

@@ -22,6 +22,7 @@
     <jee.path>/</jee.path>
     <jee.port>3333</jee.port>
     <refine.data>/tmp/refine</refine.data>
+    <surefire.version>2.22.2</surefire.version>
     <surefireArgs></surefireArgs>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
   </properties>
@@ -90,7 +91,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.22.0</version>
+        <version>${surefire.version}</version>
         <configuration>
           <argLine>${surefireArgs}</argLine>
           <suiteXmlFiles>