Merge pull request #2141 from OpenRefine/issue-1904-mock-data-extension-tests

Mock HTTP calls in data extension tests.
This commit is contained in:
Antonin Delpeuch 2019-09-12 10:41:47 +01:00 committed by GitHub
commit c35b2e154f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 261 additions and 83 deletions

View File

@ -51,6 +51,10 @@ test_script:
refine test refine test
on_failure:
- cmd: |-
dir
cache: cache:
- C:\Users\appveyor\.m2 - C:\Users\appveyor\.m2

View File

@ -64,7 +64,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version> <version>2.22.2</version>
<configuration> <configuration>
<suiteXmlFiles> <suiteXmlFiles>
<suiteXmlFile>tests/conf/tests.xml</suiteXmlFile> <suiteXmlFile>tests/conf/tests.xml</suiteXmlFile>

View File

@ -64,7 +64,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version> <version>${surefire.version}</version>
<configuration> <configuration>
<skip>true</skip> <skip>true</skip>
</configuration> </configuration>

View File

@ -66,7 +66,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version> <version>${surefire.version}</version>
<configuration> <configuration>
<suiteXmlFiles> <suiteXmlFiles>
<suiteXmlFile>tests/conf/tests.xml</suiteXmlFile> <suiteXmlFile>tests/conf/tests.xml</suiteXmlFile>

View File

@ -64,7 +64,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version> <version>${surefire.version}</version>
<configuration> <configuration>
<skip>true</skip> <skip>true</skip>
</configuration> </configuration>

View File

@ -66,7 +66,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version> <version>${surefire.version}</version>
<configuration> <configuration>
<suiteXmlFiles> <suiteXmlFiles>
<suiteXmlFile>tests/conf/tests.xml</suiteXmlFile> <suiteXmlFile>tests/conf/tests.xml</suiteXmlFile>

View File

@ -14,6 +14,10 @@
<version>3.3-SNAPSHOT</version> <version>3.3-SNAPSHOT</version>
</parent> </parent>
<properties>
<powermock.version>2.0.2</powermock.version>
</properties>
<build> <build>
<finalName>openrefine-wikidata</finalName> <finalName>openrefine-wikidata</finalName>
<resources> <resources>
@ -69,7 +73,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version> <version>${surefire.version}</version>
<configuration> <configuration>
<suiteXmlFiles> <suiteXmlFiles>
<suiteXmlFile>tests/conf/tests.xml</suiteXmlFile> <suiteXmlFile>tests/conf/tests.xml</suiteXmlFile>
@ -154,20 +158,24 @@
<version>6.9.10</version> <version>6.9.10</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.mockito</groupId> <groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId> <artifactId>mockito-core</artifactId>
<version>2.23.4</version> <version>2.23.4</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-testng</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
</project> </project>

View File

@ -34,14 +34,14 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import com.google.refine.RefineTest;
import com.google.refine.commands.Command; import com.google.refine.commands.Command;
import com.google.refine.model.Project; import com.google.refine.model.Project;
public abstract class CommandTest extends RefineTest { public abstract class CommandTest extends WikidataRefineTest {
protected Project project = null; protected Project project = null;
protected HttpServletRequest request = null; protected HttpServletRequest request = null;
@ -49,6 +49,7 @@ public abstract class CommandTest extends RefineTest {
protected StringWriter writer = null; protected StringWriter writer = null;
protected Command command = null; protected Command command = null;
@BeforeMethod(alwaysRun = true) @BeforeMethod(alwaysRun = true)
public void setUpProject() { public void setUpProject() {

View File

@ -23,11 +23,11 @@
******************************************************************************/ ******************************************************************************/
package org.openrefine.wikidata.editing; package org.openrefine.wikidata.editing;
import static org.testng.Assert.assertEquals;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -37,6 +37,7 @@ import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
@ -51,9 +52,7 @@ import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor;
import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher;
import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException;
import com.google.refine.RefineTest; public class EditBatchProcessorTest extends WikidataRefineTest {
public class EditBatchProcessorTest extends RefineTest {
private WikibaseDataFetcher fetcher = null; private WikibaseDataFetcher fetcher = null;
private WikibaseDataEditor editor = null; private WikibaseDataEditor editor = null;

View File

@ -30,6 +30,7 @@ import java.util.Collections;
import org.openrefine.wikidata.testing.JacksonSerializationTest; import org.openrefine.wikidata.testing.JacksonSerializationTest;
import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -39,7 +40,7 @@ import com.google.refine.model.Project;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
import com.google.refine.model.recon.StandardReconConfig; import com.google.refine.model.recon.StandardReconConfig;
public class NewItemLibraryTest extends RefineTest { public class NewItemLibraryTest extends WikidataRefineTest {
private NewItemLibrary library; private NewItemLibrary library;

View File

@ -33,6 +33,7 @@ import java.util.Properties;
import org.openrefine.wikidata.schema.WikibaseSchema; import org.openrefine.wikidata.schema.WikibaseSchema;
import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -48,7 +49,7 @@ import com.google.refine.RefineTest;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.model.Project; import com.google.refine.model.Project;
public class QuickStatementsExporterTest extends RefineTest { public class QuickStatementsExporterTest extends WikidataRefineTest {
private QuickStatementsExporter exporter = new QuickStatementsExporter(); private QuickStatementsExporter exporter = new QuickStatementsExporter();
private ItemIdValue newIdA = TestingData.newIdA; private ItemIdValue newIdA = TestingData.newIdA;

View File

@ -4,6 +4,7 @@ import java.io.IOException;
import java.io.StringWriter; import java.io.StringWriter;
import java.util.Properties; import java.util.Properties;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.refine.RefineTest; import com.google.refine.RefineTest;
@ -11,7 +12,7 @@ import com.google.refine.browsing.Engine;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.util.TestUtils; import com.google.refine.util.TestUtils;
public class SchemaExporterTest extends RefineTest { public class SchemaExporterTest extends WikidataRefineTest {
private SchemaExporter exporter = new SchemaExporter(); private SchemaExporter exporter = new SchemaExporter();

View File

@ -32,10 +32,10 @@ import java.io.StringReader;
import java.io.StringWriter; import java.io.StringWriter;
import java.util.Properties; import java.util.Properties;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.refine.RefineTest;
import com.google.refine.history.Change; import com.google.refine.history.Change;
import com.google.refine.model.AbstractOperation; import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project; import com.google.refine.model.Project;
@ -46,7 +46,7 @@ import com.google.refine.util.TestUtils;
import edu.mit.simile.butterfly.ButterflyModule; import edu.mit.simile.butterfly.ButterflyModule;
public abstract class OperationTest extends RefineTest { public abstract class OperationTest extends WikidataRefineTest {
protected Project project = null; protected Project project = null;
protected ButterflyModule module = null; protected ButterflyModule module = null;

View File

@ -26,13 +26,13 @@ package org.openrefine.wikidata.schema;
import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull; import static org.testng.Assert.assertNull;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.refine.RefineTest;
import com.google.refine.model.Project; import com.google.refine.model.Project;
public class ExpressionContextTest extends RefineTest { public class ExpressionContextTest extends WikidataRefineTest {
Project project = null; Project project = null;

View File

@ -29,17 +29,17 @@ import java.io.Serializable;
import org.openrefine.wikidata.qa.QAWarningStore; import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import com.google.refine.RefineTest;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.ModelException; import com.google.refine.model.ModelException;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
import com.google.refine.model.Row; import com.google.refine.model.Row;
public class WbExpressionTest<T> extends RefineTest { public class WbExpressionTest<T> extends WikidataRefineTest {
protected Project project; protected Project project;
protected Row row; protected Row row;

View File

@ -32,6 +32,7 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.testing.TestingData;
import org.openrefine.wikidata.testing.WikidataRefineTest;
import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.openrefine.wikidata.updates.ItemUpdateBuilder;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
@ -47,13 +48,12 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
import org.wikidata.wdtk.datamodel.interfaces.StringValue; import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue; import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import com.google.refine.RefineTest;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.EngineConfig;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.util.TestUtils; import com.google.refine.util.TestUtils;
public class WikibaseSchemaTest extends RefineTest { public class WikibaseSchemaTest extends WikidataRefineTest {
private ItemIdValue qid1 = Datamodel.makeWikidataItemIdValue("Q1377"); private ItemIdValue qid1 = Datamodel.makeWikidataItemIdValue("Q1377");
private ItemIdValue qid2 = Datamodel.makeWikidataItemIdValue("Q865528"); private ItemIdValue qid2 = Datamodel.makeWikidataItemIdValue("Q865528");

View File

@ -0,0 +1,63 @@
package org.openrefine.wikidata.testing;
import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import org.testng.annotations.BeforeMethod;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectManagerStub;
import com.google.refine.ProjectMetadata;
import com.google.refine.RefineServlet;
import com.google.refine.RefineServletStub;
import com.google.refine.RefineTest;
import com.google.refine.importers.SeparatorBasedImporter;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.model.Project;
public class WikidataRefineTest {
protected File workspaceDir;
protected RefineServlet servlet;
private List<Project> projects = new ArrayList<Project>();
private List<ImportingJob> importingJobs = new ArrayList<ImportingJob>();
public Project createCSVProject(String input) {
return createCSVProject("test project", input);
}
protected Project createCSVProject(String projectName, String input) {
Project project = new Project();
ProjectMetadata metadata = new ProjectMetadata();
metadata.setName(projectName);
ObjectNode options = mock(ObjectNode.class);
RefineTest.prepareImportOptions(options, ",", -1, 0, 0, 1, false, false);
ImportingJob job = ImportingManager.createJob();
SeparatorBasedImporter importer = new SeparatorBasedImporter();
List<Exception> exceptions = new ArrayList<Exception>();
importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions);
project.update();
ProjectManager.singleton.registerProject(project, metadata);
projects.add(project);
importingJobs.add(job);
return project;
}
@BeforeMethod(alwaysRun = true)
public void initServlet() {
servlet = new RefineServletStub();
ProjectManager.singleton = new ProjectManagerStub();
ImportingManager.initialize(servlet);
}
}

View File

@ -18,6 +18,7 @@
<jee.path>/</jee.path> <jee.path>/</jee.path>
<jee.port>3333</jee.port> <jee.port>3333</jee.port>
<refine.data>/tmp/refine</refine.data> <refine.data>/tmp/refine</refine.data>
<powermock.version>2.0.2</powermock.version>
</properties> </properties>
<scm> <scm>
@ -79,7 +80,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version> <version>${surefire.version}</version>
<configuration> <configuration>
<suiteXmlFiles> <suiteXmlFiles>
<suiteXmlFile>tests/server/conf/tests.xml</suiteXmlFile> <suiteXmlFile>tests/server/conf/tests.xml</suiteXmlFile>
@ -463,7 +464,18 @@
<artifactId>jaxb-api</artifactId> <artifactId>jaxb-api</artifactId>
<version>2.3.0</version> <version>2.3.0</version>
</dependency> </dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-testng</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
</project> </project>

View File

@ -45,6 +45,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.powermock.modules.testng.PowerMockTestCase;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterMethod; import org.testng.annotations.AfterMethod;
@ -75,7 +76,7 @@ import edu.mit.simile.butterfly.ButterflyModule;
/** /**
* A base class containing various utilities to help testing Refine. * A base class containing various utilities to help testing Refine.
*/ */
public class RefineTest { public class RefineTest extends PowerMockTestCase {
protected Logger logger; protected Logger logger;
@ -194,7 +195,7 @@ public class RefineTest {
* @param guessValueType * @param guessValueType
* @param ignoreQuotes * @param ignoreQuotes
*/ */
private void prepareImportOptions(ObjectNode options, public static void prepareImportOptions(ObjectNode options,
String sep, int limit, int skip, int ignoreLines, String sep, int limit, int skip, int ignoreLines,
int headerLines, boolean guessValueType, boolean ignoreQuotes) { int headerLines, boolean guessValueType, boolean ignoreQuotes) {

View File

@ -33,16 +33,26 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.recon; package com.google.refine.operations.recon;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterMethod; import org.testng.annotations.AfterMethod;
@ -50,6 +60,7 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest; import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.RefineTest; import com.google.refine.RefineTest;
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.browsing.EngineConfig; import com.google.refine.browsing.EngineConfig;
@ -63,13 +74,12 @@ import com.google.refine.model.recon.ReconciledDataExtensionJob;
import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig; import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig;
import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OperationRegistry; import com.google.refine.operations.OperationRegistry;
import com.google.refine.operations.recon.ExtendDataOperation; import com.google.refine.process.LongRunningProcessStub;
import com.google.refine.process.Process; import com.google.refine.process.Process;
import com.google.refine.process.ProcessManager;
import com.google.refine.util.ParsingUtilities; import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.TestUtils; import com.google.refine.util.TestUtils;
@PrepareForTest(ReconciledDataExtensionJob.class)
public class ExtendDataOperationTests extends RefineTest { public class ExtendDataOperationTests extends RefineTest {
static final String ENGINE_JSON_URLS = "{\"mode\":\"row-based\"}}"; static final String ENGINE_JSON_URLS = "{\"mode\":\"row-based\"}}";
@ -113,6 +123,8 @@ public class ExtendDataOperationTests extends RefineTest {
" \"status\" : \"pending\"\n" + " \"status\" : \"pending\"\n" +
" }"; " }";
private Map<JsonNode, String> mockedResponses = new HashMap<>();
static public class ReconciledDataExtensionJobStub extends ReconciledDataExtensionJob { static public class ReconciledDataExtensionJobStub extends ReconciledDataExtensionJob {
public ReconciledDataExtensionJobStub(DataExtensionConfig obj, String endpoint) { public ReconciledDataExtensionJobStub(DataExtensionConfig obj, String endpoint) {
super(obj, endpoint); super(obj, endpoint);
@ -137,6 +149,7 @@ public class ExtendDataOperationTests extends RefineTest {
EngineConfig engine_config; EngineConfig engine_config;
Engine engine; Engine engine;
@BeforeMethod @BeforeMethod
public void SetUp() throws IOException, ModelException { public void SetUp() throws IOException, ModelException {
OperationRegistry.registerOperation(getCoreModule(), "extend-reconciled-data", ExtendDataOperation.class); OperationRegistry.registerOperation(getCoreModule(), "extend-reconciled-data", ExtendDataOperation.class);
@ -209,12 +222,44 @@ public class ExtendDataOperationTests extends RefineTest {
/** /**
* Test to fetch simple strings * Test to fetch simple strings
* @throws Exception
*/ */
@BeforeMethod
public void mockHttpCalls() throws Exception {
mockStatic(ReconciledDataExtensionJob.class);
PowerMockito.spy(ReconciledDataExtensionJob.class);
Answer<InputStream> mockedResponse = new Answer<InputStream>() {
@Override
public InputStream answer(InvocationOnMock invocation) throws Throwable {
return fakeHttpCall(invocation.getArgument(0), invocation.getArgument(1));
}
};
PowerMockito.doAnswer(mockedResponse).when(ReconciledDataExtensionJob.class, "performQuery", anyString(), anyString());
}
@AfterMethod
public void cleanupHttpMocks() {
mockedResponses.clear();
}
@Test @Test
public void testFetchStrings() throws Exception { public void testFetchStrings() throws Exception {
DataExtensionConfig extension = DataExtensionConfig.reconstruct("{\"properties\":[{\"id\":\"P297\",\"name\":\"ISO 3166-1 alpha-2 code\"}]}"); DataExtensionConfig extension = DataExtensionConfig.reconstruct("{\"properties\":[{\"id\":\"P297\",\"name\":\"ISO 3166-1 alpha-2 code\"}]}");
mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P297\"}]}",
"{"
+ "\"rows\": {"
+ " \"Q794\": {\"P297\": [{\"str\": \"IR\"}]},"
+ " \"Q863\": {\"P297\": [{\"str\": \"TJ\"}]},"
+ " \"Q30\": {\"P297\": [{\"str\": \"US\"}]},"
+ " \"Q17\": {\"P297\": [{\"str\": \"JP\"}]}"
+ "},"
+ "\"meta\": ["
+ " {\"name\": \"ISO 3166-1 alpha-2 code\", \"id\": \"P297\"}"
+ "]}");
EngineDependentOperation op = new ExtendDataOperation(engine_config, EngineDependentOperation op = new ExtendDataOperation(engine_config,
"country", "country",
RECON_SERVICE, RECON_SERVICE,
@ -222,17 +267,8 @@ public class ExtendDataOperationTests extends RefineTest {
RECON_SCHEMA_SPACE, RECON_SCHEMA_SPACE,
extension, extension,
1); 1);
ProcessManager pm = project.getProcessManager(); LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options));
Process process = op.createProcess(project, options); process.run();
process.startPerforming(pm);
Assert.assertTrue(process.isRunning());
try {
// This is 10 seconds because for some reason running this test on Travis takes longer.
Thread.sleep(10000);
} catch (InterruptedException e) {
Assert.fail("Test interrupted");
}
Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected.");
// Inspect rows // Inspect rows
Assert.assertTrue("IR".equals(project.rows.get(0).getCellValue(1)), "Bad country code for Iran."); Assert.assertTrue("IR".equals(project.rows.get(0).getCellValue(1)), "Bad country code for Iran.");
@ -253,6 +289,18 @@ public class ExtendDataOperationTests extends RefineTest {
DataExtensionConfig extension = DataExtensionConfig.reconstruct( DataExtensionConfig extension = DataExtensionConfig.reconstruct(
"{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}"); "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}");
mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}",
"{"
+ "\"rows\": {"
+ " \"Q794\": {\"P38\": [{\"float\": 1}]},"
+ " \"Q863\": {\"P38\": [{\"float\": 2}]},"
+ " \"Q30\": {\"P38\": [{\"float\": 1}]},"
+ " \"Q17\": {\"P38\": [{\"float\": 1}]}"
+ "},"
+ "\"meta\": ["
+ " {\"settings\": {\"count\": \"on\", \"rank\": \"any\"}, \"name\": \"currency\", \"id\": \"P38\"}"
+ "]}");
EngineDependentOperation op = new ExtendDataOperation(engine_config, EngineDependentOperation op = new ExtendDataOperation(engine_config,
"country", "country",
RECON_SERVICE, RECON_SERVICE,
@ -260,17 +308,9 @@ public class ExtendDataOperationTests extends RefineTest {
RECON_SCHEMA_SPACE, RECON_SCHEMA_SPACE,
extension, extension,
1); 1);
ProcessManager pm = project.getProcessManager();
Process process = op.createProcess(project, options); LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options));
process.startPerforming(pm); process.run();
Assert.assertTrue(process.isRunning());
try {
// This is 10 seconds because for some reason running this test on Travis takes longer.
Thread.sleep(10000);
} catch (InterruptedException e) {
Assert.fail("Test interrupted");
}
Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected.");
// Test to be updated as countries change currencies! // Test to be updated as countries change currencies!
Assert.assertTrue(Math.round((double)project.rows.get(2).getCellValue(1)) == 2, "Incorrect number of currencies returned for Tajikistan."); Assert.assertTrue(Math.round((double)project.rows.get(2).getCellValue(1)) == 2, "Incorrect number of currencies returned for Tajikistan.");
@ -288,6 +328,16 @@ public class ExtendDataOperationTests extends RefineTest {
DataExtensionConfig extension = DataExtensionConfig.reconstruct( DataExtensionConfig extension = DataExtensionConfig.reconstruct(
"{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"best\"}}]}"); "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"best\"}}]}");
mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"rank\":\"best\"}}]}",
"{\"rows\":{"
+ " \"Q794\": {\"P38\": [{\"name\": \"Iranian rial\", \"id\": \"Q188608\"}]},"
+ " \"Q863\": {\"P38\": [{\"name\": \"Tajikistani somoni\", \"id\": \"Q199886\"}]},"
+ " \"Q30\": {\"P38\": [{\"name\": \"United States dollar\", \"id\": \"Q4917\"}]},"
+ " \"Q17\": {\"P38\": [{\"name\": \"Japanese yen\", \"id\": \"Q8146\"}]}"
+ "}, \"meta\": ["
+ " {\"settings\": {\"rank\": \"best\"}, \"name\": \"currency\", \"id\": \"P38\"}"
+ "]}");
EngineDependentOperation op = new ExtendDataOperation(engine_config, EngineDependentOperation op = new ExtendDataOperation(engine_config,
"country", "country",
RECON_SERVICE, RECON_SERVICE,
@ -295,17 +345,8 @@ public class ExtendDataOperationTests extends RefineTest {
RECON_SCHEMA_SPACE, RECON_SCHEMA_SPACE,
extension, extension,
1); 1);
ProcessManager pm = project.getProcessManager(); LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options));
Process process = op.createProcess(project, options); process.run();
process.startPerforming(pm);
Assert.assertTrue(process.isRunning());
try {
// This is 10 seconds because for some reason running this test on Travis takes longer.
Thread.sleep(10000);
} catch (InterruptedException e) {
Assert.fail("Test interrupted");
}
Assert.assertFalse(process.isRunning());
/* /*
* Tajikistan has one "preferred" currency and one "normal" one * Tajikistan has one "preferred" currency and one "normal" one
@ -329,6 +370,17 @@ public class ExtendDataOperationTests extends RefineTest {
DataExtensionConfig extension = DataExtensionConfig.reconstruct( DataExtensionConfig extension = DataExtensionConfig.reconstruct(
"{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"any\"}}]}"); "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"any\"}}]}");
mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"rank\":\"any\"}}]}",
"{\"rows\": {"
+ " \"Q794\": {\"P38\": [{\"name\": \"Iranian rial\", \"id\": \"Q188608\"}]},"
+ " \"Q863\": {\"P38\": [{\"name\": \"Tajikistani somoni\", \"id\": \"Q199886\"}, {\"name\": \"Tajikistani ruble\", \"id\": \"Q2423956\"}]},"
+ " \"Q30\": {\"P38\": [{\"name\": \"United States dollar\", \"id\": \"Q4917\"}]},"
+ " \"Q17\": {\"P38\": [{\"name\": \"Japanese yen\", \"id\": \"Q8146\"}]}"
+ "},"
+ "\"meta\": ["
+ " {\"settings\": {\"rank\": \"any\"}, \"name\": \"currency\", \"id\": \"P38\"}"
+ "]}");
EngineDependentOperation op = new ExtendDataOperation(engine_config, EngineDependentOperation op = new ExtendDataOperation(engine_config,
"country", "country",
RECON_SERVICE, RECON_SERVICE,
@ -336,17 +388,8 @@ public class ExtendDataOperationTests extends RefineTest {
RECON_SCHEMA_SPACE, RECON_SCHEMA_SPACE,
extension, extension,
1); 1);
ProcessManager pm = project.getProcessManager(); LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options));
Process process = op.createProcess(project, options); process.run();
process.startPerforming(pm);
Assert.assertTrue(process.isRunning());
try {
// This is 10 seconds because for some reason running this test on Travis takes longer.
Thread.sleep(10000);
} catch (InterruptedException e) {
Assert.fail("Test interrupted");
}
Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected.");
/* /*
* Tajikistan has one "preferred" currency and one "normal" one * Tajikistan has one "preferred" currency and one "normal" one
@ -361,5 +404,17 @@ public class ExtendDataOperationTests extends RefineTest {
// Make sure all the values are reconciled // Make sure all the values are reconciled
Assert.assertTrue(project.columnModel.getColumnByName("currency").getReconStats().matchedTopics == 5); Assert.assertTrue(project.columnModel.getColumnByName("currency").getReconStats().matchedTopics == 5);
} }
private void mockHttpCall(String query, String response) throws IOException {
mockedResponses.put(ParsingUtilities.mapper.readTree(query), response);
}
InputStream fakeHttpCall(String endpoint, String query) throws IOException {
JsonNode parsedQuery = ParsingUtilities.mapper.readTree(query);
if (mockedResponses.containsKey(parsedQuery)) {
return IOUtils.toInputStream(mockedResponses.get(parsedQuery));
} else {
throw new IllegalArgumentException("HTTP call not mocked for query: "+query);
}
}
} }

View File

@ -0,0 +1,31 @@
package com.google.refine.process;
/**
* A long running process that we can actually run in the main
* thread of the test runner, because during tests it is actually
* expected to be quick.
*
* It wraps an existing LongRunningProcess
*
* @author Antonin Delpeuch
*
*/
public class LongRunningProcessStub extends LongRunningProcess {
protected LongRunningProcess wrapped;
public LongRunningProcessStub(Process process) {
super("some description");
this.wrapped = (LongRunningProcess)process;
}
public void run() {
wrapped.getRunnable().run();
}
@Override
protected Runnable getRunnable() {
return wrapped.getRunnable();
}
}

View File

@ -22,6 +22,7 @@
<jee.path>/</jee.path> <jee.path>/</jee.path>
<jee.port>3333</jee.port> <jee.port>3333</jee.port>
<refine.data>/tmp/refine</refine.data> <refine.data>/tmp/refine</refine.data>
<surefire.version>2.22.2</surefire.version>
<surefireArgs></surefireArgs> <surefireArgs></surefireArgs>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties> </properties>
@ -90,7 +91,7 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version> <version>${surefire.version}</version>
<configuration> <configuration>
<argLine>${surefireArgs}</argLine> <argLine>${surefireArgs}</argLine>
<suiteXmlFiles> <suiteXmlFiles>