From 3e241871cc8d02cd48deed4c701af66632b249a5 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 27 Aug 2019 19:52:21 +0100 Subject: [PATCH 1/4] Mock HTTP calls in data extension tests. Closes #1904. --- main/pom.xml | 14 +- .../src/com/google/refine/RefineTest.java | 3 +- .../recon/ExtendDataOperationTests.java | 149 ++++++++++++------ .../process/LongRunningProcessStub.java | 31 ++++ 4 files changed, 148 insertions(+), 49 deletions(-) create mode 100644 main/tests/server/src/com/google/refine/process/LongRunningProcessStub.java diff --git a/main/pom.xml b/main/pom.xml index 2b0b7d90f..fd4aceee5 100644 --- a/main/pom.xml +++ b/main/pom.xml @@ -18,6 +18,7 @@ / 3333 /tmp/refine + 2.0.2 @@ -463,7 +464,18 @@ jaxb-api 2.3.0 + + org.powermock + powermock-module-testng + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + - diff --git a/main/tests/server/src/com/google/refine/RefineTest.java b/main/tests/server/src/com/google/refine/RefineTest.java index 95610a354..fc65af2e4 100644 --- a/main/tests/server/src/com/google/refine/RefineTest.java +++ b/main/tests/server/src/com/google/refine/RefineTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.io.FileUtils; +import org.powermock.modules.testng.PowerMockTestCase; import org.slf4j.Logger; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -75,7 +76,7 @@ import edu.mit.simile.butterfly.ButterflyModule; /** * A base class containing various utilities to help testing Refine. */ -public class RefineTest { +public class RefineTest extends PowerMockTestCase { protected Logger logger; diff --git a/main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java b/main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java index 95fd0836a..f72383769 100644 --- a/main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java +++ b/main/tests/server/src/com/google/refine/operations/recon/ExtendDataOperationTests.java @@ -33,16 +33,26 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.operations.recon; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.powermock.api.mockito.PowerMockito.mockStatic; import java.io.IOException; +import java.io.InputStream; import java.io.StringWriter; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; +import org.apache.commons.io.IOUtils; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -50,6 +60,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.fasterxml.jackson.databind.JsonNode; import com.google.refine.RefineTest; import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; @@ -63,13 +74,12 @@ import com.google.refine.model.recon.ReconciledDataExtensionJob; import com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtensionConfig; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.OperationRegistry; -import com.google.refine.operations.recon.ExtendDataOperation; +import com.google.refine.process.LongRunningProcessStub; import com.google.refine.process.Process; -import com.google.refine.process.ProcessManager; import com.google.refine.util.ParsingUtilities; import com.google.refine.util.TestUtils; - +@PrepareForTest(ReconciledDataExtensionJob.class) public class ExtendDataOperationTests extends RefineTest { static final String ENGINE_JSON_URLS = "{\"mode\":\"row-based\"}}"; @@ -113,6 +123,8 @@ public class ExtendDataOperationTests extends RefineTest { " \"status\" : \"pending\"\n" + " }"; + private Map mockedResponses = new HashMap<>(); + static public class ReconciledDataExtensionJobStub extends ReconciledDataExtensionJob { public ReconciledDataExtensionJobStub(DataExtensionConfig obj, String endpoint) { super(obj, endpoint); @@ -137,6 +149,7 @@ public class ExtendDataOperationTests extends RefineTest { EngineConfig engine_config; Engine engine; + @BeforeMethod public void SetUp() throws IOException, ModelException { OperationRegistry.registerOperation(getCoreModule(), "extend-reconciled-data", ExtendDataOperation.class); @@ -209,12 +222,44 @@ public class ExtendDataOperationTests extends RefineTest { /** * Test to fetch simple strings + * @throws Exception */ + + @BeforeMethod + public void mockHttpCalls() throws Exception { + mockStatic(ReconciledDataExtensionJob.class); + PowerMockito.spy(ReconciledDataExtensionJob.class); + Answer mockedResponse = new Answer() { + @Override + public InputStream answer(InvocationOnMock invocation) throws Throwable { + return fakeHttpCall(invocation.getArgument(0), invocation.getArgument(1)); + } + }; + PowerMockito.doAnswer(mockedResponse).when(ReconciledDataExtensionJob.class, "performQuery", anyString(), anyString()); + } + + @AfterMethod + public void cleanupHttpMocks() { + mockedResponses.clear(); + } @Test public void testFetchStrings() throws Exception { + DataExtensionConfig extension = DataExtensionConfig.reconstruct("{\"properties\":[{\"id\":\"P297\",\"name\":\"ISO 3166-1 alpha-2 code\"}]}"); + mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P297\"}]}", + "{" + + "\"rows\": {" + + " \"Q794\": {\"P297\": [{\"str\": \"IR\"}]}," + + " \"Q863\": {\"P297\": [{\"str\": \"TJ\"}]}," + + " \"Q30\": {\"P297\": [{\"str\": \"US\"}]}," + + " \"Q17\": {\"P297\": [{\"str\": \"JP\"}]}" + + "}," + + "\"meta\": [" + + " {\"name\": \"ISO 3166-1 alpha-2 code\", \"id\": \"P297\"}" + + "]}"); + EngineDependentOperation op = new ExtendDataOperation(engine_config, "country", RECON_SERVICE, @@ -222,17 +267,8 @@ public class ExtendDataOperationTests extends RefineTest { RECON_SCHEMA_SPACE, extension, 1); - ProcessManager pm = project.getProcessManager(); - Process process = op.createProcess(project, options); - process.startPerforming(pm); - Assert.assertTrue(process.isRunning()); - try { - // This is 10 seconds because for some reason running this test on Travis takes longer. - Thread.sleep(10000); - } catch (InterruptedException e) { - Assert.fail("Test interrupted"); - } - Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected."); + LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options)); + process.run(); // Inspect rows Assert.assertTrue("IR".equals(project.rows.get(0).getCellValue(1)), "Bad country code for Iran."); @@ -253,6 +289,18 @@ public class ExtendDataOperationTests extends RefineTest { DataExtensionConfig extension = DataExtensionConfig.reconstruct( "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}"); + mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"count\":\"on\",\"rank\":\"any\"}}]}", + "{" + + "\"rows\": {" + + " \"Q794\": {\"P38\": [{\"float\": 1}]}," + + " \"Q863\": {\"P38\": [{\"float\": 2}]}," + + " \"Q30\": {\"P38\": [{\"float\": 1}]}," + + " \"Q17\": {\"P38\": [{\"float\": 1}]}" + + "}," + + "\"meta\": [" + + " {\"settings\": {\"count\": \"on\", \"rank\": \"any\"}, \"name\": \"currency\", \"id\": \"P38\"}" + + "]}"); + EngineDependentOperation op = new ExtendDataOperation(engine_config, "country", RECON_SERVICE, @@ -260,17 +308,9 @@ public class ExtendDataOperationTests extends RefineTest { RECON_SCHEMA_SPACE, extension, 1); - ProcessManager pm = project.getProcessManager(); - Process process = op.createProcess(project, options); - process.startPerforming(pm); - Assert.assertTrue(process.isRunning()); - try { - // This is 10 seconds because for some reason running this test on Travis takes longer. - Thread.sleep(10000); - } catch (InterruptedException e) { - Assert.fail("Test interrupted"); - } - Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected."); + + LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options)); + process.run(); // Test to be updated as countries change currencies! Assert.assertTrue(Math.round((double)project.rows.get(2).getCellValue(1)) == 2, "Incorrect number of currencies returned for Tajikistan."); @@ -288,6 +328,16 @@ public class ExtendDataOperationTests extends RefineTest { DataExtensionConfig extension = DataExtensionConfig.reconstruct( "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"best\"}}]}"); + mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"rank\":\"best\"}}]}", + "{\"rows\":{" + + " \"Q794\": {\"P38\": [{\"name\": \"Iranian rial\", \"id\": \"Q188608\"}]}," + + " \"Q863\": {\"P38\": [{\"name\": \"Tajikistani somoni\", \"id\": \"Q199886\"}]}," + + " \"Q30\": {\"P38\": [{\"name\": \"United States dollar\", \"id\": \"Q4917\"}]}," + + " \"Q17\": {\"P38\": [{\"name\": \"Japanese yen\", \"id\": \"Q8146\"}]}" + + "}, \"meta\": [" + + " {\"settings\": {\"rank\": \"best\"}, \"name\": \"currency\", \"id\": \"P38\"}" + + "]}"); + EngineDependentOperation op = new ExtendDataOperation(engine_config, "country", RECON_SERVICE, @@ -295,17 +345,8 @@ public class ExtendDataOperationTests extends RefineTest { RECON_SCHEMA_SPACE, extension, 1); - ProcessManager pm = project.getProcessManager(); - Process process = op.createProcess(project, options); - process.startPerforming(pm); - Assert.assertTrue(process.isRunning()); - try { - // This is 10 seconds because for some reason running this test on Travis takes longer. - Thread.sleep(10000); - } catch (InterruptedException e) { - Assert.fail("Test interrupted"); - } - Assert.assertFalse(process.isRunning()); + LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options)); + process.run(); /* * Tajikistan has one "preferred" currency and one "normal" one @@ -329,6 +370,17 @@ public class ExtendDataOperationTests extends RefineTest { DataExtensionConfig extension = DataExtensionConfig.reconstruct( "{\"properties\":[{\"id\":\"P38\",\"name\":\"currency\",\"settings\":{\"rank\":\"any\"}}]}"); + mockHttpCall("{\"ids\":[\"Q863\",\"Q794\",\"Q17\",\"Q30\"],\"properties\":[{\"id\":\"P38\",\"settings\":{\"rank\":\"any\"}}]}", + "{\"rows\": {" + + " \"Q794\": {\"P38\": [{\"name\": \"Iranian rial\", \"id\": \"Q188608\"}]}," + + " \"Q863\": {\"P38\": [{\"name\": \"Tajikistani somoni\", \"id\": \"Q199886\"}, {\"name\": \"Tajikistani ruble\", \"id\": \"Q2423956\"}]}," + + " \"Q30\": {\"P38\": [{\"name\": \"United States dollar\", \"id\": \"Q4917\"}]}," + + " \"Q17\": {\"P38\": [{\"name\": \"Japanese yen\", \"id\": \"Q8146\"}]}" + + "}," + + "\"meta\": [" + + " {\"settings\": {\"rank\": \"any\"}, \"name\": \"currency\", \"id\": \"P38\"}" + + "]}"); + EngineDependentOperation op = new ExtendDataOperation(engine_config, "country", RECON_SERVICE, @@ -336,17 +388,8 @@ public class ExtendDataOperationTests extends RefineTest { RECON_SCHEMA_SPACE, extension, 1); - ProcessManager pm = project.getProcessManager(); - Process process = op.createProcess(project, options); - process.startPerforming(pm); - Assert.assertTrue(process.isRunning()); - try { - // This is 10 seconds because for some reason running this test on Travis takes longer. - Thread.sleep(10000); - } catch (InterruptedException e) { - Assert.fail("Test interrupted"); - } - Assert.assertFalse(process.isRunning(), "The data extension process took longer than expected."); + LongRunningProcessStub process = new LongRunningProcessStub(op.createProcess(project, options)); + process.run(); /* * Tajikistan has one "preferred" currency and one "normal" one @@ -361,5 +404,17 @@ public class ExtendDataOperationTests extends RefineTest { // Make sure all the values are reconciled Assert.assertTrue(project.columnModel.getColumnByName("currency").getReconStats().matchedTopics == 5); } + + private void mockHttpCall(String query, String response) throws IOException { + mockedResponses.put(ParsingUtilities.mapper.readTree(query), response); + } + InputStream fakeHttpCall(String endpoint, String query) throws IOException { + JsonNode parsedQuery = ParsingUtilities.mapper.readTree(query); + if (mockedResponses.containsKey(parsedQuery)) { + return IOUtils.toInputStream(mockedResponses.get(parsedQuery)); + } else { + throw new IllegalArgumentException("HTTP call not mocked for query: "+query); + } + } } diff --git a/main/tests/server/src/com/google/refine/process/LongRunningProcessStub.java b/main/tests/server/src/com/google/refine/process/LongRunningProcessStub.java new file mode 100644 index 000000000..278026e6b --- /dev/null +++ b/main/tests/server/src/com/google/refine/process/LongRunningProcessStub.java @@ -0,0 +1,31 @@ +package com.google.refine.process; + +/** + * A long running process that we can actually run in the main + * thread of the test runner, because during tests it is actually + * expected to be quick. + * + * It wraps an existing LongRunningProcess + * + * @author Antonin Delpeuch + * + */ +public class LongRunningProcessStub extends LongRunningProcess { + + protected LongRunningProcess wrapped; + + public LongRunningProcessStub(Process process) { + super("some description"); + this.wrapped = (LongRunningProcess)process; + } + + public void run() { + wrapped.getRunnable().run(); + } + + @Override + protected Runnable getRunnable() { + return wrapped.getRunnable(); + } + +} From a67bc991f9e00d3d0fa913ddcb53797030a61d8c Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 27 Aug 2019 20:01:04 +0100 Subject: [PATCH 2/4] Update extensions/wikidata/pom.xml --- extensions/wikidata/pom.xml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/extensions/wikidata/pom.xml b/extensions/wikidata/pom.xml index 6ca34de7c..ddc39d8ed 100644 --- a/extensions/wikidata/pom.xml +++ b/extensions/wikidata/pom.xml @@ -14,6 +14,10 @@ 3.3-SNAPSHOT + + 2.0.2 + + openrefine-wikidata @@ -154,20 +158,24 @@ 6.9.10 test - - junit - junit - 4.12 - test - org.mockito mockito-core 2.23.4 test - - + + org.powermock + powermock-module-testng + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + From 6c128d6692e3ae18bb8c6a105c21210081f56419 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Wed, 28 Aug 2019 10:08:22 +0100 Subject: [PATCH 3/4] Attempt to fix surefire on Appveyor --- appveyor.yml | 4 ++++ extensions/database/pom.xml | 2 +- extensions/gdata/pom.xml | 2 +- extensions/jython/pom.xml | 2 +- extensions/pc-axis/pom.xml | 2 +- extensions/phonetic/pom.xml | 2 +- extensions/wikidata/pom.xml | 2 +- main/pom.xml | 2 +- pom.xml | 3 ++- 9 files changed, 13 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 5b16ddc35..4d121e259 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -51,6 +51,10 @@ test_script: refine test +on_failure: +- cmd: |- + dir + cache: - C:\Users\appveyor\.m2 diff --git a/extensions/database/pom.xml b/extensions/database/pom.xml index 0b43d1337..04ec35734 100644 --- a/extensions/database/pom.xml +++ b/extensions/database/pom.xml @@ -64,7 +64,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.0 + 2.22.2 tests/conf/tests.xml diff --git a/extensions/gdata/pom.xml b/extensions/gdata/pom.xml index cbc5fc4bb..1c4d41d67 100644 --- a/extensions/gdata/pom.xml +++ b/extensions/gdata/pom.xml @@ -64,7 +64,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.0 + ${surefire.version} true diff --git a/extensions/jython/pom.xml b/extensions/jython/pom.xml index 9ab130ad1..730902205 100644 --- a/extensions/jython/pom.xml +++ b/extensions/jython/pom.xml @@ -66,7 +66,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.0 + ${surefire.version} tests/conf/tests.xml diff --git a/extensions/pc-axis/pom.xml b/extensions/pc-axis/pom.xml index 23c295b8b..71dc1fa79 100644 --- a/extensions/pc-axis/pom.xml +++ b/extensions/pc-axis/pom.xml @@ -64,7 +64,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.0 + ${surefire.version} true diff --git a/extensions/phonetic/pom.xml b/extensions/phonetic/pom.xml index a2b43293f..05ae761e5 100644 --- a/extensions/phonetic/pom.xml +++ b/extensions/phonetic/pom.xml @@ -66,7 +66,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.0 + ${surefire.version} tests/conf/tests.xml diff --git a/extensions/wikidata/pom.xml b/extensions/wikidata/pom.xml index ddc39d8ed..ff7326707 100644 --- a/extensions/wikidata/pom.xml +++ b/extensions/wikidata/pom.xml @@ -73,7 +73,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.0 + ${surefire.version} tests/conf/tests.xml diff --git a/main/pom.xml b/main/pom.xml index fd4aceee5..f4607e50d 100644 --- a/main/pom.xml +++ b/main/pom.xml @@ -80,7 +80,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.0 + ${surefire.version} tests/server/conf/tests.xml diff --git a/pom.xml b/pom.xml index 3e70077f6..f478d6ba1 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ / 3333 /tmp/refine + 2.22.2 UTF-8 @@ -90,7 +91,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.0 + ${surefire.version} ${surefireArgs} From fdf742b052d11fc839d9f0850534eb2aaa4367e5 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Thu, 12 Sep 2019 10:24:25 +0100 Subject: [PATCH 4/4] Unlink wikidata extension from RefineTest --- .../wikidata/commands/CommandTest.java | 5 +- .../editing/EditBatchProcessorTest.java | 7 +-- .../wikidata/editing/NewItemLibraryTest.java | 3 +- .../QuickStatementsExporterTest.java | 3 +- .../exporters/SchemaExporterTest.java | 3 +- .../wikidata/operations/OperationTest.java | 4 +- .../schema/ExpressionContextTest.java | 4 +- .../wikidata/schema/WbExpressionTest.java | 4 +- .../wikidata/schema/WikibaseSchemaTest.java | 4 +- .../wikidata/testing/WikidataRefineTest.java | 63 +++++++++++++++++++ .../src/com/google/refine/RefineTest.java | 2 +- 11 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/testing/WikidataRefineTest.java diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/CommandTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/CommandTest.java index df0265ae8..b14644a4b 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/CommandTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/commands/CommandTest.java @@ -34,14 +34,14 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.openrefine.wikidata.testing.TestingData; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.testng.Assert; import org.testng.annotations.BeforeMethod; -import com.google.refine.RefineTest; import com.google.refine.commands.Command; import com.google.refine.model.Project; -public abstract class CommandTest extends RefineTest { +public abstract class CommandTest extends WikidataRefineTest { protected Project project = null; protected HttpServletRequest request = null; @@ -49,6 +49,7 @@ public abstract class CommandTest extends RefineTest { protected StringWriter writer = null; protected Command command = null; + @BeforeMethod(alwaysRun = true) public void setUpProject() { diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java index 5bcaf5a87..d6f4be0ef 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/EditBatchProcessorTest.java @@ -23,11 +23,11 @@ ******************************************************************************/ package org.openrefine.wikidata.editing; -import static org.testng.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import java.io.IOException; import java.util.ArrayList; @@ -37,6 +37,7 @@ import java.util.Map; import java.util.stream.Collectors; import org.openrefine.wikidata.testing.TestingData; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.testng.annotations.BeforeMethod; @@ -51,9 +52,7 @@ import org.wikidata.wdtk.wikibaseapi.WikibaseDataEditor; import org.wikidata.wdtk.wikibaseapi.WikibaseDataFetcher; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; -import com.google.refine.RefineTest; - -public class EditBatchProcessorTest extends RefineTest { +public class EditBatchProcessorTest extends WikidataRefineTest { private WikibaseDataFetcher fetcher = null; private WikibaseDataEditor editor = null; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/NewItemLibraryTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/NewItemLibraryTest.java index 7cb9d8222..4774b038b 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/NewItemLibraryTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/editing/NewItemLibraryTest.java @@ -30,6 +30,7 @@ import java.util.Collections; import org.openrefine.wikidata.testing.JacksonSerializationTest; import org.openrefine.wikidata.testing.TestingData; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -39,7 +40,7 @@ import com.google.refine.model.Project; import com.google.refine.model.Recon; import com.google.refine.model.recon.StandardReconConfig; -public class NewItemLibraryTest extends RefineTest { +public class NewItemLibraryTest extends WikidataRefineTest { private NewItemLibrary library; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java index 0b4968327..e93f31349 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java @@ -33,6 +33,7 @@ import java.util.Properties; import org.openrefine.wikidata.schema.WikibaseSchema; import org.openrefine.wikidata.testing.TestingData; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.testng.annotations.Test; @@ -48,7 +49,7 @@ import com.google.refine.RefineTest; import com.google.refine.browsing.Engine; import com.google.refine.model.Project; -public class QuickStatementsExporterTest extends RefineTest { +public class QuickStatementsExporterTest extends WikidataRefineTest { private QuickStatementsExporter exporter = new QuickStatementsExporter(); private ItemIdValue newIdA = TestingData.newIdA; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/SchemaExporterTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/SchemaExporterTest.java index fe535b183..28a51169d 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/SchemaExporterTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/SchemaExporterTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.io.StringWriter; import java.util.Properties; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.testng.annotations.Test; import com.google.refine.RefineTest; @@ -11,7 +12,7 @@ import com.google.refine.browsing.Engine; import com.google.refine.model.Project; import com.google.refine.util.TestUtils; -public class SchemaExporterTest extends RefineTest { +public class SchemaExporterTest extends WikidataRefineTest { private SchemaExporter exporter = new SchemaExporter(); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java index ca9f773e7..ae6987b02 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/operations/OperationTest.java @@ -32,10 +32,10 @@ import java.io.StringReader; import java.io.StringWriter; import java.util.Properties; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import com.google.refine.RefineTest; import com.google.refine.history.Change; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; @@ -46,7 +46,7 @@ import com.google.refine.util.TestUtils; import edu.mit.simile.butterfly.ButterflyModule; -public abstract class OperationTest extends RefineTest { +public abstract class OperationTest extends WikidataRefineTest { protected Project project = null; protected ButterflyModule module = null; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java index 98486812b..c0f7b5f57 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java @@ -26,13 +26,13 @@ package org.openrefine.wikidata.schema; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import com.google.refine.RefineTest; import com.google.refine.model.Project; -public class ExpressionContextTest extends RefineTest { +public class ExpressionContextTest extends WikidataRefineTest { Project project = null; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java index 97e69a9ce..7a84a19f9 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbExpressionTest.java @@ -29,17 +29,17 @@ import java.io.Serializable; import org.openrefine.wikidata.qa.QAWarningStore; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.openrefine.wikidata.testing.TestingData; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.testng.Assert; import org.testng.annotations.BeforeMethod; -import com.google.refine.RefineTest; import com.google.refine.model.Cell; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Recon; import com.google.refine.model.Row; -public class WbExpressionTest extends RefineTest { +public class WbExpressionTest extends WikidataRefineTest { protected Project project; protected Row row; diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java index f5965c338..3bae4d83c 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WikibaseSchemaTest.java @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.List; import org.openrefine.wikidata.testing.TestingData; +import org.openrefine.wikidata.testing.WikidataRefineTest; import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.testng.annotations.BeforeMethod; @@ -47,13 +48,12 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementRank; import org.wikidata.wdtk.datamodel.interfaces.StringValue; import org.wikidata.wdtk.datamodel.interfaces.TimeValue; -import com.google.refine.RefineTest; import com.google.refine.browsing.Engine; import com.google.refine.browsing.EngineConfig; import com.google.refine.model.Project; import com.google.refine.util.TestUtils; -public class WikibaseSchemaTest extends RefineTest { +public class WikibaseSchemaTest extends WikidataRefineTest { private ItemIdValue qid1 = Datamodel.makeWikidataItemIdValue("Q1377"); private ItemIdValue qid2 = Datamodel.makeWikidataItemIdValue("Q865528"); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/WikidataRefineTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/WikidataRefineTest.java new file mode 100644 index 000000000..93245384c --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/testing/WikidataRefineTest.java @@ -0,0 +1,63 @@ +package org.openrefine.wikidata.testing; + +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import org.testng.annotations.BeforeMethod; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.refine.ProjectManager; +import com.google.refine.ProjectManagerStub; +import com.google.refine.ProjectMetadata; +import com.google.refine.RefineServlet; +import com.google.refine.RefineServletStub; +import com.google.refine.RefineTest; +import com.google.refine.importers.SeparatorBasedImporter; +import com.google.refine.importing.ImportingJob; +import com.google.refine.importing.ImportingManager; +import com.google.refine.model.Project; + +public class WikidataRefineTest { + protected File workspaceDir; + protected RefineServlet servlet; + private List projects = new ArrayList(); + private List importingJobs = new ArrayList(); + + public Project createCSVProject(String input) { + return createCSVProject("test project", input); + } + + protected Project createCSVProject(String projectName, String input) { + Project project = new Project(); + + ProjectMetadata metadata = new ProjectMetadata(); + metadata.setName(projectName); + + ObjectNode options = mock(ObjectNode.class); + RefineTest.prepareImportOptions(options, ",", -1, 0, 0, 1, false, false); + + ImportingJob job = ImportingManager.createJob(); + + SeparatorBasedImporter importer = new SeparatorBasedImporter(); + + List exceptions = new ArrayList(); + importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions); + project.update(); + ProjectManager.singleton.registerProject(project, metadata); + + projects.add(project); + importingJobs.add(job); + return project; + } + + @BeforeMethod(alwaysRun = true) + public void initServlet() { + servlet = new RefineServletStub(); + ProjectManager.singleton = new ProjectManagerStub(); + ImportingManager.initialize(servlet); + } +} diff --git a/main/tests/server/src/com/google/refine/RefineTest.java b/main/tests/server/src/com/google/refine/RefineTest.java index fc65af2e4..2a57e2371 100644 --- a/main/tests/server/src/com/google/refine/RefineTest.java +++ b/main/tests/server/src/com/google/refine/RefineTest.java @@ -195,7 +195,7 @@ public class RefineTest extends PowerMockTestCase { * @param guessValueType * @param ignoreQuotes */ - private void prepareImportOptions(ObjectNode options, + public static void prepareImportOptions(ObjectNode options, String sep, int limit, int skip, int ignoreLines, int headerLines, boolean guessValueType, boolean ignoreQuotes) {