From 5278e929a20e5c310f4d13277410da46cc05bda7 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Thu, 25 Jan 2018 23:38:06 +0000 Subject: [PATCH 1/4] Refactor CrossFunctionTests' project creating logic into RefineTest --- .../com/google/refine/tests/RefineTest.java | 99 ++++++++++++++++ .../expr/functions/CrossFunctionTests.java | 110 +++--------------- 2 files changed, 114 insertions(+), 95 deletions(-) diff --git a/main/tests/server/src/com/google/refine/tests/RefineTest.java b/main/tests/server/src/com/google/refine/tests/RefineTest.java index fe998e52b..e2f0d7392 100644 --- a/main/tests/server/src/com/google/refine/tests/RefineTest.java +++ b/main/tests/server/src/com/google/refine/tests/RefineTest.java @@ -33,30 +33,129 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.testng.Assert; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeSuite; +import com.google.refine.ProjectManager; +import com.google.refine.ProjectMetadata; +import com.google.refine.RefineServlet; +import com.google.refine.importers.SeparatorBasedImporter; +import com.google.refine.importing.ImportingJob; +import com.google.refine.importing.ImportingManager; import com.google.refine.model.Cell; import com.google.refine.model.Column; import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.util.JSONUtilities; +/** + * A base class containing various utilities to help testing Refine. + */ public class RefineTest { protected Logger logger; + + protected RefineServlet servlet; + private List projects = new ArrayList(); + private List importingJobs = new ArrayList(); @BeforeSuite public void init() { System.setProperty("log4j.configuration", "tests.log4j.properties"); + ProjectManager.singleton = new ProjectManagerStub(); + } + + /** + * Helper to create a project from a CSV encoded as a file. Not much + * control is given on the import options, because this method is intended + * to be a quick way to create a project for a test. For more control over + * the import, just call the importer directly. + * + * The projects created via this method and their importing jobs will be disposed of + * at the end of each test. + * + * @param projectName + * the name of the project to create + * @param input + * the content of the file, encoded as a CSV (with "," as a separator) + * @return + */ + protected Project createCSVProject(String projectName, String input) { + Project project = new Project(); + + ProjectMetadata metadata = new ProjectMetadata(); + metadata.setName(projectName); + + JSONObject options = mock(JSONObject.class); + prepareImportOptions(options, ",", -1, 0, 0, 1, false, false); + + servlet = new RefineServletStub(); + ImportingManager.initialize(servlet); + ImportingJob job = ImportingManager.createJob(); + + SeparatorBasedImporter importer = new SeparatorBasedImporter(); + + List exceptions = new ArrayList(); + importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions); + project.update(); + ProjectManager.singleton.registerProject(project, metadata); + + projects.add(project); + importingJobs.add(job); + return project; + } + + /** + * Initializes the importing options for the CSV importer. + * @param options + * @param sep + * @param limit + * @param skip + * @param ignoreLines + * @param headerLines + * @param guessValueType + * @param ignoreQuotes + */ + private void prepareImportOptions(JSONObject options, + String sep, int limit, int skip, int ignoreLines, + int headerLines, boolean guessValueType, boolean ignoreQuotes) { + + whenGetStringOption("separator", options, sep); + whenGetIntegerOption("limit", options, limit); + whenGetIntegerOption("skipDataLines", options, skip); + whenGetIntegerOption("ignoreLines", options, ignoreLines); + whenGetIntegerOption("headerLines", options, headerLines); + whenGetBooleanOption("guessCellValueTypes", options, guessValueType); + whenGetBooleanOption("processQuotes", options, !ignoreQuotes); + whenGetBooleanOption("storeBlankCellsAsNulls", options, true); + } + + /** + * Cleans up the projects and jobs created with createCSVProject + */ + @AfterMethod + private void cleanupProjectsAndJobs() { + for(ImportingJob job : importingJobs) { + ImportingManager.disposeJob(job.id); + } + for(Project project: projects) { + ProjectManager.singleton.deleteProject(project.id); + } + servlet = null; } /** diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/CrossFunctionTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/CrossFunctionTests.java index fbc93b5ff..4145c6ceb 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/CrossFunctionTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/CrossFunctionTests.java @@ -1,37 +1,22 @@ package com.google.refine.tests.expr.functions; -import static org.mockito.Mockito.mock; - -import java.io.StringReader; -import java.util.ArrayList; import java.util.Calendar; -import java.util.List; import java.util.Properties; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; import com.google.refine.expr.EvalError; import com.google.refine.expr.HasFieldsListImpl; import com.google.refine.expr.WrappedRow; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; -import com.google.refine.importers.SeparatorBasedImporter; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; import com.google.refine.model.Project; import com.google.refine.model.Row; -import com.google.refine.tests.ProjectManagerStub; -import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineTest; /** @@ -47,82 +32,33 @@ public class CrossFunctionTests extends RefineTest { } // dependencies - RefineServlet servlet; Project projectGift; Project projectAddress; - ProjectMetadata metadata; - ImportingJob job; - JSONObject options; - SeparatorBasedImporter importer; - + + // data from: https://github.com/OpenRefine/OpenRefine/wiki/GREL-Other-Functions @BeforeMethod public void SetUp() { bindings = new Properties(); - servlet = new RefineServletStub(); - ProjectManager.singleton = new ProjectManagerStub(); - ImportingManager.initialize(servlet); - projectAddress = new Project(); - - job = ImportingManager.createJob(); - options = mock(JSONObject.class); - importer = new SeparatorBasedImporter(); + String projectName = "My Address Book"; + String input = "friend,address\n" + + "john,120 Main St.\n" + + "mary,50 Broadway Ave.\n" + + "john,999 XXXXXX St.\n" // john's 2nd address + + "anne,17 Morning Crescent\n"; + projectAddress = createCSVProject(projectName, input); + + projectName = "Christmas Gifts"; + input = "gift,recipient\n" + + "lamp,mary\n" + + "clock,john\n"; + projectGift = createCSVProject(projectName, input); - createMyAddressBook(); - projectGift = createChristmasGifts(); bindings.put("project", projectGift); - // add a column address based on column recipient bindings.put("columnName", "recipient"); } - // data from: https://github.com/OpenRefine/OpenRefine/wiki/GREL-Other-Functions - private Project createMyAddressBook() { - String projectName = "My Address Book"; - String input = "friend;address\n" - + "john;120 Main St.\n" - + "mary;50 Broadway Ave.\n" - + "john;999 XXXXXX St.\n" // john's 2nd address - + "anne;17 Morning Crescent\n"; - return createProject(projectName, input); - } - - private Project createChristmasGifts() { - String projectName = "Christmas Gifts"; - String input = "gift;recipient\n" - + "lamp;mary\n" - + "clock;john\n"; - return createProject(projectName, input); - } - - private Project createProject(String projectName, String input) { - Project project = new Project(); - ProjectMetadata metadata = new ProjectMetadata(); - - metadata.setName(projectName); - prepareOptions(";", -1, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, metadata); - - return project; - } - - @AfterMethod - public void TearDown() { - ImportingManager.disposeJob(job.id); - ProjectManager.singleton.deleteProject(projectGift.id); - ProjectManager.singleton.deleteProject(projectAddress.id); - job = null; - metadata = null; - projectGift = null; - projectAddress = null; - options = null; - importer = null; - } - - @Test public void crossFunctionOneToOneTest() throws Exception { Row row = ((Row)((WrappedRow) ((HasFieldsListImpl) invoke("cross", "mary", "My Address Book", "friend")).get(0)).row); @@ -189,20 +125,4 @@ public class CrossFunctionTests extends RefineTest { return function.call(bindings,args); } } - - - private void prepareOptions( - String sep, int limit, int skip, int ignoreLines, - int headerLines, boolean guessValueType, boolean ignoreQuotes) { - - whenGetStringOption("separator", options, sep); - whenGetIntegerOption("limit", options, limit); - whenGetIntegerOption("skipDataLines", options, skip); - whenGetIntegerOption("ignoreLines", options, ignoreLines); - whenGetIntegerOption("headerLines", options, headerLines); - whenGetBooleanOption("guessCellValueTypes", options, guessValueType); - whenGetBooleanOption("processQuotes", options, !ignoreQuotes); - whenGetBooleanOption("storeBlankCellsAsNulls", options, true); - } - } From 8f9fc6104666fe49c67d783dbe16a1c59188b6a6 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Thu, 25 Jan 2018 23:46:08 +0000 Subject: [PATCH 2/4] Migrate TextSearchFacetTests.java to use the helpers --- .../browsing/facets/TextSearchFacetTests.java | 57 ++----------------- 1 file changed, 4 insertions(+), 53 deletions(-) diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java index 345ee166b..0a5891c0c 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java @@ -68,15 +68,10 @@ import com.google.refine.tests.util.TestUtils; public class TextSearchFacetTests extends RefineTest { // dependencies - private RefineServlet servlet; private Project project; - private ProjectMetadata pm; - private JSONObject options; - private ImportingJob job; - private SeparatorBasedImporter importer; private TextSearchFacet textfilter; - private JSONObject textsearchfacet; private RowFilter rowfilter; + private JSONObject textsearchfacet; @Override @BeforeTest @@ -86,41 +81,13 @@ public class TextSearchFacetTests extends RefineTest { @BeforeMethod public void setUp() throws JSONException, IOException, ModelException { - servlet = new RefineServletStub(); - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - pm = new ProjectMetadata(); - pm.setName("TextSearchFacet test"); - ProjectManager.singleton.registerProject(project, pm); - options = mock(JSONObject.class); - - ImportingManager.initialize(servlet); - job = ImportingManager.createJob(); - importer = new SeparatorBasedImporter(); - - String csv = "Value\n" + project = createCSVProject("TextSearchFacet test", "Value\n" + "a\n" + "b\n" + "ab\n" - + "Abc\n"; - prepareOptions(",", 10, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, pm); + + "Abc\n"); } - - @AfterMethod - public void tearDown() { - ImportingManager.disposeJob(job.id); - ProjectManager.singleton.deleteProject(project.id); - job = null; - project = null; - pm = null; - options = null; - } - + /** * Test to demonstrate the intended behaviour of the function */ @@ -245,21 +212,5 @@ public class TextSearchFacetTests extends RefineTest { Assert.assertEquals(rowfilter.filterRow(project, 2, project.rows.get(2)),false); Assert.assertEquals(rowfilter.filterRow(project, 3, project.rows.get(3)),true); } - - private void prepareOptions( - String sep, int limit, int skip, int ignoreLines, - int headerLines, boolean guessValueType, boolean ignoreQuotes) { - - whenGetStringOption("separator", options, sep); - whenGetIntegerOption("limit", options, limit); - whenGetIntegerOption("skipDataLines", options, skip); - whenGetIntegerOption("ignoreLines", options, ignoreLines); - whenGetIntegerOption("headerLines", options, headerLines); - whenGetBooleanOption("guessCellValueTypes", options, guessValueType); - whenGetBooleanOption("processQuotes", options, !ignoreQuotes); - whenGetBooleanOption("storeBlankCellsAsNulls", options, true); - } - - } From feae25a5ffb9ef7fe6680dafb463e111ec377730 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Fri, 26 Jan 2018 17:18:16 +0000 Subject: [PATCH 3/4] Migrate more tests to the createCSVProject helper --- .../com/google/refine/tests/RefineTest.java | 25 ++- .../cell/JoinMultiValuedCellsTests.java | 96 ++-------- .../cell/KeyValueColumnizeTests.java | 173 ------------------ .../cell/SplitMultiValuedCellsTests.java | 99 +--------- .../tests/operations/cell/TransposeTests.java | 147 ++++++--------- 5 files changed, 102 insertions(+), 438 deletions(-) delete mode 100644 main/tests/server/src/com/google/refine/tests/operations/cell/KeyValueColumnizeTests.java diff --git a/main/tests/server/src/com/google/refine/tests/RefineTest.java b/main/tests/server/src/com/google/refine/tests/RefineTest.java index e2f0d7392..d38a73a4e 100644 --- a/main/tests/server/src/com/google/refine/tests/RefineTest.java +++ b/main/tests/server/src/com/google/refine/tests/RefineTest.java @@ -48,6 +48,7 @@ import org.json.JSONObject; import org.slf4j.Logger; import org.testng.Assert; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeSuite; import com.google.refine.ProjectManager; @@ -76,7 +77,27 @@ public class RefineTest { @BeforeSuite public void init() { System.setProperty("log4j.configuration", "tests.log4j.properties"); + } + + @BeforeMethod + protected void initProjectManager() { + servlet = new RefineServletStub(); ProjectManager.singleton = new ProjectManagerStub(); + ImportingManager.initialize(servlet); + } + + /** + * Helper to create a project from a CSV encoded as a file. Not much + * control is given on the import options, because this method is intended + * to be a quick way to create a project for a test. For more control over + * the import, just call the importer directly. + * + * @param input + * contents of the CSV file to create the project from + * @return + */ + protected Project createCSVProject(String input) { + return createCSVProject("test project", input); } /** @@ -103,8 +124,6 @@ public class RefineTest { JSONObject options = mock(JSONObject.class); prepareImportOptions(options, ",", -1, 0, 0, 1, false, false); - servlet = new RefineServletStub(); - ImportingManager.initialize(servlet); ImportingJob job = ImportingManager.createJob(); SeparatorBasedImporter importer = new SeparatorBasedImporter(); @@ -148,7 +167,7 @@ public class RefineTest { * Cleans up the projects and jobs created with createCSVProject */ @AfterMethod - private void cleanupProjectsAndJobs() { + protected void cleanupProjectsAndJobs() { for(ImportingJob job : importingJobs) { ImportingManager.disposeJob(job.id); } diff --git a/main/tests/server/src/com/google/refine/tests/operations/cell/JoinMultiValuedCellsTests.java b/main/tests/server/src/com/google/refine/tests/operations/cell/JoinMultiValuedCellsTests.java index aa32ef2b0..e28c29339 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/cell/JoinMultiValuedCellsTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/cell/JoinMultiValuedCellsTests.java @@ -33,49 +33,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.operations.cell; -import static org.mockito.Mockito.mock; - -import java.io.File; -import java.io.IOException; -import java.io.StringReader; import java.util.Properties; -import java.util.List; -import java.util.ArrayList; -import org.json.JSONException; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.importers.SeparatorBasedImporter; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.AbstractOperation; -import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.process.Process; import com.google.refine.operations.cell.MultiValuedCellJoinOperation; -import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class JoinMultiValuedCellsTests extends RefineTest { - // dependencies - private Project project; - private ProjectMetadata pm; - private JSONObject options; - private ImportingJob job; - private SeparatorBasedImporter importer; - @Override @BeforeTest @@ -83,48 +55,18 @@ public class JoinMultiValuedCellsTests extends RefineTest { logger = LoggerFactory.getLogger(this.getClass()); } - @BeforeMethod - public void setUp() throws JSONException, IOException, ModelException { - RefineServlet servlet = new RefineServletStub(); - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - pm = new ProjectMetadata(); - pm.setName("JoinMultiValuedCells test"); - ProjectManager.singleton.registerProject(project, pm); - options = mock(JSONObject.class); - - ImportingManager.initialize(servlet); - job = ImportingManager.createJob(); - importer = new SeparatorBasedImporter(); - } - - @AfterMethod - public void tearDown() { - ImportingManager.disposeJob(job.id); - ProjectManager.singleton.deleteProject(project.id); - job = null; - project = null; - pm = null; - options = null; - } - - /** + /* * Test to demonstrate the intended behaviour of the function */ @Test public void testJoinMultiValuedCells() throws Exception { - String csv = "Key,Value\n" - + "Record_1,one\n" - + ",two\n" - + ",three\n" - + ",four\n"; - prepareOptions(",", 10, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, pm); + Project project = createCSVProject( + "Key,Value\n" + + "Record_1,one\n" + + ",two\n" + + ",three\n" + + ",four\n"); AbstractOperation op = new MultiValuedCellJoinOperation( "Value", @@ -142,16 +84,13 @@ public class JoinMultiValuedCellsTests extends RefineTest { @Test public void testJoinMultiValuedCellsMultipleSpaces() throws Exception { - String csv = "Key,Value\n" + Project project = createCSVProject( + "Key,Value\n" + "Record_1,one\n" + ",two\n" + ",three\n" - + ",four\n"; - prepareOptions(",", 10, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, pm); + + ",four\n"); + AbstractOperation op = new MultiValuedCellJoinOperation( "Value", @@ -167,19 +106,6 @@ public class JoinMultiValuedCellsTests extends RefineTest { Assert.assertEquals(project.rows.get(0).getCellValue(valueCol), "one, ,two, ,three, ,four"); } - private void prepareOptions( - String sep, int limit, int skip, int ignoreLines, - int headerLines, boolean guessValueType, boolean ignoreQuotes) { - - whenGetStringOption("separator", options, sep); - whenGetIntegerOption("limit", options, limit); - whenGetIntegerOption("skipDataLines", options, skip); - whenGetIntegerOption("ignoreLines", options, ignoreLines); - whenGetIntegerOption("headerLines", options, headerLines); - whenGetBooleanOption("guessCellValueTypes", options, guessValueType); - whenGetBooleanOption("processQuotes", options, !ignoreQuotes); - whenGetBooleanOption("storeBlankCellsAsNulls", options, true); - } } diff --git a/main/tests/server/src/com/google/refine/tests/operations/cell/KeyValueColumnizeTests.java b/main/tests/server/src/com/google/refine/tests/operations/cell/KeyValueColumnizeTests.java deleted file mode 100644 index ed54846d7..000000000 --- a/main/tests/server/src/com/google/refine/tests/operations/cell/KeyValueColumnizeTests.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -package com.google.refine.tests.operations.cell; - -import static org.mockito.Mockito.mock; - -import java.io.File; -import java.io.IOException; -import java.io.StringReader; -import java.util.Properties; -import java.util.List; -import java.util.ArrayList; - -import org.json.JSONException; -import org.json.JSONObject; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; - -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.importers.SeparatorBasedImporter; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; -import com.google.refine.io.FileProjectManager; -import com.google.refine.model.AbstractOperation; -import com.google.refine.model.ModelException; -import com.google.refine.model.Project; -import com.google.refine.process.Process; -import com.google.refine.operations.cell.KeyValueColumnizeOperation; -import com.google.refine.tests.RefineServletStub; -import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; - - -public class KeyValueColumnizeTests extends RefineTest { - // dependencies - private RefineServlet servlet; - private Project project; - private ProjectMetadata pm; - private JSONObject options; - private ImportingJob job; - private SeparatorBasedImporter importer; - - - @Override - @BeforeTest - public void init() { - logger = LoggerFactory.getLogger(this.getClass()); - } - - @BeforeMethod - public void SetUp() throws JSONException, IOException, ModelException { - servlet = new RefineServletStub(); - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - pm = new ProjectMetadata(); - pm.setName("KeyValueColumnize test"); - ProjectManager.singleton.registerProject(project, pm); - options = mock(JSONObject.class); - - ImportingManager.initialize(servlet); - job = ImportingManager.createJob(); - importer = new SeparatorBasedImporter(); - } - - @AfterMethod - public void TearDown() { - ImportingManager.disposeJob(job.id); - ProjectManager.singleton.deleteProject(project.id); - job = null; - project = null; - pm = null; - options = null; - } - - /** - * Test to demonstrate the intended behaviour of the function, for issue #1214 - * https://github.com/OpenRefine/OpenRefine/issues/1214 - */ - - @Test - public void testKeyValueColumnize() throws Exception { - String csv = "Key,Value\n" - + "merchant,Katie\n" - + "fruit,apple\n" - + "price,1.2\n" - + "fruit,pear\n" - + "price,1.5\n" - + "merchant,John\n" - + "fruit,banana\n" - + "price,3.1\n"; - prepareOptions(",", 20, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, pm); - - AbstractOperation op = new KeyValueColumnizeOperation( - "Key", - "Value", - null); - Process process = op.createProcess(project, new Properties()); - process.performImmediate(); - - int merchantCol = project.columnModel.getColumnByName("merchant").getCellIndex(); - int fruitCol = project.columnModel.getColumnByName("fruit").getCellIndex(); - int priceCol = project.columnModel.getColumnByName("price").getCellIndex(); - - Assert.assertEquals(project.rows.get(0).getCellValue(merchantCol), "Katie"); - Assert.assertEquals(project.rows.get(1).getCellValue(merchantCol), null); - Assert.assertEquals(project.rows.get(2).getCellValue(merchantCol), "John"); - Assert.assertEquals(project.rows.get(0).getCellValue(fruitCol), "apple"); - Assert.assertEquals(project.rows.get(1).getCellValue(fruitCol), "pear"); - Assert.assertEquals(project.rows.get(2).getCellValue(fruitCol), "banana"); - Assert.assertEquals(project.rows.get(0).getCellValue(priceCol), "1.2"); - Assert.assertEquals(project.rows.get(1).getCellValue(priceCol), "1.5"); - Assert.assertEquals(project.rows.get(2).getCellValue(priceCol), "3.1"); - } - - private void prepareOptions( - String sep, int limit, int skip, int ignoreLines, - int headerLines, boolean guessValueType, boolean ignoreQuotes) { - - whenGetStringOption("separator", options, sep); - whenGetIntegerOption("limit", options, limit); - whenGetIntegerOption("skipDataLines", options, skip); - whenGetIntegerOption("ignoreLines", options, ignoreLines); - whenGetIntegerOption("headerLines", options, headerLines); - whenGetBooleanOption("guessCellValueTypes", options, guessValueType); - whenGetBooleanOption("processQuotes", options, !ignoreQuotes); - whenGetBooleanOption("storeBlankCellsAsNulls", options, true); - } - - -} - diff --git a/main/tests/server/src/com/google/refine/tests/operations/cell/SplitMultiValuedCellsTests.java b/main/tests/server/src/com/google/refine/tests/operations/cell/SplitMultiValuedCellsTests.java index eb38efef4..d4d8fc6f0 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/cell/SplitMultiValuedCellsTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/cell/SplitMultiValuedCellsTests.java @@ -33,50 +33,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.operations.cell; -import static org.mockito.Mockito.mock; -import java.io.File; -import java.io.IOException; -import java.io.StringReader; import java.util.Properties; -import java.util.List; -import java.util.ArrayList; -import org.json.JSONException; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.importers.SeparatorBasedImporter; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.AbstractOperation; -import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.process.Process; import com.google.refine.operations.cell.MultiValuedCellSplitOperation; -import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class SplitMultiValuedCellsTests extends RefineTest { - // dependencies - private RefineServlet servlet; - private Project project; - private ProjectMetadata pm; - private JSONObject options; - private ImportingJob job; - private SeparatorBasedImporter importer; - @Override @BeforeTest @@ -84,32 +56,6 @@ public class SplitMultiValuedCellsTests extends RefineTest { logger = LoggerFactory.getLogger(this.getClass()); } - @BeforeMethod - public void SetUp() throws JSONException, IOException, ModelException { - servlet = new RefineServletStub(); - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - pm = new ProjectMetadata(); - pm.setName("SplitMultiValuedCells test"); - ProjectManager.singleton.registerProject(project, pm); - options = mock(JSONObject.class); - - ImportingManager.initialize(servlet); - job = ImportingManager.createJob(); - importer = new SeparatorBasedImporter(); - } - - @AfterMethod - public void TearDown() { - ImportingManager.disposeJob(job.id); - ProjectManager.singleton.deleteProject(project.id); - job = null; - project = null; - pm = null; - options = null; - } - /** * Test to demonstrate the intended behaviour of the function, for issue #1268 * https://github.com/OpenRefine/OpenRefine/issues/1268 @@ -117,13 +63,9 @@ public class SplitMultiValuedCellsTests extends RefineTest { @Test public void testSplitMultiValuedCellsTextSeparator() throws Exception { - String csv = "Key,Value\n" - + "Record_1,one:two;three four\n"; - prepareOptions(",", 10, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, pm); + Project project = createCSVProject( + "Key,Value\n" + + "Record_1,one:two;three four\n"); AbstractOperation op = new MultiValuedCellSplitOperation( "Value", @@ -144,13 +86,9 @@ public class SplitMultiValuedCellsTests extends RefineTest { @Test public void testSplitMultiValuedCellsRegExSeparator() throws Exception { - String csv = "Key,Value\n" - + "Record_1,one:two;three four\n"; - prepareOptions(",", 10, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, pm); + Project project = createCSVProject( + "Key,Value\n" + + "Record_1,one:two;three four\n"); AbstractOperation op = new MultiValuedCellSplitOperation( "Value", @@ -175,13 +113,10 @@ public class SplitMultiValuedCellsTests extends RefineTest { @Test public void testSplitMultiValuedCellsLengths() throws Exception { - String csv = "Key,Value\n" - + "Record_1,one:two;three four\n"; - prepareOptions(",", 10, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, pm); + Project project = createCSVProject( + "Key,Value\n" + + "Record_1,one:two;three four\n"); + int[] lengths = {4,4,6,4}; AbstractOperation op = new MultiValuedCellSplitOperation( @@ -204,20 +139,6 @@ public class SplitMultiValuedCellsTests extends RefineTest { Assert.assertEquals(project.rows.get(3).getCellValue(valueCol), "four"); } - private void prepareOptions( - String sep, int limit, int skip, int ignoreLines, - int headerLines, boolean guessValueType, boolean ignoreQuotes) { - - whenGetStringOption("separator", options, sep); - whenGetIntegerOption("limit", options, limit); - whenGetIntegerOption("skipDataLines", options, skip); - whenGetIntegerOption("ignoreLines", options, ignoreLines); - whenGetIntegerOption("headerLines", options, headerLines); - whenGetBooleanOption("guessCellValueTypes", options, guessValueType); - whenGetBooleanOption("processQuotes", options, !ignoreQuotes); - whenGetBooleanOption("storeBlankCellsAsNulls", options, true); - } - } diff --git a/main/tests/server/src/com/google/refine/tests/operations/cell/TransposeTests.java b/main/tests/server/src/com/google/refine/tests/operations/cell/TransposeTests.java index 947fbe668..e52fcebec 100644 --- a/main/tests/server/src/com/google/refine/tests/operations/cell/TransposeTests.java +++ b/main/tests/server/src/com/google/refine/tests/operations/cell/TransposeTests.java @@ -33,34 +33,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.operations.cell; -import static org.mockito.Mockito.mock; - -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; import java.util.Properties; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.history.HistoryEntry; -import com.google.refine.importers.SeparatorBasedImporter; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; import com.google.refine.model.AbstractOperation; import com.google.refine.model.Project; import com.google.refine.operations.cell.KeyValueColumnizeOperation; import com.google.refine.process.Process; -import com.google.refine.tests.ProjectManagerStub; -import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineTest; public class TransposeTests extends RefineTest { @@ -71,67 +54,34 @@ public class TransposeTests extends RefineTest { logger = LoggerFactory.getLogger(this.getClass()); } - // dependencies - RefineServlet servlet; - Project project; - ProjectMetadata metadata; - ImportingJob job; - JSONObject options; - SeparatorBasedImporter importer; - - @BeforeMethod - public void SetUp() { - servlet = new RefineServletStub(); - ProjectManager.singleton = new ProjectManagerStub(); - ImportingManager.initialize(servlet); - project = new Project(); - metadata = new ProjectMetadata(); - - job = ImportingManager.createJob(); - options = mock(JSONObject.class); - importer = new SeparatorBasedImporter(); - } - - @AfterMethod - public void TearDown() { - ImportingManager.disposeJob(job.id); - ProjectManager.singleton.deleteProject(project.id); - job = null; - metadata = null; - project = null; - options = null; - importer = null; - } - + /** + * Test in the case where an ID is available in the first column. + * @throws Exception + */ @Test - public void keyValueComumnize() throws Exception { - String input = "ID;Cat;Val\n" - + "1;a;1\n" - + "1;b;3\n" - + "2;b;4\n" - + "2;c;5\n" - + "3;a;2\n" - + "3;b;5\n" - + "3;d;3\n"; - - prepareOptions(";", -1, 0, 0, 1, false, false); - List exceptions = new ArrayList(); - importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions); - project.update(); - ProjectManager.singleton.registerProject(project, metadata); + public void testKeyValueColumnizeWithID() throws Exception { + Project project = createCSVProject( + "ID,Cat,Val\n" + + "1,a,1\n" + + "1,b,3\n" + + "2,b,4\n" + + "2,c,5\n" + + "3,a,2\n" + + "3,b,5\n" + + "3,d,3\n"); AbstractOperation op = new KeyValueColumnizeOperation( "Cat", "Val", null); Process process = op.createProcess(project, new Properties()); - HistoryEntry historyEntry = process.performImmediate(); + process.performImmediate(); // Expected output from the GUI. - // ID;a;b;c;d - // 1;1;3;; - // 2;;4;5; - // 3;2;5;;3 + // ID,a,b,c,d + // 1,1,3,, + // 2,,4,5, + // 3,2,5,,3 Assert.assertEquals(project.columnModel.columns.size(), 5); Assert.assertEquals(project.columnModel.columns.get(0).getName(), "ID"); Assert.assertEquals(project.columnModel.columns.get(1).getName(), "a"); @@ -142,38 +92,59 @@ public class TransposeTests extends RefineTest { // The actual row data structure has to leave the columns model untouched for redo/undo purpose. // So we have 2 empty columns(column 1,2) on the row level. - // 1;1;3;; + // 1,1,3,, Assert.assertEquals(project.rows.get(0).cells.get(0).value, "1"); Assert.assertEquals(project.rows.get(0).cells.get(3).value, "1"); Assert.assertEquals(project.rows.get(0).cells.get(4).value, "3"); - // 2;;4;5; + // 2,,4,5, Assert.assertEquals(project.rows.get(1).cells.get(0).value, "2"); Assert.assertEquals(project.rows.get(1).cells.get(4).value, "4"); Assert.assertEquals(project.rows.get(1).cells.get(5).value, "5"); - // 3;2;5;;3 + // 3,2,5,,3 Assert.assertEquals(project.rows.get(2).cells.get(0).value, "3"); Assert.assertEquals(project.rows.get(2).cells.get(3).value, "2"); Assert.assertEquals(project.rows.get(2).cells.get(4).value, "5"); Assert.assertEquals(project.rows.get(2).cells.get(6).value, "3"); } + + /** + * Test to demonstrate the intended behaviour of the function when no id is available, for issue #1214 + * https://github.com/OpenRefine/OpenRefine/issues/1214 + */ + @Test + public void testKeyValueColumnizeWithoutID() throws Exception { + Project project = createCSVProject( + "Key,Value\n" + + "merchant,Katie\n" + + "fruit,apple\n" + + "price,1.2\n" + + "fruit,pear\n" + + "price,1.5\n" + + "merchant,John\n" + + "fruit,banana\n" + + "price,3.1\n"); + AbstractOperation op = new KeyValueColumnizeOperation( + "Key", + "Value", + null); + Process process = op.createProcess(project, new Properties()); + process.performImmediate(); - - - private void prepareOptions( - String sep, int limit, int skip, int ignoreLines, - int headerLines, boolean guessValueType, boolean ignoreQuotes) { - - whenGetStringOption("separator", options, sep); - whenGetIntegerOption("limit", options, limit); - whenGetIntegerOption("skipDataLines", options, skip); - whenGetIntegerOption("ignoreLines", options, ignoreLines); - whenGetIntegerOption("headerLines", options, headerLines); - whenGetBooleanOption("guessCellValueTypes", options, guessValueType); - whenGetBooleanOption("processQuotes", options, !ignoreQuotes); - whenGetBooleanOption("storeBlankCellsAsNulls", options, true); - } - + int merchantCol = project.columnModel.getColumnByName("merchant").getCellIndex(); + int fruitCol = project.columnModel.getColumnByName("fruit").getCellIndex(); + int priceCol = project.columnModel.getColumnByName("price").getCellIndex(); + + Assert.assertEquals(project.rows.get(0).getCellValue(merchantCol), "Katie"); + Assert.assertEquals(project.rows.get(1).getCellValue(merchantCol), null); + Assert.assertEquals(project.rows.get(2).getCellValue(merchantCol), "John"); + Assert.assertEquals(project.rows.get(0).getCellValue(fruitCol), "apple"); + Assert.assertEquals(project.rows.get(1).getCellValue(fruitCol), "pear"); + Assert.assertEquals(project.rows.get(2).getCellValue(fruitCol), "banana"); + Assert.assertEquals(project.rows.get(0).getCellValue(priceCol), "1.2"); + Assert.assertEquals(project.rows.get(1).getCellValue(priceCol), "1.5"); + Assert.assertEquals(project.rows.get(2).getCellValue(priceCol), "3.1"); + } } From 0426704938be2becce6313bae94cf25c16451d81 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sat, 27 Jan 2018 13:58:06 +0000 Subject: [PATCH 4/4] Add helper createProjectWithColumns --- main/src/com/google/refine/RefineServlet.java | 3 +- .../com/google/refine/tests/RefineTest.java | 42 ++++++++++++++++- .../browsing/facets/TextSearchFacetTests.java | 19 +------- .../tests/expr/functions/FunctionTests.java | 25 +--------- .../expr/functions/booleans/BooleanTests.java | 31 ++---------- .../google/refine/tests/model/CacheTests.java | 31 +----------- .../refine/tests/model/UrlFetchingTests.java | 47 ++----------------- .../tests/model/changes/MassChangeTests.java | 12 +---- .../tests/recon/DataExtensionTests.java | 25 +--------- 9 files changed, 58 insertions(+), 177 deletions(-) diff --git a/main/src/com/google/refine/RefineServlet.java b/main/src/com/google/refine/RefineServlet.java index cb6922235..7081c0ac4 100644 --- a/main/src/com/google/refine/RefineServlet.java +++ b/main/src/com/google/refine/RefineServlet.java @@ -124,7 +124,8 @@ public class RefineServlet extends Butterfly { if (data == null) { throw new ServletException("can't find servlet init config 'refine.data', I have to give up initializing"); } - + logger.error("initializing FileProjectManager with dir"); + logger.error(data); s_dataDir = new File(data); FileProjectManager.initialize(s_dataDir); ImportingManager.initialize(this); diff --git a/main/tests/server/src/com/google/refine/tests/RefineTest.java b/main/tests/server/src/com/google/refine/tests/RefineTest.java index d38a73a4e..2384d1dbb 100644 --- a/main/tests/server/src/com/google/refine/tests/RefineTest.java +++ b/main/tests/server/src/com/google/refine/tests/RefineTest.java @@ -38,10 +38,13 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.File; +import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.List; +import org.apache.commons.io.FileUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -57,10 +60,13 @@ import com.google.refine.RefineServlet; import com.google.refine.importers.SeparatorBasedImporter; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingManager; +import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; import com.google.refine.model.Column; +import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; +import com.google.refine.tests.util.TestUtils; import com.google.refine.util.JSONUtilities; /** @@ -70,6 +76,8 @@ public class RefineTest { protected Logger logger; + boolean testFailed; + protected File workspaceDir; protected RefineServlet servlet; private List projects = new ArrayList(); private List importingJobs = new ArrayList(); @@ -77,8 +85,22 @@ public class RefineTest { @BeforeSuite public void init() { System.setProperty("log4j.configuration", "tests.log4j.properties"); + try { + workspaceDir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); + File jsonPath = new File(workspaceDir, "workspace.json"); + FileUtils.writeStringToFile(jsonPath, "{\"projectIDs\":[]\n" + + ",\"preferences\":{\"entries\":{\"scripting.starred-expressions\":" + + "{\"class\":\"com.google.refine.preference.TopList\",\"top\":2147483647," + + "\"list\":[]},\"scripting.expressions\":{\"class\":\"com.google.refine.preference.TopList\",\"top\":100,\"list\":[]}}}}"); + FileProjectManager.initialize(workspaceDir); + } catch (IOException e) { + workspaceDir = null; + e.printStackTrace(); + } + // This just keeps track of any failed test, for cleanupWorkspace + testFailed = false; } - + @BeforeMethod protected void initProjectManager() { servlet = new RefineServletStub(); @@ -86,6 +108,22 @@ public class RefineTest { ImportingManager.initialize(servlet); } + protected Project createProjectWithColumns(String projectName, String... columnNames) throws IOException, ModelException { + Project project = new Project(); + ProjectMetadata pm = new ProjectMetadata(); + pm.setName(projectName); + ProjectManager.singleton.registerProject(project, pm); + + if (columnNames != null) { + for(String columnName : columnNames) { + int index = project.columnModel.allocateNewCellIndex(); + Column column = new Column(index,columnName); + project.columnModel.addColumn(index, column, true); + } + } + return project; + } + /** * Helper to create a project from a CSV encoded as a file. Not much * control is given on the import options, because this method is intended @@ -116,6 +154,8 @@ public class RefineTest { * @return */ protected Project createCSVProject(String projectName, String input) { + + Project project = new Project(); ProjectMetadata metadata = new ProjectMetadata(); diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java index 0a5891c0c..df220906d 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java @@ -33,37 +33,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.browsing.facets; -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; -import java.io.StringReader; -import java.util.List; -import java.util.ArrayList; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.importers.SeparatorBasedImporter; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.browsing.RowFilter; import com.google.refine.browsing.facets.TextSearchFacet; -import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class TextSearchFacetTests extends RefineTest { @@ -81,7 +65,8 @@ public class TextSearchFacetTests extends RefineTest { @BeforeMethod public void setUp() throws JSONException, IOException, ModelException { - project = createCSVProject("TextSearchFacet test", "Value\n" + project = createCSVProject("TextSearchFacet", + "Value\n" + "a\n" + "b\n" + "ab\n" diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java index 3811f8d4a..dd185f47d 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java @@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.expr.functions; -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; import java.util.Properties; @@ -47,27 +44,21 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.browsing.Engine; import com.google.refine.expr.EvalError; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class FunctionTests extends RefineTest { static Properties bindings; Project project; - Properties options; JSONObject engine_config; Engine engine; @@ -80,21 +71,9 @@ public class FunctionTests extends RefineTest { @BeforeMethod public void SetUp() throws IOException, ModelException { + + project = createProjectWithColumns("FunctionTests", "Column A"); bindings = new Properties(); - - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("TNG Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"Column A"); - project.columnModel.addColumn(index, column, true); - - options = mock(Properties.class); - bindings.put("project", project); // Five rows of a's and five of 1s diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java index 387ccff91..7862688c7 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java @@ -33,13 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.expr.functions.booleans; -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; import java.util.Properties; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -47,20 +43,14 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.browsing.Engine; import com.google.refine.expr.EvalError; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class BooleanTests extends RefineTest { @@ -83,11 +73,8 @@ public class BooleanTests extends RefineTest { }; - static Properties bindings; - Project project; - Properties options; - JSONObject engine_config; - Engine engine; + static private Properties bindings; + private Project project; @Override @@ -99,19 +86,7 @@ public class BooleanTests extends RefineTest { @BeforeMethod public void SetUp() throws IOException, ModelException { bindings = new Properties(); - - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("TNG Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"Column A"); - project.columnModel.addColumn(index, column, true); - - options = mock(Properties.class); + project = createProjectWithColumns("BooleanTests", "Column A"); bindings.put("project", project); diff --git a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java index 3c3f1c33c..146f323bc 100644 --- a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java @@ -32,10 +32,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package com.google.refine.tests.model; - -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; import java.util.Properties; @@ -48,32 +44,21 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.browsing.Engine; import com.google.refine.browsing.RowVisitor; import com.google.refine.expr.functions.FacetCount; import com.google.refine.grel.Function; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.row.RowRemovalOperation; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class CacheTests extends RefineTest { - - //{project=1718051861971, engine= ...} - //engine={ "facets" : ..., "mode":"row-based"}} - //facets = [{"type":"list","name":"row","columnName":"row","expression":"facetCount(value, 'value', 'row') > 1","omitBlank":false,"omitError":false,"selection":[],"selectBlank":false,"selectError":false,"invert":false}] - - // {project=1718051861971, engine= - + // Equivalent to duplicate facet on Column A with true selected static final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"facetCount(value, 'value', 'Column A') > 1\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":true,\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}"; @@ -92,21 +77,10 @@ public class CacheTests extends RefineTest { @BeforeMethod public void SetUp() throws JSONException, IOException, ModelException { - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("TNG Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"Column A"); - project.columnModel.addColumn(index, column, true); + project = createProjectWithColumns("CacheTests", "Column A"); - options = mock(Properties.class); engine = new Engine(project); engine_config = new JSONObject(ENGINE_JSON_DUPLICATES); -// engine_config.getJSONArray("facets").getJSONObject(0).getJSONArray("selection").put(new JSONArray()); engine.initializeFromJSON(engine_config); engine.setMode(Engine.Mode.RowBased); @@ -118,7 +92,6 @@ public class CacheTests extends RefineTest { @AfterMethod public void TearDown() { project = null; - options = null; engine = null; bindings = null; } diff --git a/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java b/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java index 52ff09328..59073399c 100644 --- a/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java @@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.model; -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.util.Properties; @@ -45,18 +42,12 @@ import org.json.JSONObject; import org.json.JSONArray; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.browsing.Engine; -import com.google.refine.io.FileProjectManager; import com.google.refine.expr.ExpressionUtils; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; @@ -66,7 +57,6 @@ import com.google.refine.operations.OnError; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class UrlFetchingTests extends RefineTest { @@ -80,44 +70,15 @@ public class UrlFetchingTests extends RefineTest { } // dependencies - Project project; - Properties options; - JSONObject engine_config; - Engine engine; - Properties bindings; + private Project project; + private Properties options; + private JSONObject engine_config; @BeforeMethod public void SetUp() throws JSONException, IOException, ModelException { - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("URL Fetching Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"fruits"); - project.columnModel.addColumn(index, column, true); - - options = mock(Properties.class); - engine = new Engine(project); - engine_config = new JSONObject(ENGINE_JSON_URLS); - engine.initializeFromJSON(engine_config); - engine.setMode(Engine.Mode.RowBased); - - bindings = new Properties(); - bindings.put("project", project); - + project = createProjectWithColumns("UrlFetchingTests", "fruits"); } - @AfterMethod - public void TearDown() { - project = null; - options = null; - engine = null; - bindings = null; - } - private boolean isHostReachable(String host, int timeout){ boolean state = false; diff --git a/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java b/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java index 1de455304..b0b631340 100644 --- a/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java @@ -3,7 +3,6 @@ package com.google.refine.tests.model.changes; import static org.testng.AssertJUnit.assertTrue; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -13,17 +12,13 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.changes.CellAtRow; import com.google.refine.model.changes.ColumnAdditionChange; import com.google.refine.model.changes.MassChange; import com.google.refine.history.Change; -import com.google.refine.io.FileProjectManager; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class MassChangeTests extends RefineTest { @@ -38,12 +33,7 @@ public class MassChangeTests extends RefineTest { @BeforeMethod public void SetUp() throws IOException, ModelException { - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("TNG Test Project"); - ProjectManager.singleton.registerProject(project, pm); + project = createProjectWithColumns("MassChangeTest"); } /** diff --git a/main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java b/main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java index a8f921645..42c192a9c 100644 --- a/main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java +++ b/main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java @@ -35,7 +35,6 @@ package com.google.refine.tests.recon; import static org.mockito.Mockito.mock; -import java.io.File; import java.io.IOException; import java.util.Properties; import java.util.List; @@ -50,14 +49,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.browsing.Engine; -import com.google.refine.browsing.RowVisitor; -import com.google.refine.grel.Function; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; @@ -65,11 +58,9 @@ import com.google.refine.model.Recon; import com.google.refine.model.ReconCandidate; import com.google.refine.process.Process; import com.google.refine.process.ProcessManager; -import com.google.refine.operations.OnError; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.recon.ExtendDataOperation; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class DataExtensionTests extends RefineTest { @@ -90,29 +81,16 @@ public class DataExtensionTests extends RefineTest { Properties options; JSONObject engine_config; Engine engine; - Properties bindings; @BeforeMethod public void SetUp() throws JSONException, IOException, ModelException { - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("Data Extension Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"country"); - project.columnModel.addColumn(index, column, true); + project = createProjectWithColumns("DataExtensionTests", "country"); options = mock(Properties.class); engine = new Engine(project); engine_config = new JSONObject(ENGINE_JSON_URLS); engine.initializeFromJSON(engine_config); engine.setMode(Engine.Mode.RowBased); - - bindings = new Properties(); - bindings.put("project", project); Row row = new Row(2); row.setCell(0, reconciledCell("Iran", "Q794")); @@ -133,7 +111,6 @@ public class DataExtensionTests extends RefineTest { project = null; options = null; engine = null; - bindings = null; } static public Cell reconciledCell(String name, String id) {