From 0426704938be2becce6313bae94cf25c16451d81 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sat, 27 Jan 2018 13:58:06 +0000 Subject: [PATCH] Add helper createProjectWithColumns --- main/src/com/google/refine/RefineServlet.java | 3 +- .../com/google/refine/tests/RefineTest.java | 42 ++++++++++++++++- .../browsing/facets/TextSearchFacetTests.java | 19 +------- .../tests/expr/functions/FunctionTests.java | 25 +--------- .../expr/functions/booleans/BooleanTests.java | 31 ++---------- .../google/refine/tests/model/CacheTests.java | 31 +----------- .../refine/tests/model/UrlFetchingTests.java | 47 ++----------------- .../tests/model/changes/MassChangeTests.java | 12 +---- .../tests/recon/DataExtensionTests.java | 25 +--------- 9 files changed, 58 insertions(+), 177 deletions(-) diff --git a/main/src/com/google/refine/RefineServlet.java b/main/src/com/google/refine/RefineServlet.java index cb6922235..7081c0ac4 100644 --- a/main/src/com/google/refine/RefineServlet.java +++ b/main/src/com/google/refine/RefineServlet.java @@ -124,7 +124,8 @@ public class RefineServlet extends Butterfly { if (data == null) { throw new ServletException("can't find servlet init config 'refine.data', I have to give up initializing"); } - + logger.error("initializing FileProjectManager with dir"); + logger.error(data); s_dataDir = new File(data); FileProjectManager.initialize(s_dataDir); ImportingManager.initialize(this); diff --git a/main/tests/server/src/com/google/refine/tests/RefineTest.java b/main/tests/server/src/com/google/refine/tests/RefineTest.java index d38a73a4e..2384d1dbb 100644 --- a/main/tests/server/src/com/google/refine/tests/RefineTest.java +++ b/main/tests/server/src/com/google/refine/tests/RefineTest.java @@ -38,10 +38,13 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.File; +import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.List; +import org.apache.commons.io.FileUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -57,10 +60,13 @@ import com.google.refine.RefineServlet; import com.google.refine.importers.SeparatorBasedImporter; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingManager; +import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; import com.google.refine.model.Column; +import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; +import com.google.refine.tests.util.TestUtils; import com.google.refine.util.JSONUtilities; /** @@ -70,6 +76,8 @@ public class RefineTest { protected Logger logger; + boolean testFailed; + protected File workspaceDir; protected RefineServlet servlet; private List projects = new ArrayList(); private List importingJobs = new ArrayList(); @@ -77,8 +85,22 @@ public class RefineTest { @BeforeSuite public void init() { System.setProperty("log4j.configuration", "tests.log4j.properties"); + try { + workspaceDir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); + File jsonPath = new File(workspaceDir, "workspace.json"); + FileUtils.writeStringToFile(jsonPath, "{\"projectIDs\":[]\n" + + ",\"preferences\":{\"entries\":{\"scripting.starred-expressions\":" + + "{\"class\":\"com.google.refine.preference.TopList\",\"top\":2147483647," + + "\"list\":[]},\"scripting.expressions\":{\"class\":\"com.google.refine.preference.TopList\",\"top\":100,\"list\":[]}}}}"); + FileProjectManager.initialize(workspaceDir); + } catch (IOException e) { + workspaceDir = null; + e.printStackTrace(); + } + // This just keeps track of any failed test, for cleanupWorkspace + testFailed = false; } - + @BeforeMethod protected void initProjectManager() { servlet = new RefineServletStub(); @@ -86,6 +108,22 @@ public class RefineTest { ImportingManager.initialize(servlet); } + protected Project createProjectWithColumns(String projectName, String... columnNames) throws IOException, ModelException { + Project project = new Project(); + ProjectMetadata pm = new ProjectMetadata(); + pm.setName(projectName); + ProjectManager.singleton.registerProject(project, pm); + + if (columnNames != null) { + for(String columnName : columnNames) { + int index = project.columnModel.allocateNewCellIndex(); + Column column = new Column(index,columnName); + project.columnModel.addColumn(index, column, true); + } + } + return project; + } + /** * Helper to create a project from a CSV encoded as a file. Not much * control is given on the import options, because this method is intended @@ -116,6 +154,8 @@ public class RefineTest { * @return */ protected Project createCSVProject(String projectName, String input) { + + Project project = new Project(); ProjectMetadata metadata = new ProjectMetadata(); diff --git a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java index 0a5891c0c..df220906d 100644 --- a/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java +++ b/main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java @@ -33,37 +33,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.browsing.facets; -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; -import java.io.StringReader; -import java.util.List; -import java.util.ArrayList; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.RefineServlet; -import com.google.refine.importers.SeparatorBasedImporter; -import com.google.refine.importing.ImportingJob; -import com.google.refine.importing.ImportingManager; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.browsing.RowFilter; import com.google.refine.browsing.facets.TextSearchFacet; -import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class TextSearchFacetTests extends RefineTest { @@ -81,7 +65,8 @@ public class TextSearchFacetTests extends RefineTest { @BeforeMethod public void setUp() throws JSONException, IOException, ModelException { - project = createCSVProject("TextSearchFacet test", "Value\n" + project = createCSVProject("TextSearchFacet", + "Value\n" + "a\n" + "b\n" + "ab\n" diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java index 3811f8d4a..dd185f47d 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java @@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.expr.functions; -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; import java.util.Properties; @@ -47,27 +44,21 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.browsing.Engine; import com.google.refine.expr.EvalError; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class FunctionTests extends RefineTest { static Properties bindings; Project project; - Properties options; JSONObject engine_config; Engine engine; @@ -80,21 +71,9 @@ public class FunctionTests extends RefineTest { @BeforeMethod public void SetUp() throws IOException, ModelException { + + project = createProjectWithColumns("FunctionTests", "Column A"); bindings = new Properties(); - - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("TNG Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"Column A"); - project.columnModel.addColumn(index, column, true); - - options = mock(Properties.class); - bindings.put("project", project); // Five rows of a's and five of 1s diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java index 387ccff91..7862688c7 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java @@ -33,13 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.expr.functions.booleans; -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; import java.util.Properties; -import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -47,20 +43,14 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.browsing.Engine; import com.google.refine.expr.EvalError; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class BooleanTests extends RefineTest { @@ -83,11 +73,8 @@ public class BooleanTests extends RefineTest { }; - static Properties bindings; - Project project; - Properties options; - JSONObject engine_config; - Engine engine; + static private Properties bindings; + private Project project; @Override @@ -99,19 +86,7 @@ public class BooleanTests extends RefineTest { @BeforeMethod public void SetUp() throws IOException, ModelException { bindings = new Properties(); - - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("TNG Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"Column A"); - project.columnModel.addColumn(index, column, true); - - options = mock(Properties.class); + project = createProjectWithColumns("BooleanTests", "Column A"); bindings.put("project", project); diff --git a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java index 3c3f1c33c..146f323bc 100644 --- a/main/tests/server/src/com/google/refine/tests/model/CacheTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/CacheTests.java @@ -32,10 +32,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ package com.google.refine.tests.model; - -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; import java.util.Properties; @@ -48,32 +44,21 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.browsing.Engine; import com.google.refine.browsing.RowVisitor; import com.google.refine.expr.functions.FacetCount; import com.google.refine.grel.Function; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.row.RowRemovalOperation; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class CacheTests extends RefineTest { - - //{project=1718051861971, engine= ...} - //engine={ "facets" : ..., "mode":"row-based"}} - //facets = [{"type":"list","name":"row","columnName":"row","expression":"facetCount(value, 'value', 'row') > 1","omitBlank":false,"omitError":false,"selection":[],"selectBlank":false,"selectError":false,"invert":false}] - - // {project=1718051861971, engine= - + // Equivalent to duplicate facet on Column A with true selected static final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"facetCount(value, 'value', 'Column A') > 1\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":true,\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}"; @@ -92,21 +77,10 @@ public class CacheTests extends RefineTest { @BeforeMethod public void SetUp() throws JSONException, IOException, ModelException { - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("TNG Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"Column A"); - project.columnModel.addColumn(index, column, true); + project = createProjectWithColumns("CacheTests", "Column A"); - options = mock(Properties.class); engine = new Engine(project); engine_config = new JSONObject(ENGINE_JSON_DUPLICATES); -// engine_config.getJSONArray("facets").getJSONObject(0).getJSONArray("selection").put(new JSONArray()); engine.initializeFromJSON(engine_config); engine.setMode(Engine.Mode.RowBased); @@ -118,7 +92,6 @@ public class CacheTests extends RefineTest { @AfterMethod public void TearDown() { project = null; - options = null; engine = null; bindings = null; } diff --git a/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java b/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java index 52ff09328..59073399c 100644 --- a/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java @@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.tests.model; -import static org.mockito.Mockito.mock; - -import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.util.Properties; @@ -45,18 +42,12 @@ import org.json.JSONObject; import org.json.JSONArray; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; -import com.google.refine.browsing.Engine; -import com.google.refine.io.FileProjectManager; import com.google.refine.expr.ExpressionUtils; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; @@ -66,7 +57,6 @@ import com.google.refine.operations.OnError; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class UrlFetchingTests extends RefineTest { @@ -80,44 +70,15 @@ public class UrlFetchingTests extends RefineTest { } // dependencies - Project project; - Properties options; - JSONObject engine_config; - Engine engine; - Properties bindings; + private Project project; + private Properties options; + private JSONObject engine_config; @BeforeMethod public void SetUp() throws JSONException, IOException, ModelException { - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("URL Fetching Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"fruits"); - project.columnModel.addColumn(index, column, true); - - options = mock(Properties.class); - engine = new Engine(project); - engine_config = new JSONObject(ENGINE_JSON_URLS); - engine.initializeFromJSON(engine_config); - engine.setMode(Engine.Mode.RowBased); - - bindings = new Properties(); - bindings.put("project", project); - + project = createProjectWithColumns("UrlFetchingTests", "fruits"); } - @AfterMethod - public void TearDown() { - project = null; - options = null; - engine = null; - bindings = null; - } - private boolean isHostReachable(String host, int timeout){ boolean state = false; diff --git a/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java b/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java index 1de455304..b0b631340 100644 --- a/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java +++ b/main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java @@ -3,7 +3,6 @@ package com.google.refine.tests.model.changes; import static org.testng.AssertJUnit.assertTrue; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -13,17 +12,13 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.changes.CellAtRow; import com.google.refine.model.changes.ColumnAdditionChange; import com.google.refine.model.changes.MassChange; import com.google.refine.history.Change; -import com.google.refine.io.FileProjectManager; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class MassChangeTests extends RefineTest { @@ -38,12 +33,7 @@ public class MassChangeTests extends RefineTest { @BeforeMethod public void SetUp() throws IOException, ModelException { - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("TNG Test Project"); - ProjectManager.singleton.registerProject(project, pm); + project = createProjectWithColumns("MassChangeTest"); } /** diff --git a/main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java b/main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java index a8f921645..42c192a9c 100644 --- a/main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java +++ b/main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java @@ -35,7 +35,6 @@ package com.google.refine.tests.recon; import static org.mockito.Mockito.mock; -import java.io.File; import java.io.IOException; import java.util.Properties; import java.util.List; @@ -50,14 +49,8 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.browsing.Engine; -import com.google.refine.browsing.RowVisitor; -import com.google.refine.grel.Function; -import com.google.refine.io.FileProjectManager; import com.google.refine.model.Cell; -import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; import com.google.refine.model.Row; @@ -65,11 +58,9 @@ import com.google.refine.model.Recon; import com.google.refine.model.ReconCandidate; import com.google.refine.process.Process; import com.google.refine.process.ProcessManager; -import com.google.refine.operations.OnError; import com.google.refine.operations.EngineDependentOperation; import com.google.refine.operations.recon.ExtendDataOperation; import com.google.refine.tests.RefineTest; -import com.google.refine.tests.util.TestUtils; public class DataExtensionTests extends RefineTest { @@ -90,29 +81,16 @@ public class DataExtensionTests extends RefineTest { Properties options; JSONObject engine_config; Engine engine; - Properties bindings; @BeforeMethod public void SetUp() throws JSONException, IOException, ModelException { - File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir"); - FileProjectManager.initialize(dir); - project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - pm.setName("Data Extension Test Project"); - ProjectManager.singleton.registerProject(project, pm); - - int index = project.columnModel.allocateNewCellIndex(); - Column column = new Column(index,"country"); - project.columnModel.addColumn(index, column, true); + project = createProjectWithColumns("DataExtensionTests", "country"); options = mock(Properties.class); engine = new Engine(project); engine_config = new JSONObject(ENGINE_JSON_URLS); engine.initializeFromJSON(engine_config); engine.setMode(Engine.Mode.RowBased); - - bindings = new Properties(); - bindings.put("project", project); Row row = new Row(2); row.setCell(0, reconciledCell("Iran", "Q794")); @@ -133,7 +111,6 @@ public class DataExtensionTests extends RefineTest { project = null; options = null; engine = null; - bindings = null; } static public Cell reconciledCell(String name, String id) {