Add helper createProjectWithColumns

main/src/com/google/refine/RefineServlet.java

@@ -124,7 +124,8 @@ public class RefineServlet extends Butterfly {
         if (data == null) {
             throw new ServletException("can't find servlet init config 'refine.data', I have to give up initializing");
         }
-
+        logger.error("initializing FileProjectManager with dir");
+        logger.error(data);
         s_dataDir = new File(data);
         FileProjectManager.initialize(s_dataDir);
         ImportingManager.initialize(this);

main/tests/server/src/com/google/refine/tests/RefineTest.java

@@ -38,10 +38,13 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.io.File;
+import java.io.IOException;
 import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.io.FileUtils;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -57,10 +60,13 @@ import com.google.refine.RefineServlet;
 import com.google.refine.importers.SeparatorBasedImporter;
 import com.google.refine.importing.ImportingJob;
 import com.google.refine.importing.ImportingManager;
+import com.google.refine.io.FileProjectManager;
 import com.google.refine.model.Cell;
 import com.google.refine.model.Column;
+import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.model.Row;
+import com.google.refine.tests.util.TestUtils;
 import com.google.refine.util.JSONUtilities;
 
 /**
@@ -70,6 +76,8 @@ public class RefineTest {
 
     protected Logger logger;
     
+    boolean testFailed;
+    protected File workspaceDir;
     protected RefineServlet servlet;
     private List<Project> projects = new ArrayList<Project>();
     private List<ImportingJob> importingJobs = new ArrayList<ImportingJob>();
@@ -77,8 +85,22 @@ public class RefineTest {
     @BeforeSuite
     public void init() {
         System.setProperty("log4j.configuration", "tests.log4j.properties");
+        try {
+            workspaceDir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
+            File jsonPath = new File(workspaceDir, "workspace.json");
+            FileUtils.writeStringToFile(jsonPath, "{\"projectIDs\":[]\n" + 
+                    ",\"preferences\":{\"entries\":{\"scripting.starred-expressions\":" +
+                    "{\"class\":\"com.google.refine.preference.TopList\",\"top\":2147483647," +
+                    "\"list\":[]},\"scripting.expressions\":{\"class\":\"com.google.refine.preference.TopList\",\"top\":100,\"list\":[]}}}}");
+            FileProjectManager.initialize(workspaceDir);
+        } catch (IOException e) {
+            workspaceDir = null;
+            e.printStackTrace();
+        }
+        // This just keeps track of any failed test, for cleanupWorkspace
+        testFailed = false;
     }
-    
+
     @BeforeMethod
     protected void initProjectManager() {
         servlet = new RefineServletStub();
@@ -86,6 +108,22 @@ public class RefineTest {
         ImportingManager.initialize(servlet);
     }
     
+    protected Project createProjectWithColumns(String projectName, String... columnNames) throws IOException, ModelException {
+        Project project = new Project();
+        ProjectMetadata pm = new ProjectMetadata();
+        pm.setName(projectName);
+        ProjectManager.singleton.registerProject(project, pm);
+
+        if (columnNames != null) {
+            for(String columnName : columnNames) {
+                int index = project.columnModel.allocateNewCellIndex();
+                Column column = new Column(index,columnName);
+                project.columnModel.addColumn(index, column, true);
+            }
+        }
+        return project;
+    }
+    
     /**
      * Helper to create a project from a CSV encoded as a file. Not much
      * control is given on the import options, because this method is intended
@@ -116,6 +154,8 @@ public class RefineTest {
      * @return
      */
     protected Project createCSVProject(String projectName, String input) {
+
+        
         Project project = new Project();
         
         ProjectMetadata metadata = new ProjectMetadata();

main/tests/server/src/com/google/refine/tests/browsing/facets/TextSearchFacetTests.java

@@ -33,37 +33,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.tests.browsing.facets;
 
-import static org.mockito.Mockito.mock;
-
-import java.io.File;
 import java.io.IOException;
-import java.io.StringReader;
-import java.util.List;
-import java.util.ArrayList;
 
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.refine.ProjectManager;
-import com.google.refine.ProjectMetadata;
-import com.google.refine.RefineServlet;
-import com.google.refine.importers.SeparatorBasedImporter;
-import com.google.refine.importing.ImportingJob;
-import com.google.refine.importing.ImportingManager;
-import com.google.refine.io.FileProjectManager;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.browsing.RowFilter;
 import com.google.refine.browsing.facets.TextSearchFacet;
-import com.google.refine.tests.RefineServletStub;
 import com.google.refine.tests.RefineTest;
-import com.google.refine.tests.util.TestUtils;
 
 
 public class TextSearchFacetTests extends RefineTest {
@@ -81,7 +65,8 @@ public class TextSearchFacetTests extends RefineTest {
 
     @BeforeMethod
     public void setUp() throws JSONException, IOException, ModelException {
-        project = createCSVProject("TextSearchFacet test", "Value\n"
+        project = createCSVProject("TextSearchFacet",
+             "Value\n"
             + "a\n"
             + "b\n"
             + "ab\n"

main/tests/server/src/com/google/refine/tests/expr/functions/FunctionTests.java

@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  
 package com.google.refine.tests.expr.functions;
 
-import static org.mockito.Mockito.mock;
-
-import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
 
@@ -47,27 +44,21 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.refine.ProjectManager;
-import com.google.refine.ProjectMetadata;
 import com.google.refine.browsing.Engine;
 import com.google.refine.expr.EvalError;
 import com.google.refine.grel.ControlFunctionRegistry;
 import com.google.refine.grel.Function;
-import com.google.refine.io.FileProjectManager;
 import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.model.Row;
 import com.google.refine.tests.RefineTest;
-import com.google.refine.tests.util.TestUtils;
 
 
 public class FunctionTests extends RefineTest {
 
     static Properties bindings;
     Project project;
-    Properties options;
     JSONObject engine_config;
     Engine engine;
 
@@ -80,21 +71,9 @@ public class FunctionTests extends RefineTest {
     
     @BeforeMethod
     public void SetUp() throws IOException, ModelException {
+        
+        project = createProjectWithColumns("FunctionTests", "Column A");
         bindings = new Properties();
-        
-        File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
-        FileProjectManager.initialize(dir);
-        project = new Project();
-        ProjectMetadata pm = new ProjectMetadata();
-        pm.setName("TNG Test Project");
-        ProjectManager.singleton.registerProject(project, pm);
-
-        int index = project.columnModel.allocateNewCellIndex();
-        Column column = new Column(index,"Column A");
-        project.columnModel.addColumn(index, column, true);
-        
-        options = mock(Properties.class);
-      
         bindings.put("project", project);
         
         // Five rows of a's and five of 1s

main/tests/server/src/com/google/refine/tests/expr/functions/booleans/BooleanTests.java

@@ -33,13 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.tests.expr.functions.booleans;
 
-import static org.mockito.Mockito.mock;
-
-import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
 
-import org.json.JSONObject;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
@@ -47,20 +43,14 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.refine.ProjectManager;
-import com.google.refine.ProjectMetadata;
-import com.google.refine.browsing.Engine;
 import com.google.refine.expr.EvalError;
 import com.google.refine.grel.ControlFunctionRegistry;
 import com.google.refine.grel.Function;
-import com.google.refine.io.FileProjectManager;
 import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.model.Row;
 import com.google.refine.tests.RefineTest;
-import com.google.refine.tests.util.TestUtils;
 
 
 public class BooleanTests extends RefineTest {
@@ -83,11 +73,8 @@ public class BooleanTests extends RefineTest {
     };
 
 
-    static Properties bindings;
+    static private Properties bindings;
-    Project project;
+    private Project project;
-    Properties options;
-    JSONObject engine_config;
-    Engine engine;
 
 
     @Override
@@ -99,19 +86,7 @@ public class BooleanTests extends RefineTest {
     @BeforeMethod
     public void SetUp() throws IOException, ModelException {
         bindings = new Properties();
-
+        project = createProjectWithColumns("BooleanTests", "Column A");
-        File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
-        FileProjectManager.initialize(dir);
-        project = new Project();
-        ProjectMetadata pm = new ProjectMetadata();
-        pm.setName("TNG Test Project");
-        ProjectManager.singleton.registerProject(project, pm);
-
-        int index = project.columnModel.allocateNewCellIndex();
-        Column column = new Column(index,"Column A");
-        project.columnModel.addColumn(index, column, true);
-
-        options = mock(Properties.class);
 
         bindings.put("project", project);
 

main/tests/server/src/com/google/refine/tests/model/CacheTests.java

@@ -32,10 +32,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
 package com.google.refine.tests.model;
-
-import static org.mockito.Mockito.mock;
-
-import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
 
@@ -48,32 +44,21 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.refine.ProjectManager;
-import com.google.refine.ProjectMetadata;
 import com.google.refine.browsing.Engine;
 import com.google.refine.browsing.RowVisitor;
 import com.google.refine.expr.functions.FacetCount;
 import com.google.refine.grel.Function;
-import com.google.refine.io.FileProjectManager;
 import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.model.Row;
 import com.google.refine.operations.EngineDependentOperation;
 import com.google.refine.operations.row.RowRemovalOperation;
 import com.google.refine.tests.RefineTest;
-import com.google.refine.tests.util.TestUtils;
 
 
 public class CacheTests extends RefineTest {
-
+    
-    //{project=1718051861971, engine= ...}
-    //engine={ "facets" : ..., "mode":"row-based"}}
-    //facets = [{"type":"list","name":"row","columnName":"row","expression":"facetCount(value, 'value', 'row') > 1","omitBlank":false,"omitError":false,"selection":[],"selectBlank":false,"selectError":false,"invert":false}]
-
-    // {project=1718051861971, engine=
-
     // Equivalent to duplicate facet on Column A with true selected
     static final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"facetCount(value, 'value', 'Column A') > 1\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":true,\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}";
 
@@ -92,21 +77,10 @@ public class CacheTests extends RefineTest {
 
     @BeforeMethod
     public void SetUp() throws JSONException, IOException, ModelException {
-        File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
+        project = createProjectWithColumns("CacheTests", "Column A");
-        FileProjectManager.initialize(dir);
-        project = new Project();
-        ProjectMetadata pm = new ProjectMetadata();
-        pm.setName("TNG Test Project");
-        ProjectManager.singleton.registerProject(project, pm);
-
-        int index = project.columnModel.allocateNewCellIndex();
-        Column column = new Column(index,"Column A");
-        project.columnModel.addColumn(index, column, true);
         
-        options = mock(Properties.class);
         engine = new Engine(project);
         engine_config = new JSONObject(ENGINE_JSON_DUPLICATES);
-//        engine_config.getJSONArray("facets").getJSONObject(0).getJSONArray("selection").put(new JSONArray());
         engine.initializeFromJSON(engine_config);
         engine.setMode(Engine.Mode.RowBased);
         
@@ -118,7 +92,6 @@ public class CacheTests extends RefineTest {
     @AfterMethod
     public void TearDown() {
         project = null;
-        options = null;
         engine = null;
         bindings = null;
     }

main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java

@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.tests.model;
 
-import static org.mockito.Mockito.mock;
-
-import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.Properties;
@@ -45,18 +42,12 @@ import org.json.JSONObject;
 import org.json.JSONArray;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.refine.ProjectManager;
-import com.google.refine.ProjectMetadata;
-import com.google.refine.browsing.Engine;
-import com.google.refine.io.FileProjectManager;
 import com.google.refine.expr.ExpressionUtils;
 import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.model.Row;
@@ -66,7 +57,6 @@ import com.google.refine.operations.OnError;
 import com.google.refine.operations.EngineDependentOperation;
 import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation;
 import com.google.refine.tests.RefineTest;
-import com.google.refine.tests.util.TestUtils;
 
 
 public class UrlFetchingTests extends RefineTest {
@@ -80,44 +70,15 @@ public class UrlFetchingTests extends RefineTest {
     }
 
     // dependencies
-    Project project;
+    private Project project;
-    Properties options;
+    private Properties options;
-    JSONObject engine_config;
+    private JSONObject engine_config;
-    Engine engine;
-    Properties bindings;
 
     @BeforeMethod
     public void SetUp() throws JSONException, IOException, ModelException {
-        File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
+        project = createProjectWithColumns("UrlFetchingTests", "fruits");       
-        FileProjectManager.initialize(dir);
-        project = new Project();
-        ProjectMetadata pm = new ProjectMetadata();
-        pm.setName("URL Fetching Test Project");
-        ProjectManager.singleton.registerProject(project, pm);
-
-        int index = project.columnModel.allocateNewCellIndex();
-        Column column = new Column(index,"fruits");
-        project.columnModel.addColumn(index, column, true);
-        
-        options = mock(Properties.class);
-        engine = new Engine(project);
-        engine_config = new JSONObject(ENGINE_JSON_URLS);
-        engine.initializeFromJSON(engine_config);
-        engine.setMode(Engine.Mode.RowBased);
-        
-        bindings = new Properties();
-        bindings.put("project", project);
-        
     }
 
-    @AfterMethod
-    public void TearDown() {
-        project = null;
-        options = null;
-        engine = null;
-        bindings = null;
-    }
-    
     private boolean isHostReachable(String host, int timeout){
         boolean state = false;
 

main/tests/server/src/com/google/refine/tests/model/changes/MassChangeTests.java

@@ -3,7 +3,6 @@ package com.google.refine.tests.model.changes;
 
 import static org.testng.AssertJUnit.assertTrue;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -13,17 +12,13 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.refine.ProjectManager;
-import com.google.refine.ProjectMetadata;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.model.changes.CellAtRow;
 import com.google.refine.model.changes.ColumnAdditionChange;
 import com.google.refine.model.changes.MassChange;
 import com.google.refine.history.Change;
-import com.google.refine.io.FileProjectManager;
 import com.google.refine.tests.RefineTest;
-import com.google.refine.tests.util.TestUtils;
 
 public class MassChangeTests extends RefineTest {
 
@@ -38,12 +33,7 @@ public class MassChangeTests extends RefineTest {
     @BeforeMethod
     public void SetUp()
             throws IOException, ModelException {
-        File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
+        project = createProjectWithColumns("MassChangeTest");
-        FileProjectManager.initialize(dir);
-        project = new Project();
-        ProjectMetadata pm = new ProjectMetadata();
-        pm.setName("TNG Test Project");
-        ProjectManager.singleton.registerProject(project, pm);
     }
 
     /**

main/tests/server/src/com/google/refine/tests/recon/DataExtensionTests.java

@@ -35,7 +35,6 @@ package com.google.refine.tests.recon;
 
 import static org.mockito.Mockito.mock;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
 import java.util.List;
@@ -50,14 +49,8 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.refine.ProjectManager;
-import com.google.refine.ProjectMetadata;
 import com.google.refine.browsing.Engine;
-import com.google.refine.browsing.RowVisitor;
-import com.google.refine.grel.Function;
-import com.google.refine.io.FileProjectManager;
 import com.google.refine.model.Cell;
-import com.google.refine.model.Column;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
 import com.google.refine.model.Row;
@@ -65,11 +58,9 @@ import com.google.refine.model.Recon;
 import com.google.refine.model.ReconCandidate;
 import com.google.refine.process.Process;
 import com.google.refine.process.ProcessManager;
-import com.google.refine.operations.OnError;
 import com.google.refine.operations.EngineDependentOperation;
 import com.google.refine.operations.recon.ExtendDataOperation;
 import com.google.refine.tests.RefineTest;
-import com.google.refine.tests.util.TestUtils;
 
 
 public class DataExtensionTests extends RefineTest {
@@ -90,29 +81,16 @@ public class DataExtensionTests extends RefineTest {
     Properties options;
     JSONObject engine_config;
     Engine engine;
-    Properties bindings;
 
     @BeforeMethod
     public void SetUp() throws JSONException, IOException, ModelException {
-        File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
+        project = createProjectWithColumns("DataExtensionTests", "country");
-        FileProjectManager.initialize(dir);
-        project = new Project();
-        ProjectMetadata pm = new ProjectMetadata();
-        pm.setName("Data Extension Test Project");
-        ProjectManager.singleton.registerProject(project, pm);
-
-        int index = project.columnModel.allocateNewCellIndex();
-        Column column = new Column(index,"country");
-        project.columnModel.addColumn(index, column, true);
         
         options = mock(Properties.class);
         engine = new Engine(project);
         engine_config = new JSONObject(ENGINE_JSON_URLS);
         engine.initializeFromJSON(engine_config);
         engine.setMode(Engine.Mode.RowBased);
-        
-        bindings = new Properties();
-        bindings.put("project", project);
 
                Row row = new Row(2);
         row.setCell(0, reconciledCell("Iran", "Q794"));
@@ -133,7 +111,6 @@ public class DataExtensionTests extends RefineTest {
         project = null;
         options = null;
         engine = null;
-        bindings = null;
     }
 
     static public Cell reconciledCell(String name, String id) {