Add helper createProjectWithColumns

This commit is contained in:
Antonin Delpeuch 2018-01-27 13:58:06 +00:00
parent feae25a5ff
commit 0426704938
9 changed files with 58 additions and 177 deletions

View File

@ -124,7 +124,8 @@ public class RefineServlet extends Butterfly {
if (data == null) {
throw new ServletException("can't find servlet init config 'refine.data', I have to give up initializing");
}
logger.error("initializing FileProjectManager with dir");
logger.error(data);
s_dataDir = new File(data);
FileProjectManager.initialize(s_dataDir);
ImportingManager.initialize(this);

View File

@ -38,10 +38,13 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@ -57,10 +60,13 @@ import com.google.refine.RefineServlet;
import com.google.refine.importers.SeparatorBasedImporter;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.tests.util.TestUtils;
import com.google.refine.util.JSONUtilities;
/**
@ -70,6 +76,8 @@ public class RefineTest {
protected Logger logger;
boolean testFailed;
protected File workspaceDir;
protected RefineServlet servlet;
private List<Project> projects = new ArrayList<Project>();
private List<ImportingJob> importingJobs = new ArrayList<ImportingJob>();
@ -77,6 +85,20 @@ public class RefineTest {
@BeforeSuite
public void init() {
System.setProperty("log4j.configuration", "tests.log4j.properties");
try {
workspaceDir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
File jsonPath = new File(workspaceDir, "workspace.json");
FileUtils.writeStringToFile(jsonPath, "{\"projectIDs\":[]\n" +
",\"preferences\":{\"entries\":{\"scripting.starred-expressions\":" +
"{\"class\":\"com.google.refine.preference.TopList\",\"top\":2147483647," +
"\"list\":[]},\"scripting.expressions\":{\"class\":\"com.google.refine.preference.TopList\",\"top\":100,\"list\":[]}}}}");
FileProjectManager.initialize(workspaceDir);
} catch (IOException e) {
workspaceDir = null;
e.printStackTrace();
}
// This just keeps track of any failed test, for cleanupWorkspace
testFailed = false;
}
@BeforeMethod
@ -86,6 +108,22 @@ public class RefineTest {
ImportingManager.initialize(servlet);
}
protected Project createProjectWithColumns(String projectName, String... columnNames) throws IOException, ModelException {
Project project = new Project();
ProjectMetadata pm = new ProjectMetadata();
pm.setName(projectName);
ProjectManager.singleton.registerProject(project, pm);
if (columnNames != null) {
for(String columnName : columnNames) {
int index = project.columnModel.allocateNewCellIndex();
Column column = new Column(index,columnName);
project.columnModel.addColumn(index, column, true);
}
}
return project;
}
/**
* Helper to create a project from a CSV encoded as a file. Not much
* control is given on the import options, because this method is intended
@ -116,6 +154,8 @@ public class RefineTest {
* @return
*/
protected Project createCSVProject(String projectName, String input) {
Project project = new Project();
ProjectMetadata metadata = new ProjectMetadata();

View File

@ -33,37 +33,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.tests.browsing.facets;
import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.List;
import java.util.ArrayList;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.RefineServlet;
import com.google.refine.importers.SeparatorBasedImporter;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.browsing.RowFilter;
import com.google.refine.browsing.facets.TextSearchFacet;
import com.google.refine.tests.RefineServletStub;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class TextSearchFacetTests extends RefineTest {
@ -81,7 +65,8 @@ public class TextSearchFacetTests extends RefineTest {
@BeforeMethod
public void setUp() throws JSONException, IOException, ModelException {
project = createCSVProject("TextSearchFacet test", "Value\n"
project = createCSVProject("TextSearchFacet",
"Value\n"
+ "a\n"
+ "b\n"
+ "ab\n"

View File

@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.tests.expr.functions;
import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.util.Properties;
@ -47,27 +44,21 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.browsing.Engine;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class FunctionTests extends RefineTest {
static Properties bindings;
Project project;
Properties options;
JSONObject engine_config;
Engine engine;
@ -80,21 +71,9 @@ public class FunctionTests extends RefineTest {
@BeforeMethod
public void SetUp() throws IOException, ModelException {
project = createProjectWithColumns("FunctionTests", "Column A");
bindings = new Properties();
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
FileProjectManager.initialize(dir);
project = new Project();
ProjectMetadata pm = new ProjectMetadata();
pm.setName("TNG Test Project");
ProjectManager.singleton.registerProject(project, pm);
int index = project.columnModel.allocateNewCellIndex();
Column column = new Column(index,"Column A");
project.columnModel.addColumn(index, column, true);
options = mock(Properties.class);
bindings.put("project", project);
// Five rows of a's and five of 1s

View File

@ -33,13 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.tests.expr.functions.booleans;
import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.util.Properties;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
@ -47,20 +43,14 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.browsing.Engine;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class BooleanTests extends RefineTest {
@ -83,11 +73,8 @@ public class BooleanTests extends RefineTest {
};
static Properties bindings;
Project project;
Properties options;
JSONObject engine_config;
Engine engine;
static private Properties bindings;
private Project project;
@Override
@ -99,19 +86,7 @@ public class BooleanTests extends RefineTest {
@BeforeMethod
public void SetUp() throws IOException, ModelException {
bindings = new Properties();
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
FileProjectManager.initialize(dir);
project = new Project();
ProjectMetadata pm = new ProjectMetadata();
pm.setName("TNG Test Project");
ProjectManager.singleton.registerProject(project, pm);
int index = project.columnModel.allocateNewCellIndex();
Column column = new Column(index,"Column A");
project.columnModel.addColumn(index, column, true);
options = mock(Properties.class);
project = createProjectWithColumns("BooleanTests", "Column A");
bindings.put("project", project);

View File

@ -32,10 +32,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.tests.model;
import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.util.Properties;
@ -48,32 +44,21 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.expr.functions.FacetCount;
import com.google.refine.grel.Function;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.row.RowRemovalOperation;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class CacheTests extends RefineTest {
//{project=1718051861971, engine= ...}
//engine={ "facets" : ..., "mode":"row-based"}}
//facets = [{"type":"list","name":"row","columnName":"row","expression":"facetCount(value, 'value', 'row') > 1","omitBlank":false,"omitError":false,"selection":[],"selectBlank":false,"selectError":false,"invert":false}]
// {project=1718051861971, engine=
// Equivalent to duplicate facet on Column A with true selected
static final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"facetCount(value, 'value', 'Column A') > 1\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":true,\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}";
@ -92,21 +77,10 @@ public class CacheTests extends RefineTest {
@BeforeMethod
public void SetUp() throws JSONException, IOException, ModelException {
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
FileProjectManager.initialize(dir);
project = new Project();
ProjectMetadata pm = new ProjectMetadata();
pm.setName("TNG Test Project");
ProjectManager.singleton.registerProject(project, pm);
project = createProjectWithColumns("CacheTests", "Column A");
int index = project.columnModel.allocateNewCellIndex();
Column column = new Column(index,"Column A");
project.columnModel.addColumn(index, column, true);
options = mock(Properties.class);
engine = new Engine(project);
engine_config = new JSONObject(ENGINE_JSON_DUPLICATES);
// engine_config.getJSONArray("facets").getJSONObject(0).getJSONArray("selection").put(new JSONArray());
engine.initializeFromJSON(engine_config);
engine.setMode(Engine.Mode.RowBased);
@ -118,7 +92,6 @@ public class CacheTests extends RefineTest {
@AfterMethod
public void TearDown() {
project = null;
options = null;
engine = null;
bindings = null;
}

View File

@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.tests.model;
import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.util.Properties;
@ -45,18 +42,12 @@ import org.json.JSONObject;
import org.json.JSONArray;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.browsing.Engine;
import com.google.refine.io.FileProjectManager;
import com.google.refine.expr.ExpressionUtils;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
@ -66,7 +57,6 @@ import com.google.refine.operations.OnError;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class UrlFetchingTests extends RefineTest {
@ -80,42 +70,13 @@ public class UrlFetchingTests extends RefineTest {
}
// dependencies
Project project;
Properties options;
JSONObject engine_config;
Engine engine;
Properties bindings;
private Project project;
private Properties options;
private JSONObject engine_config;
@BeforeMethod
public void SetUp() throws JSONException, IOException, ModelException {
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
FileProjectManager.initialize(dir);
project = new Project();
ProjectMetadata pm = new ProjectMetadata();
pm.setName("URL Fetching Test Project");
ProjectManager.singleton.registerProject(project, pm);
int index = project.columnModel.allocateNewCellIndex();
Column column = new Column(index,"fruits");
project.columnModel.addColumn(index, column, true);
options = mock(Properties.class);
engine = new Engine(project);
engine_config = new JSONObject(ENGINE_JSON_URLS);
engine.initializeFromJSON(engine_config);
engine.setMode(Engine.Mode.RowBased);
bindings = new Properties();
bindings.put("project", project);
}
@AfterMethod
public void TearDown() {
project = null;
options = null;
engine = null;
bindings = null;
project = createProjectWithColumns("UrlFetchingTests", "fruits");
}
private boolean isHostReachable(String host, int timeout){

View File

@ -3,7 +3,6 @@ package com.google.refine.tests.model.changes;
import static org.testng.AssertJUnit.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@ -13,17 +12,13 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.changes.CellAtRow;
import com.google.refine.model.changes.ColumnAdditionChange;
import com.google.refine.model.changes.MassChange;
import com.google.refine.history.Change;
import com.google.refine.io.FileProjectManager;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class MassChangeTests extends RefineTest {
@ -38,12 +33,7 @@ public class MassChangeTests extends RefineTest {
@BeforeMethod
public void SetUp()
throws IOException, ModelException {
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
FileProjectManager.initialize(dir);
project = new Project();
ProjectMetadata pm = new ProjectMetadata();
pm.setName("TNG Test Project");
ProjectManager.singleton.registerProject(project, pm);
project = createProjectWithColumns("MassChangeTest");
}
/**

View File

@ -35,7 +35,6 @@ package com.google.refine.tests.recon;
import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.util.Properties;
import java.util.List;
@ -50,14 +49,8 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.grel.Function;
import com.google.refine.io.FileProjectManager;
import com.google.refine.model.Cell;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
@ -65,11 +58,9 @@ import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.process.Process;
import com.google.refine.process.ProcessManager;
import com.google.refine.operations.OnError;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.recon.ExtendDataOperation;
import com.google.refine.tests.RefineTest;
import com.google.refine.tests.util.TestUtils;
public class DataExtensionTests extends RefineTest {
@ -90,20 +81,10 @@ public class DataExtensionTests extends RefineTest {
Properties options;
JSONObject engine_config;
Engine engine;
Properties bindings;
@BeforeMethod
public void SetUp() throws JSONException, IOException, ModelException {
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
FileProjectManager.initialize(dir);
project = new Project();
ProjectMetadata pm = new ProjectMetadata();
pm.setName("Data Extension Test Project");
ProjectManager.singleton.registerProject(project, pm);
int index = project.columnModel.allocateNewCellIndex();
Column column = new Column(index,"country");
project.columnModel.addColumn(index, column, true);
project = createProjectWithColumns("DataExtensionTests", "country");
options = mock(Properties.class);
engine = new Engine(project);
@ -111,9 +92,6 @@ public class DataExtensionTests extends RefineTest {
engine.initializeFromJSON(engine_config);
engine.setMode(Engine.Mode.RowBased);
bindings = new Properties();
bindings.put("project", project);
Row row = new Row(2);
row.setCell(0, reconciledCell("Iran", "Q794"));
project.rows.add(row);
@ -133,7 +111,6 @@ public class DataExtensionTests extends RefineTest {
project = null;
options = null;
engine = null;
bindings = null;
}
static public Cell reconciledCell(String name, String id) {