Merge pull request #1442 from OpenRefine/refactor-test-initialization
Simplify project creation in unit tests
This commit is contained in:
Antonin Delpeuch
GitHub
commit
3f56a4eee9
@ -124,7 +124,8 @@ public class RefineServlet extends Butterfly {
|
||||
if (data == null) {
|
||||
throw new ServletException("can't find servlet init config 'refine.data', I have to give up initializing");
|
||||
}
|
||||
|
||||
logger.error("initializing FileProjectManager with dir");
|
||||
logger.error(data);
|
||||
s_dataDir = new File(data);
|
||||
FileProjectManager.initialize(s_dataDir);
|
||||
ImportingManager.initialize(this);
|
||||
|
||||
@ -33,30 +33,188 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeSuite;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
import com.google.refine.util.JSONUtilities;
|
||||
|
||||
/**
|
||||
* A base class containing various utilities to help testing Refine.
|
||||
*/
|
||||
public class RefineTest {
|
||||
|
||||
protected Logger logger;
|
||||
|
||||
boolean testFailed;
|
||||
protected File workspaceDir;
|
||||
protected RefineServlet servlet;
|
||||
private List<Project> projects = new ArrayList<Project>();
|
||||
private List<ImportingJob> importingJobs = new ArrayList<ImportingJob>();
|
||||
|
||||
@BeforeSuite
|
||||
public void init() {
|
||||
System.setProperty("log4j.configuration", "tests.log4j.properties");
|
||||
try {
|
||||
workspaceDir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
File jsonPath = new File(workspaceDir, "workspace.json");
|
||||
FileUtils.writeStringToFile(jsonPath, "{\"projectIDs\":[]\n" +
|
||||
",\"preferences\":{\"entries\":{\"scripting.starred-expressions\":" +
|
||||
"{\"class\":\"com.google.refine.preference.TopList\",\"top\":2147483647," +
|
||||
"\"list\":[]},\"scripting.expressions\":{\"class\":\"com.google.refine.preference.TopList\",\"top\":100,\"list\":[]}}}}");
|
||||
FileProjectManager.initialize(workspaceDir);
|
||||
} catch (IOException e) {
|
||||
workspaceDir = null;
|
||||
e.printStackTrace();
|
||||
}
|
||||
// This just keeps track of any failed test, for cleanupWorkspace
|
||||
testFailed = false;
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
protected void initProjectManager() {
|
||||
servlet = new RefineServletStub();
|
||||
ProjectManager.singleton = new ProjectManagerStub();
|
||||
ImportingManager.initialize(servlet);
|
||||
}
|
||||
|
||||
protected Project createProjectWithColumns(String projectName, String... columnNames) throws IOException, ModelException {
|
||||
Project project = new Project();
|
||||
ProjectMetadata pm = new ProjectMetadata();
|
||||
pm.setName(projectName);
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
|
||||
if (columnNames != null) {
|
||||
for(String columnName : columnNames) {
|
||||
int index = project.columnModel.allocateNewCellIndex();
|
||||
Column column = new Column(index,columnName);
|
||||
project.columnModel.addColumn(index, column, true);
|
||||
}
|
||||
}
|
||||
return project;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a project from a CSV encoded as a file. Not much
|
||||
* control is given on the import options, because this method is intended
|
||||
* to be a quick way to create a project for a test. For more control over
|
||||
* the import, just call the importer directly.
|
||||
*
|
||||
* @param input
|
||||
* contents of the CSV file to create the project from
|
||||
* @return
|
||||
*/
|
||||
protected Project createCSVProject(String input) {
|
||||
return createCSVProject("test project", input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a project from a CSV encoded as a file. Not much
|
||||
* control is given on the import options, because this method is intended
|
||||
* to be a quick way to create a project for a test. For more control over
|
||||
* the import, just call the importer directly.
|
||||
*
|
||||
* The projects created via this method and their importing jobs will be disposed of
|
||||
* at the end of each test.
|
||||
*
|
||||
* @param projectName
|
||||
* the name of the project to create
|
||||
* @param input
|
||||
* the content of the file, encoded as a CSV (with "," as a separator)
|
||||
* @return
|
||||
*/
|
||||
protected Project createCSVProject(String projectName, String input) {
|
||||
|
||||
|
||||
Project project = new Project();
|
||||
|
||||
ProjectMetadata metadata = new ProjectMetadata();
|
||||
metadata.setName(projectName);
|
||||
|
||||
JSONObject options = mock(JSONObject.class);
|
||||
prepareImportOptions(options, ",", -1, 0, 0, 1, false, false);
|
||||
|
||||
ImportingJob job = ImportingManager.createJob();
|
||||
|
||||
SeparatorBasedImporter importer = new SeparatorBasedImporter();
|
||||
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, metadata);
|
||||
|
||||
projects.add(project);
|
||||
importingJobs.add(job);
|
||||
return project;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the importing options for the CSV importer.
|
||||
* @param options
|
||||
* @param sep
|
||||
* @param limit
|
||||
* @param skip
|
||||
* @param ignoreLines
|
||||
* @param headerLines
|
||||
* @param guessValueType
|
||||
* @param ignoreQuotes
|
||||
*/
|
||||
private void prepareImportOptions(JSONObject options,
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans up the projects and jobs created with createCSVProject
|
||||
*/
|
||||
@AfterMethod
|
||||
protected void cleanupProjectsAndJobs() {
|
||||
for(ImportingJob job : importingJobs) {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
}
|
||||
for(Project project: projects) {
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
}
|
||||
servlet = null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -33,50 +33,29 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.browsing.facets;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.browsing.RowFilter;
|
||||
import com.google.refine.browsing.facets.TextSearchFacet;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class TextSearchFacetTests extends RefineTest {
|
||||
// dependencies
|
||||
private RefineServlet servlet;
|
||||
private Project project;
|
||||
private ProjectMetadata pm;
|
||||
private JSONObject options;
|
||||
private ImportingJob job;
|
||||
private SeparatorBasedImporter importer;
|
||||
private TextSearchFacet textfilter;
|
||||
private JSONObject textsearchfacet;
|
||||
private RowFilter rowfilter;
|
||||
private JSONObject textsearchfacet;
|
||||
|
||||
@Override
|
||||
@BeforeTest
|
||||
@ -86,41 +65,14 @@ public class TextSearchFacetTests extends RefineTest {
|
||||
|
||||
@BeforeMethod
|
||||
public void setUp() throws JSONException, IOException, ModelException {
|
||||
servlet = new RefineServletStub();
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
pm = new ProjectMetadata();
|
||||
pm.setName("TextSearchFacet test");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
options = mock(JSONObject.class);
|
||||
|
||||
ImportingManager.initialize(servlet);
|
||||
job = ImportingManager.createJob();
|
||||
importer = new SeparatorBasedImporter();
|
||||
|
||||
String csv = "Value\n"
|
||||
project = createCSVProject("TextSearchFacet",
|
||||
"Value\n"
|
||||
+ "a\n"
|
||||
+ "b\n"
|
||||
+ "ab\n"
|
||||
+ "Abc\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
+ "Abc\n");
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void tearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
project = null;
|
||||
pm = null;
|
||||
options = null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Test to demonstrate the intended behaviour of the function
|
||||
*/
|
||||
@ -245,21 +197,5 @@ public class TextSearchFacetTests extends RefineTest {
|
||||
Assert.assertEquals(rowfilter.filterRow(project, 2, project.rows.get(2)),false);
|
||||
Assert.assertEquals(rowfilter.filterRow(project, 3, project.rows.get(3)),true);
|
||||
}
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -1,37 +1,22 @@
|
||||
|
||||
package com.google.refine.tests.expr.functions;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.expr.EvalError;
|
||||
import com.google.refine.expr.HasFieldsListImpl;
|
||||
import com.google.refine.expr.WrappedRow;
|
||||
import com.google.refine.grel.ControlFunctionRegistry;
|
||||
import com.google.refine.grel.Function;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
import com.google.refine.tests.ProjectManagerStub;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
|
||||
/**
|
||||
@ -47,82 +32,33 @@ public class CrossFunctionTests extends RefineTest {
|
||||
}
|
||||
|
||||
// dependencies
|
||||
RefineServlet servlet;
|
||||
Project projectGift;
|
||||
Project projectAddress;
|
||||
ProjectMetadata metadata;
|
||||
ImportingJob job;
|
||||
JSONObject options;
|
||||
SeparatorBasedImporter importer;
|
||||
|
||||
|
||||
// data from: https://github.com/OpenRefine/OpenRefine/wiki/GREL-Other-Functions
|
||||
@BeforeMethod
|
||||
public void SetUp() {
|
||||
bindings = new Properties();
|
||||
|
||||
servlet = new RefineServletStub();
|
||||
ProjectManager.singleton = new ProjectManagerStub();
|
||||
ImportingManager.initialize(servlet);
|
||||
projectAddress = new Project();
|
||||
|
||||
job = ImportingManager.createJob();
|
||||
options = mock(JSONObject.class);
|
||||
importer = new SeparatorBasedImporter();
|
||||
String projectName = "My Address Book";
|
||||
String input = "friend,address\n"
|
||||
+ "john,120 Main St.\n"
|
||||
+ "mary,50 Broadway Ave.\n"
|
||||
+ "john,999 XXXXXX St.\n" // john's 2nd address
|
||||
+ "anne,17 Morning Crescent\n";
|
||||
projectAddress = createCSVProject(projectName, input);
|
||||
|
||||
projectName = "Christmas Gifts";
|
||||
input = "gift,recipient\n"
|
||||
+ "lamp,mary\n"
|
||||
+ "clock,john\n";
|
||||
projectGift = createCSVProject(projectName, input);
|
||||
|
||||
createMyAddressBook();
|
||||
projectGift = createChristmasGifts();
|
||||
bindings.put("project", projectGift);
|
||||
|
||||
// add a column address based on column recipient
|
||||
bindings.put("columnName", "recipient");
|
||||
}
|
||||
|
||||
// data from: https://github.com/OpenRefine/OpenRefine/wiki/GREL-Other-Functions
|
||||
private Project createMyAddressBook() {
|
||||
String projectName = "My Address Book";
|
||||
String input = "friend;address\n"
|
||||
+ "john;120 Main St.\n"
|
||||
+ "mary;50 Broadway Ave.\n"
|
||||
+ "john;999 XXXXXX St.\n" // john's 2nd address
|
||||
+ "anne;17 Morning Crescent\n";
|
||||
return createProject(projectName, input);
|
||||
}
|
||||
|
||||
private Project createChristmasGifts() {
|
||||
String projectName = "Christmas Gifts";
|
||||
String input = "gift;recipient\n"
|
||||
+ "lamp;mary\n"
|
||||
+ "clock;john\n";
|
||||
return createProject(projectName, input);
|
||||
}
|
||||
|
||||
private Project createProject(String projectName, String input) {
|
||||
Project project = new Project();
|
||||
ProjectMetadata metadata = new ProjectMetadata();
|
||||
|
||||
metadata.setName(projectName);
|
||||
prepareOptions(";", -1, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, metadata);
|
||||
|
||||
return project;
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(projectGift.id);
|
||||
ProjectManager.singleton.deleteProject(projectAddress.id);
|
||||
job = null;
|
||||
metadata = null;
|
||||
projectGift = null;
|
||||
projectAddress = null;
|
||||
options = null;
|
||||
importer = null;
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void crossFunctionOneToOneTest() throws Exception {
|
||||
Row row = ((Row)((WrappedRow) ((HasFieldsListImpl) invoke("cross", "mary", "My Address Book", "friend")).get(0)).row);
|
||||
@ -189,20 +125,4 @@ public class CrossFunctionTests extends RefineTest {
|
||||
return function.call(bindings,args);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.expr.functions;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
@ -47,27 +44,21 @@ import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.browsing.Engine;
|
||||
import com.google.refine.expr.EvalError;
|
||||
import com.google.refine.grel.ControlFunctionRegistry;
|
||||
import com.google.refine.grel.Function;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class FunctionTests extends RefineTest {
|
||||
|
||||
static Properties bindings;
|
||||
Project project;
|
||||
Properties options;
|
||||
JSONObject engine_config;
|
||||
Engine engine;
|
||||
|
||||
@ -80,21 +71,9 @@ public class FunctionTests extends RefineTest {
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() throws IOException, ModelException {
|
||||
|
||||
project = createProjectWithColumns("FunctionTests", "Column A");
|
||||
bindings = new Properties();
|
||||
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
ProjectMetadata pm = new ProjectMetadata();
|
||||
pm.setName("TNG Test Project");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
|
||||
int index = project.columnModel.allocateNewCellIndex();
|
||||
Column column = new Column(index,"Column A");
|
||||
project.columnModel.addColumn(index, column, true);
|
||||
|
||||
options = mock(Properties.class);
|
||||
|
||||
bindings.put("project", project);
|
||||
|
||||
// Five rows of a's and five of 1s
|
||||
|
||||
@ -33,13 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.expr.functions.booleans;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
@ -47,20 +43,14 @@ import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.browsing.Engine;
|
||||
import com.google.refine.expr.EvalError;
|
||||
import com.google.refine.grel.ControlFunctionRegistry;
|
||||
import com.google.refine.grel.Function;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class BooleanTests extends RefineTest {
|
||||
@ -83,11 +73,8 @@ public class BooleanTests extends RefineTest {
|
||||
};
|
||||
|
||||
|
||||
static Properties bindings;
|
||||
Project project;
|
||||
Properties options;
|
||||
JSONObject engine_config;
|
||||
Engine engine;
|
||||
static private Properties bindings;
|
||||
private Project project;
|
||||
|
||||
|
||||
@Override
|
||||
@ -99,19 +86,7 @@ public class BooleanTests extends RefineTest {
|
||||
@BeforeMethod
|
||||
public void SetUp() throws IOException, ModelException {
|
||||
bindings = new Properties();
|
||||
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
ProjectMetadata pm = new ProjectMetadata();
|
||||
pm.setName("TNG Test Project");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
|
||||
int index = project.columnModel.allocateNewCellIndex();
|
||||
Column column = new Column(index,"Column A");
|
||||
project.columnModel.addColumn(index, column, true);
|
||||
|
||||
options = mock(Properties.class);
|
||||
project = createProjectWithColumns("BooleanTests", "Column A");
|
||||
|
||||
bindings.put("project", project);
|
||||
|
||||
|
||||
@ -32,10 +32,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
package com.google.refine.tests.model;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
|
||||
@ -48,32 +44,21 @@ import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.browsing.Engine;
|
||||
import com.google.refine.browsing.RowVisitor;
|
||||
import com.google.refine.expr.functions.FacetCount;
|
||||
import com.google.refine.grel.Function;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
import com.google.refine.operations.EngineDependentOperation;
|
||||
import com.google.refine.operations.row.RowRemovalOperation;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class CacheTests extends RefineTest {
|
||||
|
||||
//{project=1718051861971, engine= ...}
|
||||
//engine={ "facets" : ..., "mode":"row-based"}}
|
||||
//facets = [{"type":"list","name":"row","columnName":"row","expression":"facetCount(value, 'value', 'row') > 1","omitBlank":false,"omitError":false,"selection":[],"selectBlank":false,"selectError":false,"invert":false}]
|
||||
|
||||
// {project=1718051861971, engine=
|
||||
|
||||
|
||||
// Equivalent to duplicate facet on Column A with true selected
|
||||
static final String ENGINE_JSON_DUPLICATES = "{\"facets\":[{\"type\":\"list\",\"name\":\"facet A\",\"columnName\":\"Column A\",\"expression\":\"facetCount(value, 'value', 'Column A') > 1\",\"omitBlank\":false,\"omitError\":false,\"selection\":[{\"v\":{\"v\":true,\"l\":\"true\"}}],\"selectBlank\":false,\"selectError\":false,\"invert\":false}],\"mode\":\"row-based\"}}";
|
||||
|
||||
@ -92,21 +77,10 @@ public class CacheTests extends RefineTest {
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() throws JSONException, IOException, ModelException {
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
ProjectMetadata pm = new ProjectMetadata();
|
||||
pm.setName("TNG Test Project");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
|
||||
int index = project.columnModel.allocateNewCellIndex();
|
||||
Column column = new Column(index,"Column A");
|
||||
project.columnModel.addColumn(index, column, true);
|
||||
project = createProjectWithColumns("CacheTests", "Column A");
|
||||
|
||||
options = mock(Properties.class);
|
||||
engine = new Engine(project);
|
||||
engine_config = new JSONObject(ENGINE_JSON_DUPLICATES);
|
||||
// engine_config.getJSONArray("facets").getJSONObject(0).getJSONArray("selection").put(new JSONArray());
|
||||
engine.initializeFromJSON(engine_config);
|
||||
engine.setMode(Engine.Mode.RowBased);
|
||||
|
||||
@ -118,7 +92,6 @@ public class CacheTests extends RefineTest {
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
project = null;
|
||||
options = null;
|
||||
engine = null;
|
||||
bindings = null;
|
||||
}
|
||||
|
||||
@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.model;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.util.Properties;
|
||||
@ -45,18 +42,12 @@ import org.json.JSONObject;
|
||||
import org.json.JSONArray;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.browsing.Engine;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.expr.ExpressionUtils;
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
@ -66,7 +57,6 @@ import com.google.refine.operations.OnError;
|
||||
import com.google.refine.operations.EngineDependentOperation;
|
||||
import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class UrlFetchingTests extends RefineTest {
|
||||
@ -80,44 +70,15 @@ public class UrlFetchingTests extends RefineTest {
|
||||
}
|
||||
|
||||
// dependencies
|
||||
Project project;
|
||||
Properties options;
|
||||
JSONObject engine_config;
|
||||
Engine engine;
|
||||
Properties bindings;
|
||||
private Project project;
|
||||
private Properties options;
|
||||
private JSONObject engine_config;
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() throws JSONException, IOException, ModelException {
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
ProjectMetadata pm = new ProjectMetadata();
|
||||
pm.setName("URL Fetching Test Project");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
|
||||
int index = project.columnModel.allocateNewCellIndex();
|
||||
Column column = new Column(index,"fruits");
|
||||
project.columnModel.addColumn(index, column, true);
|
||||
|
||||
options = mock(Properties.class);
|
||||
engine = new Engine(project);
|
||||
engine_config = new JSONObject(ENGINE_JSON_URLS);
|
||||
engine.initializeFromJSON(engine_config);
|
||||
engine.setMode(Engine.Mode.RowBased);
|
||||
|
||||
bindings = new Properties();
|
||||
bindings.put("project", project);
|
||||
|
||||
project = createProjectWithColumns("UrlFetchingTests", "fruits");
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
project = null;
|
||||
options = null;
|
||||
engine = null;
|
||||
bindings = null;
|
||||
}
|
||||
|
||||
private boolean isHostReachable(String host, int timeout){
|
||||
boolean state = false;
|
||||
|
||||
|
||||
@ -3,7 +3,6 @@ package com.google.refine.tests.model.changes;
|
||||
|
||||
import static org.testng.AssertJUnit.assertTrue;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@ -13,17 +12,13 @@ import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.changes.CellAtRow;
|
||||
import com.google.refine.model.changes.ColumnAdditionChange;
|
||||
import com.google.refine.model.changes.MassChange;
|
||||
import com.google.refine.history.Change;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
public class MassChangeTests extends RefineTest {
|
||||
|
||||
@ -38,12 +33,7 @@ public class MassChangeTests extends RefineTest {
|
||||
@BeforeMethod
|
||||
public void SetUp()
|
||||
throws IOException, ModelException {
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
ProjectMetadata pm = new ProjectMetadata();
|
||||
pm.setName("TNG Test Project");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
project = createProjectWithColumns("MassChangeTest");
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -33,49 +33,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.operations.cell;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.Properties;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.operations.cell.MultiValuedCellJoinOperation;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class JoinMultiValuedCellsTests extends RefineTest {
|
||||
// dependencies
|
||||
private Project project;
|
||||
private ProjectMetadata pm;
|
||||
private JSONObject options;
|
||||
private ImportingJob job;
|
||||
private SeparatorBasedImporter importer;
|
||||
|
||||
|
||||
@Override
|
||||
@BeforeTest
|
||||
@ -83,48 +55,18 @@ public class JoinMultiValuedCellsTests extends RefineTest {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
public void setUp() throws JSONException, IOException, ModelException {
|
||||
RefineServlet servlet = new RefineServletStub();
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
pm = new ProjectMetadata();
|
||||
pm.setName("JoinMultiValuedCells test");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
options = mock(JSONObject.class);
|
||||
|
||||
ImportingManager.initialize(servlet);
|
||||
job = ImportingManager.createJob();
|
||||
importer = new SeparatorBasedImporter();
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void tearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
project = null;
|
||||
pm = null;
|
||||
options = null;
|
||||
}
|
||||
|
||||
/**
|
||||
/*
|
||||
* Test to demonstrate the intended behaviour of the function
|
||||
*/
|
||||
|
||||
@Test
|
||||
public void testJoinMultiValuedCells() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "Record_1,one\n"
|
||||
+ ",two\n"
|
||||
+ ",three\n"
|
||||
+ ",four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one\n"
|
||||
+ ",two\n"
|
||||
+ ",three\n"
|
||||
+ ",four\n");
|
||||
|
||||
AbstractOperation op = new MultiValuedCellJoinOperation(
|
||||
"Value",
|
||||
@ -142,16 +84,13 @@ public class JoinMultiValuedCellsTests extends RefineTest {
|
||||
|
||||
@Test
|
||||
public void testJoinMultiValuedCellsMultipleSpaces() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one\n"
|
||||
+ ",two\n"
|
||||
+ ",three\n"
|
||||
+ ",four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
+ ",four\n");
|
||||
|
||||
|
||||
AbstractOperation op = new MultiValuedCellJoinOperation(
|
||||
"Value",
|
||||
@ -167,19 +106,6 @@ public class JoinMultiValuedCellsTests extends RefineTest {
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(valueCol), "one, ,two, ,three, ,four");
|
||||
}
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -1,173 +0,0 @@
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
package com.google.refine.tests.operations.cell;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.Properties;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.operations.cell.KeyValueColumnizeOperation;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class KeyValueColumnizeTests extends RefineTest {
|
||||
// dependencies
|
||||
private RefineServlet servlet;
|
||||
private Project project;
|
||||
private ProjectMetadata pm;
|
||||
private JSONObject options;
|
||||
private ImportingJob job;
|
||||
private SeparatorBasedImporter importer;
|
||||
|
||||
|
||||
@Override
|
||||
@BeforeTest
|
||||
public void init() {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() throws JSONException, IOException, ModelException {
|
||||
servlet = new RefineServletStub();
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
pm = new ProjectMetadata();
|
||||
pm.setName("KeyValueColumnize test");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
options = mock(JSONObject.class);
|
||||
|
||||
ImportingManager.initialize(servlet);
|
||||
job = ImportingManager.createJob();
|
||||
importer = new SeparatorBasedImporter();
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
project = null;
|
||||
pm = null;
|
||||
options = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test to demonstrate the intended behaviour of the function, for issue #1214
|
||||
* https://github.com/OpenRefine/OpenRefine/issues/1214
|
||||
*/
|
||||
|
||||
@Test
|
||||
public void testKeyValueColumnize() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "merchant,Katie\n"
|
||||
+ "fruit,apple\n"
|
||||
+ "price,1.2\n"
|
||||
+ "fruit,pear\n"
|
||||
+ "price,1.5\n"
|
||||
+ "merchant,John\n"
|
||||
+ "fruit,banana\n"
|
||||
+ "price,3.1\n";
|
||||
prepareOptions(",", 20, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
|
||||
AbstractOperation op = new KeyValueColumnizeOperation(
|
||||
"Key",
|
||||
"Value",
|
||||
null);
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
process.performImmediate();
|
||||
|
||||
int merchantCol = project.columnModel.getColumnByName("merchant").getCellIndex();
|
||||
int fruitCol = project.columnModel.getColumnByName("fruit").getCellIndex();
|
||||
int priceCol = project.columnModel.getColumnByName("price").getCellIndex();
|
||||
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(merchantCol), "Katie");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(merchantCol), null);
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(merchantCol), "John");
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(fruitCol), "apple");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(fruitCol), "pear");
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(fruitCol), "banana");
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(priceCol), "1.2");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(priceCol), "1.5");
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(priceCol), "3.1");
|
||||
}
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -33,50 +33,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.operations.cell;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.Properties;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.operations.cell.MultiValuedCellSplitOperation;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
// dependencies
|
||||
private RefineServlet servlet;
|
||||
private Project project;
|
||||
private ProjectMetadata pm;
|
||||
private JSONObject options;
|
||||
private ImportingJob job;
|
||||
private SeparatorBasedImporter importer;
|
||||
|
||||
|
||||
@Override
|
||||
@BeforeTest
|
||||
@ -84,32 +56,6 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() throws JSONException, IOException, ModelException {
|
||||
servlet = new RefineServletStub();
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
pm = new ProjectMetadata();
|
||||
pm.setName("SplitMultiValuedCells test");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
options = mock(JSONObject.class);
|
||||
|
||||
ImportingManager.initialize(servlet);
|
||||
job = ImportingManager.createJob();
|
||||
importer = new SeparatorBasedImporter();
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
project = null;
|
||||
pm = null;
|
||||
options = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test to demonstrate the intended behaviour of the function, for issue #1268
|
||||
* https://github.com/OpenRefine/OpenRefine/issues/1268
|
||||
@ -117,13 +63,9 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
|
||||
@Test
|
||||
public void testSplitMultiValuedCellsTextSeparator() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n");
|
||||
|
||||
AbstractOperation op = new MultiValuedCellSplitOperation(
|
||||
"Value",
|
||||
@ -144,13 +86,9 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
|
||||
@Test
|
||||
public void testSplitMultiValuedCellsRegExSeparator() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n");
|
||||
|
||||
AbstractOperation op = new MultiValuedCellSplitOperation(
|
||||
"Value",
|
||||
@ -175,13 +113,10 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
|
||||
@Test
|
||||
public void testSplitMultiValuedCellsLengths() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n");
|
||||
|
||||
int[] lengths = {4,4,6,4};
|
||||
|
||||
AbstractOperation op = new MultiValuedCellSplitOperation(
|
||||
@ -204,20 +139,6 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
Assert.assertEquals(project.rows.get(3).getCellValue(valueCol), "four");
|
||||
}
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -33,34 +33,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.operations.cell;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.history.HistoryEntry;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.operations.cell.KeyValueColumnizeOperation;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.tests.ProjectManagerStub;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
|
||||
public class TransposeTests extends RefineTest {
|
||||
@ -71,67 +54,34 @@ public class TransposeTests extends RefineTest {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
}
|
||||
|
||||
// dependencies
|
||||
RefineServlet servlet;
|
||||
Project project;
|
||||
ProjectMetadata metadata;
|
||||
ImportingJob job;
|
||||
JSONObject options;
|
||||
SeparatorBasedImporter importer;
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() {
|
||||
servlet = new RefineServletStub();
|
||||
ProjectManager.singleton = new ProjectManagerStub();
|
||||
ImportingManager.initialize(servlet);
|
||||
project = new Project();
|
||||
metadata = new ProjectMetadata();
|
||||
|
||||
job = ImportingManager.createJob();
|
||||
options = mock(JSONObject.class);
|
||||
importer = new SeparatorBasedImporter();
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
metadata = null;
|
||||
project = null;
|
||||
options = null;
|
||||
importer = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test in the case where an ID is available in the first column.
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test
|
||||
public void keyValueComumnize() throws Exception {
|
||||
String input = "ID;Cat;Val\n"
|
||||
+ "1;a;1\n"
|
||||
+ "1;b;3\n"
|
||||
+ "2;b;4\n"
|
||||
+ "2;c;5\n"
|
||||
+ "3;a;2\n"
|
||||
+ "3;b;5\n"
|
||||
+ "3;d;3\n";
|
||||
|
||||
prepareOptions(";", -1, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, metadata);
|
||||
public void testKeyValueColumnizeWithID() throws Exception {
|
||||
Project project = createCSVProject(
|
||||
"ID,Cat,Val\n"
|
||||
+ "1,a,1\n"
|
||||
+ "1,b,3\n"
|
||||
+ "2,b,4\n"
|
||||
+ "2,c,5\n"
|
||||
+ "3,a,2\n"
|
||||
+ "3,b,5\n"
|
||||
+ "3,d,3\n");
|
||||
|
||||
AbstractOperation op = new KeyValueColumnizeOperation(
|
||||
"Cat", "Val", null);
|
||||
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
|
||||
HistoryEntry historyEntry = process.performImmediate();
|
||||
process.performImmediate();
|
||||
|
||||
// Expected output from the GUI.
|
||||
// ID;a;b;c;d
|
||||
// 1;1;3;;
|
||||
// 2;;4;5;
|
||||
// 3;2;5;;3
|
||||
// ID,a,b,c,d
|
||||
// 1,1,3,,
|
||||
// 2,,4,5,
|
||||
// 3,2,5,,3
|
||||
Assert.assertEquals(project.columnModel.columns.size(), 5);
|
||||
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "ID");
|
||||
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "a");
|
||||
@ -142,38 +92,59 @@ public class TransposeTests extends RefineTest {
|
||||
|
||||
// The actual row data structure has to leave the columns model untouched for redo/undo purpose.
|
||||
// So we have 2 empty columns(column 1,2) on the row level.
|
||||
// 1;1;3;;
|
||||
// 1,1,3,,
|
||||
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "1");
|
||||
Assert.assertEquals(project.rows.get(0).cells.get(3).value, "1");
|
||||
Assert.assertEquals(project.rows.get(0).cells.get(4).value, "3");
|
||||
|
||||
// 2;;4;5;
|
||||
// 2,,4,5,
|
||||
Assert.assertEquals(project.rows.get(1).cells.get(0).value, "2");
|
||||
Assert.assertEquals(project.rows.get(1).cells.get(4).value, "4");
|
||||
Assert.assertEquals(project.rows.get(1).cells.get(5).value, "5");
|
||||
|
||||
// 3;2;5;;3
|
||||
// 3,2,5,,3
|
||||
Assert.assertEquals(project.rows.get(2).cells.get(0).value, "3");
|
||||
Assert.assertEquals(project.rows.get(2).cells.get(3).value, "2");
|
||||
Assert.assertEquals(project.rows.get(2).cells.get(4).value, "5");
|
||||
Assert.assertEquals(project.rows.get(2).cells.get(6).value, "3");
|
||||
}
|
||||
|
||||
/**
|
||||
* Test to demonstrate the intended behaviour of the function when no id is available, for issue #1214
|
||||
* https://github.com/OpenRefine/OpenRefine/issues/1214
|
||||
*/
|
||||
@Test
|
||||
public void testKeyValueColumnizeWithoutID() throws Exception {
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "merchant,Katie\n"
|
||||
+ "fruit,apple\n"
|
||||
+ "price,1.2\n"
|
||||
+ "fruit,pear\n"
|
||||
+ "price,1.5\n"
|
||||
+ "merchant,John\n"
|
||||
+ "fruit,banana\n"
|
||||
+ "price,3.1\n");
|
||||
|
||||
AbstractOperation op = new KeyValueColumnizeOperation(
|
||||
"Key",
|
||||
"Value",
|
||||
null);
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
process.performImmediate();
|
||||
|
||||
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
int merchantCol = project.columnModel.getColumnByName("merchant").getCellIndex();
|
||||
int fruitCol = project.columnModel.getColumnByName("fruit").getCellIndex();
|
||||
int priceCol = project.columnModel.getColumnByName("price").getCellIndex();
|
||||
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(merchantCol), "Katie");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(merchantCol), null);
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(merchantCol), "John");
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(fruitCol), "apple");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(fruitCol), "pear");
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(fruitCol), "banana");
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(priceCol), "1.2");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(priceCol), "1.5");
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(priceCol), "3.1");
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,7 +35,6 @@ package com.google.refine.tests.recon;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Properties;
|
||||
import java.util.List;
|
||||
@ -50,14 +49,8 @@ import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.browsing.Engine;
|
||||
import com.google.refine.browsing.RowVisitor;
|
||||
import com.google.refine.grel.Function;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
@ -65,11 +58,9 @@ import com.google.refine.model.Recon;
|
||||
import com.google.refine.model.ReconCandidate;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.process.ProcessManager;
|
||||
import com.google.refine.operations.OnError;
|
||||
import com.google.refine.operations.EngineDependentOperation;
|
||||
import com.google.refine.operations.recon.ExtendDataOperation;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class DataExtensionTests extends RefineTest {
|
||||
@ -90,29 +81,16 @@ public class DataExtensionTests extends RefineTest {
|
||||
Properties options;
|
||||
JSONObject engine_config;
|
||||
Engine engine;
|
||||
Properties bindings;
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() throws JSONException, IOException, ModelException {
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
ProjectMetadata pm = new ProjectMetadata();
|
||||
pm.setName("Data Extension Test Project");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
|
||||
int index = project.columnModel.allocateNewCellIndex();
|
||||
Column column = new Column(index,"country");
|
||||
project.columnModel.addColumn(index, column, true);
|
||||
project = createProjectWithColumns("DataExtensionTests", "country");
|
||||
|
||||
options = mock(Properties.class);
|
||||
engine = new Engine(project);
|
||||
engine_config = new JSONObject(ENGINE_JSON_URLS);
|
||||
engine.initializeFromJSON(engine_config);
|
||||
engine.setMode(Engine.Mode.RowBased);
|
||||
|
||||
bindings = new Properties();
|
||||
bindings.put("project", project);
|
||||
|
||||
Row row = new Row(2);
|
||||
row.setCell(0, reconciledCell("Iran", "Q794"));
|
||||
@ -133,7 +111,6 @@ public class DataExtensionTests extends RefineTest {
|
||||
project = null;
|
||||
options = null;
|
||||
engine = null;
|
||||
bindings = null;
|
||||
}
|
||||
|
||||
static public Cell reconciledCell(String name, String id) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user