Migrate more tests to the createCSVProject helper
This commit is contained in:
Antonin Delpeuch
parent
8f9fc61046
commit
feae25a5ff
@ -48,6 +48,7 @@ import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeSuite;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
@ -76,7 +77,27 @@ public class RefineTest {
|
||||
@BeforeSuite
|
||||
public void init() {
|
||||
System.setProperty("log4j.configuration", "tests.log4j.properties");
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
protected void initProjectManager() {
|
||||
servlet = new RefineServletStub();
|
||||
ProjectManager.singleton = new ProjectManagerStub();
|
||||
ImportingManager.initialize(servlet);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a project from a CSV encoded as a file. Not much
|
||||
* control is given on the import options, because this method is intended
|
||||
* to be a quick way to create a project for a test. For more control over
|
||||
* the import, just call the importer directly.
|
||||
*
|
||||
* @param input
|
||||
* contents of the CSV file to create the project from
|
||||
* @return
|
||||
*/
|
||||
protected Project createCSVProject(String input) {
|
||||
return createCSVProject("test project", input);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -103,8 +124,6 @@ public class RefineTest {
|
||||
JSONObject options = mock(JSONObject.class);
|
||||
prepareImportOptions(options, ",", -1, 0, 0, 1, false, false);
|
||||
|
||||
servlet = new RefineServletStub();
|
||||
ImportingManager.initialize(servlet);
|
||||
ImportingJob job = ImportingManager.createJob();
|
||||
|
||||
SeparatorBasedImporter importer = new SeparatorBasedImporter();
|
||||
@ -148,7 +167,7 @@ public class RefineTest {
|
||||
* Cleans up the projects and jobs created with createCSVProject
|
||||
*/
|
||||
@AfterMethod
|
||||
private void cleanupProjectsAndJobs() {
|
||||
protected void cleanupProjectsAndJobs() {
|
||||
for(ImportingJob job : importingJobs) {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
}
|
||||
|
||||
@ -33,49 +33,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.operations.cell;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.Properties;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.operations.cell.MultiValuedCellJoinOperation;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class JoinMultiValuedCellsTests extends RefineTest {
|
||||
// dependencies
|
||||
private Project project;
|
||||
private ProjectMetadata pm;
|
||||
private JSONObject options;
|
||||
private ImportingJob job;
|
||||
private SeparatorBasedImporter importer;
|
||||
|
||||
|
||||
@Override
|
||||
@BeforeTest
|
||||
@ -83,48 +55,18 @@ public class JoinMultiValuedCellsTests extends RefineTest {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
public void setUp() throws JSONException, IOException, ModelException {
|
||||
RefineServlet servlet = new RefineServletStub();
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
pm = new ProjectMetadata();
|
||||
pm.setName("JoinMultiValuedCells test");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
options = mock(JSONObject.class);
|
||||
|
||||
ImportingManager.initialize(servlet);
|
||||
job = ImportingManager.createJob();
|
||||
importer = new SeparatorBasedImporter();
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void tearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
project = null;
|
||||
pm = null;
|
||||
options = null;
|
||||
}
|
||||
|
||||
/**
|
||||
/*
|
||||
* Test to demonstrate the intended behaviour of the function
|
||||
*/
|
||||
|
||||
@Test
|
||||
public void testJoinMultiValuedCells() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "Record_1,one\n"
|
||||
+ ",two\n"
|
||||
+ ",three\n"
|
||||
+ ",four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one\n"
|
||||
+ ",two\n"
|
||||
+ ",three\n"
|
||||
+ ",four\n");
|
||||
|
||||
AbstractOperation op = new MultiValuedCellJoinOperation(
|
||||
"Value",
|
||||
@ -142,16 +84,13 @@ public class JoinMultiValuedCellsTests extends RefineTest {
|
||||
|
||||
@Test
|
||||
public void testJoinMultiValuedCellsMultipleSpaces() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one\n"
|
||||
+ ",two\n"
|
||||
+ ",three\n"
|
||||
+ ",four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
+ ",four\n");
|
||||
|
||||
|
||||
AbstractOperation op = new MultiValuedCellJoinOperation(
|
||||
"Value",
|
||||
@ -167,19 +106,6 @@ public class JoinMultiValuedCellsTests extends RefineTest {
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(valueCol), "one, ,two, ,three, ,four");
|
||||
}
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -1,173 +0,0 @@
|
||||
/*
|
||||
|
||||
Copyright 2010, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
package com.google.refine.tests.operations.cell;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.Properties;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.operations.cell.KeyValueColumnizeOperation;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class KeyValueColumnizeTests extends RefineTest {
|
||||
// dependencies
|
||||
private RefineServlet servlet;
|
||||
private Project project;
|
||||
private ProjectMetadata pm;
|
||||
private JSONObject options;
|
||||
private ImportingJob job;
|
||||
private SeparatorBasedImporter importer;
|
||||
|
||||
|
||||
@Override
|
||||
@BeforeTest
|
||||
public void init() {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() throws JSONException, IOException, ModelException {
|
||||
servlet = new RefineServletStub();
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
pm = new ProjectMetadata();
|
||||
pm.setName("KeyValueColumnize test");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
options = mock(JSONObject.class);
|
||||
|
||||
ImportingManager.initialize(servlet);
|
||||
job = ImportingManager.createJob();
|
||||
importer = new SeparatorBasedImporter();
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
project = null;
|
||||
pm = null;
|
||||
options = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test to demonstrate the intended behaviour of the function, for issue #1214
|
||||
* https://github.com/OpenRefine/OpenRefine/issues/1214
|
||||
*/
|
||||
|
||||
@Test
|
||||
public void testKeyValueColumnize() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "merchant,Katie\n"
|
||||
+ "fruit,apple\n"
|
||||
+ "price,1.2\n"
|
||||
+ "fruit,pear\n"
|
||||
+ "price,1.5\n"
|
||||
+ "merchant,John\n"
|
||||
+ "fruit,banana\n"
|
||||
+ "price,3.1\n";
|
||||
prepareOptions(",", 20, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
|
||||
AbstractOperation op = new KeyValueColumnizeOperation(
|
||||
"Key",
|
||||
"Value",
|
||||
null);
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
process.performImmediate();
|
||||
|
||||
int merchantCol = project.columnModel.getColumnByName("merchant").getCellIndex();
|
||||
int fruitCol = project.columnModel.getColumnByName("fruit").getCellIndex();
|
||||
int priceCol = project.columnModel.getColumnByName("price").getCellIndex();
|
||||
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(merchantCol), "Katie");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(merchantCol), null);
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(merchantCol), "John");
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(fruitCol), "apple");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(fruitCol), "pear");
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(fruitCol), "banana");
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(priceCol), "1.2");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(priceCol), "1.5");
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(priceCol), "3.1");
|
||||
}
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -33,50 +33,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.operations.cell;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.Properties;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.ModelException;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.operations.cell.MultiValuedCellSplitOperation;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
import com.google.refine.tests.util.TestUtils;
|
||||
|
||||
|
||||
public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
// dependencies
|
||||
private RefineServlet servlet;
|
||||
private Project project;
|
||||
private ProjectMetadata pm;
|
||||
private JSONObject options;
|
||||
private ImportingJob job;
|
||||
private SeparatorBasedImporter importer;
|
||||
|
||||
|
||||
@Override
|
||||
@BeforeTest
|
||||
@ -84,32 +56,6 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() throws JSONException, IOException, ModelException {
|
||||
servlet = new RefineServletStub();
|
||||
File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
|
||||
FileProjectManager.initialize(dir);
|
||||
project = new Project();
|
||||
pm = new ProjectMetadata();
|
||||
pm.setName("SplitMultiValuedCells test");
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
options = mock(JSONObject.class);
|
||||
|
||||
ImportingManager.initialize(servlet);
|
||||
job = ImportingManager.createJob();
|
||||
importer = new SeparatorBasedImporter();
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
project = null;
|
||||
pm = null;
|
||||
options = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test to demonstrate the intended behaviour of the function, for issue #1268
|
||||
* https://github.com/OpenRefine/OpenRefine/issues/1268
|
||||
@ -117,13 +63,9 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
|
||||
@Test
|
||||
public void testSplitMultiValuedCellsTextSeparator() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n");
|
||||
|
||||
AbstractOperation op = new MultiValuedCellSplitOperation(
|
||||
"Value",
|
||||
@ -144,13 +86,9 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
|
||||
@Test
|
||||
public void testSplitMultiValuedCellsRegExSeparator() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n");
|
||||
|
||||
AbstractOperation op = new MultiValuedCellSplitOperation(
|
||||
"Value",
|
||||
@ -175,13 +113,10 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
|
||||
@Test
|
||||
public void testSplitMultiValuedCellsLengths() throws Exception {
|
||||
String csv = "Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n";
|
||||
prepareOptions(",", 10, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, pm);
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "Record_1,one:two;three four\n");
|
||||
|
||||
int[] lengths = {4,4,6,4};
|
||||
|
||||
AbstractOperation op = new MultiValuedCellSplitOperation(
|
||||
@ -204,20 +139,6 @@ public class SplitMultiValuedCellsTests extends RefineTest {
|
||||
Assert.assertEquals(project.rows.get(3).getCellValue(valueCol), "four");
|
||||
}
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -33,34 +33,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package com.google.refine.tests.operations.cell;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.ProjectMetadata;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.history.HistoryEntry;
|
||||
import com.google.refine.importers.SeparatorBasedImporter;
|
||||
import com.google.refine.importing.ImportingJob;
|
||||
import com.google.refine.importing.ImportingManager;
|
||||
import com.google.refine.model.AbstractOperation;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.operations.cell.KeyValueColumnizeOperation;
|
||||
import com.google.refine.process.Process;
|
||||
import com.google.refine.tests.ProjectManagerStub;
|
||||
import com.google.refine.tests.RefineServletStub;
|
||||
import com.google.refine.tests.RefineTest;
|
||||
|
||||
public class TransposeTests extends RefineTest {
|
||||
@ -71,67 +54,34 @@ public class TransposeTests extends RefineTest {
|
||||
logger = LoggerFactory.getLogger(this.getClass());
|
||||
}
|
||||
|
||||
// dependencies
|
||||
RefineServlet servlet;
|
||||
Project project;
|
||||
ProjectMetadata metadata;
|
||||
ImportingJob job;
|
||||
JSONObject options;
|
||||
SeparatorBasedImporter importer;
|
||||
|
||||
@BeforeMethod
|
||||
public void SetUp() {
|
||||
servlet = new RefineServletStub();
|
||||
ProjectManager.singleton = new ProjectManagerStub();
|
||||
ImportingManager.initialize(servlet);
|
||||
project = new Project();
|
||||
metadata = new ProjectMetadata();
|
||||
|
||||
job = ImportingManager.createJob();
|
||||
options = mock(JSONObject.class);
|
||||
importer = new SeparatorBasedImporter();
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void TearDown() {
|
||||
ImportingManager.disposeJob(job.id);
|
||||
ProjectManager.singleton.deleteProject(project.id);
|
||||
job = null;
|
||||
metadata = null;
|
||||
project = null;
|
||||
options = null;
|
||||
importer = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test in the case where an ID is available in the first column.
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test
|
||||
public void keyValueComumnize() throws Exception {
|
||||
String input = "ID;Cat;Val\n"
|
||||
+ "1;a;1\n"
|
||||
+ "1;b;3\n"
|
||||
+ "2;b;4\n"
|
||||
+ "2;c;5\n"
|
||||
+ "3;a;2\n"
|
||||
+ "3;b;5\n"
|
||||
+ "3;d;3\n";
|
||||
|
||||
prepareOptions(";", -1, 0, 0, 1, false, false);
|
||||
List<Exception> exceptions = new ArrayList<Exception>();
|
||||
importer.parseOneFile(project, metadata, job, "filesource", new StringReader(input), -1, options, exceptions);
|
||||
project.update();
|
||||
ProjectManager.singleton.registerProject(project, metadata);
|
||||
public void testKeyValueColumnizeWithID() throws Exception {
|
||||
Project project = createCSVProject(
|
||||
"ID,Cat,Val\n"
|
||||
+ "1,a,1\n"
|
||||
+ "1,b,3\n"
|
||||
+ "2,b,4\n"
|
||||
+ "2,c,5\n"
|
||||
+ "3,a,2\n"
|
||||
+ "3,b,5\n"
|
||||
+ "3,d,3\n");
|
||||
|
||||
AbstractOperation op = new KeyValueColumnizeOperation(
|
||||
"Cat", "Val", null);
|
||||
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
|
||||
HistoryEntry historyEntry = process.performImmediate();
|
||||
process.performImmediate();
|
||||
|
||||
// Expected output from the GUI.
|
||||
// ID;a;b;c;d
|
||||
// 1;1;3;;
|
||||
// 2;;4;5;
|
||||
// 3;2;5;;3
|
||||
// ID,a,b,c,d
|
||||
// 1,1,3,,
|
||||
// 2,,4,5,
|
||||
// 3,2,5,,3
|
||||
Assert.assertEquals(project.columnModel.columns.size(), 5);
|
||||
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "ID");
|
||||
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "a");
|
||||
@ -142,38 +92,59 @@ public class TransposeTests extends RefineTest {
|
||||
|
||||
// The actual row data structure has to leave the columns model untouched for redo/undo purpose.
|
||||
// So we have 2 empty columns(column 1,2) on the row level.
|
||||
// 1;1;3;;
|
||||
// 1,1,3,,
|
||||
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "1");
|
||||
Assert.assertEquals(project.rows.get(0).cells.get(3).value, "1");
|
||||
Assert.assertEquals(project.rows.get(0).cells.get(4).value, "3");
|
||||
|
||||
// 2;;4;5;
|
||||
// 2,,4,5,
|
||||
Assert.assertEquals(project.rows.get(1).cells.get(0).value, "2");
|
||||
Assert.assertEquals(project.rows.get(1).cells.get(4).value, "4");
|
||||
Assert.assertEquals(project.rows.get(1).cells.get(5).value, "5");
|
||||
|
||||
// 3;2;5;;3
|
||||
// 3,2,5,,3
|
||||
Assert.assertEquals(project.rows.get(2).cells.get(0).value, "3");
|
||||
Assert.assertEquals(project.rows.get(2).cells.get(3).value, "2");
|
||||
Assert.assertEquals(project.rows.get(2).cells.get(4).value, "5");
|
||||
Assert.assertEquals(project.rows.get(2).cells.get(6).value, "3");
|
||||
}
|
||||
|
||||
/**
|
||||
* Test to demonstrate the intended behaviour of the function when no id is available, for issue #1214
|
||||
* https://github.com/OpenRefine/OpenRefine/issues/1214
|
||||
*/
|
||||
@Test
|
||||
public void testKeyValueColumnizeWithoutID() throws Exception {
|
||||
Project project = createCSVProject(
|
||||
"Key,Value\n"
|
||||
+ "merchant,Katie\n"
|
||||
+ "fruit,apple\n"
|
||||
+ "price,1.2\n"
|
||||
+ "fruit,pear\n"
|
||||
+ "price,1.5\n"
|
||||
+ "merchant,John\n"
|
||||
+ "fruit,banana\n"
|
||||
+ "price,3.1\n");
|
||||
|
||||
AbstractOperation op = new KeyValueColumnizeOperation(
|
||||
"Key",
|
||||
"Value",
|
||||
null);
|
||||
Process process = op.createProcess(project, new Properties());
|
||||
process.performImmediate();
|
||||
|
||||
|
||||
|
||||
private void prepareOptions(
|
||||
String sep, int limit, int skip, int ignoreLines,
|
||||
int headerLines, boolean guessValueType, boolean ignoreQuotes) {
|
||||
|
||||
whenGetStringOption("separator", options, sep);
|
||||
whenGetIntegerOption("limit", options, limit);
|
||||
whenGetIntegerOption("skipDataLines", options, skip);
|
||||
whenGetIntegerOption("ignoreLines", options, ignoreLines);
|
||||
whenGetIntegerOption("headerLines", options, headerLines);
|
||||
whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
|
||||
whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
|
||||
whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
|
||||
}
|
||||
|
||||
int merchantCol = project.columnModel.getColumnByName("merchant").getCellIndex();
|
||||
int fruitCol = project.columnModel.getColumnByName("fruit").getCellIndex();
|
||||
int priceCol = project.columnModel.getColumnByName("price").getCellIndex();
|
||||
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(merchantCol), "Katie");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(merchantCol), null);
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(merchantCol), "John");
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(fruitCol), "apple");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(fruitCol), "pear");
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(fruitCol), "banana");
|
||||
Assert.assertEquals(project.rows.get(0).getCellValue(priceCol), "1.2");
|
||||
Assert.assertEquals(project.rows.get(1).getCellValue(priceCol), "1.5");
|
||||
Assert.assertEquals(project.rows.get(2).getCellValue(priceCol), "3.1");
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user