s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed tsv/csv tests.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2276 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2011-10-06 06:22:30 +00:00
parent 3c6358d518
commit 1c5dc32b88
4 changed files with 23 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
main/src/com/google/refine/importers/SeparatorBasedImporter.java
View File

@ -92,6 +92,9 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
) { ) {
// String lineSeparator = JSONUtilities.getString(options, "lineSeparator", "\n"); // String lineSeparator = JSONUtilities.getString(options, "lineSeparator", "\n");
String sep = JSONUtilities.getString(options, "separator", "\t"); String sep = JSONUtilities.getString(options, "separator", "\t");
if (sep == null) {
sep = "\t";
}
boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true); boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true);
final CSVParser parser = new CSVParser( final CSVParser parser = new CSVParser(

2
main/src/com/google/refine/model/ColumnModel.java
View File

@ -75,7 +75,7 @@ public class ColumnModel implements Jsonizable {
} }
synchronized public int allocateNewCellIndex() { synchronized public int allocateNewCellIndex() {
return ++_maxCellIndex; return _maxCellIndex++;
} }
public void setKeyColumnIndex(int keyColumnIndex) { public void setKeyColumnIndex(int keyColumnIndex) {

8
main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java
View File

@ -1,16 +1,11 @@
package com.google.refine.tests.importers; package com.google.refine.tests.importers;
import static org.mockito.Mockito.mock;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.Reader; import java.io.Reader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import com.google.refine.ProjectMetadata; import com.google.refine.ProjectMetadata;
@ -24,7 +19,6 @@ import com.google.refine.importing.ImportingManager;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineServletStub;
import com.google.refine.tests.RefineTest; import com.google.refine.tests.RefineTest;
import com.google.refine.util.JSONUtilities;
abstract class ImporterTest extends RefineTest { abstract class ImporterTest extends RefineTest {
//mock dependencies //mock dependencies
@ -44,7 +38,7 @@ abstract class ImporterTest extends RefineTest {
metadata = new ProjectMetadata(); metadata = new ProjectMetadata();
job = ImportingManager.createJob(); job = ImportingManager.createJob();
options = mock(JSONObject.class); options = new JSONObject();
} }
public void TearDown(){ public void TearDown(){

37
main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java
View File

@ -81,7 +81,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readJustColumns(String sep){ public void readJustColumns(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3"; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3";
try { try {
@ -99,7 +99,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readUnseperatedData(String sep){ public void readUnseperatedData(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "value1" + inputSeparator + "value2" + inputSeparator + "value3"; String input = "value1" + inputSeparator + "value2" + inputSeparator + "value3";
try { try {
@ -118,7 +118,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readSimpleData_CSV_1Header_1Row(String sep){ public void readSimpleData_CSV_1Header_1Row(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"data1" + inputSeparator + "data2" + inputSeparator + "data3"; "data1" + inputSeparator + "data2" + inputSeparator + "data3";
@ -144,7 +144,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readSimpleData_CSV_1Header_1Row_GuessValues(String sep){ public void readSimpleData_CSV_1Header_1Row_GuessValues(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"data1" + inputSeparator + "234" + inputSeparator + "data3"; "data1" + inputSeparator + "234" + inputSeparator + "data3";
@ -169,7 +169,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readSimpleData_0Header_1Row(String sep){ public void readSimpleData_0Header_1Row(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "data1" + inputSeparator + "data2" + inputSeparator + "data3"; String input = "data1" + inputSeparator + "data2" + inputSeparator + "data3";
try { try {
@ -192,7 +192,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine")
public void readDoesNotTrimLeadingTrailingWhitespace(String sep){ public void readDoesNotTrimLeadingTrailingWhitespace(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = " data1 " + inputSeparator + " 3.4 " + inputSeparator + " data3 "; String input = " data1 " + inputSeparator + " 3.4 " + inputSeparator + " data3 ";
try { try {
@ -212,7 +212,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readDoesNotTrimLeadingWhitespace(String sep){ public void readDoesNotTrimLeadingWhitespace(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = " data1" + inputSeparator + " 12" + inputSeparator + " data3"; String input = " data1" + inputSeparator + " 12" + inputSeparator + " data3";
try { try {
@ -232,7 +232,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readCanAddNull(String sep){ public void readCanAddNull(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = " data1" + inputSeparator + inputSeparator + " data3"; String input = " data1" + inputSeparator + inputSeparator + " data3";
try { try {
@ -252,7 +252,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readSimpleData_2Header_1Row(String sep){ public void readSimpleData_2Header_1Row(String sep){
//create input to test with //create input to test with
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"sub1" + inputSeparator + "sub2" + inputSeparator + "sub3\n" + "sub1" + inputSeparator + "sub2" + inputSeparator + "sub3\n" +
"data1" + inputSeparator + "data2" + inputSeparator + "data3"; "data1" + inputSeparator + "data2" + inputSeparator + "data3";
@ -277,7 +277,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readSimpleData_RowLongerThanHeader(String sep){ public void readSimpleData_RowLongerThanHeader(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"data1" + inputSeparator + "data2" + inputSeparator + "data3" + inputSeparator + "data4" + inputSeparator + "data5" + inputSeparator + "data6"; "data1" + inputSeparator + "data2" + inputSeparator + "data3" + inputSeparator + "data4" + inputSeparator + "data5" + inputSeparator + "data6";
@ -307,7 +307,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine")
public void readQuotedData(String sep){ public void readQuotedData(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"\"\"\"To Be\"\" is often followed by \"\"or not To Be\"\"\"" + inputSeparator + "data2"; "\"\"\"To Be\"\" is often followed by \"\"or not To Be\"\"\"" + inputSeparator + "data2";
@ -330,7 +330,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readIgnoreFirstLine(String sep){ public void readIgnoreFirstLine(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "ignore1\n" + String input = "ignore1\n" +
"col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"data1" + inputSeparator + "data2" + inputSeparator + "data3"; "data1" + inputSeparator + "data2" + inputSeparator + "data3";
@ -355,7 +355,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readSkipFirstDataLine(String sep){ public void readSkipFirstDataLine(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"skip1\n" + "skip1\n" +
"data1" + inputSeparator + "data2" + inputSeparator + "data3"; "data1" + inputSeparator + "data2" + inputSeparator + "data3";
@ -380,7 +380,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void readIgnore3_Header2_Skip1(String sep){ public void readIgnore3_Header2_Skip1(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "ignore1\n" + String input = "ignore1\n" +
"ignore2\n" + "ignore2\n" +
"ignore3\n" + "ignore3\n" +
@ -409,7 +409,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine")
public void readIgnore3_Header2_Skip2_limit2(String sep){ public void readIgnore3_Header2_Skip2_limit2(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "ignore1\n" + String input = "ignore1\n" +
"ignore2\n" + "ignore2\n" +
"ignore3\n" + "ignore3\n" +
@ -445,9 +445,8 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(dataProvider = "CSV-TSV-AutoDetermine") @Test(dataProvider = "CSV-TSV-AutoDetermine")
public void ignoreQuotes(String sep){ public void ignoreQuotes(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "data1" + inputSeparator + "data2\"" + inputSeparator + "data3" + inputSeparator + "data4"; String input = "data1" + inputSeparator + "data2\"" + inputSeparator + "data3" + inputSeparator + "data4";
try { try {
prepareOptions(sep, -1, 0, 0, 0, false, true, true); prepareOptions(sep, -1, 0, 0, 0, false, true, true);
parseOneFile(SUT, new StringReader(input)); parseOneFile(SUT, new StringReader(input));
@ -465,7 +464,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine")
public void readWithMultiLinedQuotedData(String sep){ public void readWithMultiLinedQuotedData(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"\"\"\"To\n Be\"\" is often followed by \"\"or not To\n Be\"\"\"" + inputSeparator + "data2"; "\"\"\"To\n Be\"\" is often followed by \"\"or not To\n Be\"\"\"" + inputSeparator + "data2";
@ -488,7 +487,7 @@ public class TsvCsvImporterTests extends ImporterTest {
@Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine")
public void readWithMultiLinedQuotedDataAndBlankLines(String sep){ public void readWithMultiLinedQuotedDataAndBlankLines(String sep){
//create input //create input
String inputSeparator = sep == "\t" ? "\t" : ","; String inputSeparator = sep == null ? "\t" : sep;
String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" +
"\"A line with many \n\n\n\n\n empty lines\"" + inputSeparator + "data2"; "\"A line with many \n\n\n\n\n empty lines\"" + inputSeparator + "data2";