From 1c5dc32b88add2f4650ebcc07ed3951b30c83223 Mon Sep 17 00:00:00 2001 From: David Huynh Date: Thu, 6 Oct 2011 06:22:30 +0000 Subject: [PATCH] Fixed tsv/csv tests. git-svn-id: http://google-refine.googlecode.com/svn/trunk@2276 7d457c2a-affb-35e4-300a-418c747d4874 --- .../importers/SeparatorBasedImporter.java | 3 ++ .../com/google/refine/model/ColumnModel.java | 2 +- .../refine/tests/importers/ImporterTest.java | 8 +--- .../tests/importers/TsvCsvImporterTests.java | 37 +++++++++---------- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/main/src/com/google/refine/importers/SeparatorBasedImporter.java b/main/src/com/google/refine/importers/SeparatorBasedImporter.java index 4cdec4928..6576149b3 100644 --- a/main/src/com/google/refine/importers/SeparatorBasedImporter.java +++ b/main/src/com/google/refine/importers/SeparatorBasedImporter.java @@ -92,6 +92,9 @@ public class SeparatorBasedImporter extends TabularImportingParserBase { ) { // String lineSeparator = JSONUtilities.getString(options, "lineSeparator", "\n"); String sep = JSONUtilities.getString(options, "separator", "\t"); + if (sep == null) { + sep = "\t"; + } boolean processQuotes = JSONUtilities.getBoolean(options, "processQuotes", true); final CSVParser parser = new CSVParser( diff --git a/main/src/com/google/refine/model/ColumnModel.java b/main/src/com/google/refine/model/ColumnModel.java index 11e2de7b1..fb8e78c5a 100644 --- a/main/src/com/google/refine/model/ColumnModel.java +++ b/main/src/com/google/refine/model/ColumnModel.java @@ -75,7 +75,7 @@ public class ColumnModel implements Jsonizable { } synchronized public int allocateNewCellIndex() { - return ++_maxCellIndex; + return _maxCellIndex++; } public void setKeyColumnIndex(int keyColumnIndex) { diff --git a/main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java b/main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java index d7f1a1326..61591647e 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java +++ b/main/tests/server/src/com/google/refine/tests/importers/ImporterTest.java @@ -1,16 +1,11 @@ package com.google.refine.tests.importers; -import static org.mockito.Mockito.mock; - import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import org.json.JSONArray; import org.json.JSONObject; import com.google.refine.ProjectMetadata; @@ -24,7 +19,6 @@ import com.google.refine.importing.ImportingManager; import com.google.refine.model.Project; import com.google.refine.tests.RefineServletStub; import com.google.refine.tests.RefineTest; -import com.google.refine.util.JSONUtilities; abstract class ImporterTest extends RefineTest { //mock dependencies @@ -44,7 +38,7 @@ abstract class ImporterTest extends RefineTest { metadata = new ProjectMetadata(); job = ImportingManager.createJob(); - options = mock(JSONObject.class); + options = new JSONObject(); } public void TearDown(){ diff --git a/main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java b/main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java index 4e4c97151..8a655db82 100644 --- a/main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java +++ b/main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java @@ -81,7 +81,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readJustColumns(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3"; try { @@ -99,7 +99,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readUnseperatedData(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "value1" + inputSeparator + "value2" + inputSeparator + "value3"; try { @@ -118,7 +118,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readSimpleData_CSV_1Header_1Row(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "data1" + inputSeparator + "data2" + inputSeparator + "data3"; @@ -144,7 +144,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readSimpleData_CSV_1Header_1Row_GuessValues(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "data1" + inputSeparator + "234" + inputSeparator + "data3"; @@ -169,7 +169,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readSimpleData_0Header_1Row(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "data1" + inputSeparator + "data2" + inputSeparator + "data3"; try { @@ -192,7 +192,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") public void readDoesNotTrimLeadingTrailingWhitespace(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = " data1 " + inputSeparator + " 3.4 " + inputSeparator + " data3 "; try { @@ -212,7 +212,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readDoesNotTrimLeadingWhitespace(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = " data1" + inputSeparator + " 12" + inputSeparator + " data3"; try { @@ -232,7 +232,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readCanAddNull(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = " data1" + inputSeparator + inputSeparator + " data3"; try { @@ -252,7 +252,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readSimpleData_2Header_1Row(String sep){ //create input to test with - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "sub1" + inputSeparator + "sub2" + inputSeparator + "sub3\n" + "data1" + inputSeparator + "data2" + inputSeparator + "data3"; @@ -277,7 +277,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readSimpleData_RowLongerThanHeader(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "data1" + inputSeparator + "data2" + inputSeparator + "data3" + inputSeparator + "data4" + inputSeparator + "data5" + inputSeparator + "data6"; @@ -307,7 +307,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") public void readQuotedData(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "\"\"\"To Be\"\" is often followed by \"\"or not To Be\"\"\"" + inputSeparator + "data2"; @@ -330,7 +330,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readIgnoreFirstLine(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "ignore1\n" + "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "data1" + inputSeparator + "data2" + inputSeparator + "data3"; @@ -355,7 +355,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readSkipFirstDataLine(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "skip1\n" + "data1" + inputSeparator + "data2" + inputSeparator + "data3"; @@ -380,7 +380,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void readIgnore3_Header2_Skip1(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "ignore1\n" + "ignore2\n" + "ignore3\n" + @@ -409,7 +409,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") public void readIgnore3_Header2_Skip2_limit2(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "ignore1\n" + "ignore2\n" + "ignore3\n" + @@ -445,9 +445,8 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(dataProvider = "CSV-TSV-AutoDetermine") public void ignoreQuotes(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "data1" + inputSeparator + "data2\"" + inputSeparator + "data3" + inputSeparator + "data4"; - try { prepareOptions(sep, -1, 0, 0, 0, false, true, true); parseOneFile(SUT, new StringReader(input)); @@ -465,7 +464,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") public void readWithMultiLinedQuotedData(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "\"\"\"To\n Be\"\" is often followed by \"\"or not To\n Be\"\"\"" + inputSeparator + "data2"; @@ -488,7 +487,7 @@ public class TsvCsvImporterTests extends ImporterTest { @Test(groups = { }, dataProvider = "CSV-TSV-AutoDetermine") public void readWithMultiLinedQuotedDataAndBlankLines(String sep){ //create input - String inputSeparator = sep == "\t" ? "\t" : ","; + String inputSeparator = sep == null ? "\t" : sep; String input = "col1" + inputSeparator + "col2" + inputSeparator + "col3\n" + "\"A line with many \n\n\n\n\n empty lines\"" + inputSeparator + "data2";