diff --git a/src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java b/src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java index 59d26b482..c23617a99 100644 --- a/src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java +++ b/src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java @@ -29,12 +29,14 @@ public class TsvCsvImporter implements Importer { int skip = ImporterUtilities.getIntegerOption("skip",options,0); boolean guessValueType = ImporterUtilities.getBooleanOption("guess-value-type", options, true); - - LineNumberReader lnReader = new LineNumberReader(reader); RowParser parser = (sep != null && sep.length() > 0 && splitIntoColumns) ? new SeparatorRowParser(sep) : null; - read(parser, lnReader, project, sep, limit, skip, ignoreLines, headerLines, guessValueType, splitIntoColumns); + + read(parser, lnReader, project, sep, + limit, skip, ignoreLines, headerLines, + guessValueType, splitIntoColumns + ); } public void read(RowParser parser, LineNumberReader lnReader, Project project, String sep, int limit, int skip, int ignoreLines, int headerLines, boolean guessValueType, boolean splitIntoColumns ) throws IOException{ diff --git a/src/main/java/com/metaweb/gridworks/importers/parsers/CSVRowParser.java b/src/main/java/com/metaweb/gridworks/importers/parsers/CSVRowParser.java index e1f5a332b..b74fe181c 100644 --- a/src/main/java/com/metaweb/gridworks/importers/parsers/CSVRowParser.java +++ b/src/main/java/com/metaweb/gridworks/importers/parsers/CSVRowParser.java @@ -27,6 +27,7 @@ public class CSVRowParser extends RowParser { int quote = line.indexOf('"', start); if (quote < 0) { sb.append(line.substring(start)); + sb.append('\n'); start = 0; try { diff --git a/tests/java/src/com/metaweb/gridworks/tests/importers/TsvCsvImporterTests.java b/tests/java/src/com/metaweb/gridworks/tests/importers/TsvCsvImporterTests.java index 5bfa867e8..6c481cea5 100644 --- a/tests/java/src/com/metaweb/gridworks/tests/importers/TsvCsvImporterTests.java +++ b/tests/java/src/com/metaweb/gridworks/tests/importers/TsvCsvImporterTests.java @@ -17,8 +17,6 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import au.com.bytecode.opencsv.CSVReader; - import com.metaweb.gridworks.importers.TsvCsvImporter; import com.metaweb.gridworks.importers.parsers.CSVRowParser; import com.metaweb.gridworks.importers.parsers.RowParser; @@ -96,16 +94,15 @@ public class TsvCsvImporterTests { Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3"); } - @Test(enabled=false) + // dfhuynh + @Test public void readSimpleData_TSV_1Header_1Row(){ String input = "col1\tcol2\tcol3\n" + "data1\tdata2\tdata3"; - //CSVReader reader = new CSVReader(new StringReader(input),'\t'); - RowParser parser = new CSVRowParser(); LineNumberReader lnReader = new LineNumberReader(new StringReader(input)); try { //SUT.read(reader, project, -1, 0, 0, 1, false); - SUT.read(parser, lnReader, project, "\t", -1, 0, 0, 1, false, true); + SUT.read(null, lnReader, project, "\t", -1, 0, 0, 1, false, true); } catch (IOException e) { Assert.fail(); } @@ -258,16 +255,14 @@ public class TsvCsvImporterTests { Assert.assertEquals(project.rows.get(0).cells.get(5).value, "data6"); } - @Test(enabled=false) + @Test public void readQuotedData(){ String input = "col1,col2,col3\n" + - "\"\"To Be\"\" is often followed by \"\"or not To Be\"\",data2"; - //CSVReader reader = new CSVReader(new StringReader(input)); - RowParser parser = new CSVRowParser(); + "\"\"\"To Be\"\" is often followed by \"\"or not To Be\"\"\",data2"; + LineNumberReader lnReader = new LineNumberReader(new StringReader(input)); try { - //SUT.read(reader, project, -1, 0, 0, 1, false); - SUT.read(parser, lnReader, project, ",", -1, 0, 0, 1, false, true); + SUT.read(null, lnReader, project, ",", -1, 0, 0, 1, false, true); } catch (IOException e) { Assert.fail(); } @@ -360,7 +355,7 @@ public class TsvCsvImporterTests { Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3"); } - @Test(enabled=false) + @Test public void readIgnore3_Header2_Skip2_limit2(){ String input = "ignore1\n" + "ignore2\n" + @@ -372,12 +367,10 @@ public class TsvCsvImporterTests { "data-row1-cell1,data-row1-cell2,data-row1-cell3\n" + "data-row2-cell1,data-row2-cell2,\n" + //missing last data point of this row on purpose "data-row3-cell1,data-row3-cell2,data-row1-cell3"; - //CSVReader reader = new CSVReader(new StringReader(input)); - RowParser parser = new CSVRowParser(); + LineNumberReader lnReader = new LineNumberReader(new StringReader(input)); try { - //SUT.read(reader, project, 2, 2, 3, 2, false); - SUT.read(parser, lnReader, project, ",", 2, 2, 3, 2, false, true); + SUT.read(null, lnReader, project, ",", 2, 2, 3, 2, false, true); } catch (IOException e) { Assert.fail(); } @@ -390,21 +383,19 @@ public class TsvCsvImporterTests { Assert.assertEquals(project.rows.get(0).cells.get(0).value, "data-row1-cell1"); Assert.assertEquals(project.rows.get(0).cells.get(1).value, "data-row1-cell2"); Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data-row1-cell3"); - Assert.assertEquals(project.rows.get(1).cells.size(), 3); + Assert.assertEquals(project.rows.get(1).cells.size(), 2); Assert.assertEquals(project.rows.get(1).cells.get(0).value, "data-row2-cell1"); Assert.assertEquals(project.rows.get(1).cells.get(1).value, "data-row2-cell2"); } - @Test(enabled=false) + @Test public void readWithMultiLinedQuotedData(){ String input = "col1,col2,col3\n" + - "\"\"\"To\n Be\"\" is often followed by \"\"or not To\n Be\"\"\",data2"; - //CSVReader reader = new CSVReader(new StringReader(input)); - RowParser parser = new CSVRowParser(); + "\"\"\"To\n Be\"\" is often followed by \"\"or not To\n Be\"\"\",data2"; + LineNumberReader lnReader = new LineNumberReader(new StringReader(input)); try { - //SUT.read(reader, project, -1, 0, 0, 1, false); - SUT.read(parser, lnReader, project, ",", -1, 0, 0, 1, false, true); + SUT.read(null, lnReader, project, ",", -1, 0, 0, 1, false, true); } catch (IOException e) { Assert.fail(); }