s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Got disabled tests in Issue 41 to pass.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@788 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-05-17 05:26:29 +00:00
parent 22fb3a1585
commit 68fbd3320b
3 changed files with 21 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java
View File

@ -29,12 +29,14 @@ public class TsvCsvImporter implements Importer {
int skip = ImporterUtilities.getIntegerOption("skip",options,0); int skip = ImporterUtilities.getIntegerOption("skip",options,0);
boolean guessValueType = ImporterUtilities.getBooleanOption("guess-value-type", options, true); boolean guessValueType = ImporterUtilities.getBooleanOption("guess-value-type", options, true);
LineNumberReader lnReader = new LineNumberReader(reader); LineNumberReader lnReader = new LineNumberReader(reader);
RowParser parser = (sep != null && sep.length() > 0 && splitIntoColumns) ? RowParser parser = (sep != null && sep.length() > 0 && splitIntoColumns) ?
new SeparatorRowParser(sep) : null; new SeparatorRowParser(sep) : null;
read(parser, lnReader, project, sep, limit, skip, ignoreLines, headerLines, guessValueType, splitIntoColumns);
read(parser, lnReader, project, sep,
limit, skip, ignoreLines, headerLines,
guessValueType, splitIntoColumns
);
} }
public void read(RowParser parser, LineNumberReader lnReader, Project project, String sep, int limit, int skip, int ignoreLines, int headerLines, boolean guessValueType, boolean splitIntoColumns ) throws IOException{ public void read(RowParser parser, LineNumberReader lnReader, Project project, String sep, int limit, int skip, int ignoreLines, int headerLines, boolean guessValueType, boolean splitIntoColumns ) throws IOException{

1
src/main/java/com/metaweb/gridworks/importers/parsers/CSVRowParser.java
View File

@ -27,6 +27,7 @@ public class CSVRowParser extends RowParser {
int quote = line.indexOf('"', start); int quote = line.indexOf('"', start);
if (quote < 0) { if (quote < 0) {
sb.append(line.substring(start)); sb.append(line.substring(start));
sb.append('\n');
start = 0; start = 0;
try { try {

39
tests/java/src/com/metaweb/gridworks/tests/importers/TsvCsvImporterTests.java
View File

@ -17,8 +17,6 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import au.com.bytecode.opencsv.CSVReader;
import com.metaweb.gridworks.importers.TsvCsvImporter; import com.metaweb.gridworks.importers.TsvCsvImporter;
import com.metaweb.gridworks.importers.parsers.CSVRowParser; import com.metaweb.gridworks.importers.parsers.CSVRowParser;
import com.metaweb.gridworks.importers.parsers.RowParser; import com.metaweb.gridworks.importers.parsers.RowParser;
@ -96,16 +94,15 @@ public class TsvCsvImporterTests {
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3"); Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3");
} }
@Test(enabled=false) // dfhuynh
@Test
public void readSimpleData_TSV_1Header_1Row(){ public void readSimpleData_TSV_1Header_1Row(){
String input = "col1\tcol2\tcol3\n" + String input = "col1\tcol2\tcol3\n" +
"data1\tdata2\tdata3"; "data1\tdata2\tdata3";
//CSVReader reader = new CSVReader(new StringReader(input),'\t');
RowParser parser = new CSVRowParser();
LineNumberReader lnReader = new LineNumberReader(new StringReader(input)); LineNumberReader lnReader = new LineNumberReader(new StringReader(input));
try { try {
//SUT.read(reader, project, -1, 0, 0, 1, false); //SUT.read(reader, project, -1, 0, 0, 1, false);
SUT.read(parser, lnReader, project, "\t", -1, 0, 0, 1, false, true); SUT.read(null, lnReader, project, "\t", -1, 0, 0, 1, false, true);
} catch (IOException e) { } catch (IOException e) {
Assert.fail(); Assert.fail();
} }
@ -258,16 +255,14 @@ public class TsvCsvImporterTests {
Assert.assertEquals(project.rows.get(0).cells.get(5).value, "data6"); Assert.assertEquals(project.rows.get(0).cells.get(5).value, "data6");
} }
@Test(enabled=false) @Test
public void readQuotedData(){ public void readQuotedData(){
String input = "col1,col2,col3\n" + String input = "col1,col2,col3\n" +
"\"\"To Be\"\" is often followed by \"\"or not To Be\"\",data2"; "\"\"\"To Be\"\" is often followed by \"\"or not To Be\"\"\",data2";
//CSVReader reader = new CSVReader(new StringReader(input));
RowParser parser = new CSVRowParser();
LineNumberReader lnReader = new LineNumberReader(new StringReader(input)); LineNumberReader lnReader = new LineNumberReader(new StringReader(input));
try { try {
//SUT.read(reader, project, -1, 0, 0, 1, false); SUT.read(null, lnReader, project, ",", -1, 0, 0, 1, false, true);
SUT.read(parser, lnReader, project, ",", -1, 0, 0, 1, false, true);
} catch (IOException e) { } catch (IOException e) {
Assert.fail(); Assert.fail();
} }
@ -360,7 +355,7 @@ public class TsvCsvImporterTests {
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3"); Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3");
} }
@Test(enabled=false) @Test
public void readIgnore3_Header2_Skip2_limit2(){ public void readIgnore3_Header2_Skip2_limit2(){
String input = "ignore1\n" + String input = "ignore1\n" +
"ignore2\n" + "ignore2\n" +
@ -372,12 +367,10 @@ public class TsvCsvImporterTests {
"data-row1-cell1,data-row1-cell2,data-row1-cell3\n" + "data-row1-cell1,data-row1-cell2,data-row1-cell3\n" +
"data-row2-cell1,data-row2-cell2,\n" + //missing last data point of this row on purpose "data-row2-cell1,data-row2-cell2,\n" + //missing last data point of this row on purpose
"data-row3-cell1,data-row3-cell2,data-row1-cell3"; "data-row3-cell1,data-row3-cell2,data-row1-cell3";
//CSVReader reader = new CSVReader(new StringReader(input));
RowParser parser = new CSVRowParser();
LineNumberReader lnReader = new LineNumberReader(new StringReader(input)); LineNumberReader lnReader = new LineNumberReader(new StringReader(input));
try { try {
//SUT.read(reader, project, 2, 2, 3, 2, false); SUT.read(null, lnReader, project, ",", 2, 2, 3, 2, false, true);
SUT.read(parser, lnReader, project, ",", 2, 2, 3, 2, false, true);
} catch (IOException e) { } catch (IOException e) {
Assert.fail(); Assert.fail();
} }
@ -390,21 +383,19 @@ public class TsvCsvImporterTests {
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "data-row1-cell1"); Assert.assertEquals(project.rows.get(0).cells.get(0).value, "data-row1-cell1");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "data-row1-cell2"); Assert.assertEquals(project.rows.get(0).cells.get(1).value, "data-row1-cell2");
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data-row1-cell3"); Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data-row1-cell3");
Assert.assertEquals(project.rows.get(1).cells.size(), 3); Assert.assertEquals(project.rows.get(1).cells.size(), 2);
Assert.assertEquals(project.rows.get(1).cells.get(0).value, "data-row2-cell1"); Assert.assertEquals(project.rows.get(1).cells.get(0).value, "data-row2-cell1");
Assert.assertEquals(project.rows.get(1).cells.get(1).value, "data-row2-cell2"); Assert.assertEquals(project.rows.get(1).cells.get(1).value, "data-row2-cell2");
} }
@Test(enabled=false) @Test
public void readWithMultiLinedQuotedData(){ public void readWithMultiLinedQuotedData(){
String input = "col1,col2,col3\n" + String input = "col1,col2,col3\n" +
"\"\"\"To\n Be\"\" is often followed by \"\"or not To\n Be\"\"\",data2"; "\"\"\"To\n Be\"\" is often followed by \"\"or not To\n Be\"\"\",data2";
//CSVReader reader = new CSVReader(new StringReader(input));
RowParser parser = new CSVRowParser();
LineNumberReader lnReader = new LineNumberReader(new StringReader(input)); LineNumberReader lnReader = new LineNumberReader(new StringReader(input));
try { try {
//SUT.read(reader, project, -1, 0, 0, 1, false); SUT.read(null, lnReader, project, ",", -1, 0, 0, 1, false, true);
SUT.read(parser, lnReader, project, ",", -1, 0, 0, 1, false, true);
} catch (IOException e) { } catch (IOException e) {
Assert.fail(); Assert.fail();
} }