s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Made csv/tsv importer not trim whitespace even if "guess cells' types" is checked (for cells that are strings).

Browse Source 
Updated csv tests to expect un-trimmed cells.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1557 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-10-15 05:30:15 +00:00
parent 35def004ac
commit 73042712ed
3 changed files with 27 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
main/src/com/google/refine/importers/ImporterUtilities.java
View File

@ -18,19 +18,21 @@ public class ImporterUtilities {
return text.substring(1, text.length() - 1);
}
try {
return Long.parseLong(text);
} catch (NumberFormatException e) {
}
try {
double d = Double.parseDouble(text);
if (!Double.isInfinite(d) && !Double.isNaN(d)) {
return d;
String text2 = text.trim();
if (text2.length() > 0) {
try {
return Long.parseLong(text2);
} catch (NumberFormatException e) {
}
try {
double d = Double.parseDouble(text2);
if (!Double.isInfinite(d) && !Double.isNaN(d)) {
return d;
}
} catch (NumberFormatException e) {
}
} catch (NumberFormatException e) {
}
text = text.trim();
}
return text;
}

2
main/src/com/google/refine/importers/TsvCsvImporter.java
View File

@ -136,7 +136,7 @@ public class TsvCsvImporter implements ReaderImporter,StreamImporter {
//add parsed data to row
for(String s : cells){
if (ExpressionUtils.isNonBlankData(s)) {
Serializable value = guessValueType ? ImporterUtilities.parseCellValue(s.trim()) : s;
Serializable value = guessValueType ? ImporterUtilities.parseCellValue(s) : s;
row.cells.add(new Cell(value, null));
}else{
row.cells.add(null);

26
main/tests/server/src/com/google/refine/tests/importers/TsvCsvImporterTests.java
View File

@ -173,8 +173,8 @@ public class TsvCsvImporterTests extends RefineTest {
}
@Test(groups = { }, dataProvider = "CSV-or-null")
public void readDoesTrimsLeadingTrailingWhitespace(String sep){
String input = " data1 , data2 , data3 ";
public void readDoesNotTrimLeadingTrailingWhitespace(String sep){
String input = " data1 , 3.4 , data3 ";
LineNumberReader lnReader = new LineNumberReader(new StringReader(input));
try {
SUT.read(lnReader, project, sep, -1, 0, 0, 0, false, true, false);
@ -184,14 +184,14 @@ public class TsvCsvImporterTests extends RefineTest {
Assert.assertEquals(project.columnModel.columns.size(), 3);
Assert.assertEquals(project.rows.size(), 1);
Assert.assertEquals(project.rows.get(0).cells.size(), 3);
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "data1");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "data2");
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3");
Assert.assertEquals(project.rows.get(0).cells.get(0).value, " data1 ");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, " 3.4 ");
Assert.assertEquals(project.rows.get(0).cells.get(2).value, " data3 ");
}
@Test(dataProvider = "CSV-or-null")
public void readTrimsLeadingTrailingWhitespace(String sep){
String input = " data1, data2, data3";
public void readDoesNotTrimLeadingWhitespace(String sep){
String input = " data1, 12, data3";
LineNumberReader lnReader = new LineNumberReader(new StringReader(input));
try {
SUT.read(lnReader, project, sep, -1, 0, 0, 0, true, true, false);
@ -201,14 +201,14 @@ public class TsvCsvImporterTests extends RefineTest {
Assert.assertEquals(project.columnModel.columns.size(), 3);
Assert.assertEquals(project.rows.size(), 1);
Assert.assertEquals(project.rows.get(0).cells.size(), 3);
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "data1");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "data2");
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3");
Assert.assertEquals(project.rows.get(0).cells.get(0).value, " data1");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, 12L);
Assert.assertEquals(project.rows.get(0).cells.get(2).value, " data3");
}
@Test(dataProvider = "CSV-or-null")
public void readCanAddNull(String sep){
String input = " data1, , data3";
String input = " data1,, data3";
LineNumberReader lnReader = new LineNumberReader(new StringReader(input));
try {
SUT.read(lnReader, project, sep, -1, 0, 0, 0, true, true, false);
@ -218,9 +218,9 @@ public class TsvCsvImporterTests extends RefineTest {
Assert.assertEquals(project.columnModel.columns.size(), 3);
Assert.assertEquals(project.rows.size(), 1);
Assert.assertEquals(project.rows.get(0).cells.size(), 3);
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "data1");
Assert.assertEquals(project.rows.get(0).cells.get(0).value, " data1");
Assert.assertNull(project.rows.get(0).cells.get(1));
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "data3");
Assert.assertEquals(project.rows.get(0).cells.get(2).value, " data3");
}
@Test(dataProvider = "CSV-or-null")