s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Removed unnecessary files following refactor of TsvExporter and TsvCsvImporter.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@859 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Iain Sproat 2010-05-25 16:50:44 +00:00
parent 2f3980f969
commit 76cb65bfb8
6 changed files with 1 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/com/metaweb/gridworks/commands/project/ExportRowsCommand.java
View File

@ -18,7 +18,6 @@ import com.metaweb.gridworks.exporters.Exporter;
import com.metaweb.gridworks.exporters.HtmlTableExporter;
import com.metaweb.gridworks.exporters.ProtographTransposeExporter.TripleLoaderExporter;
import com.metaweb.gridworks.exporters.ProtographTransposeExporter.MqlwriteLikeExporter;
import com.metaweb.gridworks.exporters.TsvExporter;
import com.metaweb.gridworks.exporters.XlsExporter;
import com.metaweb.gridworks.model.Project;

5
src/main/java/com/metaweb/gridworks/exporters/CsvExporter.java
View File

@ -50,11 +50,6 @@ public class CsvExporter implements Exporter{
boolean printColumnHeader = true;
boolean isFirstRow = true; //the first row should also add the column headers
public RowVisitor init(CSVWriter writer) {
this.csvWriter = writer;
return this;
}
public RowVisitor init(CSVWriter writer, boolean printColumnHeader){
this.csvWriter = writer;
this.printColumnHeader = printColumnHeader;

100
src/main/java/com/metaweb/gridworks/exporters/TsvExporter.java
View File

@ -1,100 +0,0 @@
package com.metaweb.gridworks.exporters;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.util.Properties;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
public class TsvExporter implements Exporter {
public String getContentType() {
return "application/x-unknown";
}
public boolean takeWriter() {
return true;
}
public void export(Project project, Properties options, Engine engine,
OutputStream outputStream) throws IOException {
throw new RuntimeException("Not implemented");
}
public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException {
boolean first = true;
for (Column column : project.columnModel.columns) {
if (first) {
first = false;
} else {
writer.write("\t");
}
writer.write(column.getName());
}
writer.write("\n");
{
RowVisitor visitor = new RowVisitor() {
Writer writer;
public RowVisitor init(Writer writer) {
this.writer = writer;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
boolean first = true;
try {
for (Column column : project.columnModel.columns) {
if (first) {
first = false;
} else {
writer.write("\t");
}
int cellIndex = column.getCellIndex();
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
if (cell != null && cell.value != null) {
Object v = cell.value;
String s = v instanceof String ? ((String) v) : v.toString();
s = s.replace("\\", "\\\\")
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
writer.write(s);
}
}
}
writer.write("\n");
} catch (IOException e) {
// ignore
}
return false;
}
}.init(writer);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, visitor);
}
}
}

96
src/main/java/com/metaweb/gridworks/importers/parsers/TsvCsvRowParser.java
View File

@ -1,96 +0,0 @@
package com.metaweb.gridworks.importers.parsers;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.importers.ImporterUtilities;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Row;
public class TsvCsvRowParser extends RowParser {
final protected char _sep;
public TsvCsvRowParser(char sep) {
_sep = sep;
}
public List<String> split(String line, LineNumberReader lineReader) {
List<String> results = new ArrayList<String>();
int start = 0;
while (start < line.length()) {
String text = null;
if (line.charAt(start) == '"') {
StringBuffer sb = new StringBuffer();
start++; // skip over "
while (start < line.length()) {
int quote = line.indexOf('"', start);
if (quote < 0) {
sb.append(line.substring(start));
sb.append('\n');
start = 0;
try {
line = lineReader.readLine();
} catch (IOException e) {
line = "";
break;
}
} else {
if (quote < line.length() - 1 && line.charAt(quote + 1) == '"') {
sb.append(line.substring(start, quote + 1)); // include " as well
start = quote + 2;
} else {
sb.append(line.substring(start, quote));
start = quote + 1;
if (start < line.length() && line.charAt(start) == _sep) {
start++; // skip separator
}
break;
}
}
}
text = sb.toString();
} else {
int next = line.indexOf(_sep, start);
if (next < 0) {
text = line.substring(start);
start = line.length();
} else {
text = line.substring(start, next);
start = next + 1;
}
}
results.add(text);
}
return results;
}
public boolean parseRow(Row row, String line, boolean guessValueType, LineNumberReader lineReader) {
boolean hasData = false;
List<String> strings = split(line, lineReader);
for (String s : strings) {
Serializable value = guessValueType ? ImporterUtilities.parseCellValue(s) : s;
if (ExpressionUtils.isNonBlankData(value)) {
row.cells.add(new Cell(value, null));
hasData = true;
} else {
row.cells.add(null);
}
}
return hasData;
}
}

3
tests/java/src/com/metaweb/gridworks/tests/exporters/TsvExporterTests.java
View File

@ -16,7 +16,6 @@ import org.testng.annotations.Test;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.exporters.CsvExporter;
import com.metaweb.gridworks.exporters.TsvExporter;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.ModelException;
@ -66,7 +65,7 @@ public class TsvExporterTests {
"row1cell0\trow1cell1\n");
}
@Test
public void exportSimpleTsvNoHeader(){
CreateGrid(2, 2);

108
tests/java/src/com/metaweb/gridworks/tests/importers/parsers/TsvCsvRowParserTests.java
View File

@ -1,108 +0,0 @@
package com.metaweb.gridworks.tests.importers.parsers;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.LineNumberReader;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.metaweb.gridworks.importers.parsers.TsvCsvRowParser;
public class TsvCsvRowParserTests {
// logging
final static protected Logger logger = LoggerFactory.getLogger("CSVRowParserTests");
//constants
String SAMPLE_ROW = "NDB_No,Shrt_Desc,Water";
String ROW_WITH_QUOTED_COMMA = "01001,\"BUTTER,WITH SALT\",15.87";
String UNCLOSED_QUOTED_ROW = "\"Open quoted value, with commas";
String LEADING_QUOTE_WITH_COMMA = "value1, \"\"\"Open quoted with commas\"\", and close quote but no comma\", value3";
String QUOTED = "value1, \"value2 with \"\"quote\"\" in middle\", value3";
String SAMPLE_CSV = SAMPLE_ROW + "\n" + ROW_WITH_QUOTED_COMMA; //Unix line endings?
//System Under Test
TsvCsvRowParser SUT = null;
//mocked dependencies
LineNumberReader lineReader = null;
@BeforeMethod
public void SetUp(){
lineReader = mock(LineNumberReader.class);
SUT = new TsvCsvRowParser(',');
}
@AfterMethod
public void TearDown(){
lineReader = null;
SUT = null;
}
//------------split tests-------------------------
@Test
public void split(){
List<String> splitLine = SUT.split(SAMPLE_ROW, lineReader);
Assert.assertEquals(3, splitLine.size());
Assert.assertEquals("NDB_No", splitLine.get(0));
Assert.assertEquals("Shrt_Desc", splitLine.get(1));
Assert.assertEquals("Water", splitLine.get(2));
}
@Test
public void splitWithQuotedComma(){
List<String> splitLine = SUT.split(ROW_WITH_QUOTED_COMMA, lineReader);
Assert.assertEquals(splitLine.size(), 3);
Assert.assertEquals(splitLine.get(0), "01001");
Assert.assertEquals(splitLine.get(1), "BUTTER,WITH SALT");
Assert.assertEquals(splitLine.get(2), "15.87");
}
@Test
public void splitWithUnclosedQuote(){
try {
when(lineReader.readLine()).thenReturn(" continuation of row above, with comma\",value2");
} catch (IOException e) {
Assert.fail();
}
List<String> splitLine = SUT.split(UNCLOSED_QUOTED_ROW, lineReader);
Assert.assertEquals(splitLine.size(), 2);
Assert.assertEquals(splitLine.get(0), "Open quoted value, with commas\n continuation of row above, with comma");
Assert.assertEquals(splitLine.get(1), "value2");
try {
verify(lineReader, times(1)).readLine();
} catch (IOException e) {
Assert.fail();
}
}
@Test(groups = { "broken" })
public void splitWithLeadingQuoteWithComma(){
List<String> splitLine = SUT.split(LEADING_QUOTE_WITH_COMMA, lineReader);
Assert.assertEquals(splitLine.size(), 3);
Assert.assertEquals(splitLine.get(0), "value1");
Assert.assertEquals(splitLine.get(1), "\"Open quoted with commas\", and close quote but no comma");
Assert.assertEquals(splitLine.get(2), "value3");
}
@Test(groups = { "broken" })
public void splitWithQuoteInsideValue(){
List<String> splitLine = SUT.split(QUOTED, lineReader);
Assert.assertEquals(splitLine.size(), 3);
Assert.assertEquals(splitLine.get(0), "value1");
Assert.assertEquals(splitLine.get(1), "value2 with \"quote\" in middle");
Assert.assertEquals(splitLine.get(2), "value3");
}
}