s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Excel importer now supports "header lines" parameter.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1125 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-08-01 04:22:45 +00:00
parent 7bb6674e5b
commit 4ad31ffcde
1 changed files with 181 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
main/src/com/metaweb/gridworks/importers/ExcelImporter.java
View File

@ -6,9 +6,11 @@ import java.io.Reader;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.poi.common.usermodel.Hyperlink;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
@ -39,8 +41,9 @@ public class ExcelImporter implements Importer {
public void read(InputStream inputStream, Project project, Properties options) throws Exception {
int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1);
int limit = ImporterUtilities.getIntegerOption("limit",options,-1);
int skip = ImporterUtilities.getIntegerOption("skip",options,0);
int headerLines = ImporterUtilities.getIntegerOption("header-lines", options, 1);
int limit = ImporterUtilities.getIntegerOption("limit", options, -1);
int skip = ImporterUtilities.getIntegerOption("skip", options, 0);
Workbook wb = null;
try {
@ -59,15 +62,15 @@ public class ExcelImporter implements Importer {
int firstRow = sheet.getFirstRowNum();
int lastRow = sheet.getLastRowNum();
int r = firstRow;
List<Integer> nonBlankIndices = null;
List<String> nonBlankHeaderStrings = null;
List<String> columnNames = new ArrayList<String>();
Set<String> columnNameSet = new HashSet<String>();
Map<String, Integer> columnRootNameToIndex = new HashMap<String, Integer>();
/*
* Find the header row
*/
for (; r <= lastRow; r++) {
int rowsWithData = 0;
Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (int r = firstRow; r <= lastRow; r++) {
org.apache.poi.ss.usermodel.Row row = sheet.getRow(r);
if (row == null) {
continue;
@ -78,83 +81,124 @@ public class ExcelImporter implements Importer {
short firstCell = row.getFirstCellNum();
short lastCell = row.getLastCellNum();
if (firstCell >= 0 && firstCell <= lastCell) {
nonBlankIndices = new ArrayList<Integer>(lastCell - firstCell + 1);
nonBlankHeaderStrings = new ArrayList<String>(lastCell - firstCell + 1);
if (firstCell < 0 || firstCell > lastCell) {
continue;
}
/*
* Still processing header lines
*/
if (headerLines > 0) {
headerLines--;
for (int c = firstCell; c <= lastCell; c++) {
org.apache.poi.ss.usermodel.Cell cell = row.getCell(c);
if (cell != null) {
String text = cell.getStringCellValue().trim();
if (text.length() > 0) {
nonBlankIndices.add((int) c);
nonBlankHeaderStrings.add(text);
while (columnNames.size() < c + 1) {
columnNames.add(null);
}
String existingName = columnNames.get(c);
String name = (existingName == null) ? text : (existingName + " " + text);
columnNames.set(c, name);
}
}
}
if (nonBlankIndices.size() > 0) {
r++;
break;
}
}
}
if (nonBlankIndices == null || nonBlankIndices.size() == 0) {
return;
}
/*
* Create columns
*/
Map<String, Integer> nameToIndex = new HashMap<String, Integer>();
for (int c = 0; c < nonBlankIndices.size(); c++) {
String cell = nonBlankHeaderStrings.get(c);
if (nameToIndex.containsKey(cell)) {
int index = nameToIndex.get(cell);
nameToIndex.put(cell, index + 1);
cell = cell.contains(" ") ? (cell + " " + index) : (cell + index);
} else {
nameToIndex.put(cell, 2);
}
Column column = new Column(c, cell);
project.columnModel.columns.add(column);
}
/*
* Now process the data rows
*/
int rowsWithData = 0;
Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (; r <= lastRow; r++) {
org.apache.poi.ss.usermodel.Row row = sheet.getRow(r);
if (row == null) {
if (headerLines == 0) {
for (int i = 0; i < columnNames.size(); i++) {
String rootName = columnNames.get(i);
if (rootName == null) {
continue;
}
setUnduplicatedColumnName(rootName, columnNames, i, columnNameSet, columnRootNameToIndex);
}
}
short firstCell = row.getFirstCellNum();
short lastCell = row.getLastCellNum();
if (firstCell >= 0 && firstCell <= lastCell) {
Row newRow = new Row(nonBlankIndices.size());
/*
* Processing data rows
*/
} else {
Row newRow = new Row(columnNames.size());
boolean hasData = false;
for (int c = 0; c < nonBlankIndices.size(); c++) {
if (c < firstCell || c > lastCell) {
continue;
}
for (int c = firstCell; c <= lastCell; c++) {
org.apache.poi.ss.usermodel.Cell cell = row.getCell(c);
if (cell == null) {
continue;
}
Cell ourCell = extractCell(cell, reconMap);
if (ourCell != null) {
while (columnNames.size() < c + 1) {
columnNames.add(null);
}
if (columnNames.get(c) == null) {
setUnduplicatedColumnName("Column", columnNames, c, columnNameSet, columnRootNameToIndex);
}
newRow.setCell(c, ourCell);
hasData = true;
}
}
if (hasData) {
rowsWithData++;
if (skip <= 0 || rowsWithData > skip) {
project.rows.add(newRow);
project.columnModel.setMaxCellIndex(newRow.cells.size());
if (limit > 0 && project.rows.size() >= limit) {
break;
}
}
}
}
}
/*
* Create columns
*/
for (int c = 0; c < columnNames.size(); c++) {
String name = columnNames.get(c);
if (name != null) {
Column column = new Column(c, name);
project.columnModel.columns.add(column);
}
}
}
protected void setUnduplicatedColumnName(
String rootName, List<String> columnNames, int index, Set<String> columnNameSet, Map<String, Integer> columnRootNameToIndex) {
if (columnNameSet.contains(rootName)) {
int startIndex = columnRootNameToIndex.containsKey(rootName) ? columnRootNameToIndex.get(rootName) : 2;
while (true) {
String name = rootName + " " + startIndex;
if (columnNameSet.contains(name)) {
startIndex++;
} else {
columnNames.set(index, name);
columnNameSet.add(name);
break;
}
}
columnRootNameToIndex.put(rootName, startIndex + 1);
} else {
columnNames.set(index, rootName);
columnNameSet.add(rootName);
}
}
protected Cell extractCell(org.apache.poi.ss.usermodel.Cell cell, Map<String, Recon> reconMap) {
int cellType = cell.getCellType();
if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_ERROR ||
cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BLANK) {
continue;
return null;
}
if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_FORMULA) {
cellType = cell.getCachedFormulaResultType();
@ -223,24 +267,9 @@ public class ExcelImporter implements Importer {
}
}
newRow.setCell(c, new Cell(value, recon));
hasData = true;
}
}
if (hasData) {
rowsWithData++;
if (skip <= 0 || rowsWithData > skip) {
project.rows.add(newRow);
project.columnModel.setMaxCellIndex(newRow.cells.size());
if (limit > 0 && project.rows.size() >= limit) {
break;
}
}
}
}
return new Cell(value, recon);
} else {
return null;
}
}