Excel importer now supports "header lines" parameter.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1125 7d457c2a-affb-35e4-300a-418c747d4874

main/src/com/metaweb/gridworks/importers/ExcelImporter.java

@@ -6,9 +6,11 @@ import java.io.Reader;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 
 import org.apache.poi.common.usermodel.Hyperlink;
 import org.apache.poi.hssf.usermodel.HSSFDateUtil;
@@ -39,8 +41,9 @@ public class ExcelImporter implements Importer {
     
     public void read(InputStream inputStream, Project project, Properties options) throws Exception {
         int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1);
-        int limit = ImporterUtilities.getIntegerOption("limit",options,-1);
+        int headerLines = ImporterUtilities.getIntegerOption("header-lines", options, 1);
-        int skip = ImporterUtilities.getIntegerOption("skip",options,0);
+        int limit = ImporterUtilities.getIntegerOption("limit", options, -1);
+        int skip = ImporterUtilities.getIntegerOption("skip", options, 0);
 
         Workbook wb = null;
         try {
@@ -59,15 +62,15 @@ public class ExcelImporter implements Importer {
         
         int firstRow = sheet.getFirstRowNum();
         int lastRow = sheet.getLastRowNum();
-        int r = firstRow;
         
-        List<Integer>    nonBlankIndices = null;
+        List<String>         columnNames = new ArrayList<String>();
-        List<String>     nonBlankHeaderStrings = null;
+        Set<String>          columnNameSet = new HashSet<String>();
+        Map<String, Integer> columnRootNameToIndex = new HashMap<String, Integer>();
         
-        /*
+        int                  rowsWithData = 0;
-         *  Find the header row
+        Map<String, Recon>   reconMap = new HashMap<String, Recon>();
-         */
+        
-        for (; r <= lastRow; r++) {
+        for (int r = firstRow; r <= lastRow; r++) {
             org.apache.poi.ss.usermodel.Row row = sheet.getRow(r);
             if (row == null) {
                 continue;
@@ -78,83 +81,124 @@ public class ExcelImporter implements Importer {
             
             short firstCell = row.getFirstCellNum();
             short lastCell = row.getLastCellNum();
-            if (firstCell >= 0 && firstCell <= lastCell) {
+            if (firstCell < 0 || firstCell > lastCell) {
-                nonBlankIndices = new ArrayList<Integer>(lastCell - firstCell + 1);
+                continue;
-                nonBlankHeaderStrings = new ArrayList<String>(lastCell - firstCell + 1);
+            }
+            
+            /*
+             *  Still processing header lines
+             */
+            if (headerLines > 0) {
+                headerLines--;
                 
                 for (int c = firstCell; c <= lastCell; c++) {
                     org.apache.poi.ss.usermodel.Cell cell = row.getCell(c);
                     if (cell != null) {
                         String text = cell.getStringCellValue().trim();
                         if (text.length() > 0) {
-                            nonBlankIndices.add((int) c);
+                            while (columnNames.size() < c + 1) {
-                            nonBlankHeaderStrings.add(text);
+                                columnNames.add(null);
+                            }
+                            
+                            String existingName = columnNames.get(c);
+                            String name = (existingName == null) ? text : (existingName + " " + text);
+                            
+                            columnNames.set(c, name);
                         }
                     }
                 }
                 
-                if (nonBlankIndices.size() > 0) {
+                if (headerLines == 0) {
-                    r++;
+                    for (int i = 0; i < columnNames.size(); i++) {
-                    break;
+                        String rootName = columnNames.get(i);
-                }
+                        if (rootName == null) {
-            }
-        }
-
-        if (nonBlankIndices == null || nonBlankIndices.size() == 0) {
-            return;
-        }
-
-        /*
-         *  Create columns
-         */
-        Map<String, Integer> nameToIndex = new HashMap<String, Integer>();
-        for (int c = 0; c < nonBlankIndices.size(); c++) {
-            String cell = nonBlankHeaderStrings.get(c);
-            if (nameToIndex.containsKey(cell)) {
-                int index = nameToIndex.get(cell);
-                nameToIndex.put(cell, index + 1);
-
-                cell = cell.contains(" ") ? (cell + " " + index) : (cell + index);
-            } else {
-                nameToIndex.put(cell, 2);
-            }
-
-            Column column = new Column(c, cell);
-            project.columnModel.columns.add(column);
-        }
-
-        /*
-         *  Now process the data rows
-         */
-        int rowsWithData = 0;
-        Map<String, Recon> reconMap = new HashMap<String, Recon>();
-
-        for (; r <= lastRow; r++) {
-            org.apache.poi.ss.usermodel.Row row = sheet.getRow(r);
-            if (row == null) {
                             continue;
                         }
+                        setUnduplicatedColumnName(rootName, columnNames, i, columnNameSet, columnRootNameToIndex);
+                    }
+                }
                 
-            short firstCell = row.getFirstCellNum();
+            /*
-            short lastCell = row.getLastCellNum();
+             *  Processing data rows
-            if (firstCell >= 0 && firstCell <= lastCell) {
+             */
-                Row newRow = new Row(nonBlankIndices.size());
+            } else {
+                Row newRow = new Row(columnNames.size());
                 boolean hasData = false;
                 
-                for (int c = 0; c < nonBlankIndices.size(); c++) {
+                for (int c = firstCell; c <= lastCell; c++) {
-                    if (c < firstCell || c > lastCell) {
-                        continue;
-                    }
-
                     org.apache.poi.ss.usermodel.Cell cell = row.getCell(c);
                     if (cell == null) {
                         continue;
                     }
                     
+                    Cell ourCell = extractCell(cell, reconMap);
+                    if (ourCell != null) {
+                        while (columnNames.size() < c + 1) {
+                            columnNames.add(null);
+                        }
+                        if (columnNames.get(c) == null) {
+                            setUnduplicatedColumnName("Column", columnNames, c, columnNameSet, columnRootNameToIndex);
+                        }
+                        
+                        newRow.setCell(c, ourCell);
+                        hasData = true;
+                    }
+                }
+                
+                if (hasData) {
+                    rowsWithData++;
+                    
+                    if (skip <= 0 || rowsWithData > skip) {
+                        project.rows.add(newRow);
+                        project.columnModel.setMaxCellIndex(newRow.cells.size());
+                        
+                        if (limit > 0 && project.rows.size() >= limit) {
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        
+        /*
+         *  Create columns
+         */
+        for (int c = 0; c < columnNames.size(); c++) {
+            String name = columnNames.get(c);
+            if (name != null) {
+                Column column = new Column(c, name);
+                project.columnModel.columns.add(column);
+            }
+        }
+    }
+    
+    protected void setUnduplicatedColumnName(
+        String rootName, List<String> columnNames, int index, Set<String> columnNameSet, Map<String, Integer> columnRootNameToIndex) {
+        if (columnNameSet.contains(rootName)) {
+            int startIndex = columnRootNameToIndex.containsKey(rootName) ? columnRootNameToIndex.get(rootName) : 2;
+            while (true) {
+                String name = rootName + " " + startIndex;
+                if (columnNameSet.contains(name)) {
+                    startIndex++;
+                } else {
+                    columnNames.set(index, name);
+                    columnNameSet.add(name);
+                    break;
+                }
+            }
+            
+            columnRootNameToIndex.put(rootName, startIndex + 1);
+        } else {
+            columnNames.set(index, rootName);
+            columnNameSet.add(rootName);
+        }
+    }
+    
+    protected Cell extractCell(org.apache.poi.ss.usermodel.Cell cell, Map<String, Recon> reconMap) {
         int cellType = cell.getCellType();
         if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_ERROR ||
             cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BLANK) {
-                        continue;
+            return null;
         }
         if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_FORMULA) {
             cellType = cell.getCachedFormulaResultType();
@@ -223,24 +267,9 @@ public class ExcelImporter implements Importer {
                 }
             }
             
-                        newRow.setCell(c, new Cell(value, recon));
+            return new Cell(value, recon);
-                        hasData = true;
+        } else {
-                    }
+            return null;
-                }
-
-                if (hasData) {
-                    rowsWithData++;
-
-                    if (skip <= 0 || rowsWithData > skip) {
-                        project.rows.add(newRow);
-                        project.columnModel.setMaxCellIndex(newRow.cells.size());
-
-                        if (limit > 0 && project.rows.size() >= limit) {
-                            break;
-                        }
-                    }
-                }
-            }
         }
     }