From 6f42df95e2536f7bb0763579bd8396e7ade95441 Mon Sep 17 00:00:00 2001
From: Jacky <jackyq2015@gmail.com>
Date: Wed, 15 Nov 2017 22:58:47 -0500
Subject: [PATCH 1/2] fix import multiple excel with mulitple sheets issue
 #1328

---
 .../refine/importers/ExcelImporter.java       | 38 +++++++++++++------
 .../parser-interfaces/excel-parser-ui.js      |  6 ++-
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/main/src/com/google/refine/importers/ExcelImporter.java b/main/src/com/google/refine/importers/ExcelImporter.java
index 5838d7a87..2cd9cfbd2 100644
--- a/main/src/com/google/refine/importers/ExcelImporter.java
+++ b/main/src/com/google/refine/importers/ExcelImporter.java
@@ -44,6 +44,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.poi.POIXMLDocument;
 import org.apache.poi.POIXMLException;
 import org.apache.poi.common.usermodel.Hyperlink;
@@ -54,6 +55,7 @@ import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -83,9 +85,9 @@ public class ExcelImporter extends TabularImportingParserBase {
         JSONArray sheetRecords = new JSONArray();
         JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
         try {
-            if (fileRecords.size() > 0) {
-                JSONObject firstFileRecord = fileRecords.get(0);
-                File file = ImportingUtilities.getFile(job, firstFileRecord);
+            for (int index = 0;index < fileRecords.size();index++) {
+                JSONObject fileRecord = fileRecords.get(index);
+                File file = ImportingUtilities.getFile(job, fileRecord);
                 InputStream is = new FileInputStream(file);
 
                 if (!is.markSupported()) {
@@ -98,19 +100,18 @@ public class ExcelImporter extends TabularImportingParserBase {
                                 new HSSFWorkbook(new POIFSFileSystem(is));
 
                             int sheetCount = wb.getNumberOfSheets();
-                            boolean hasData = false;
                             for (int i = 0; i < sheetCount; i++) {
                                 Sheet sheet = wb.getSheetAt(i);
                                 int rows = sheet.getLastRowNum() - sheet.getFirstRowNum() + 1;
 
                                 JSONObject sheetRecord = new JSONObject();
-                                JSONUtilities.safePut(sheetRecord, "name", sheet.getSheetName());
+                                JSONUtilities.safePut(sheetRecord, "name",  file.getName() + "#" + sheet.getSheetName());
+                                JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i);
                                 JSONUtilities.safePut(sheetRecord, "rows", rows);
-                                if (hasData) {
-                                    JSONUtilities.safePut(sheetRecord, "selected", false);
-                                } else if (rows > 1) {
+                                if (rows > 1) {
                                     JSONUtilities.safePut(sheetRecord, "selected", true);
-                                    hasData = true;
+                                } else {
+                                    JSONUtilities.safePut(sheetRecord, "selected", false);
                                 }
                                 JSONUtilities.append(sheetRecords, sheetRecord);
                             }
@@ -181,9 +182,22 @@ public class ExcelImporter extends TabularImportingParserBase {
                 return;
         }
         
-        int[] sheets = JSONUtilities.getIntArray(options, "sheets");
-        for (int sheetIndex : sheets) {
-            final Sheet sheet = wb.getSheetAt(sheetIndex);
+        JSONArray sheets = JSONUtilities.getArray(options, "sheets");
+        
+        for(int i=0;i<sheets.length();i++)  {
+            String[] fileNameAndSheetIndex = new String[2];
+            try {
+                JSONObject sheetObj = sheets.getJSONObject(i);
+                // value is fileName#sheetIndex
+                fileNameAndSheetIndex = sheetObj.getString("fileNameAndSheetIndex").split("#");
+            } catch (JSONException e) {
+                logger.error(ExceptionUtils.getFullStackTrace(e));
+            }
+            
+            if (!fileNameAndSheetIndex[0].equals(fileSource))
+                continue;
+            
+            final Sheet sheet = wb.getSheetAt(Integer.parseInt(fileNameAndSheetIndex[1]));
             final int lastRow = sheet.getLastRowNum();
             
             TableDataReader dataReader = new TableDataReader() {
diff --git a/main/webapp/modules/core/scripts/index/parser-interfaces/excel-parser-ui.js b/main/webapp/modules/core/scripts/index/parser-interfaces/excel-parser-ui.js
index fd24278e3..3f935e423 100644
--- a/main/webapp/modules/core/scripts/index/parser-interfaces/excel-parser-ui.js
+++ b/main/webapp/modules/core/scripts/index/parser-interfaces/excel-parser-ui.js
@@ -76,10 +76,12 @@ Refine.ExcelParserUI.prototype.getOptions = function() {
     }
     return def;
   };
-
+  
+  var self = this;
+  
   this._optionContainerElmts.sheetRecordContainer.find('input').each(function() {
     if (this.checked) {
-      options.sheets.push(parseInt(this.getAttribute('index'),10));
+        options.sheets.push(self._config.sheetRecords[parseInt(this.getAttribute('index'))]);
     }
   });
 

From 1d2998f37463193b5016a4298017ab4584721430 Mon Sep 17 00:00:00 2001
From: Jacky <jackyq2015@gmail.com>
Date: Thu, 16 Nov 2017 10:29:16 -0500
Subject: [PATCH 2/2] fix the unit test support multi files with multi tables
 for open office

---
 .../google/refine/importers/OdsImporter.java  | 65 ++++++++++++-------
 .../tests/importers/ExcelImporterTests.java   |  8 +--
 2 files changed, 44 insertions(+), 29 deletions(-)

diff --git a/main/src/com/google/refine/importers/OdsImporter.java b/main/src/com/google/refine/importers/OdsImporter.java
index 47f69f864..b6e5147dd 100644
--- a/main/src/com/google/refine/importers/OdsImporter.java
+++ b/main/src/com/google/refine/importers/OdsImporter.java
@@ -44,7 +44,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang.exception.ExceptionUtils;
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.odftoolkit.odfdom.doc.OdfDocument;
 import org.odftoolkit.odfdom.doc.table.OdfTable;
@@ -59,8 +61,8 @@ import com.google.refine.importing.ImportingUtilities;
 import com.google.refine.model.Cell;
 import com.google.refine.model.Project;
 import com.google.refine.model.Recon;
-import com.google.refine.model.ReconCandidate;
 import com.google.refine.model.Recon.Judgment;
+import com.google.refine.model.ReconCandidate;
 import com.google.refine.util.JSONUtilities;
 
 
@@ -81,28 +83,29 @@ public class OdsImporter extends TabularImportingParserBase {
         JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
         OdfDocument odfDoc = null;
         try {
-            JSONObject firstFileRecord = fileRecords.get(0);
-            File file = ImportingUtilities.getFile(job, firstFileRecord);
-            InputStream is = new FileInputStream(file);
-            odfDoc = OdfDocument.loadDocument(is);
-            List<OdfTable> tables = odfDoc.getTableList();
-            int sheetCount = tables.size();
-
-            boolean hasData = false;
-            for (int i = 0; i < sheetCount; i++) {
-                OdfTable sheet = tables.get(i);
-                int rows = sheet.getRowCount();
-
-                JSONObject sheetRecord = new JSONObject();
-                JSONUtilities.safePut(sheetRecord, "name", sheet.getTableName());
-                JSONUtilities.safePut(sheetRecord, "rows", rows);
-                if (hasData) {
-                    JSONUtilities.safePut(sheetRecord, "selected", false);
-                } else if (rows > 0) {
-                    JSONUtilities.safePut(sheetRecord, "selected", true);
-                    hasData = true;
+            for (int index = 0;index < fileRecords.size();index++) {
+                JSONObject fileRecord = fileRecords.get(index);
+                File file = ImportingUtilities.getFile(job, fileRecord);
+                InputStream is = new FileInputStream(file);
+                odfDoc = OdfDocument.loadDocument(is);
+                List<OdfTable> tables = odfDoc.getTableList();
+                int sheetCount = tables.size();
+    
+                for (int i = 0; i < sheetCount; i++) {
+                    OdfTable sheet = tables.get(i);
+                    int rows = sheet.getRowCount();
+    
+                    JSONObject sheetRecord = new JSONObject();
+                    JSONUtilities.safePut(sheetRecord, "name",  file.getName() + "#" + sheet.getTableName());
+                    JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i);
+                    JSONUtilities.safePut(sheetRecord, "rows", rows);
+                    if (rows > 0) {
+                        JSONUtilities.safePut(sheetRecord, "selected", true);
+                    } else {
+                        JSONUtilities.safePut(sheetRecord, "selected", false);
+                    }
+                    JSONUtilities.append(sheetRecords, sheetRecord);
                 }
-                JSONUtilities.append(sheetRecords, sheetRecord);
             }
         } catch (FileNotFoundException e) {
             logger.info("File not found",e);
@@ -139,9 +142,21 @@ public class OdsImporter extends TabularImportingParserBase {
 
         List<OdfTable> tables = odfDoc.getTableList();
 
-        int[] sheets = JSONUtilities.getIntArray(options, "sheets");
-        for (int sheetIndex : sheets) {
-            final OdfTable table = tables.get(sheetIndex);
+        JSONArray sheets = JSONUtilities.getArray(options, "sheets");
+        for(int i=0;i<sheets.length();i++)  {
+            String[] fileNameAndSheetIndex = new String[2];
+            try {
+                JSONObject sheetObj = sheets.getJSONObject(i);
+                // value is fileName#sheetIndex
+                fileNameAndSheetIndex = sheetObj.getString("fileNameAndSheetIndex").split("#");
+            } catch (JSONException e) {
+                logger.error(ExceptionUtils.getFullStackTrace(e));
+            }
+            
+            if (!fileNameAndSheetIndex[0].equals(fileSource))
+                continue;
+            
+            final OdfTable table = tables.get(Integer.parseInt(fileNameAndSheetIndex[1]));
             final int lastRow = table.getRowCount();
 
             TableDataReader dataReader = new TableDataReader() {
diff --git a/main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java b/main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java
index 40a8c2032..efc36c8d9 100644
--- a/main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java
+++ b/main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java
@@ -55,6 +55,7 @@ import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
 import org.json.JSONArray;
 import org.json.JSONException;
+import org.json.JSONObject;
 import org.slf4j.LoggerFactory;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;
@@ -100,12 +101,11 @@ public class ExcelImporterTests extends ImporterTest {
     
     //---------------------read tests------------------------
     @Test
-    public void readXls() throws FileNotFoundException{
+    public void readXls() throws FileNotFoundException, JSONException{
 
         JSONArray sheets = new JSONArray();
-        JSONUtilities.append(sheets, 0);
-//        JSONUtilities.append(sheets, 1);
-//        JSONUtilities.append(sheets, 2);
+        JSONUtilities.append(sheets, 
+                new JSONObject("{name: \"file-source#Test Sheet 0\", fileNameAndSheetIndex: \"file-source#0\", rows: 31, selected: true}"));
         whenGetArrayOption("sheets", options, sheets);
         
         whenGetIntegerOption("ignoreLines", options, 0);