s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1334 from jackyq2015/issue/1328

Browse Source 
fix import multiple excel with mulitple sheets issue #1328
This commit is contained in:
Jacky 2017-11-16 21:30:24 -05:00 committed by GitHub
commit 8d2c8b521b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 74 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
main/src/com/google/refine/importers/ExcelImporter.java
View File

@ -44,6 +44,7 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.poi.POIXMLDocument; import org.apache.poi.POIXMLDocument;
import org.apache.poi.POIXMLException; import org.apache.poi.POIXMLException;
import org.apache.poi.common.usermodel.Hyperlink; import org.apache.poi.common.usermodel.Hyperlink;
@ -54,6 +55,7 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -83,9 +85,9 @@ public class ExcelImporter extends TabularImportingParserBase {
JSONArray sheetRecords = new JSONArray(); JSONArray sheetRecords = new JSONArray();
JSONUtilities.safePut(options, "sheetRecords", sheetRecords); JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
try { try {
if (fileRecords.size() > 0) { for (int index = 0;index < fileRecords.size();index++) {
JSONObject firstFileRecord = fileRecords.get(0); JSONObject fileRecord = fileRecords.get(index);
File file = ImportingUtilities.getFile(job, firstFileRecord); File file = ImportingUtilities.getFile(job, fileRecord);
InputStream is = new FileInputStream(file); InputStream is = new FileInputStream(file);
if (!is.markSupported()) { if (!is.markSupported()) {
@ -98,19 +100,18 @@ public class ExcelImporter extends TabularImportingParserBase {
new HSSFWorkbook(new POIFSFileSystem(is)); new HSSFWorkbook(new POIFSFileSystem(is));
int sheetCount = wb.getNumberOfSheets(); int sheetCount = wb.getNumberOfSheets();
boolean hasData = false;
for (int i = 0; i < sheetCount; i++) { for (int i = 0; i < sheetCount; i++) {
Sheet sheet = wb.getSheetAt(i); Sheet sheet = wb.getSheetAt(i);
int rows = sheet.getLastRowNum() - sheet.getFirstRowNum() + 1; int rows = sheet.getLastRowNum() - sheet.getFirstRowNum() + 1;
JSONObject sheetRecord = new JSONObject(); JSONObject sheetRecord = new JSONObject();
JSONUtilities.safePut(sheetRecord, "name", sheet.getSheetName()); JSONUtilities.safePut(sheetRecord, "name", file.getName() + "#" + sheet.getSheetName());
JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i);
JSONUtilities.safePut(sheetRecord, "rows", rows); JSONUtilities.safePut(sheetRecord, "rows", rows);
if (hasData) { if (rows > 1) {
JSONUtilities.safePut(sheetRecord, "selected", false);
} else if (rows > 1) {
JSONUtilities.safePut(sheetRecord, "selected", true); JSONUtilities.safePut(sheetRecord, "selected", true);
hasData = true; } else {
JSONUtilities.safePut(sheetRecord, "selected", false);
} }
JSONUtilities.append(sheetRecords, sheetRecord); JSONUtilities.append(sheetRecords, sheetRecord);
} }
@ -181,9 +182,22 @@ public class ExcelImporter extends TabularImportingParserBase {
return; return;
} }
int[] sheets = JSONUtilities.getIntArray(options, "sheets"); JSONArray sheets = JSONUtilities.getArray(options, "sheets");
for (int sheetIndex : sheets) {
final Sheet sheet = wb.getSheetAt(sheetIndex); for(int i=0;i<sheets.length();i++) {
String[] fileNameAndSheetIndex = new String[2];
try {
JSONObject sheetObj = sheets.getJSONObject(i);
// value is fileName#sheetIndex
fileNameAndSheetIndex = sheetObj.getString("fileNameAndSheetIndex").split("#");
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
if (!fileNameAndSheetIndex[0].equals(fileSource))
continue;
final Sheet sheet = wb.getSheetAt(Integer.parseInt(fileNameAndSheetIndex[1]));
final int lastRow = sheet.getLastRowNum(); final int lastRow = sheet.getLastRowNum();
TableDataReader dataReader = new TableDataReader() { TableDataReader dataReader = new TableDataReader() {

39
main/src/com/google/refine/importers/OdsImporter.java
View File

@ -44,7 +44,9 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.odftoolkit.odfdom.doc.OdfDocument; import org.odftoolkit.odfdom.doc.OdfDocument;
import org.odftoolkit.odfdom.doc.table.OdfTable; import org.odftoolkit.odfdom.doc.table.OdfTable;
@ -59,8 +61,8 @@ import com.google.refine.importing.ImportingUtilities;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Recon.Judgment; import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.ReconCandidate;
import com.google.refine.util.JSONUtilities; import com.google.refine.util.JSONUtilities;
@ -81,29 +83,30 @@ public class OdsImporter extends TabularImportingParserBase {
JSONUtilities.safePut(options, "sheetRecords", sheetRecords); JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
OdfDocument odfDoc = null; OdfDocument odfDoc = null;
try { try {
JSONObject firstFileRecord = fileRecords.get(0); for (int index = 0;index < fileRecords.size();index++) {
File file = ImportingUtilities.getFile(job, firstFileRecord); JSONObject fileRecord = fileRecords.get(index);
File file = ImportingUtilities.getFile(job, fileRecord);
InputStream is = new FileInputStream(file); InputStream is = new FileInputStream(file);
odfDoc = OdfDocument.loadDocument(is); odfDoc = OdfDocument.loadDocument(is);
List<OdfTable> tables = odfDoc.getTableList(); List<OdfTable> tables = odfDoc.getTableList();
int sheetCount = tables.size(); int sheetCount = tables.size();
boolean hasData = false;
for (int i = 0; i < sheetCount; i++) { for (int i = 0; i < sheetCount; i++) {
OdfTable sheet = tables.get(i); OdfTable sheet = tables.get(i);
int rows = sheet.getRowCount(); int rows = sheet.getRowCount();
JSONObject sheetRecord = new JSONObject(); JSONObject sheetRecord = new JSONObject();
JSONUtilities.safePut(sheetRecord, "name", sheet.getTableName()); JSONUtilities.safePut(sheetRecord, "name", file.getName() + "#" + sheet.getTableName());
JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i);
JSONUtilities.safePut(sheetRecord, "rows", rows); JSONUtilities.safePut(sheetRecord, "rows", rows);
if (hasData) { if (rows > 0) {
JSONUtilities.safePut(sheetRecord, "selected", false);
} else if (rows > 0) {
JSONUtilities.safePut(sheetRecord, "selected", true); JSONUtilities.safePut(sheetRecord, "selected", true);
hasData = true; } else {
JSONUtilities.safePut(sheetRecord, "selected", false);
} }
JSONUtilities.append(sheetRecords, sheetRecord); JSONUtilities.append(sheetRecords, sheetRecord);
} }
}
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
logger.info("File not found",e); logger.info("File not found",e);
} catch (Exception e) { } catch (Exception e) {
@ -139,9 +142,21 @@ public class OdsImporter extends TabularImportingParserBase {
List<OdfTable> tables = odfDoc.getTableList(); List<OdfTable> tables = odfDoc.getTableList();
int[] sheets = JSONUtilities.getIntArray(options, "sheets"); JSONArray sheets = JSONUtilities.getArray(options, "sheets");
for (int sheetIndex : sheets) { for(int i=0;i<sheets.length();i++) {
final OdfTable table = tables.get(sheetIndex); String[] fileNameAndSheetIndex = new String[2];
try {
JSONObject sheetObj = sheets.getJSONObject(i);
// value is fileName#sheetIndex
fileNameAndSheetIndex = sheetObj.getString("fileNameAndSheetIndex").split("#");
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
if (!fileNameAndSheetIndex[0].equals(fileSource))
continue;
final OdfTable table = tables.get(Integer.parseInt(fileNameAndSheetIndex[1]));
final int lastRow = table.getRowCount(); final int lastRow = table.getRowCount();
TableDataReader dataReader = new TableDataReader() { TableDataReader dataReader = new TableDataReader() {

8
main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java
View File

@ -55,6 +55,7 @@ import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterMethod; import org.testng.annotations.AfterMethod;
@ -100,12 +101,11 @@ public class ExcelImporterTests extends ImporterTest {
//---------------------read tests------------------------ //---------------------read tests------------------------
@Test @Test
public void readXls() throws FileNotFoundException{ public void readXls() throws FileNotFoundException, JSONException{
JSONArray sheets = new JSONArray(); JSONArray sheets = new JSONArray();
JSONUtilities.append(sheets, 0); JSONUtilities.append(sheets,
// JSONUtilities.append(sheets, 1); new JSONObject("{name: \"file-source#Test Sheet 0\", fileNameAndSheetIndex: \"file-source#0\", rows: 31, selected: true}"));
// JSONUtilities.append(sheets, 2);
whenGetArrayOption("sheets", options, sheets); whenGetArrayOption("sheets", options, sheets);
whenGetIntegerOption("ignoreLines", options, 0); whenGetIntegerOption("ignoreLines", options, 0);

4
main/webapp/modules/core/scripts/index/parser-interfaces/excel-parser-ui.js
View File

@ -77,9 +77,11 @@ Refine.ExcelParserUI.prototype.getOptions = function() {
return def; return def;
}; };
var self = this;
this._optionContainerElmts.sheetRecordContainer.find('input').each(function() { this._optionContainerElmts.sheetRecordContainer.find('input').each(function() {
if (this.checked) { if (this.checked) {
options.sheets.push(parseInt(this.getAttribute('index'),10)); options.sheets.push(self._config.sheetRecords[parseInt(this.getAttribute('index'))]);
} }
}); });