s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1334 from jackyq2015/issue/1328

Browse Source 
fix import multiple excel with mulitple sheets issue #1328
This commit is contained in:
Jacky 2017-11-16 21:30:24 -05:00 committed by GitHub
commit 8d2c8b521b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 74 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
main/src/com/google/refine/importers/ExcelImporter.java
View File

@ -44,6 +44,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.poi.POIXMLDocument;
import org.apache.poi.POIXMLException;
import org.apache.poi.common.usermodel.Hyperlink;
@ -54,6 +55,7 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -83,9 +85,9 @@ public class ExcelImporter extends TabularImportingParserBase {
JSONArray sheetRecords = new JSONArray();
JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
try {
if (fileRecords.size() > 0) {
JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord);
for (int index = 0;index < fileRecords.size();index++) {
JSONObject fileRecord = fileRecords.get(index);
File file = ImportingUtilities.getFile(job, fileRecord);
InputStream is = new FileInputStream(file);
if (!is.markSupported()) {
@ -98,19 +100,18 @@ public class ExcelImporter extends TabularImportingParserBase {
new HSSFWorkbook(new POIFSFileSystem(is));
int sheetCount = wb.getNumberOfSheets();
boolean hasData = false;
for (int i = 0; i < sheetCount; i++) {
Sheet sheet = wb.getSheetAt(i);
int rows = sheet.getLastRowNum() - sheet.getFirstRowNum() + 1;
JSONObject sheetRecord = new JSONObject();
JSONUtilities.safePut(sheetRecord, "name", sheet.getSheetName());
JSONUtilities.safePut(sheetRecord, "name", file.getName() + "#" + sheet.getSheetName());
JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i);
JSONUtilities.safePut(sheetRecord, "rows", rows);
if (hasData) {
JSONUtilities.safePut(sheetRecord, "selected", false);
} else if (rows > 1) {
if (rows > 1) {
JSONUtilities.safePut(sheetRecord, "selected", true);
hasData = true;
} else {
JSONUtilities.safePut(sheetRecord, "selected", false);
}
JSONUtilities.append(sheetRecords, sheetRecord);
}
@ -181,9 +182,22 @@ public class ExcelImporter extends TabularImportingParserBase {
return;
}
int[] sheets = JSONUtilities.getIntArray(options, "sheets");
for (int sheetIndex : sheets) {
final Sheet sheet = wb.getSheetAt(sheetIndex);
JSONArray sheets = JSONUtilities.getArray(options, "sheets");
for(int i=0;i<sheets.length();i++) {
String[] fileNameAndSheetIndex = new String[2];
try {
JSONObject sheetObj = sheets.getJSONObject(i);
// value is fileName#sheetIndex
fileNameAndSheetIndex = sheetObj.getString("fileNameAndSheetIndex").split("#");
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
if (!fileNameAndSheetIndex[0].equals(fileSource))
continue;
final Sheet sheet = wb.getSheetAt(Integer.parseInt(fileNameAndSheetIndex[1]));
final int lastRow = sheet.getLastRowNum();
TableDataReader dataReader = new TableDataReader() {

65
main/src/com/google/refine/importers/OdsImporter.java
View File

@ -44,7 +44,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.odftoolkit.odfdom.doc.OdfDocument;
import org.odftoolkit.odfdom.doc.table.OdfTable;
@ -59,8 +61,8 @@ import com.google.refine.importing.ImportingUtilities;
import com.google.refine.model.Cell;
import com.google.refine.model.Project;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.ReconCandidate;
import com.google.refine.util.JSONUtilities;
@ -81,28 +83,29 @@ public class OdsImporter extends TabularImportingParserBase {
JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
OdfDocument odfDoc = null;
try {
JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord);
InputStream is = new FileInputStream(file);
odfDoc = OdfDocument.loadDocument(is);
List<OdfTable> tables = odfDoc.getTableList();
int sheetCount = tables.size();
boolean hasData = false;
for (int i = 0; i < sheetCount; i++) {
OdfTable sheet = tables.get(i);
int rows = sheet.getRowCount();
JSONObject sheetRecord = new JSONObject();
JSONUtilities.safePut(sheetRecord, "name", sheet.getTableName());
JSONUtilities.safePut(sheetRecord, "rows", rows);
if (hasData) {
JSONUtilities.safePut(sheetRecord, "selected", false);
} else if (rows > 0) {
JSONUtilities.safePut(sheetRecord, "selected", true);
hasData = true;
for (int index = 0;index < fileRecords.size();index++) {
JSONObject fileRecord = fileRecords.get(index);
File file = ImportingUtilities.getFile(job, fileRecord);
InputStream is = new FileInputStream(file);
odfDoc = OdfDocument.loadDocument(is);
List<OdfTable> tables = odfDoc.getTableList();
int sheetCount = tables.size();
for (int i = 0; i < sheetCount; i++) {
OdfTable sheet = tables.get(i);
int rows = sheet.getRowCount();
JSONObject sheetRecord = new JSONObject();
JSONUtilities.safePut(sheetRecord, "name", file.getName() + "#" + sheet.getTableName());
JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i);
JSONUtilities.safePut(sheetRecord, "rows", rows);
if (rows > 0) {
JSONUtilities.safePut(sheetRecord, "selected", true);
} else {
JSONUtilities.safePut(sheetRecord, "selected", false);
}
JSONUtilities.append(sheetRecords, sheetRecord);
}
JSONUtilities.append(sheetRecords, sheetRecord);
}
} catch (FileNotFoundException e) {
logger.info("File not found",e);
@ -139,9 +142,21 @@ public class OdsImporter extends TabularImportingParserBase {
List<OdfTable> tables = odfDoc.getTableList();
int[] sheets = JSONUtilities.getIntArray(options, "sheets");
for (int sheetIndex : sheets) {
final OdfTable table = tables.get(sheetIndex);
JSONArray sheets = JSONUtilities.getArray(options, "sheets");
for(int i=0;i<sheets.length();i++) {
String[] fileNameAndSheetIndex = new String[2];
try {
JSONObject sheetObj = sheets.getJSONObject(i);
// value is fileName#sheetIndex
fileNameAndSheetIndex = sheetObj.getString("fileNameAndSheetIndex").split("#");
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
if (!fileNameAndSheetIndex[0].equals(fileSource))
continue;
final OdfTable table = tables.get(Integer.parseInt(fileNameAndSheetIndex[1]));
final int lastRow = table.getRowCount();
TableDataReader dataReader = new TableDataReader() {

8
main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java
View File

@ -55,6 +55,7 @@ import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
@ -100,12 +101,11 @@ public class ExcelImporterTests extends ImporterTest {
//---------------------read tests------------------------
@Test
public void readXls() throws FileNotFoundException{
public void readXls() throws FileNotFoundException, JSONException{
JSONArray sheets = new JSONArray();
JSONUtilities.append(sheets, 0);
// JSONUtilities.append(sheets, 1);
// JSONUtilities.append(sheets, 2);
JSONUtilities.append(sheets,
new JSONObject("{name: \"file-source#Test Sheet 0\", fileNameAndSheetIndex: \"file-source#0\", rows: 31, selected: true}"));
whenGetArrayOption("sheets", options, sheets);
whenGetIntegerOption("ignoreLines", options, 0);

6
main/webapp/modules/core/scripts/index/parser-interfaces/excel-parser-ui.js
View File

@ -76,10 +76,12 @@ Refine.ExcelParserUI.prototype.getOptions = function() {
}
return def;
};
var self = this;
this._optionContainerElmts.sheetRecordContainer.find('input').each(function() {
if (this.checked) {
options.sheets.push(parseInt(this.getAttribute('index'),10));
options.sheets.push(self._config.sheetRecords[parseInt(this.getAttribute('index'))]);
}
});