s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix the unit test

Browse Source 
support multi files with multi tables for open office
This commit is contained in:
Jacky 2017-11-16 10:29:16 -05:00
parent 6f42df95e2
commit 1d2998f374
2 changed files with 44 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
main/src/com/google/refine/importers/OdsImporter.java
View File

@ -44,7 +44,9 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.odftoolkit.odfdom.doc.OdfDocument; import org.odftoolkit.odfdom.doc.OdfDocument;
import org.odftoolkit.odfdom.doc.table.OdfTable; import org.odftoolkit.odfdom.doc.table.OdfTable;
@ -59,8 +61,8 @@ import com.google.refine.importing.ImportingUtilities;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
import com.google.refine.model.Recon.Judgment; import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.ReconCandidate;
import com.google.refine.util.JSONUtilities; import com.google.refine.util.JSONUtilities;
@ -81,28 +83,29 @@ public class OdsImporter extends TabularImportingParserBase {
JSONUtilities.safePut(options, "sheetRecords", sheetRecords); JSONUtilities.safePut(options, "sheetRecords", sheetRecords);
OdfDocument odfDoc = null; OdfDocument odfDoc = null;
try { try {
JSONObject firstFileRecord = fileRecords.get(0); for (int index = 0;index < fileRecords.size();index++) {
File file = ImportingUtilities.getFile(job, firstFileRecord); JSONObject fileRecord = fileRecords.get(index);
InputStream is = new FileInputStream(file); File file = ImportingUtilities.getFile(job, fileRecord);
odfDoc = OdfDocument.loadDocument(is); InputStream is = new FileInputStream(file);
List<OdfTable> tables = odfDoc.getTableList(); odfDoc = OdfDocument.loadDocument(is);
int sheetCount = tables.size(); List<OdfTable> tables = odfDoc.getTableList();
int sheetCount = tables.size();
boolean hasData = false;
for (int i = 0; i < sheetCount; i++) { for (int i = 0; i < sheetCount; i++) {
OdfTable sheet = tables.get(i); OdfTable sheet = tables.get(i);
int rows = sheet.getRowCount(); int rows = sheet.getRowCount();
JSONObject sheetRecord = new JSONObject(); JSONObject sheetRecord = new JSONObject();
JSONUtilities.safePut(sheetRecord, "name", sheet.getTableName()); JSONUtilities.safePut(sheetRecord, "name", file.getName() + "#" + sheet.getTableName());
JSONUtilities.safePut(sheetRecord, "rows", rows); JSONUtilities.safePut(sheetRecord, "fileNameAndSheetIndex", file.getName() + "#" + i);
if (hasData) { JSONUtilities.safePut(sheetRecord, "rows", rows);
JSONUtilities.safePut(sheetRecord, "selected", false); if (rows > 0) {
} else if (rows > 0) { JSONUtilities.safePut(sheetRecord, "selected", true);
JSONUtilities.safePut(sheetRecord, "selected", true); } else {
hasData = true; JSONUtilities.safePut(sheetRecord, "selected", false);
}
JSONUtilities.append(sheetRecords, sheetRecord);
} }
JSONUtilities.append(sheetRecords, sheetRecord);
} }
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
logger.info("File not found",e); logger.info("File not found",e);
@ -139,9 +142,21 @@ public class OdsImporter extends TabularImportingParserBase {
List<OdfTable> tables = odfDoc.getTableList(); List<OdfTable> tables = odfDoc.getTableList();
int[] sheets = JSONUtilities.getIntArray(options, "sheets"); JSONArray sheets = JSONUtilities.getArray(options, "sheets");
for (int sheetIndex : sheets) { for(int i=0;i<sheets.length();i++) {
final OdfTable table = tables.get(sheetIndex); String[] fileNameAndSheetIndex = new String[2];
try {
JSONObject sheetObj = sheets.getJSONObject(i);
// value is fileName#sheetIndex
fileNameAndSheetIndex = sheetObj.getString("fileNameAndSheetIndex").split("#");
} catch (JSONException e) {
logger.error(ExceptionUtils.getFullStackTrace(e));
}
if (!fileNameAndSheetIndex[0].equals(fileSource))
continue;
final OdfTable table = tables.get(Integer.parseInt(fileNameAndSheetIndex[1]));
final int lastRow = table.getRowCount(); final int lastRow = table.getRowCount();
TableDataReader dataReader = new TableDataReader() { TableDataReader dataReader = new TableDataReader() {

8
main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java
View File

@ -55,6 +55,7 @@ import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterMethod; import org.testng.annotations.AfterMethod;
@ -100,12 +101,11 @@ public class ExcelImporterTests extends ImporterTest {
//---------------------read tests------------------------ //---------------------read tests------------------------
@Test @Test
public void readXls() throws FileNotFoundException{ public void readXls() throws FileNotFoundException, JSONException{
JSONArray sheets = new JSONArray(); JSONArray sheets = new JSONArray();
JSONUtilities.append(sheets, 0); JSONUtilities.append(sheets,
// JSONUtilities.append(sheets, 1); new JSONObject("{name: \"file-source#Test Sheet 0\", fileNameAndSheetIndex: \"file-source#0\", rows: 31, selected: true}"));
// JSONUtilities.append(sheets, 2);
whenGetArrayOption("sheets", options, sheets); whenGetArrayOption("sheets", options, sheets);
whenGetIntegerOption("ignoreLines", options, 0); whenGetIntegerOption("ignoreLines", options, 0);