Merge pull request #2011 from ostephens/poi-update

Fix XLSX import and export

main/pom.xml

@@ -348,12 +348,12 @@
     <dependency>
       <groupId>org.apache.poi</groupId>
       <artifactId>poi</artifactId>
-      <version>4.0.1</version>
+      <version>4.1.0</version>
     </dependency>
     <dependency>
       <groupId>org.apache.poi</groupId>
       <artifactId>poi-ooxml</artifactId>
-      <version>3.8</version> <!-- in classpath: 3.13-20150929 -->
+      <version>4.1.0</version> <!-- in classpath: 3.13-20150929 -->
     </dependency>
     <dependency>
       <groupId>org.apache.odftoolkit</groupId>

main/src/com/google/refine/exporters/XlsExporter.java

@@ -142,6 +142,7 @@ public class XlsExporter implements StreamExporter {
         
         wb.write(outputStream);
         outputStream.flush();
+        wb.close();
     }
 
 }

main/src/com/google/refine/importers/ExcelImporter.java

@@ -35,6 +35,7 @@ package com.google.refine.importers;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PushbackInputStream;
@@ -44,8 +45,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.poi.POIXMLDocument;
+import org.apache.poi.ooxml.POIXMLException;
-import org.apache.poi.POIXMLException;
 import org.apache.poi.common.usermodel.Hyperlink;
 import org.apache.poi.hssf.usermodel.HSSFDateUtil;
 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
@@ -54,6 +54,8 @@ import org.apache.poi.ss.usermodel.CellType;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.apache.poi.poifs.filesystem.FileMagic;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -91,11 +93,11 @@ public class ExcelImporter extends TabularImportingParserBase {
                 InputStream is = new FileInputStream(file);
 
                 if (!is.markSupported()) {
-                  is = new PushbackInputStream(is, 8);
+                  is = new BufferedInputStream(is);
                 }
 
                 try {
-                    Workbook wb = POIXMLDocument.hasOOXMLHeader(is) ?
+                    Workbook wb = FileMagic.valueOf(is) == FileMagic.OOXML ?
                             new XSSFWorkbook(is) :
                                 new HSSFWorkbook(new POIFSFileSystem(is));
 
@@ -115,6 +117,7 @@ public class ExcelImporter extends TabularImportingParserBase {
                                 }
                                 JSONUtilities.append(sheetRecords, sheetRecord);
                             }
+                            wb.close();
                 } finally {
                     is.close();
                 }
@@ -143,11 +146,11 @@ public class ExcelImporter extends TabularImportingParserBase {
     ) {
         Workbook wb = null;
         if (!inputStream.markSupported()) {
-          inputStream = new PushbackInputStream(inputStream, 8);
+          inputStream = new BufferedInputStream(inputStream);
         }
         
         try {
-            wb = POIXMLDocument.hasOOXMLHeader(inputStream) ?
+            wb = FileMagic.valueOf(inputStream) == FileMagic.OOXML ?
                 new XSSFWorkbook(inputStream) :
                 new HSSFWorkbook(new POIFSFileSystem(inputStream));
         } catch (IOException e) {
@@ -235,7 +238,7 @@ public class ExcelImporter extends TabularImportingParserBase {
                 exceptions
             );
         }
-        
+
         super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
     }
     

main/tests/server/src/com/google/refine/tests/exporters/XlsxExporterTests.java

@@ -0,0 +1,194 @@
+/*
+
+Copyright 2010, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+package com.google.refine.tests.exporters;
+
+import java.io.ByteArrayInputStream;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+import org.apache.poi.xssf.usermodel.XSSFRow;
+import org.apache.poi.xssf.usermodel.XSSFCell;
+
+import static org.mockito.Mockito.mock;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.util.Properties;
+
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.google.refine.ProjectManager;
+import com.google.refine.ProjectMetadata;
+import com.google.refine.browsing.Engine;
+import com.google.refine.exporters.StreamExporter;
+import com.google.refine.exporters.XlsExporter;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Column;
+import com.google.refine.model.ModelException;
+import com.google.refine.model.Project;
+import com.google.refine.model.Row;
+import com.google.refine.tests.ProjectManagerStub;
+import com.google.refine.tests.RefineTest;
+
+public class XlsxExporterTests extends RefineTest {
+
+    private static final String TEST_PROJECT_NAME = "xlsx exporter test project";
+    
+    @Override
+    @BeforeTest
+    public void init() {
+        logger = LoggerFactory.getLogger(this.getClass());
+    }
+
+    //dependencies
+    ByteArrayOutputStream stream;
+    ProjectMetadata projectMetadata;
+    Project project;
+    Engine engine;
+    Properties options;
+
+    //System Under Test
+    StreamExporter SUT;
+
+    @BeforeMethod
+    public void SetUp(){
+        SUT = new XlsExporter(true);
+        stream = new ByteArrayOutputStream();
+        ProjectManager.singleton = new ProjectManagerStub();
+        projectMetadata = new ProjectMetadata();
+        project = new Project();
+        projectMetadata.setName(TEST_PROJECT_NAME);
+        ProjectManager.singleton.registerProject(project, projectMetadata);
+        engine = new Engine(project);
+        options = mock(Properties.class);
+    }
+
+    @AfterMethod
+    public void TearDown(){
+        SUT = null;
+        stream = null;
+        ProjectManager.singleton.deleteProject(project.id);
+        project = null;
+        engine = null;
+        options = null;
+    }
+
+    @Test
+    public void exportSimpleXlsx(){
+        CreateGrid(2, 2);
+
+        try {
+            SUT.export(project, options, engine, stream);
+        } catch (IOException e) {
+            Assert.fail();
+        }
+
+        ByteArrayInputStream inStream = new ByteArrayInputStream( stream.toByteArray() );
+        try {
+            XSSFWorkbook wb = new XSSFWorkbook(inStream);
+            XSSFSheet ws = wb.getSheetAt(0);
+            XSSFRow row1 = ws.getRow(1);
+            XSSFCell cell0 = row1.getCell(0);
+            Assert.assertEquals(cell0.toString(),"row0cell0");
+            wb.close();
+        } catch (IOException e) {
+            Assert.fail();
+        }
+    }
+    
+    @Test
+    public void exportXlsxDateType() throws IOException{
+        OffsetDateTime odt =  OffsetDateTime.parse("2019-04-09T12:00+00:00");
+        createDateGrid(2, 2, odt);
+
+        try {
+            SUT.export(project, options, engine, stream);
+        } catch (IOException e) {
+            Assert.fail();
+        }
+        
+        ByteArrayInputStream inStream = new ByteArrayInputStream( stream.toByteArray() );
+        try {
+            XSSFWorkbook wb = new XSSFWorkbook(inStream);
+            XSSFSheet ws = wb.getSheetAt(0);
+            XSSFRow row1 = ws.getRow(1);
+            XSSFCell cell0 = row1.getCell(0);
+            Assert.assertEquals(cell0.toString(),"09-Apr-2019");
+            wb.close();
+        } catch (IOException e) {
+            Assert.fail();
+        }
+    }
+
+    //helper methods
+
+    protected void CreateColumns(int noOfColumns){
+        for(int i = 0; i < noOfColumns; i++){
+            try {
+                project.columnModel.addColumn(i, new Column(i, "column" + i), true);
+            } catch (ModelException e1) {
+                Assert.fail("Could not create column");
+            }
+        }
+    }
+
+    protected void CreateGrid(int noOfRows, int noOfColumns){
+        CreateColumns(noOfColumns);
+
+        for(int i = 0; i < noOfRows; i++){
+            Row row = new Row(noOfColumns);
+            for(int j = 0; j < noOfColumns; j++){
+                row.cells.add(new Cell("row" + i + "cell" + j, null));
+            }
+            project.rows.add(row);
+        }
+    }
+    
+    private void createDateGrid(int noOfRows, int noOfColumns, OffsetDateTime now){
+        CreateColumns(noOfColumns);
+
+        for(int i = 0; i < noOfRows; i++){
+            Row row = new Row(noOfColumns);
+            for(int j = 0; j < noOfColumns; j++){
+                row.cells.add(new Cell(now, null));
+            }
+            project.rows.add(row);
+        }
+    }
+}

main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java

@@ -73,6 +73,7 @@ public class ExcelImporterTests extends ImporterTest {
     
     //private static final File xlsxFile = createSpreadsheet(true);
     private static final File xlsFile = createSpreadsheet(false);
+    private static final File xlsxFile = createSpreadsheet(true);
     
     @Override
     @BeforeTest
@@ -137,6 +138,45 @@ public class ExcelImporterTests extends ImporterTest {
         verify(options, times(1)).get("storeBlankCellsAsNulls");
     }
     
+    @Test
+    public void readXlsx() throws FileNotFoundException, IOException{
+
+        ArrayNode sheets = ParsingUtilities.mapper.createArrayNode();
+        sheets.add(ParsingUtilities.mapper.readTree("{name: \"file-source#Test Sheet 0\", fileNameAndSheetIndex: \"file-source#0\", rows: 31, selected: true}"));
+        whenGetArrayOption("sheets", options, sheets);
+        
+        whenGetIntegerOption("ignoreLines", options, 0);
+        whenGetIntegerOption("headerLines", options, 0);
+        whenGetIntegerOption("skipDataLines", options, 0);
+        whenGetIntegerOption("limit", options, -1);
+        whenGetBooleanOption("storeBlankCellsAsNulls",options,true);
+        
+        InputStream stream = new FileInputStream(xlsxFile);
+        
+        try {
+            parseOneFile(SUT, stream);
+        } catch (Exception e) {
+            Assert.fail(e.getMessage());
+        }
+        
+        Assert.assertEquals(project.rows.size(), ROWS);
+        Assert.assertEquals(project.rows.get(1).cells.size(), COLUMNS);
+        Assert.assertEquals(((Number)project.rows.get(1).getCellValue(0)).doubleValue(),1.1, EPSILON);
+        Assert.assertEquals(((Number)project.rows.get(2).getCellValue(0)).doubleValue(),2.2, EPSILON);
+
+        Assert.assertFalse((Boolean)project.rows.get(1).getCellValue(1));
+        Assert.assertTrue((Boolean)project.rows.get(2).getCellValue(1));
+        
+        Assert.assertEquals((String)project.rows.get(1).getCellValue(4)," Row 1 Col 5");
+        Assert.assertNull((String)project.rows.get(1).getCellValue(5));
+
+        verify(options, times(1)).get("ignoreLines");
+        verify(options, times(1)).get("headerLines");
+        verify(options, times(1)).get("skipDataLines");
+        verify(options, times(1)).get("limit");
+        verify(options, times(1)).get("storeBlankCellsAsNulls");
+    }
+    
     private static File createSpreadsheet(boolean xml) {
 
         final Workbook wb = xml ? new XSSFWorkbook() : new HSSFWorkbook();
@@ -182,6 +222,7 @@ public class ExcelImporterTests extends ImporterTest {
             wb.write(outputStream);
             outputStream.flush();
             outputStream.close();
+            wb.close();
         } catch (IOException e) {
             return null;
         }