From 18c18e587ef072af6aa6b32902a6b492a6926c36 Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Thu, 11 Jun 2020 10:39:51 -0400 Subject: [PATCH] Replace Apache Ant with Commons Compress (#2691) NOTE: Changes the public API where some of the old types were embedded which means that any extensions that extend these interfaces will have to be updated. Fixes #2690. --- main/pom.xml | 6 ++-- .../src/com/google/refine/ProjectManager.java | 4 +-- .../refine/importing/ImportingUtilities.java | 22 ++++++------- .../google/refine/io/FileProjectManager.java | 32 +++++++++---------- .../com/google/refine/ProjectManagerStub.java | 4 +-- 5 files changed, 33 insertions(+), 35 deletions(-) diff --git a/main/pom.xml b/main/pom.xml index 2df1fa916..bf0204300 100644 --- a/main/pom.xml +++ b/main/pom.xml @@ -236,9 +236,9 @@ 2.4 - org.apache.ant - ant - 1.8.0 + org.apache.commons + commons-compress + 1.20 org.jsoup diff --git a/main/src/com/google/refine/ProjectManager.java b/main/src/com/google/refine/ProjectManager.java index 86b4bdd5a..60ce1d695 100644 --- a/main/src/com/google/refine/ProjectManager.java +++ b/main/src/com/google/refine/ProjectManager.java @@ -46,7 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import org.apache.tools.tar.TarOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -178,7 +178,7 @@ public abstract class ProjectManager { * @param tos * @throws IOException */ - public abstract void exportProject(long projectId, TarOutputStream tos) throws IOException; + public abstract void exportProject(long projectId, TarArchiveOutputStream tos) throws IOException; /** diff --git a/main/src/com/google/refine/importing/ImportingUtilities.java b/main/src/com/google/refine/importing/ImportingUtilities.java index 8000a0b25..d01d64066 100644 --- a/main/src/com/google/refine/importing/ImportingUtilities.java +++ b/main/src/com/google/refine/importing/ImportingUtilities.java @@ -60,6 +60,9 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.ProgressListener; import org.apache.commons.fileupload.disk.DiskFileItemFactory; @@ -73,9 +76,6 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DecompressingHttpClient; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; -import org.apache.tools.bzip2.CBZip2InputStream; -import org.apache.tools.tar.TarEntry; -import org.apache.tools.tar.TarInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -592,11 +592,11 @@ public class ImportingUtilities { String fileName = file.getName(); try { if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) { - return new TarInputStream(new GZIPInputStream(new FileInputStream(file))); + return new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(file))); } else if (fileName.endsWith(".tar.bz2")) { - return new TarInputStream(new CBZip2InputStream(new FileInputStream(file))); + return new TarArchiveInputStream(new BZip2CompressorInputStream(new FileInputStream(file))); } else if (fileName.endsWith(".tar") || "application/x-tar".equals(contentType)) { - return new TarInputStream(new FileInputStream(file)); + return new TarArchiveInputStream(new FileInputStream(file)); } else if (fileName.endsWith(".zip") || "application/x-zip-compressed".equals(contentType) || "application/zip".equals(contentType) @@ -618,11 +618,11 @@ public class ImportingUtilities { ArrayNode fileRecords, final Progress progress ) { - if (archiveIS instanceof TarInputStream) { - TarInputStream tis = (TarInputStream) archiveIS; + if (archiveIS instanceof TarArchiveInputStream) { + TarArchiveInputStream tis = (TarArchiveInputStream) archiveIS; try { - TarEntry te; - while (!progress.isCanceled() && (te = tis.getNextEntry()) != null) { + TarArchiveEntry te; + while (!progress.isCanceled() && (te = tis.getNextTarEntry()) != null) { if (!te.isDirectory()) { String fileName2 = te.getName(); File file2 = allocateFile(rawDataDir, fileName2); @@ -702,7 +702,7 @@ public class ImportingUtilities { // No BZ prefix as appended by command line tools. Reset and hope for the best is.reset(); } - return new CBZip2InputStream(is); + return new BZip2CompressorInputStream(is); } } catch (IOException e) { logger.warn("Something that looked like a compressed file gave an error on open: "+file,e); diff --git a/main/src/com/google/refine/io/FileProjectManager.java b/main/src/com/google/refine/io/FileProjectManager.java index 75e4fa231..e8b2d74c2 100644 --- a/main/src/com/google/refine/io/FileProjectManager.java +++ b/main/src/com/google/refine/io/FileProjectManager.java @@ -36,12 +36,9 @@ package com.google.refine.io; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map.Entry; @@ -49,9 +46,10 @@ import java.util.Set; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import org.apache.tools.tar.TarEntry; -import org.apache.tools.tar.TarInputStream; -import org.apache.tools.tar.TarOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.poi.util.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -162,10 +160,10 @@ public class FileProjectManager extends ProjectManager { } protected void untar(File destDir, InputStream inputStream) throws IOException { - TarInputStream tin = new TarInputStream(inputStream); - TarEntry tarEntry = null; + TarArchiveInputStream tin = new TarArchiveInputStream(inputStream); + TarArchiveEntry tarEntry = null; - while ((tarEntry = tin.getNextEntry()) != null) { + while ((tarEntry = tin.getNextTarEntry()) != null) { File destEntry = new File(destDir, tarEntry.getName()); File parent = destEntry.getParentFile(); @@ -178,7 +176,7 @@ public class FileProjectManager extends ProjectManager { } else { FileOutputStream fout = new FileOutputStream(destEntry); try { - tin.copyEntryContents(fout); + IOUtils.copy(tin, fout); } finally { fout.close(); } @@ -189,12 +187,12 @@ public class FileProjectManager extends ProjectManager { } @Override - public void exportProject(long projectId, TarOutputStream tos) throws IOException { + public void exportProject(long projectId, TarArchiveOutputStream tos) throws IOException { File dir = this.getProjectDir(projectId); this.tarDir("", dir, tos); } - protected void tarDir(String relative, File dir, TarOutputStream tos) throws IOException{ + protected void tarDir(String relative, File dir, TarArchiveOutputStream tos) throws IOException{ File[] files = dir.listFiles(); if (files == null) return; for (File file : files) { @@ -205,17 +203,17 @@ public class FileProjectManager extends ProjectManager { if (file.isDirectory()) { tarDir(path + File.separator, file, tos); } else { - TarEntry entry = new TarEntry(path); + TarArchiveEntry entry = new TarArchiveEntry(path); - entry.setMode(TarEntry.DEFAULT_FILE_MODE); + entry.setMode(TarArchiveEntry.DEFAULT_FILE_MODE); entry.setSize(file.length()); entry.setModTime(file.lastModified()); - tos.putNextEntry(entry); + tos.putArchiveEntry(entry); copyFile(file, tos); - tos.closeEntry(); + tos.closeArchiveEntry(); } } } @@ -427,7 +425,7 @@ public class FileProjectManager extends ProjectManager { public static void gzipTarToOutputStream(Project project, OutputStream os) throws IOException { GZIPOutputStream gos = new GZIPOutputStream(os); - TarOutputStream tos = new TarOutputStream(gos); + TarArchiveOutputStream tos = new TarArchiveOutputStream(gos); try { ProjectManager.singleton.exportProject(project.id, tos); } finally { diff --git a/main/tests/server/src/com/google/refine/ProjectManagerStub.java b/main/tests/server/src/com/google/refine/ProjectManagerStub.java index 9c111d568..4671c86bc 100644 --- a/main/tests/server/src/com/google/refine/ProjectManagerStub.java +++ b/main/tests/server/src/com/google/refine/ProjectManagerStub.java @@ -36,7 +36,7 @@ package com.google.refine; import java.io.IOException; import java.io.InputStream; -import org.apache.tools.tar.TarOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import com.google.refine.ProjectManager; import com.google.refine.ProjectMetadata; @@ -56,7 +56,7 @@ public class ProjectManagerStub extends ProjectManager { } @Override - public void exportProject(long projectId, TarOutputStream tos) throws IOException { + public void exportProject(long projectId, TarArchiveOutputStream tos) throws IOException { // empty }