s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Replace Apache Ant with Commons Compress (#2691)

Browse Source 
NOTE: Changes the public API where some of the old types were
embedded which means that any extensions that extend these
interfaces will have to be updated.

Fixes #2690.
This commit is contained in:
Tom Morris 2020-06-11 10:39:51 -04:00 committed by GitHub
parent 7c9b78d076
commit 18c18e587e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 33 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
main/pom.xml
View File

@ -236,9 +236,9 @@
<version>2.4</version> <version>2.4</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.ant</groupId> <groupId>org.apache.commons</groupId>
<artifactId>ant</artifactId> <!-- for org.apache.tools.bzip2 --> <artifactId>commons-compress</artifactId>
<version>1.8.0</version> <version>1.20</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jsoup</groupId> <groupId>org.jsoup</groupId>

4
main/src/com/google/refine/ProjectManager.java
View File

@ -46,7 +46,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import org.apache.tools.tar.TarOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -178,7 +178,7 @@ public abstract class ProjectManager {
* @param tos * @param tos
* @throws IOException * @throws IOException
*/ */
public abstract void exportProject(long projectId, TarOutputStream tos) throws IOException; public abstract void exportProject(long projectId, TarArchiveOutputStream tos) throws IOException;
/** /**

22
main/src/com/google/refine/importing/ImportingUtilities.java
View File

@ -60,6 +60,9 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.ProgressListener; import org.apache.commons.fileupload.ProgressListener;
import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.disk.DiskFileItemFactory;
@ -73,9 +76,6 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DecompressingHttpClient; import org.apache.http.impl.client.DecompressingHttpClient;
import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.apache.tools.bzip2.CBZip2InputStream;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarInputStream;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -592,11 +592,11 @@ public class ImportingUtilities {
String fileName = file.getName(); String fileName = file.getName();
try { try {
if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) { if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
return new TarInputStream(new GZIPInputStream(new FileInputStream(file))); return new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(file)));
} else if (fileName.endsWith(".tar.bz2")) { } else if (fileName.endsWith(".tar.bz2")) {
return new TarInputStream(new CBZip2InputStream(new FileInputStream(file))); return new TarArchiveInputStream(new BZip2CompressorInputStream(new FileInputStream(file)));
} else if (fileName.endsWith(".tar") || "application/x-tar".equals(contentType)) { } else if (fileName.endsWith(".tar") || "application/x-tar".equals(contentType)) {
return new TarInputStream(new FileInputStream(file)); return new TarArchiveInputStream(new FileInputStream(file));
} else if (fileName.endsWith(".zip") } else if (fileName.endsWith(".zip")
|| "application/x-zip-compressed".equals(contentType) || "application/x-zip-compressed".equals(contentType)
|| "application/zip".equals(contentType) || "application/zip".equals(contentType)
@ -618,11 +618,11 @@ public class ImportingUtilities {
ArrayNode fileRecords, ArrayNode fileRecords,
final Progress progress final Progress progress
) { ) {
if (archiveIS instanceof TarInputStream) { if (archiveIS instanceof TarArchiveInputStream) {
TarInputStream tis = (TarInputStream) archiveIS; TarArchiveInputStream tis = (TarArchiveInputStream) archiveIS;
try { try {
TarEntry te; TarArchiveEntry te;
while (!progress.isCanceled() && (te = tis.getNextEntry()) != null) { while (!progress.isCanceled() && (te = tis.getNextTarEntry()) != null) {
if (!te.isDirectory()) { if (!te.isDirectory()) {
String fileName2 = te.getName(); String fileName2 = te.getName();
File file2 = allocateFile(rawDataDir, fileName2); File file2 = allocateFile(rawDataDir, fileName2);
@ -702,7 +702,7 @@ public class ImportingUtilities {
// No BZ prefix as appended by command line tools. Reset and hope for the best // No BZ prefix as appended by command line tools. Reset and hope for the best
is.reset(); is.reset();
} }
return new CBZip2InputStream(is); return new BZip2CompressorInputStream(is);
} }
} catch (IOException e) { } catch (IOException e) {
logger.warn("Something that looked like a compressed file gave an error on open: "+file,e); logger.warn("Something that looked like a compressed file gave an error on open: "+file,e);

32
main/src/com/google/refine/io/FileProjectManager.java
View File

@ -36,12 +36,9 @@ package com.google.refine.io;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map.Entry; import java.util.Map.Entry;
@ -49,9 +46,10 @@ import java.util.Set;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
import org.apache.tools.tar.TarEntry; import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.tools.tar.TarInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.tools.tar.TarOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.poi.util.IOUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -162,10 +160,10 @@ public class FileProjectManager extends ProjectManager {
} }
protected void untar(File destDir, InputStream inputStream) throws IOException { protected void untar(File destDir, InputStream inputStream) throws IOException {
TarInputStream tin = new TarInputStream(inputStream); TarArchiveInputStream tin = new TarArchiveInputStream(inputStream);
TarEntry tarEntry = null; TarArchiveEntry tarEntry = null;
while ((tarEntry = tin.getNextEntry()) != null) { while ((tarEntry = tin.getNextTarEntry()) != null) {
File destEntry = new File(destDir, tarEntry.getName()); File destEntry = new File(destDir, tarEntry.getName());
File parent = destEntry.getParentFile(); File parent = destEntry.getParentFile();
@ -178,7 +176,7 @@ public class FileProjectManager extends ProjectManager {
} else { } else {
FileOutputStream fout = new FileOutputStream(destEntry); FileOutputStream fout = new FileOutputStream(destEntry);
try { try {
tin.copyEntryContents(fout); IOUtils.copy(tin, fout);
} finally { } finally {
fout.close(); fout.close();
} }
@ -189,12 +187,12 @@ public class FileProjectManager extends ProjectManager {
} }
@Override @Override
public void exportProject(long projectId, TarOutputStream tos) throws IOException { public void exportProject(long projectId, TarArchiveOutputStream tos) throws IOException {
File dir = this.getProjectDir(projectId); File dir = this.getProjectDir(projectId);
this.tarDir("", dir, tos); this.tarDir("", dir, tos);
} }
protected void tarDir(String relative, File dir, TarOutputStream tos) throws IOException{ protected void tarDir(String relative, File dir, TarArchiveOutputStream tos) throws IOException{
File[] files = dir.listFiles(); File[] files = dir.listFiles();
if (files == null) return; if (files == null) return;
for (File file : files) { for (File file : files) {
@ -205,17 +203,17 @@ public class FileProjectManager extends ProjectManager {
if (file.isDirectory()) { if (file.isDirectory()) {
tarDir(path + File.separator, file, tos); tarDir(path + File.separator, file, tos);
} else { } else {
TarEntry entry = new TarEntry(path); TarArchiveEntry entry = new TarArchiveEntry(path);
entry.setMode(TarEntry.DEFAULT_FILE_MODE); entry.setMode(TarArchiveEntry.DEFAULT_FILE_MODE);
entry.setSize(file.length()); entry.setSize(file.length());
entry.setModTime(file.lastModified()); entry.setModTime(file.lastModified());
tos.putNextEntry(entry); tos.putArchiveEntry(entry);
copyFile(file, tos); copyFile(file, tos);
tos.closeEntry(); tos.closeArchiveEntry();
} }
} }
} }
@ -427,7 +425,7 @@ public class FileProjectManager extends ProjectManager {
public static void gzipTarToOutputStream(Project project, OutputStream os) throws IOException { public static void gzipTarToOutputStream(Project project, OutputStream os) throws IOException {
GZIPOutputStream gos = new GZIPOutputStream(os); GZIPOutputStream gos = new GZIPOutputStream(os);
TarOutputStream tos = new TarOutputStream(gos); TarArchiveOutputStream tos = new TarArchiveOutputStream(gos);
try { try {
ProjectManager.singleton.exportProject(project.id, tos); ProjectManager.singleton.exportProject(project.id, tos);
} finally { } finally {

4
main/tests/server/src/com/google/refine/ProjectManagerStub.java
View File

@ -36,7 +36,7 @@ package com.google.refine;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import org.apache.tools.tar.TarOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata; import com.google.refine.ProjectMetadata;
@ -56,7 +56,7 @@ public class ProjectManagerStub extends ProjectManager {
} }
@Override @Override
public void exportProject(long projectId, TarOutputStream tos) throws IOException { public void exportProject(long projectId, TarArchiveOutputStream tos) throws IOException {
// empty // empty
} }