s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed - issue 544,600,618: Clean up handling of compressed files & archives with multi-segment paths

Browse Source 
http://code.google.com/p/google-refine/issues/detail?id=600


git-svn-id: http://google-refine.googlecode.com/svn/trunk@2569 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2012-09-22 18:08:56 +00:00
parent 748e205ae8
commit 2c52a00f55
1 changed files with 12 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
main/src/com/google/refine/importing/ImportingUtilities.java
View File

@ -313,7 +313,7 @@ public class ImportingUtilities {
progress.setProgress("Saving " + urlString + " locally", progress.setProgress("Saving " + urlString + " locally",
calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize)); calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
if (postProcessRetrievedFile(file, fileRecord, fileRecords, progress)) { if (postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress)) {
archiveCount++; archiveCount++;
} }
@ -348,7 +348,7 @@ public class ImportingUtilities {
calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize)); calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null)); JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
if (postProcessRetrievedFile(file, fileRecord, fileRecords, progress)) { if (postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress)) {
archiveCount++; archiveCount++;
} }
@ -441,7 +441,7 @@ public class ImportingUtilities {
long length = 0; long length = 0;
FileOutputStream fos = new FileOutputStream(file); FileOutputStream fos = new FileOutputStream(file);
try { try {
byte[] bytes = new byte[4096]; byte[] bytes = new byte[16*1024];
int c; int c;
while ((update == null || !update.isCanceled()) && (c = stream.read(bytes)) > 0) { while ((update == null || !update.isCanceled()) && (c = stream.read(bytes)) > 0) {
fos.write(bytes, 0, c); fos.write(bytes, 0, c);
@ -459,11 +459,10 @@ public class ImportingUtilities {
} }
static public boolean postProcessRetrievedFile( static public boolean postProcessRetrievedFile(
File file, JSONObject fileRecord, JSONArray fileRecords, final Progress progress) { File rawDataDir, File file, JSONObject fileRecord, JSONArray fileRecords, final Progress progress) {
String mimeType = JSONUtilities.getString(fileRecord, "declaredMimeType", null); String mimeType = JSONUtilities.getString(fileRecord, "declaredMimeType", null);
String contentEncoding = JSONUtilities.getString(fileRecord, "declaredEncoding", null); String contentEncoding = JSONUtilities.getString(fileRecord, "declaredEncoding", null);
File rawDataDir = file.getParentFile();
InputStream archiveIS = tryOpenAsArchive(file, mimeType, contentEncoding); InputStream archiveIS = tryOpenAsArchive(file, mimeType, contentEncoding);
if (archiveIS != null) { if (archiveIS != null) {
@ -651,7 +650,13 @@ public class ImportingUtilities {
JSONObject fileRecord, JSONObject fileRecord,
final Progress progress final Progress progress
) throws IOException { ) throws IOException {
String fileName = JSONUtilities.getString(fileRecord, "fileName", "unknown"); String fileName = JSONUtilities.getString(fileRecord, "location", "unknown");
for (String ext : new String[] {".gz",".bz2"}) {
if (fileName.endsWith(ext)) {
fileName = fileName.substring(0, fileName.length()-ext.length());
break;
}
}
File file2 = allocateFile(rawDataDir, fileName); File file2 = allocateFile(rawDataDir, fileName);
progress.setProgress("Uncompressing " + fileName, -1); progress.setProgress("Uncompressing " + fileName, -1);
@ -659,11 +664,8 @@ public class ImportingUtilities {
saveStreamToFile(uncompressedIS, file2, null); saveStreamToFile(uncompressedIS, file2, null);
JSONUtilities.safePut(fileRecord, "declaredEncoding", (String) null); JSONUtilities.safePut(fileRecord, "declaredEncoding", (String) null);
// TODO: Why is MIME type cleared here?
JSONUtilities.safePut(fileRecord, "declaredMimeType", (String) null); JSONUtilities.safePut(fileRecord, "declaredMimeType", (String) null);
String location = JSONUtilities.getString(fileRecord, "location", ""); JSONUtilities.safePut(fileRecord, "location", getRelativePath(file2, rawDataDir));
location = location.substring(0,location.lastIndexOf('/')) + "/" + file2;
JSONUtilities.safePut(fileRecord, "location", location);
return file2; return file2;
} }