From 7b5b549113d0d25ba8feef5c80d94562890df140 Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Mon, 5 Aug 2013 14:13:56 -0400 Subject: [PATCH] More project saving changes for #528 - reduce project retention in memory from 1 hr to 15 min. - free all unmodified projects if we get an error on save (we could be running low on memory) - make sure exceptions propagate up to where they can be usefully handled --- .../src/com/google/refine/ProjectManager.java | 30 ++++++++++++++----- .../google/refine/io/FileProjectManager.java | 2 +- .../google/refine/io/ProjectUtilities.java | 15 ++++++---- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/main/src/com/google/refine/ProjectManager.java b/main/src/com/google/refine/ProjectManager.java index c6a744e8f..0be3d87ae 100644 --- a/main/src/com/google/refine/ProjectManager.java +++ b/main/src/com/google/refine/ProjectManager.java @@ -119,7 +119,6 @@ public abstract class ProjectManager { _projectsMetadata.put(project.id, projectMetadata); } } - //----------Load from data store to memory---------------- /** * Load project metadata from data storage @@ -135,7 +134,6 @@ public abstract class ProjectManager { */ protected abstract Project loadProject(long id); - //------------Import and Export from Refine archive----------------- /** * Import project from a Refine archive * @param projectID @@ -154,7 +152,6 @@ public abstract class ProjectManager { public abstract void exportProject(long projectId, TarOutputStream tos) throws IOException; - //------------Save to record store------------ /** * Saves a project and its metadata to the data store * @param id @@ -194,8 +191,9 @@ public abstract class ProjectManager { /** * Save project to the data store * @param project + * @throws IOException */ - protected abstract void saveProject(Project project); + protected abstract void saveProject(Project project) throws IOException; /** * Save workspace and all projects to data store @@ -204,6 +202,7 @@ public abstract class ProjectManager { public void save(boolean allModified) { if (allModified || _busy == 0) { saveProjects(allModified); + // TODO: Only save workspace if it's dirty saveWorkspace(); } } @@ -227,7 +226,7 @@ public abstract class ProjectManager { } } - static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour + static protected final int s_projectFlushDelay = 1000 * 60 * 15; // 15 minutes static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs /** @@ -296,12 +295,29 @@ public abstract class ProjectManager { saveProject(records.get(i).project); } catch (Exception e) { e.printStackTrace(); + // In case we're running low on memory, free as much as we can + disposeUnmodifiedProjects(); + } + } + } + } + + /** + * Flush all unmodified projects from memory. + */ + protected void disposeUnmodifiedProjects() { + synchronized (this) { + for (long id : _projectsMetadata.keySet()) { + ProjectMetadata metadata = getProjectMetadata(id); + Project project = _projects.get(id); + if (project != null && !project.getProcessManager().hasPending() + && metadata.getModified().getTime() < project.getLastSave().getTime()) { + _projects.remove(id).dispose(); } } } } - //--------------Get from memory-------------- /** * Gets the InterProjectModel from memory */ @@ -405,7 +421,6 @@ public abstract class ProjectManager { */ public abstract HistoryEntryManager getHistoryEntryManager(); - //-------------remove project----------- /** * Remove the project from the data store @@ -434,7 +449,6 @@ public abstract class ProjectManager { } } - //--------------Miscellaneous----------- /** * Sets the flag for long running operations. This will prevent diff --git a/main/src/com/google/refine/io/FileProjectManager.java b/main/src/com/google/refine/io/FileProjectManager.java index 472c1df2d..1fa219215 100644 --- a/main/src/com/google/refine/io/FileProjectManager.java +++ b/main/src/com/google/refine/io/FileProjectManager.java @@ -217,7 +217,7 @@ public class FileProjectManager extends ProjectManager { } @Override - protected void saveProject(Project project){ + protected void saveProject(Project project) throws IOException{ ProjectUtilities.save(project); } diff --git a/main/src/com/google/refine/io/ProjectUtilities.java b/main/src/com/google/refine/io/ProjectUtilities.java index 0599d5f26..4088cfeb9 100644 --- a/main/src/com/google/refine/io/ProjectUtilities.java +++ b/main/src/com/google/refine/io/ProjectUtilities.java @@ -35,6 +35,7 @@ package com.google.refine.io; import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; @@ -50,7 +51,7 @@ import com.google.refine.util.Pool; public class ProjectUtilities { final static Logger logger = LoggerFactory.getLogger("project_utilities"); - synchronized public static void save(Project project) { + synchronized public static void save(Project project) throws IOException { synchronized (project) { long id = project.id; File dir = ((FileProjectManager)ProjectManager.singleton).getProjectDir(id); @@ -58,11 +59,15 @@ public class ProjectUtilities { File tempFile = new File(dir, "data.temp.zip"); try { saveToFile(project, tempFile); - } catch (Exception e) { + } catch (IOException e) { e.printStackTrace(); - logger.warn("Failed to save project {}", id); - return; + try { + tempFile.delete(); + } catch (Exception e2) { + // just ignore - file probably was never created. + } + throw e; } File file = new File(dir, "data.zip"); @@ -83,7 +88,7 @@ public class ProjectUtilities { } } - protected static void saveToFile(Project project, File file) throws Exception { + protected static void saveToFile(Project project, File file) throws IOException { ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file)); try { Pool pool = new Pool();