From a1ec0ea8df0b5b4afb62774a4444e237c9e95aa9 Mon Sep 17 00:00:00 2001 From: David Huynh Date: Mon, 8 Mar 2010 00:37:06 +0000 Subject: [PATCH] When saving projects, save only modified ones. Save projects and workspace periodically. git-svn-id: http://google-refine.googlecode.com/svn/trunk@232 7d457c2a-affb-35e4-300a-418c747d4874 --- .../metaweb/gridworks/GridworksServlet.java | 24 ++- .../com/metaweb/gridworks/ProjectManager.java | 165 +++++++++++++----- .../com/metaweb/gridworks/model/Project.java | 58 +++--- 3 files changed, 166 insertions(+), 81 deletions(-) diff --git a/src/main/java/com/metaweb/gridworks/GridworksServlet.java b/src/main/java/com/metaweb/gridworks/GridworksServlet.java index d902c2771..8ea2b1e86 100644 --- a/src/main/java/com/metaweb/gridworks/GridworksServlet.java +++ b/src/main/java/com/metaweb/gridworks/GridworksServlet.java @@ -3,6 +3,8 @@ package com.metaweb.gridworks; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -54,6 +56,7 @@ public class GridworksServlet extends HttpServlet { private static final long serialVersionUID = 2386057901503517403L; static protected Map _commands = new HashMap(); + static protected Timer _timer = new Timer(); static { _commands.put("create-project-from-upload", new CreateProjectCommand()); @@ -111,13 +114,26 @@ public class GridworksServlet extends HttpServlet { @Override public void init() throws ServletException { super.init(); + + ProjectManager.initialize(); + + long period = 1000 * 60 * 5; // 5 minutes + _timer.scheduleAtFixedRate(new TimerTask() { + + @Override + public void run() { + ProjectManager.singleton.save(false); + } + }, period, period); } @Override public void destroy() { + _timer.cancel(); + _timer = null; + if (ProjectManager.singleton != null) { - ProjectManager.singleton.saveAllProjects(); - ProjectManager.singleton.save(); + ProjectManager.singleton.save(true); ProjectManager.singleton = null; } @@ -125,8 +141,6 @@ public class GridworksServlet extends HttpServlet { } protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - ProjectManager.initialize(); - Command command = _commands.get(getCommandName(request)); if (command != null) { command.doPost(request, response); @@ -134,8 +148,6 @@ public class GridworksServlet extends HttpServlet { } protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - ProjectManager.initialize(); - Command command = _commands.get(getCommandName(request)); if (command != null) { command.doGet(request, response); diff --git a/src/main/java/com/metaweb/gridworks/ProjectManager.java b/src/main/java/com/metaweb/gridworks/ProjectManager.java index c790aadd8..75ba90f8f 100644 --- a/src/main/java/com/metaweb/gridworks/ProjectManager.java +++ b/src/main/java/com/metaweb/gridworks/ProjectManager.java @@ -4,6 +4,10 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -28,14 +32,14 @@ public class ProjectManager { protected Map _projectsMetadata; protected List _expressions; - transient protected Map _projects; + transient protected Map _projects; static public ProjectManager singleton; static public void initialize() { if (singleton == null) { File dir = getProjectLocation(); - Gridworks.log("Using data directory: " + dir.getAbsolutePath()); + Gridworks.log("Using workspace directory: " + dir.getAbsolutePath()); singleton = new ProjectManager(dir); } @@ -107,8 +111,10 @@ public class ProjectManager { } public void registerProject(Project project, ProjectMetadata projectMetadata) { - _projects.put(project.id, project); - _projectsMetadata.put(project.id, projectMetadata); + synchronized (this) { + _projects.put(project.id, project); + _projectsMetadata.put(project.id, projectMetadata); + } } public ProjectMetadata getProjectMetadata(long id) { @@ -143,32 +149,37 @@ public class ProjectManager { return _expressions; } - public void save() { - Gridworks.log("Saving all projects' metadata ..."); - - File tempFile = new File(_workspaceDir, "workspace.temp.json"); - try { - saveToFile(tempFile); - } catch (Exception e) { - e.printStackTrace(); + public void save(boolean allModified) { + saveProjects(allModified); + saveWorkspace(); + } + + protected void saveWorkspace() { + synchronized (this) { + File tempFile = new File(_workspaceDir, "workspace.temp.json"); + try { + saveToFile(tempFile); + } catch (Exception e) { + e.printStackTrace(); + + Gridworks.log("Failed to save workspace."); + return; + } + + File file = new File(_workspaceDir, "workspace.json"); + File oldFile = new File(_workspaceDir, "workspace.old.json"); - Gridworks.log("Failed to save projects' metadata."); - return; + if (file.exists()) { + file.renameTo(oldFile); + } + + tempFile.renameTo(file); + if (oldFile.exists()) { + oldFile.delete(); + } + + Gridworks.log("Saved workspace."); } - - File file = new File(_workspaceDir, "workspace.json"); - File oldFile = new File(_workspaceDir, "workspace.old.json"); - - if (file.exists()) { - file.renameTo(oldFile); - } - - tempFile.renameTo(file); - if (oldFile.exists()) { - oldFile.delete(); - } - - Gridworks.log("All projects' metadata saved."); } protected void saveToFile(File file) throws IOException, JSONException { @@ -198,31 +209,91 @@ public class ProjectManager { } } - public void saveAllProjects() { - Gridworks.log("Saving all projects ..."); - for (Project project : _projects.values()) { - try { - project.save(); - } catch (Exception e) { - e.printStackTrace(); + static protected class SaveRecord { + final Project project; + final long overdue; + + SaveRecord(Project project, long overdue) { + this.project = project; + this.overdue = overdue; + } + } + + protected void saveProjects(boolean allModified) { + List records = new ArrayList(); + Date now = new Date(); + + synchronized (this) { + for (long id : _projectsMetadata.keySet()) { + ProjectMetadata metadata = _projectsMetadata.get(id); + Project project = _projects.get(id); + + if (project != null) { + boolean hasUnsavedChanges = metadata.getModified().getTime() > project.lastSave.getTime(); + + if (hasUnsavedChanges) { + long msecsOverdue = now.getTime() - project.lastSave.getTime(); + + records.add(new SaveRecord(project, msecsOverdue)); + + } else if (now.getTime() - project.lastSave.getTime() > 1000 * 60 * 60) { + /* + * It's been 1 hour since the project was last saved and it hasn't + * been modified. We can safely remove it from the cache to save some memory. + */ + _projects.remove(id); + } + } + } + } + + if (records.size() > 0) { + Collections.sort(records, new Comparator() { + public int compare(SaveRecord o1, SaveRecord o2) { + if (o1.overdue < o2.overdue) { + return 1; + } else if (o1.overdue > o2.overdue) { + return -1; + } else { + return 0; + } + } + }); + + Gridworks.log(allModified ? + "Saving all modified projects ..." : + "Saving some modified projects ..." + ); + + for (int i = 0; + i < records.size() && + (allModified || (new Date().getTime() - now.getTime() < 30000)); i++) { + + try { + records.get(i).project.save(); + } catch (Exception e) { + e.printStackTrace(); + } } } } public void deleteProject(Project project) { - if (_projectsMetadata.containsKey(project.id)) { - _projectsMetadata.remove(project.id); - } - if (_projects.containsKey(project.id)) { - _projects.remove(project.id); + synchronized (this) { + if (_projectsMetadata.containsKey(project.id)) { + _projectsMetadata.remove(project.id); + } + if (_projects.containsKey(project.id)) { + _projects.remove(project.id); + } + + File dir = getProjectDir(project.id); + if (dir.exists()) { + dir.delete(); + } } - File dir = getProjectDir(project.id); - if (dir.exists()) { - dir.delete(); - } - - save(); + saveWorkspace(); } protected void load() { @@ -246,7 +317,7 @@ public class ProjectManager { } protected void loadFromFile(File file) throws IOException, JSONException { - Gridworks.log("Loading project metadata from " + file.getAbsolutePath()); + Gridworks.log("Loading workspace from " + file.getAbsolutePath()); _projectsMetadata.clear(); _expressions.clear(); diff --git a/src/main/java/com/metaweb/gridworks/model/Project.java b/src/main/java/com/metaweb/gridworks/model/Project.java index 5d2ffcd8a..ec1812e96 100644 --- a/src/main/java/com/metaweb/gridworks/model/Project.java +++ b/src/main/java/com/metaweb/gridworks/model/Project.java @@ -11,6 +11,7 @@ import java.io.Writer; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.List; import java.util.Properties; import java.util.zip.ZipEntry; @@ -34,18 +35,17 @@ public class Project { public Protograph protograph; - transient public ProcessManager processManager; + transient public ProcessManager processManager = new ProcessManager(); + transient public Date lastSave = new Date(); public Project() { id = System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L); history = new History(this); - processManager = new ProcessManager(); } protected Project(long id) { this.id = id; this.history = new History(this); - this.processManager = new ProcessManager(); } public ProjectMetadata getMetadata() { @@ -53,33 +53,35 @@ public class Project { } public void save() { - Gridworks.log("Saving project " + id + " ..."); - - File dir = ProjectManager.singleton.getProjectDir(id); - - File tempFile = new File(dir, "data.temp.zip"); - try { - saveToFile(tempFile); - } catch (Exception e) { - e.printStackTrace(); + synchronized (this) { + File dir = ProjectManager.singleton.getProjectDir(id); - Gridworks.log("Failed to save project data"); - return; + File tempFile = new File(dir, "data.temp.zip"); + try { + saveToFile(tempFile); + } catch (Exception e) { + e.printStackTrace(); + + Gridworks.log("Failed to save project " + id); + return; + } + + File file = new File(dir, "data.zip"); + File oldFile = new File(dir, "data.old.zip"); + + if (file.exists()) { + file.renameTo(oldFile); + } + + tempFile.renameTo(file); + if (oldFile.exists()) { + oldFile.delete(); + } + + lastSave = new Date(); + + Gridworks.log("Saved project " + id + "."); } - - File file = new File(dir, "data.zip"); - File oldFile = new File(dir, "data.old.zip"); - - if (file.exists()) { - file.renameTo(oldFile); - } - - tempFile.renameTo(file); - if (oldFile.exists()) { - oldFile.delete(); - } - - Gridworks.log("Project saved."); } protected void saveToFile(File file) throws Exception {