When saving projects, save only modified ones.

Save projects and workspace periodically.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@232 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/GridworksServlet.java

@@ -3,6 +3,8 @@ package com.metaweb.gridworks;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
@@ -54,6 +56,7 @@ public class GridworksServlet extends HttpServlet {
     private static final long serialVersionUID = 2386057901503517403L;
     
     static protected Map<String, Command> _commands = new HashMap<String, Command>();
+    static protected Timer _timer = new Timer();
     
     static {
         _commands.put("create-project-from-upload", new CreateProjectCommand());
@@ -111,13 +114,26 @@ public class GridworksServlet extends HttpServlet {
     @Override
     public void init() throws ServletException {
         super.init();
+        
+        ProjectManager.initialize();
+        
+        long period = 1000 * 60 * 5; // 5 minutes
+        _timer.scheduleAtFixedRate(new TimerTask() {
+            
+            @Override
+            public void run() {
+                ProjectManager.singleton.save(false);
+            }
+        }, period, period);
     }
     
     @Override
     public void destroy() {
+        _timer.cancel();
+        _timer = null;
+        
         if (ProjectManager.singleton != null) {
-            ProjectManager.singleton.saveAllProjects();
+            ProjectManager.singleton.save(true);
-            ProjectManager.singleton.save();
             ProjectManager.singleton = null;
         }
         
@@ -125,8 +141,6 @@ public class GridworksServlet extends HttpServlet {
     }
     
     protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
-        ProjectManager.initialize();
-        
         Command command = _commands.get(getCommandName(request));
         if (command != null) {
             command.doPost(request, response);
@@ -134,8 +148,6 @@ public class GridworksServlet extends HttpServlet {
     }
     
     protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
-        ProjectManager.initialize();
-        
         Command command = _commands.get(getCommandName(request));
         if (command != null) {
             command.doGet(request, response);

src/main/java/com/metaweb/gridworks/ProjectManager.java

@@ -4,6 +4,10 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -35,7 +39,7 @@ public class ProjectManager {
     static public void initialize() {
         if (singleton == null) {
             File dir = getProjectLocation();
-            Gridworks.log("Using data directory: " + dir.getAbsolutePath());
+            Gridworks.log("Using workspace directory: " + dir.getAbsolutePath());
             
             singleton = new ProjectManager(dir);
         }
@@ -107,9 +111,11 @@ public class ProjectManager {
     }
     
     public void registerProject(Project project, ProjectMetadata projectMetadata) {
+        synchronized (this) {
             _projects.put(project.id, project);
             _projectsMetadata.put(project.id, projectMetadata);
         }
+    }
     
     public ProjectMetadata getProjectMetadata(long id) {
         return _projectsMetadata.get(id);
@@ -143,16 +149,20 @@ public class ProjectManager {
     	return _expressions;
     }
     
-    public void save() {
+    public void save(boolean allModified) {
-    	Gridworks.log("Saving all projects' metadata ...");
+        saveProjects(allModified);
+        saveWorkspace();
+    }
     
+    protected void saveWorkspace() {
+        synchronized (this) {
         	File tempFile = new File(_workspaceDir, "workspace.temp.json");
         	try {
                 saveToFile(tempFile);
             } catch (Exception e) {
                 e.printStackTrace();
                 
-            Gridworks.log("Failed to save projects' metadata.");
+                Gridworks.log("Failed to save workspace.");
                 return;
             }
         	
@@ -168,7 +178,8 @@ public class ProjectManager {
                 oldFile.delete();
             }
             
-        Gridworks.log("All projects' metadata saved.");
+            Gridworks.log("Saved workspace.");
+        }
     }
     
     protected void saveToFile(File file) throws IOException, JSONException {
@@ -198,18 +209,77 @@ public class ProjectManager {
         }
     }
     
-    public void saveAllProjects() {
+    static protected class SaveRecord {
-        Gridworks.log("Saving all projects ...");
+        final Project project;
-        for (Project project : _projects.values()) {
+        final long overdue;
+        
+        SaveRecord(Project project, long overdue) {
+            this.project = project;
+            this.overdue = overdue;
+        }
+    }
+    
+    protected void saveProjects(boolean allModified) {
+        List<SaveRecord> records = new ArrayList<SaveRecord>();
+        Date now = new Date();
+        
+        synchronized (this) {
+            for (long id : _projectsMetadata.keySet()) {
+                ProjectMetadata metadata = _projectsMetadata.get(id);
+                Project project = _projects.get(id);
+                
+                if (project != null) {
+                    boolean hasUnsavedChanges = metadata.getModified().getTime() > project.lastSave.getTime();
+                    
+                    if (hasUnsavedChanges) {
+                        long msecsOverdue = now.getTime() - project.lastSave.getTime();
+                        
+                        records.add(new SaveRecord(project, msecsOverdue));
+                        
+                    } else if (now.getTime() - project.lastSave.getTime() > 1000 * 60 * 60) {
+                        /*
+                         *  It's been 1 hour since the project was last saved and it hasn't
+                         *  been modified. We can safely remove it from the cache to save some memory.
+                         */
+                        _projects.remove(id);
+                    }
+                }
+            }
+        }
+        
+        if (records.size() > 0) {
+            Collections.sort(records, new Comparator<SaveRecord>() {
+                public int compare(SaveRecord o1, SaveRecord o2) {
+                    if (o1.overdue < o2.overdue) {
+                        return 1;
+                    } else if (o1.overdue > o2.overdue) {
+                        return -1;
+                    } else {
+                        return 0;
+                    }
+                }
+            });
+            
+            Gridworks.log(allModified ?
+                "Saving all modified projects ..." :
+                "Saving some modified projects ..."
+            );
+            
+            for (int i = 0; 
+                 i < records.size() && 
+                    (allModified || (new Date().getTime() - now.getTime() < 30000)); i++) {
+                
                 try {
-                project.save();
+                    records.get(i).project.save();
                 } catch (Exception e) {
                     e.printStackTrace();
                 }
             }
         }
+    }
     
     public void deleteProject(Project project) {
+        synchronized (this) {
             if (_projectsMetadata.containsKey(project.id)) {
                 _projectsMetadata.remove(project.id);
             }
@@ -221,8 +291,9 @@ public class ProjectManager {
             if (dir.exists()) {
                 dir.delete();
             }
+        }
         
-        save();
+        saveWorkspace();
     }
     
     protected void load() {
@@ -246,7 +317,7 @@ public class ProjectManager {
     }
     
     protected void loadFromFile(File file) throws IOException, JSONException {
-        Gridworks.log("Loading project metadata from " + file.getAbsolutePath());
+        Gridworks.log("Loading workspace from " + file.getAbsolutePath());
         
         _projectsMetadata.clear();
         _expressions.clear();

src/main/java/com/metaweb/gridworks/model/Project.java

@@ -11,6 +11,7 @@ import java.io.Writer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Date;
 import java.util.List;
 import java.util.Properties;
 import java.util.zip.ZipEntry;
@@ -34,18 +35,17 @@ public class Project {
     
     public Protograph            protograph;
     
-    transient public ProcessManager processManager;
+    transient public ProcessManager processManager = new ProcessManager();
+    transient public Date lastSave = new Date();
     
     public Project() {
         id = System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L);
         history = new History(this);
-        processManager = new ProcessManager();
     }
     
     protected Project(long id) {
         this.id = id;
         this.history = new History(this);
-        this.processManager = new ProcessManager();
     }
     
     public ProjectMetadata getMetadata() {
@@ -53,8 +53,7 @@ public class Project {
     }
     
     public void save() {
-        Gridworks.log("Saving project " + id + " ...");
+        synchronized (this) {
-        
             File dir = ProjectManager.singleton.getProjectDir(id);
             
             File tempFile = new File(dir, "data.temp.zip");
@@ -63,7 +62,7 @@ public class Project {
             } catch (Exception e) {
                 e.printStackTrace();
                 
-            Gridworks.log("Failed to save project data");
+                Gridworks.log("Failed to save project " + id);
                 return;
             }
             
@@ -79,7 +78,10 @@ public class Project {
                 oldFile.delete();
             }
             
-        Gridworks.log("Project saved.");
+            lastSave = new Date();
+            
+            Gridworks.log("Saved project " + id + ".");
+        }
     }
     
     protected void saveToFile(File file) throws Exception {