When saving projects, save only modified ones.

Save projects and workspace periodically.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@232 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/GridworksServlet.java

@@ -3,6 +3,8 @@ package com.metaweb.gridworks;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
@@ -54,6 +56,7 @@ public class GridworksServlet extends HttpServlet {
     private static final long serialVersionUID = 2386057901503517403L;
     
     static protected Map<String, Command> _commands = new HashMap<String, Command>();
+    static protected Timer _timer = new Timer();
     
     static {
         _commands.put("create-project-from-upload", new CreateProjectCommand());
@@ -111,13 +114,26 @@ public class GridworksServlet extends HttpServlet {
     @Override
     public void init() throws ServletException {
         super.init();
+        
+        ProjectManager.initialize();
+        
+        long period = 1000 * 60 * 5; // 5 minutes
+        _timer.scheduleAtFixedRate(new TimerTask() {
+            
+            @Override
+            public void run() {
+                ProjectManager.singleton.save(false);
+            }
+        }, period, period);
     }
     
     @Override
     public void destroy() {
+        _timer.cancel();
+        _timer = null;
+        
         if (ProjectManager.singleton != null) {
-            ProjectManager.singleton.saveAllProjects();
+            ProjectManager.singleton.save(true);
-            ProjectManager.singleton.save();
             ProjectManager.singleton = null;
         }
         
@@ -125,8 +141,6 @@ public class GridworksServlet extends HttpServlet {
     }
     
     protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
-        ProjectManager.initialize();
-        
         Command command = _commands.get(getCommandName(request));
         if (command != null) {
             command.doPost(request, response);
@@ -134,8 +148,6 @@ public class GridworksServlet extends HttpServlet {
     }
     
     protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
-        ProjectManager.initialize();
-        
         Command command = _commands.get(getCommandName(request));
         if (command != null) {
             command.doGet(request, response);

src/main/java/com/metaweb/gridworks/ProjectManager.java

@@ -4,6 +4,10 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -28,14 +32,14 @@ public class ProjectManager {
     protected Map<Long, ProjectMetadata> _projectsMetadata;
     protected List<String> 				 _expressions;
     
-    transient protected Map<Long, Project> _projects;
+    transient protected Map<Long, Project>  _projects;
     
     static public ProjectManager singleton;
     
     static public void initialize() {
         if (singleton == null) {
             File dir = getProjectLocation();
-            Gridworks.log("Using data directory: " + dir.getAbsolutePath());
+            Gridworks.log("Using workspace directory: " + dir.getAbsolutePath());
             
             singleton = new ProjectManager(dir);
         }
@@ -107,8 +111,10 @@ public class ProjectManager {
     }
     
     public void registerProject(Project project, ProjectMetadata projectMetadata) {
-        _projects.put(project.id, project);
+        synchronized (this) {
-        _projectsMetadata.put(project.id, projectMetadata);
+            _projects.put(project.id, project);
+            _projectsMetadata.put(project.id, projectMetadata);
+        }
     }
     
     public ProjectMetadata getProjectMetadata(long id) {
@@ -143,32 +149,37 @@ public class ProjectManager {
     	return _expressions;
     }
     
-    public void save() {
+    public void save(boolean allModified) {
-    	Gridworks.log("Saving all projects' metadata ...");
+        saveProjects(allModified);
-    	
+        saveWorkspace();
-    	File tempFile = new File(_workspaceDir, "workspace.temp.json");
+    }
-    	try {
+    
-            saveToFile(tempFile);
+    protected void saveWorkspace() {
-        } catch (Exception e) {
+        synchronized (this) {
-            e.printStackTrace();
+        	File tempFile = new File(_workspaceDir, "workspace.temp.json");
+        	try {
+                saveToFile(tempFile);
+            } catch (Exception e) {
+                e.printStackTrace();
+                
+                Gridworks.log("Failed to save workspace.");
+                return;
+            }
+        	
+            File file = new File(_workspaceDir, "workspace.json");
+            File oldFile = new File(_workspaceDir, "workspace.old.json");
             
-            Gridworks.log("Failed to save projects' metadata.");
+            if (file.exists()) {
-            return;
+                file.renameTo(oldFile);
+            }
+            
+            tempFile.renameTo(file);
+            if (oldFile.exists()) {
+                oldFile.delete();
+            }
+            
+            Gridworks.log("Saved workspace.");
         }
-    	
-        File file = new File(_workspaceDir, "workspace.json");
-        File oldFile = new File(_workspaceDir, "workspace.old.json");
-        
-        if (file.exists()) {
-            file.renameTo(oldFile);
-        }
-        
-        tempFile.renameTo(file);
-        if (oldFile.exists()) {
-            oldFile.delete();
-        }
-        
-        Gridworks.log("All projects' metadata saved.");
     }
     
     protected void saveToFile(File file) throws IOException, JSONException {
@@ -198,31 +209,91 @@ public class ProjectManager {
         }
     }
     
-    public void saveAllProjects() {
+    static protected class SaveRecord {
-        Gridworks.log("Saving all projects ...");
+        final Project project;
-        for (Project project : _projects.values()) {
+        final long overdue;
-            try {
+        
-                project.save();
+        SaveRecord(Project project, long overdue) {
-            } catch (Exception e) {
+            this.project = project;
-                e.printStackTrace();
+            this.overdue = overdue;
+        }
+    }
+    
+    protected void saveProjects(boolean allModified) {
+        List<SaveRecord> records = new ArrayList<SaveRecord>();
+        Date now = new Date();
+        
+        synchronized (this) {
+            for (long id : _projectsMetadata.keySet()) {
+                ProjectMetadata metadata = _projectsMetadata.get(id);
+                Project project = _projects.get(id);
+                
+                if (project != null) {
+                    boolean hasUnsavedChanges = metadata.getModified().getTime() > project.lastSave.getTime();
+                    
+                    if (hasUnsavedChanges) {
+                        long msecsOverdue = now.getTime() - project.lastSave.getTime();
+                        
+                        records.add(new SaveRecord(project, msecsOverdue));
+                        
+                    } else if (now.getTime() - project.lastSave.getTime() > 1000 * 60 * 60) {
+                        /*
+                         *  It's been 1 hour since the project was last saved and it hasn't
+                         *  been modified. We can safely remove it from the cache to save some memory.
+                         */
+                        _projects.remove(id);
+                    }
+                }
+            }
+        }
+        
+        if (records.size() > 0) {
+            Collections.sort(records, new Comparator<SaveRecord>() {
+                public int compare(SaveRecord o1, SaveRecord o2) {
+                    if (o1.overdue < o2.overdue) {
+                        return 1;
+                    } else if (o1.overdue > o2.overdue) {
+                        return -1;
+                    } else {
+                        return 0;
+                    }
+                }
+            });
+            
+            Gridworks.log(allModified ?
+                "Saving all modified projects ..." :
+                "Saving some modified projects ..."
+            );
+            
+            for (int i = 0; 
+                 i < records.size() && 
+                    (allModified || (new Date().getTime() - now.getTime() < 30000)); i++) {
+                
+                try {
+                    records.get(i).project.save();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
             }
         }
     }
     
     public void deleteProject(Project project) {
-        if (_projectsMetadata.containsKey(project.id)) {
+        synchronized (this) {
-            _projectsMetadata.remove(project.id);
+            if (_projectsMetadata.containsKey(project.id)) {
-        }
+                _projectsMetadata.remove(project.id);
-        if (_projects.containsKey(project.id)) {
+            }
-            _projects.remove(project.id);
+            if (_projects.containsKey(project.id)) {
+                _projects.remove(project.id);
+            }
+            
+            File dir = getProjectDir(project.id);
+            if (dir.exists()) {
+                dir.delete();
+            }
         }
         
-        File dir = getProjectDir(project.id);
+        saveWorkspace();
-        if (dir.exists()) {
-            dir.delete();
-        }
-        
-        save();
     }
     
     protected void load() {
@@ -246,7 +317,7 @@ public class ProjectManager {
     }
     
     protected void loadFromFile(File file) throws IOException, JSONException {
-        Gridworks.log("Loading project metadata from " + file.getAbsolutePath());
+        Gridworks.log("Loading workspace from " + file.getAbsolutePath());
         
         _projectsMetadata.clear();
         _expressions.clear();

src/main/java/com/metaweb/gridworks/model/Project.java

@@ -11,6 +11,7 @@ import java.io.Writer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Date;
 import java.util.List;
 import java.util.Properties;
 import java.util.zip.ZipEntry;
@@ -34,18 +35,17 @@ public class Project {
     
     public Protograph            protograph;
     
-    transient public ProcessManager processManager;
+    transient public ProcessManager processManager = new ProcessManager();
+    transient public Date lastSave = new Date();
     
     public Project() {
         id = System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L);
         history = new History(this);
-        processManager = new ProcessManager();
     }
     
     protected Project(long id) {
         this.id = id;
         this.history = new History(this);
-        this.processManager = new ProcessManager();
     }
     
     public ProjectMetadata getMetadata() {
@@ -53,33 +53,35 @@ public class Project {
     }
     
     public void save() {
-        Gridworks.log("Saving project " + id + " ...");
+        synchronized (this) {
-        
+            File dir = ProjectManager.singleton.getProjectDir(id);
-        File dir = ProjectManager.singleton.getProjectDir(id);
-        
-        File tempFile = new File(dir, "data.temp.zip");
-        try {
-            saveToFile(tempFile);
-        } catch (Exception e) {
-            e.printStackTrace();
             
-            Gridworks.log("Failed to save project data");
+            File tempFile = new File(dir, "data.temp.zip");
-            return;
+            try {
+                saveToFile(tempFile);
+            } catch (Exception e) {
+                e.printStackTrace();
+                
+                Gridworks.log("Failed to save project " + id);
+                return;
+            }
+            
+            File file = new File(dir, "data.zip");
+            File oldFile = new File(dir, "data.old.zip");
+            
+            if (file.exists()) {
+                file.renameTo(oldFile);
+            }
+            
+            tempFile.renameTo(file);
+            if (oldFile.exists()) {
+                oldFile.delete();
+            }
+            
+            lastSave = new Date();
+            
+            Gridworks.log("Saved project " + id + ".");
         }
-        
-        File file = new File(dir, "data.zip");
-        File oldFile = new File(dir, "data.old.zip");
-        
-        if (file.exists()) {
-            file.renameTo(oldFile);
-        }
-        
-        tempFile.renameTo(file);
-        if (oldFile.exists()) {
-            oldFile.delete();
-        }
-        
-        Gridworks.log("Project saved.");
     }
     
     protected void saveToFile(File file) throws Exception {