When saving projects, save only modified ones.

Save projects and workspace periodically.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@232 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-03-08 00:37:06 +00:00
parent 404883da92
commit a1ec0ea8df
3 changed files with 166 additions and 81 deletions

View File

@ -3,6 +3,8 @@ package com.metaweb.gridworks;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServlet;
@ -54,6 +56,7 @@ public class GridworksServlet extends HttpServlet {
private static final long serialVersionUID = 2386057901503517403L; private static final long serialVersionUID = 2386057901503517403L;
static protected Map<String, Command> _commands = new HashMap<String, Command>(); static protected Map<String, Command> _commands = new HashMap<String, Command>();
static protected Timer _timer = new Timer();
static { static {
_commands.put("create-project-from-upload", new CreateProjectCommand()); _commands.put("create-project-from-upload", new CreateProjectCommand());
@ -111,13 +114,26 @@ public class GridworksServlet extends HttpServlet {
@Override @Override
public void init() throws ServletException { public void init() throws ServletException {
super.init(); super.init();
ProjectManager.initialize();
long period = 1000 * 60 * 5; // 5 minutes
_timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
ProjectManager.singleton.save(false);
}
}, period, period);
} }
@Override @Override
public void destroy() { public void destroy() {
_timer.cancel();
_timer = null;
if (ProjectManager.singleton != null) { if (ProjectManager.singleton != null) {
ProjectManager.singleton.saveAllProjects(); ProjectManager.singleton.save(true);
ProjectManager.singleton.save();
ProjectManager.singleton = null; ProjectManager.singleton = null;
} }
@ -125,8 +141,6 @@ public class GridworksServlet extends HttpServlet {
} }
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
ProjectManager.initialize();
Command command = _commands.get(getCommandName(request)); Command command = _commands.get(getCommandName(request));
if (command != null) { if (command != null) {
command.doPost(request, response); command.doPost(request, response);
@ -134,8 +148,6 @@ public class GridworksServlet extends HttpServlet {
} }
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
ProjectManager.initialize();
Command command = _commands.get(getCommandName(request)); Command command = _commands.get(getCommandName(request));
if (command != null) { if (command != null) {
command.doGet(request, response); command.doGet(request, response);

View File

@ -4,6 +4,10 @@ import java.io.File;
import java.io.FileReader; import java.io.FileReader;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
@ -35,7 +39,7 @@ public class ProjectManager {
static public void initialize() { static public void initialize() {
if (singleton == null) { if (singleton == null) {
File dir = getProjectLocation(); File dir = getProjectLocation();
Gridworks.log("Using data directory: " + dir.getAbsolutePath()); Gridworks.log("Using workspace directory: " + dir.getAbsolutePath());
singleton = new ProjectManager(dir); singleton = new ProjectManager(dir);
} }
@ -107,9 +111,11 @@ public class ProjectManager {
} }
public void registerProject(Project project, ProjectMetadata projectMetadata) { public void registerProject(Project project, ProjectMetadata projectMetadata) {
synchronized (this) {
_projects.put(project.id, project); _projects.put(project.id, project);
_projectsMetadata.put(project.id, projectMetadata); _projectsMetadata.put(project.id, projectMetadata);
} }
}
public ProjectMetadata getProjectMetadata(long id) { public ProjectMetadata getProjectMetadata(long id) {
return _projectsMetadata.get(id); return _projectsMetadata.get(id);
@ -143,16 +149,20 @@ public class ProjectManager {
return _expressions; return _expressions;
} }
public void save() { public void save(boolean allModified) {
Gridworks.log("Saving all projects' metadata ..."); saveProjects(allModified);
saveWorkspace();
}
protected void saveWorkspace() {
synchronized (this) {
File tempFile = new File(_workspaceDir, "workspace.temp.json"); File tempFile = new File(_workspaceDir, "workspace.temp.json");
try { try {
saveToFile(tempFile); saveToFile(tempFile);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
Gridworks.log("Failed to save projects' metadata."); Gridworks.log("Failed to save workspace.");
return; return;
} }
@ -168,7 +178,8 @@ public class ProjectManager {
oldFile.delete(); oldFile.delete();
} }
Gridworks.log("All projects' metadata saved."); Gridworks.log("Saved workspace.");
}
} }
protected void saveToFile(File file) throws IOException, JSONException { protected void saveToFile(File file) throws IOException, JSONException {
@ -198,18 +209,77 @@ public class ProjectManager {
} }
} }
public void saveAllProjects() { static protected class SaveRecord {
Gridworks.log("Saving all projects ..."); final Project project;
for (Project project : _projects.values()) { final long overdue;
SaveRecord(Project project, long overdue) {
this.project = project;
this.overdue = overdue;
}
}
protected void saveProjects(boolean allModified) {
List<SaveRecord> records = new ArrayList<SaveRecord>();
Date now = new Date();
synchronized (this) {
for (long id : _projectsMetadata.keySet()) {
ProjectMetadata metadata = _projectsMetadata.get(id);
Project project = _projects.get(id);
if (project != null) {
boolean hasUnsavedChanges = metadata.getModified().getTime() > project.lastSave.getTime();
if (hasUnsavedChanges) {
long msecsOverdue = now.getTime() - project.lastSave.getTime();
records.add(new SaveRecord(project, msecsOverdue));
} else if (now.getTime() - project.lastSave.getTime() > 1000 * 60 * 60) {
/*
* It's been 1 hour since the project was last saved and it hasn't
* been modified. We can safely remove it from the cache to save some memory.
*/
_projects.remove(id);
}
}
}
}
if (records.size() > 0) {
Collections.sort(records, new Comparator<SaveRecord>() {
public int compare(SaveRecord o1, SaveRecord o2) {
if (o1.overdue < o2.overdue) {
return 1;
} else if (o1.overdue > o2.overdue) {
return -1;
} else {
return 0;
}
}
});
Gridworks.log(allModified ?
"Saving all modified projects ..." :
"Saving some modified projects ..."
);
for (int i = 0;
i < records.size() &&
(allModified || (new Date().getTime() - now.getTime() < 30000)); i++) {
try { try {
project.save(); records.get(i).project.save();
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
}
public void deleteProject(Project project) { public void deleteProject(Project project) {
synchronized (this) {
if (_projectsMetadata.containsKey(project.id)) { if (_projectsMetadata.containsKey(project.id)) {
_projectsMetadata.remove(project.id); _projectsMetadata.remove(project.id);
} }
@ -221,8 +291,9 @@ public class ProjectManager {
if (dir.exists()) { if (dir.exists()) {
dir.delete(); dir.delete();
} }
}
save(); saveWorkspace();
} }
protected void load() { protected void load() {
@ -246,7 +317,7 @@ public class ProjectManager {
} }
protected void loadFromFile(File file) throws IOException, JSONException { protected void loadFromFile(File file) throws IOException, JSONException {
Gridworks.log("Loading project metadata from " + file.getAbsolutePath()); Gridworks.log("Loading workspace from " + file.getAbsolutePath());
_projectsMetadata.clear(); _projectsMetadata.clear();
_expressions.clear(); _expressions.clear();

View File

@ -11,6 +11,7 @@ import java.io.Writer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
@ -34,18 +35,17 @@ public class Project {
public Protograph protograph; public Protograph protograph;
transient public ProcessManager processManager; transient public ProcessManager processManager = new ProcessManager();
transient public Date lastSave = new Date();
public Project() { public Project() {
id = System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L); id = System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L);
history = new History(this); history = new History(this);
processManager = new ProcessManager();
} }
protected Project(long id) { protected Project(long id) {
this.id = id; this.id = id;
this.history = new History(this); this.history = new History(this);
this.processManager = new ProcessManager();
} }
public ProjectMetadata getMetadata() { public ProjectMetadata getMetadata() {
@ -53,8 +53,7 @@ public class Project {
} }
public void save() { public void save() {
Gridworks.log("Saving project " + id + " ..."); synchronized (this) {
File dir = ProjectManager.singleton.getProjectDir(id); File dir = ProjectManager.singleton.getProjectDir(id);
File tempFile = new File(dir, "data.temp.zip"); File tempFile = new File(dir, "data.temp.zip");
@ -63,7 +62,7 @@ public class Project {
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
Gridworks.log("Failed to save project data"); Gridworks.log("Failed to save project " + id);
return; return;
} }
@ -79,7 +78,10 @@ public class Project {
oldFile.delete(); oldFile.delete();
} }
Gridworks.log("Project saved."); lastSave = new Date();
Gridworks.log("Saved project " + id + ".");
}
} }
protected void saveToFile(File file) throws Exception { protected void saveToFile(File file) throws Exception {