s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

When saving projects, save only modified ones.

Browse Source 
Save projects and workspace periodically.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@232 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-03-08 00:37:06 +00:00
parent 404883da92
commit a1ec0ea8df
3 changed files with 166 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/main/java/com/metaweb/gridworks/GridworksServlet.java
View File

@ -3,6 +3,8 @@ package com.metaweb.gridworks;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
@ -54,6 +56,7 @@ public class GridworksServlet extends HttpServlet {
private static final long serialVersionUID = 2386057901503517403L;
static protected Map<String, Command> _commands = new HashMap<String, Command>();
static protected Timer _timer = new Timer();
static {
_commands.put("create-project-from-upload", new CreateProjectCommand());
@ -111,13 +114,26 @@ public class GridworksServlet extends HttpServlet {
@Override
public void init() throws ServletException {
super.init();
ProjectManager.initialize();
long period = 1000 * 60 * 5; // 5 minutes
_timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
ProjectManager.singleton.save(false);
}
}, period, period);
}
@Override
public void destroy() {
_timer.cancel();
_timer = null;
if (ProjectManager.singleton != null) {
ProjectManager.singleton.saveAllProjects();
ProjectManager.singleton.save();
ProjectManager.singleton.save(true);
ProjectManager.singleton = null;
}
@ -125,8 +141,6 @@ public class GridworksServlet extends HttpServlet {
}
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
ProjectManager.initialize();
Command command = _commands.get(getCommandName(request));
if (command != null) {
command.doPost(request, response);
@ -134,8 +148,6 @@ public class GridworksServlet extends HttpServlet {
}
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
ProjectManager.initialize();
Command command = _commands.get(getCommandName(request));
if (command != null) {
command.doGet(request, response);

165
src/main/java/com/metaweb/gridworks/ProjectManager.java
View File

@ -4,6 +4,10 @@ import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@ -28,14 +32,14 @@ public class ProjectManager {
protected Map<Long, ProjectMetadata> _projectsMetadata;
protected List<String> _expressions;
transient protected Map<Long, Project> _projects;
transient protected Map<Long, Project> _projects;
static public ProjectManager singleton;
static public void initialize() {
if (singleton == null) {
File dir = getProjectLocation();
Gridworks.log("Using data directory: " + dir.getAbsolutePath());
Gridworks.log("Using workspace directory: " + dir.getAbsolutePath());
singleton = new ProjectManager(dir);
}
@ -107,8 +111,10 @@ public class ProjectManager {
}
public void registerProject(Project project, ProjectMetadata projectMetadata) {
_projects.put(project.id, project);
_projectsMetadata.put(project.id, projectMetadata);
synchronized (this) {
_projects.put(project.id, project);
_projectsMetadata.put(project.id, projectMetadata);
}
}
public ProjectMetadata getProjectMetadata(long id) {
@ -143,32 +149,37 @@ public class ProjectManager {
return _expressions;
}
public void save() {
Gridworks.log("Saving all projects' metadata ...");
File tempFile = new File(_workspaceDir, "workspace.temp.json");
try {
saveToFile(tempFile);
} catch (Exception e) {
e.printStackTrace();
public void save(boolean allModified) {
saveProjects(allModified);
saveWorkspace();
}
protected void saveWorkspace() {
synchronized (this) {
File tempFile = new File(_workspaceDir, "workspace.temp.json");
try {
saveToFile(tempFile);
} catch (Exception e) {
e.printStackTrace();
Gridworks.log("Failed to save workspace.");
return;
}
File file = new File(_workspaceDir, "workspace.json");
File oldFile = new File(_workspaceDir, "workspace.old.json");
Gridworks.log("Failed to save projects' metadata.");
return;
if (file.exists()) {
file.renameTo(oldFile);
}
tempFile.renameTo(file);
if (oldFile.exists()) {
oldFile.delete();
}
Gridworks.log("Saved workspace.");
}
File file = new File(_workspaceDir, "workspace.json");
File oldFile = new File(_workspaceDir, "workspace.old.json");
if (file.exists()) {
file.renameTo(oldFile);
}
tempFile.renameTo(file);
if (oldFile.exists()) {
oldFile.delete();
}
Gridworks.log("All projects' metadata saved.");
}
protected void saveToFile(File file) throws IOException, JSONException {
@ -198,31 +209,91 @@ public class ProjectManager {
}
}
public void saveAllProjects() {
Gridworks.log("Saving all projects ...");
for (Project project : _projects.values()) {
try {
project.save();
} catch (Exception e) {
e.printStackTrace();
static protected class SaveRecord {
final Project project;
final long overdue;
SaveRecord(Project project, long overdue) {
this.project = project;
this.overdue = overdue;
}
}
protected void saveProjects(boolean allModified) {
List<SaveRecord> records = new ArrayList<SaveRecord>();
Date now = new Date();
synchronized (this) {
for (long id : _projectsMetadata.keySet()) {
ProjectMetadata metadata = _projectsMetadata.get(id);
Project project = _projects.get(id);
if (project != null) {
boolean hasUnsavedChanges = metadata.getModified().getTime() > project.lastSave.getTime();
if (hasUnsavedChanges) {
long msecsOverdue = now.getTime() - project.lastSave.getTime();
records.add(new SaveRecord(project, msecsOverdue));
} else if (now.getTime() - project.lastSave.getTime() > 1000 * 60 * 60) {
/*
* It's been 1 hour since the project was last saved and it hasn't
* been modified. We can safely remove it from the cache to save some memory.
*/
_projects.remove(id);
}
}
}
}
if (records.size() > 0) {
Collections.sort(records, new Comparator<SaveRecord>() {
public int compare(SaveRecord o1, SaveRecord o2) {
if (o1.overdue < o2.overdue) {
return 1;
} else if (o1.overdue > o2.overdue) {
return -1;
} else {
return 0;
}
}
});
Gridworks.log(allModified ?
"Saving all modified projects ..." :
"Saving some modified projects ..."
);
for (int i = 0;
i < records.size() &&
(allModified || (new Date().getTime() - now.getTime() < 30000)); i++) {
try {
records.get(i).project.save();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
public void deleteProject(Project project) {
if (_projectsMetadata.containsKey(project.id)) {
_projectsMetadata.remove(project.id);
}
if (_projects.containsKey(project.id)) {
_projects.remove(project.id);
synchronized (this) {
if (_projectsMetadata.containsKey(project.id)) {
_projectsMetadata.remove(project.id);
}
if (_projects.containsKey(project.id)) {
_projects.remove(project.id);
}
File dir = getProjectDir(project.id);
if (dir.exists()) {
dir.delete();
}
}
File dir = getProjectDir(project.id);
if (dir.exists()) {
dir.delete();
}
save();
saveWorkspace();
}
protected void load() {
@ -246,7 +317,7 @@ public class ProjectManager {
}
protected void loadFromFile(File file) throws IOException, JSONException {
Gridworks.log("Loading project metadata from " + file.getAbsolutePath());
Gridworks.log("Loading workspace from " + file.getAbsolutePath());
_projectsMetadata.clear();
_expressions.clear();

58
src/main/java/com/metaweb/gridworks/model/Project.java
View File

@ -11,6 +11,7 @@ import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.zip.ZipEntry;
@ -34,18 +35,17 @@ public class Project {
public Protograph protograph;
transient public ProcessManager processManager;
transient public ProcessManager processManager = new ProcessManager();
transient public Date lastSave = new Date();
public Project() {
id = System.currentTimeMillis() + Math.round(Math.random() * 1000000000000L);
history = new History(this);
processManager = new ProcessManager();
}
protected Project(long id) {
this.id = id;
this.history = new History(this);
this.processManager = new ProcessManager();
}
public ProjectMetadata getMetadata() {
@ -53,33 +53,35 @@ public class Project {
}
public void save() {
Gridworks.log("Saving project " + id + " ...");
File dir = ProjectManager.singleton.getProjectDir(id);
File tempFile = new File(dir, "data.temp.zip");
try {
saveToFile(tempFile);
} catch (Exception e) {
e.printStackTrace();
synchronized (this) {
File dir = ProjectManager.singleton.getProjectDir(id);
Gridworks.log("Failed to save project data");
return;
File tempFile = new File(dir, "data.temp.zip");
try {
saveToFile(tempFile);
} catch (Exception e) {
e.printStackTrace();
Gridworks.log("Failed to save project " + id);
return;
}
File file = new File(dir, "data.zip");
File oldFile = new File(dir, "data.old.zip");
if (file.exists()) {
file.renameTo(oldFile);
}
tempFile.renameTo(file);
if (oldFile.exists()) {
oldFile.delete();
}
lastSave = new Date();
Gridworks.log("Saved project " + id + ".");
}
File file = new File(dir, "data.zip");
File oldFile = new File(dir, "data.old.zip");
if (file.exists()) {
file.renameTo(oldFile);
}
tempFile.renameTo(file);
if (oldFile.exists()) {
oldFile.delete();
}
Gridworks.log("Project saved.");
}
protected void saveToFile(File file) throws Exception {