From d90e75dff1b4b8a38fb0ca43dfbe34198f96c9b1 Mon Sep 17 00:00:00 2001 From: David Huynh Date: Sat, 20 Mar 2010 23:56:28 +0000 Subject: [PATCH] Started a round of documentation. git-svn-id: http://google-refine.googlecode.com/svn/trunk@329 7d457c2a-affb-35e4-300a-418c747d4874 --- .../metaweb/gridworks/GridworksServlet.java | 20 ++- .../com/metaweb/gridworks/Jsonizable.java | 6 + .../com/metaweb/gridworks/ProjectManager.java | 122 +++++++++++++----- .../metaweb/gridworks/ProjectMetadata.java | 24 ++-- 4 files changed, 121 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/metaweb/gridworks/GridworksServlet.java b/src/main/java/com/metaweb/gridworks/GridworksServlet.java index 9d2114685..bc50648f3 100644 --- a/src/main/java/com/metaweb/gridworks/GridworksServlet.java +++ b/src/main/java/com/metaweb/gridworks/GridworksServlet.java @@ -60,6 +60,8 @@ public class GridworksServlet extends HttpServlet { private static final long serialVersionUID = 2386057901503517403L; static protected Map _commands = new HashMap(); + + // timer for periodically saving projects static protected Timer _timer = new Timer(); static { @@ -128,21 +130,22 @@ public class GridworksServlet extends HttpServlet { long period = 1000 * 60 * 5; // 5 minutes _timer.scheduleAtFixedRate(new TimerTask() { - @Override public void run() { - ProjectManager.singleton.save(false); + ProjectManager.singleton.save(false); // quick, potentially incomplete save } }, period, period); } @Override public void destroy() { - _timer.cancel(); - _timer = null; - + // cancel automatic periodic saving and force a complete save. + if (_timer != null) { + _timer.cancel(); + _timer = null; + } if (ProjectManager.singleton != null) { - ProjectManager.singleton.save(true); + ProjectManager.singleton.save(true); // complete save ProjectManager.singleton = null; } @@ -164,6 +167,11 @@ public class GridworksServlet extends HttpServlet { } protected String getCommandName(HttpServletRequest request) { + /* + * Remove extraneous path segments that might be there for other purposes, + * e.g., for /export-rows/filename.ext, export-rows is the command while + * filename.ext is only for the browser to prompt a convenient filename. + */ String commandName = request.getPathInfo().substring(1); int slash = commandName.indexOf('/'); return slash > 0 ? commandName.substring(0, slash) : commandName; diff --git a/src/main/java/com/metaweb/gridworks/Jsonizable.java b/src/main/java/com/metaweb/gridworks/Jsonizable.java index 27b5286b5..b11931f9a 100644 --- a/src/main/java/com/metaweb/gridworks/Jsonizable.java +++ b/src/main/java/com/metaweb/gridworks/Jsonizable.java @@ -5,6 +5,12 @@ import java.util.Properties; import org.json.JSONException; import org.json.JSONWriter; +/** + * Interface for streaming out JSON, either into HTTP responses or + * serialization files. + * + * @author dfhuynh + */ public interface Jsonizable { public void write(JSONWriter writer, Properties options) throws JSONException; } diff --git a/src/main/java/com/metaweb/gridworks/ProjectManager.java b/src/main/java/com/metaweb/gridworks/ProjectManager.java index 72f2202a4..cc4e90b80 100644 --- a/src/main/java/com/metaweb/gridworks/ProjectManager.java +++ b/src/main/java/com/metaweb/gridworks/ProjectManager.java @@ -26,13 +26,20 @@ import com.metaweb.gridworks.util.JSONUtilities; public class ProjectManager { - private static final int s_expressionHistoryMax = 100; // last n expressions used across all projects + // last n expressions used across all projects + static protected final int s_expressionHistoryMax = 100; protected File _workspaceDir; protected Map _projectsMetadata; protected List _expressions; - transient protected Map _projects; + /** + * While each project's metadata is loaded completely at start-up, each project's raw data + * is loaded only when the project is accessed by the user. This is because project + * metadata is tiny compared to raw project data. This hash map from project ID to project + * is more like a last accessed-last out cache. + */ + transient protected Map _projects; static public ProjectManager singleton; @@ -57,25 +64,37 @@ public class ProjectManager { // NOTE(SM): finding the "local data app" in windows from java is actually a PITA // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java // so we're using a library that uses JNI to ask directly the win32 APIs, - // it's not elegant but it's the safest bet + // it's not elegant but it's the safest bet. + DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks"); File data = new File(localDataPath.getPath()); data.mkdirs(); return data; } catch (Error e) { - Gridworks.log("Failed to use jdatapath to detect user data path. Resorting to environment variables."); - - String appData = System.getenv("APPDATA"); - File parentDir = null; - if (appData != null && appData.length() > 0) { - parentDir = new File(appData); - } else { - String userProfile = System.getenv("USERPROFILE"); - if (userProfile != null && userProfile.length() > 0) { - parentDir = new File(userProfile); - } - } + /* + * The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll + * we include is compiled for 32-bit. In this case, we just have to dig up + * environment variables and try our best to find a user-specific path. + */ + + Gridworks.log( + "Failed to use jdatapath to detect user data path. " + + "Resorting to environment variables."); + File parentDir = null; + { + String appData = System.getenv("APPDATA"); + if (appData != null && appData.length() > 0) { + // e.g., C:\Users\[userid]\AppData\Roaming + parentDir = new File(appData); + } else { + String userProfile = System.getenv("USERPROFILE"); + if (userProfile != null && userProfile.length() > 0) { + // e.g., C:\Users\[userid] + parentDir = new File(userProfile); + } + } + } if (parentDir == null) { parentDir = new File("."); } @@ -141,6 +160,12 @@ public class ProjectManager { } } + /** + * Import an external project that has been received as a .tar file, expanded, and + * copied into our workspace directory. + * + * @param projectID + */ public void importProject(long projectID) { synchronized (this) { ProjectMetadata metadata = ProjectMetadata.load(getProjectDir(projectID)); @@ -149,6 +174,12 @@ public class ProjectManager { } } + /** + * Make sure that a project's metadata and data are saved to file. For example, + * this method is called before the project is exported to a .tar file. + * + * @param id + */ public void ensureProjectSaved(long id) { synchronized (this) { File projectDir = getProjectDir(id); @@ -182,23 +213,27 @@ public class ProjectManager { } public Project getProject(long id) { - if (_projects.containsKey(id)) { - return _projects.get(id); - } else { - Project project = Project.load(getProjectDir(id), id); - - _projects.put(id, project); - - return project; + synchronized (this) { + if (_projects.containsKey(id)) { + return _projects.get(id); + } else { + Project project = Project.load(getProjectDir(id), id); + + _projects.put(id, project); + + return project; + } } } public void addLatestExpression(String s) { - _expressions.remove(s); - _expressions.add(0, s); - while (_expressions.size() > s_expressionHistoryMax) { - _expressions.remove(_expressions.size() - 1); - } + synchronized (this) { + _expressions.remove(s); + _expressions.add(0, s); + while (_expressions.size() > s_expressionHistoryMax) { + _expressions.remove(_expressions.size() - 1); + } + } } public List getExpressions() { @@ -210,6 +245,10 @@ public class ProjectManager { saveWorkspace(); } + /** + * Save the workspace's data out to file in a safe way: save to a temporary file first + * and rename it to the real file. + */ protected void saveWorkspace() { synchronized (this) { File tempFile = new File(_workspaceDir, "workspace.temp.json"); @@ -265,6 +304,10 @@ public class ProjectManager { } } + /** + * A utility class to prioritize projects for saving, depending on how long ago + * they have been changed but have not been saved. + */ static protected class SaveRecord { final Project project; final long overdue; @@ -275,29 +318,37 @@ public class ProjectManager { } } + static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour + static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs + protected void saveProjects(boolean allModified) { List records = new ArrayList(); Date now = new Date(); + boolean gc = false; + synchronized (this) { for (long id : _projectsMetadata.keySet()) { ProjectMetadata metadata = _projectsMetadata.get(id); Project project = _projects.get(id); if (project != null) { - boolean hasUnsavedChanges = metadata.getModified().getTime() > project.lastSave.getTime(); + boolean hasUnsavedChanges = + metadata.getModified().getTime() > project.lastSave.getTime(); if (hasUnsavedChanges) { long msecsOverdue = now.getTime() - project.lastSave.getTime(); records.add(new SaveRecord(project, msecsOverdue)); - } else if (now.getTime() - project.lastSave.getTime() > 1000 * 60 * 60) { + } else if (now.getTime() - project.lastSave.getTime() > s_projectFlushDelay) { /* - * It's been 1 hour since the project was last saved and it hasn't - * been modified. We can safely remove it from the cache to save some memory. + * It's been a while since the project was last saved and it hasn't been + * modified. We can safely remove it from the cache to save some memory. */ _projects.remove(id); + + gc = true; } } } @@ -323,7 +374,8 @@ public class ProjectManager { for (int i = 0; i < records.size() && - (allModified || (new Date().getTime() - now.getTime() < 30000)); i++) { + (allModified || (new Date().getTime() - now.getTime() < s_quickSaveTimeout)); + i++) { try { records.get(i).project.save(); @@ -332,6 +384,10 @@ public class ProjectManager { } } } + + if (gc) { + System.gc(); + } } public void deleteProject(Project project) { diff --git a/src/main/java/com/metaweb/gridworks/ProjectMetadata.java b/src/main/java/com/metaweb/gridworks/ProjectMetadata.java index 4b5628fad..ac9e03428 100644 --- a/src/main/java/com/metaweb/gridworks/ProjectMetadata.java +++ b/src/main/java/com/metaweb/gridworks/ProjectMetadata.java @@ -57,8 +57,8 @@ public class ProjectMetadata implements Jsonizable { writer.endObject(); } - public void save(File dir) throws Exception { - File tempFile = new File(dir, "metadata.temp.json"); + public void save(File projectDir) throws Exception { + File tempFile = new File(projectDir, "metadata.temp.json"); try { saveToFile(tempFile); } catch (Exception e) { @@ -68,8 +68,8 @@ public class ProjectMetadata implements Jsonizable { return; } - File file = new File(dir, "metadata.json"); - File oldFile = new File(dir, "metadata.old.json"); + File file = new File(projectDir, "metadata.json"); + File oldFile = new File(projectDir, "metadata.old.json"); if (file.exists()) { file.renameTo(oldFile); @@ -81,8 +81,8 @@ public class ProjectMetadata implements Jsonizable { } } - protected void saveToFile(File file) throws Exception { - Writer writer = new OutputStreamWriter(new FileOutputStream(file)); + protected void saveToFile(File metadataFile) throws Exception { + Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile)); try { Properties options = new Properties(); options.setProperty("mode", "save"); @@ -95,27 +95,27 @@ public class ProjectMetadata implements Jsonizable { } } - static public ProjectMetadata load(File dir) { + static public ProjectMetadata load(File projectDir) { try { - return loadFromFile(new File(dir, "metadata.json")); + return loadFromFile(new File(projectDir, "metadata.json")); } catch (Exception e) { } try { - return loadFromFile(new File(dir, "metadata.temp.json")); + return loadFromFile(new File(projectDir, "metadata.temp.json")); } catch (Exception e) { } try { - return loadFromFile(new File(dir, "metadata.old.json")); + return loadFromFile(new File(projectDir, "metadata.old.json")); } catch (Exception e) { } return null; } - static protected ProjectMetadata loadFromFile(File file) throws Exception { - FileReader reader = new FileReader(file); + static protected ProjectMetadata loadFromFile(File metadataFile) throws Exception { + FileReader reader = new FileReader(metadataFile); try { JSONTokener tokener = new JSONTokener(reader); JSONObject obj = (JSONObject) tokener.nextValue();