s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Started a round of documentation.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@329 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-03-20 23:56:28 +00:00
parent a43b2a72c1
commit d90e75dff1
4 changed files with 121 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/com/metaweb/gridworks/GridworksServlet.java
View File

@ -60,6 +60,8 @@ public class GridworksServlet extends HttpServlet {
private static final long serialVersionUID = 2386057901503517403L; private static final long serialVersionUID = 2386057901503517403L;
static protected Map<String, Command> _commands = new HashMap<String, Command>(); static protected Map<String, Command> _commands = new HashMap<String, Command>();
// timer for periodically saving projects
static protected Timer _timer = new Timer(); static protected Timer _timer = new Timer();
static { static {
@ -128,21 +130,22 @@ public class GridworksServlet extends HttpServlet {
long period = 1000 * 60 * 5; // 5 minutes long period = 1000 * 60 * 5; // 5 minutes
_timer.scheduleAtFixedRate(new TimerTask() { _timer.scheduleAtFixedRate(new TimerTask() {
@Override @Override
public void run() { public void run() {
ProjectManager.singleton.save(false); ProjectManager.singleton.save(false); // quick, potentially incomplete save
} }
}, period, period); }, period, period);
} }
@Override @Override
public void destroy() { public void destroy() {
_timer.cancel(); // cancel automatic periodic saving and force a complete save.
_timer = null; if (_timer != null) {
_timer.cancel();
_timer = null;
}
if (ProjectManager.singleton != null) { if (ProjectManager.singleton != null) {
ProjectManager.singleton.save(true); ProjectManager.singleton.save(true); // complete save
ProjectManager.singleton = null; ProjectManager.singleton = null;
} }
@ -164,6 +167,11 @@ public class GridworksServlet extends HttpServlet {
} }
protected String getCommandName(HttpServletRequest request) { protected String getCommandName(HttpServletRequest request) {
/*
* Remove extraneous path segments that might be there for other purposes,
* e.g., for /export-rows/filename.ext, export-rows is the command while
* filename.ext is only for the browser to prompt a convenient filename.
*/
String commandName = request.getPathInfo().substring(1); String commandName = request.getPathInfo().substring(1);
int slash = commandName.indexOf('/'); int slash = commandName.indexOf('/');
return slash > 0 ? commandName.substring(0, slash) : commandName; return slash > 0 ? commandName.substring(0, slash) : commandName;

6
src/main/java/com/metaweb/gridworks/Jsonizable.java
View File

@ -5,6 +5,12 @@ import java.util.Properties;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONWriter; import org.json.JSONWriter;
/**
* Interface for streaming out JSON, either into HTTP responses or
* serialization files.
*
* @author dfhuynh
*/
public interface Jsonizable { public interface Jsonizable {
public void write(JSONWriter writer, Properties options) throws JSONException; public void write(JSONWriter writer, Properties options) throws JSONException;
} }

122
src/main/java/com/metaweb/gridworks/ProjectManager.java
View File

@ -26,13 +26,20 @@ import com.metaweb.gridworks.util.JSONUtilities;
public class ProjectManager { public class ProjectManager {
private static final int s_expressionHistoryMax = 100; // last n expressions used across all projects // last n expressions used across all projects
static protected final int s_expressionHistoryMax = 100;
protected File _workspaceDir; protected File _workspaceDir;
protected Map<Long, ProjectMetadata> _projectsMetadata; protected Map<Long, ProjectMetadata> _projectsMetadata;
protected List<String> _expressions; protected List<String> _expressions;
transient protected Map<Long, Project> _projects; /**
* While each project's metadata is loaded completely at start-up, each project's raw data
* is loaded only when the project is accessed by the user. This is because project
* metadata is tiny compared to raw project data. This hash map from project ID to project
* is more like a last accessed-last out cache.
*/
transient protected Map<Long, Project> _projects;
static public ProjectManager singleton; static public ProjectManager singleton;
@ -57,25 +64,37 @@ public class ProjectManager {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA // NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs, // so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet // it's not elegant but it's the safest bet.
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks"); DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(localDataPath.getPath()); File data = new File(localDataPath.getPath());
data.mkdirs(); data.mkdirs();
return data; return data;
} catch (Error e) { } catch (Error e) {
Gridworks.log("Failed to use jdatapath to detect user data path. Resorting to environment variables."); /*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
String appData = System.getenv("APPDATA"); * we include is compiled for 32-bit. In this case, we just have to dig up
File parentDir = null; * environment variables and try our best to find a user-specific path.
if (appData != null && appData.length() > 0) { */
parentDir = new File(appData);
} else { Gridworks.log(
String userProfile = System.getenv("USERPROFILE"); "Failed to use jdatapath to detect user data path. " +
if (userProfile != null && userProfile.length() > 0) { "Resorting to environment variables.");
parentDir = new File(userProfile);
}
}
File parentDir = null;
{
String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) {
// e.g., C:\Users\[userid]\AppData\Roaming
parentDir = new File(appData);
} else {
String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid]
parentDir = new File(userProfile);
}
}
}
if (parentDir == null) { if (parentDir == null) {
parentDir = new File("."); parentDir = new File(".");
} }
@ -141,6 +160,12 @@ public class ProjectManager {
} }
} }
/**
* Import an external project that has been received as a .tar file, expanded, and
* copied into our workspace directory.
*
* @param projectID
*/
public void importProject(long projectID) { public void importProject(long projectID) {
synchronized (this) { synchronized (this) {
ProjectMetadata metadata = ProjectMetadata.load(getProjectDir(projectID)); ProjectMetadata metadata = ProjectMetadata.load(getProjectDir(projectID));
@ -149,6 +174,12 @@ public class ProjectManager {
} }
} }
/**
* Make sure that a project's metadata and data are saved to file. For example,
* this method is called before the project is exported to a .tar file.
*
* @param id
*/
public void ensureProjectSaved(long id) { public void ensureProjectSaved(long id) {
synchronized (this) { synchronized (this) {
File projectDir = getProjectDir(id); File projectDir = getProjectDir(id);
@ -182,23 +213,27 @@ public class ProjectManager {
} }
public Project getProject(long id) { public Project getProject(long id) {
if (_projects.containsKey(id)) { synchronized (this) {
return _projects.get(id); if (_projects.containsKey(id)) {
} else { return _projects.get(id);
Project project = Project.load(getProjectDir(id), id); } else {
Project project = Project.load(getProjectDir(id), id);
_projects.put(id, project);
_projects.put(id, project);
return project;
return project;
}
} }
} }
public void addLatestExpression(String s) { public void addLatestExpression(String s) {
_expressions.remove(s); synchronized (this) {
_expressions.add(0, s); _expressions.remove(s);
while (_expressions.size() > s_expressionHistoryMax) { _expressions.add(0, s);
_expressions.remove(_expressions.size() - 1); while (_expressions.size() > s_expressionHistoryMax) {
} _expressions.remove(_expressions.size() - 1);
}
}
} }
public List<String> getExpressions() { public List<String> getExpressions() {
@ -210,6 +245,10 @@ public class ProjectManager {
saveWorkspace(); saveWorkspace();
} }
/**
* Save the workspace's data out to file in a safe way: save to a temporary file first
* and rename it to the real file.
*/
protected void saveWorkspace() { protected void saveWorkspace() {
synchronized (this) { synchronized (this) {
File tempFile = new File(_workspaceDir, "workspace.temp.json"); File tempFile = new File(_workspaceDir, "workspace.temp.json");
@ -265,6 +304,10 @@ public class ProjectManager {
} }
} }
/**
* A utility class to prioritize projects for saving, depending on how long ago
* they have been changed but have not been saved.
*/
static protected class SaveRecord { static protected class SaveRecord {
final Project project; final Project project;
final long overdue; final long overdue;
@ -275,29 +318,37 @@ public class ProjectManager {
} }
} }
static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour
static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs
protected void saveProjects(boolean allModified) { protected void saveProjects(boolean allModified) {
List<SaveRecord> records = new ArrayList<SaveRecord>(); List<SaveRecord> records = new ArrayList<SaveRecord>();
Date now = new Date(); Date now = new Date();
boolean gc = false;
synchronized (this) { synchronized (this) {
for (long id : _projectsMetadata.keySet()) { for (long id : _projectsMetadata.keySet()) {
ProjectMetadata metadata = _projectsMetadata.get(id); ProjectMetadata metadata = _projectsMetadata.get(id);
Project project = _projects.get(id); Project project = _projects.get(id);
if (project != null) { if (project != null) {
boolean hasUnsavedChanges = metadata.getModified().getTime() > project.lastSave.getTime(); boolean hasUnsavedChanges =
metadata.getModified().getTime() > project.lastSave.getTime();
if (hasUnsavedChanges) { if (hasUnsavedChanges) {
long msecsOverdue = now.getTime() - project.lastSave.getTime(); long msecsOverdue = now.getTime() - project.lastSave.getTime();
records.add(new SaveRecord(project, msecsOverdue)); records.add(new SaveRecord(project, msecsOverdue));
} else if (now.getTime() - project.lastSave.getTime() > 1000 * 60 * 60) { } else if (now.getTime() - project.lastSave.getTime() > s_projectFlushDelay) {
/* /*
* It's been 1 hour since the project was last saved and it hasn't * It's been a while since the project was last saved and it hasn't been
* been modified. We can safely remove it from the cache to save some memory. * modified. We can safely remove it from the cache to save some memory.
*/ */
_projects.remove(id); _projects.remove(id);
gc = true;
} }
} }
} }
@ -323,7 +374,8 @@ public class ProjectManager {
for (int i = 0; for (int i = 0;
i < records.size() && i < records.size() &&
(allModified || (new Date().getTime() - now.getTime() < 30000)); i++) { (allModified || (new Date().getTime() - now.getTime() < s_quickSaveTimeout));
i++) {
try { try {
records.get(i).project.save(); records.get(i).project.save();
@ -332,6 +384,10 @@ public class ProjectManager {
} }
} }
} }
if (gc) {
System.gc();
}
} }
public void deleteProject(Project project) { public void deleteProject(Project project) {

24
src/main/java/com/metaweb/gridworks/ProjectMetadata.java
View File

@ -57,8 +57,8 @@ public class ProjectMetadata implements Jsonizable {
writer.endObject(); writer.endObject();
} }
public void save(File dir) throws Exception { public void save(File projectDir) throws Exception {
File tempFile = new File(dir, "metadata.temp.json"); File tempFile = new File(projectDir, "metadata.temp.json");
try { try {
saveToFile(tempFile); saveToFile(tempFile);
} catch (Exception e) { } catch (Exception e) {
@ -68,8 +68,8 @@ public class ProjectMetadata implements Jsonizable {
return; return;
} }
File file = new File(dir, "metadata.json"); File file = new File(projectDir, "metadata.json");
File oldFile = new File(dir, "metadata.old.json"); File oldFile = new File(projectDir, "metadata.old.json");
if (file.exists()) { if (file.exists()) {
file.renameTo(oldFile); file.renameTo(oldFile);
@ -81,8 +81,8 @@ public class ProjectMetadata implements Jsonizable {
} }
} }
protected void saveToFile(File file) throws Exception { protected void saveToFile(File metadataFile) throws Exception {
Writer writer = new OutputStreamWriter(new FileOutputStream(file)); Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile));
try { try {
Properties options = new Properties(); Properties options = new Properties();
options.setProperty("mode", "save"); options.setProperty("mode", "save");
@ -95,27 +95,27 @@ public class ProjectMetadata implements Jsonizable {
} }
} }
static public ProjectMetadata load(File dir) { static public ProjectMetadata load(File projectDir) {
try { try {
return loadFromFile(new File(dir, "metadata.json")); return loadFromFile(new File(projectDir, "metadata.json"));
} catch (Exception e) { } catch (Exception e) {
} }
try { try {
return loadFromFile(new File(dir, "metadata.temp.json")); return loadFromFile(new File(projectDir, "metadata.temp.json"));
} catch (Exception e) { } catch (Exception e) {
} }
try { try {
return loadFromFile(new File(dir, "metadata.old.json")); return loadFromFile(new File(projectDir, "metadata.old.json"));
} catch (Exception e) { } catch (Exception e) {
} }
return null; return null;
} }
static protected ProjectMetadata loadFromFile(File file) throws Exception { static protected ProjectMetadata loadFromFile(File metadataFile) throws Exception {
FileReader reader = new FileReader(file); FileReader reader = new FileReader(metadataFile);
try { try {
JSONTokener tokener = new JSONTokener(reader); JSONTokener tokener = new JSONTokener(reader);
JSONObject obj = (JSONObject) tokener.nextValue(); JSONObject obj = (JSONObject) tokener.nextValue();