s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Started a round of documentation.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@329 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-03-20 23:56:28 +00:00
parent a43b2a72c1
commit d90e75dff1
4 changed files with 121 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/com/metaweb/gridworks/GridworksServlet.java
View File

@ -60,6 +60,8 @@ public class GridworksServlet extends HttpServlet {
private static final long serialVersionUID = 2386057901503517403L;
static protected Map<String, Command> _commands = new HashMap<String, Command>();
// timer for periodically saving projects
static protected Timer _timer = new Timer();
static {
@ -128,21 +130,22 @@ public class GridworksServlet extends HttpServlet {
long period = 1000 * 60 * 5; // 5 minutes
_timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
ProjectManager.singleton.save(false);
ProjectManager.singleton.save(false); // quick, potentially incomplete save
}
}, period, period);
}
@Override
public void destroy() {
// cancel automatic periodic saving and force a complete save.
if (_timer != null) {
_timer.cancel();
_timer = null;
}
if (ProjectManager.singleton != null) {
ProjectManager.singleton.save(true);
ProjectManager.singleton.save(true); // complete save
ProjectManager.singleton = null;
}
@ -164,6 +167,11 @@ public class GridworksServlet extends HttpServlet {
}
protected String getCommandName(HttpServletRequest request) {
/*
* Remove extraneous path segments that might be there for other purposes,
* e.g., for /export-rows/filename.ext, export-rows is the command while
* filename.ext is only for the browser to prompt a convenient filename.
*/
String commandName = request.getPathInfo().substring(1);
int slash = commandName.indexOf('/');
return slash > 0 ? commandName.substring(0, slash) : commandName;

6
src/main/java/com/metaweb/gridworks/Jsonizable.java
View File

@ -5,6 +5,12 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
/**
* Interface for streaming out JSON, either into HTTP responses or
* serialization files.
*
* @author dfhuynh
*/
public interface Jsonizable {
public void write(JSONWriter writer, Properties options) throws JSONException;
}

76
src/main/java/com/metaweb/gridworks/ProjectManager.java
View File

@ -26,12 +26,19 @@ import com.metaweb.gridworks.util.JSONUtilities;
public class ProjectManager {
private static final int s_expressionHistoryMax = 100; // last n expressions used across all projects
// last n expressions used across all projects
static protected final int s_expressionHistoryMax = 100;
protected File _workspaceDir;
protected Map<Long, ProjectMetadata> _projectsMetadata;
protected List<String> _expressions;
/**
* While each project's metadata is loaded completely at start-up, each project's raw data
* is loaded only when the project is accessed by the user. This is because project
* metadata is tiny compared to raw project data. This hash map from project ID to project
* is more like a last accessed-last out cache.
*/
transient protected Map<Long, Project> _projects;
static public ProjectManager singleton;
@ -57,25 +64,37 @@ public class ProjectManager {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet
// it's not elegant but it's the safest bet.
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(localDataPath.getPath());
data.mkdirs();
return data;
} catch (Error e) {
Gridworks.log("Failed to use jdatapath to detect user data path. Resorting to environment variables.");
/*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
* we include is compiled for 32-bit. In this case, we just have to dig up
* environment variables and try our best to find a user-specific path.
*/
Gridworks.log(
"Failed to use jdatapath to detect user data path. " +
"Resorting to environment variables.");
String appData = System.getenv("APPDATA");
File parentDir = null;
{
String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) {
// e.g., C:\Users\[userid]\AppData\Roaming
parentDir = new File(appData);
} else {
String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid]
parentDir = new File(userProfile);
}
}
}
if (parentDir == null) {
parentDir = new File(".");
}
@ -141,6 +160,12 @@ public class ProjectManager {
}
}
/**
* Import an external project that has been received as a .tar file, expanded, and
* copied into our workspace directory.
*
* @param projectID
*/
public void importProject(long projectID) {
synchronized (this) {
ProjectMetadata metadata = ProjectMetadata.load(getProjectDir(projectID));
@ -149,6 +174,12 @@ public class ProjectManager {
}
}
/**
* Make sure that a project's metadata and data are saved to file. For example,
* this method is called before the project is exported to a .tar file.
*
* @param id
*/
public void ensureProjectSaved(long id) {
synchronized (this) {
File projectDir = getProjectDir(id);
@ -182,6 +213,7 @@ public class ProjectManager {
}
public Project getProject(long id) {
synchronized (this) {
if (_projects.containsKey(id)) {
return _projects.get(id);
} else {
@ -192,14 +224,17 @@ public class ProjectManager {
return project;
}
}
}
public void addLatestExpression(String s) {
synchronized (this) {
_expressions.remove(s);
_expressions.add(0, s);
while (_expressions.size() > s_expressionHistoryMax) {
_expressions.remove(_expressions.size() - 1);
}
}
}
public List<String> getExpressions() {
return _expressions;
@ -210,6 +245,10 @@ public class ProjectManager {
saveWorkspace();
}
/**
* Save the workspace's data out to file in a safe way: save to a temporary file first
* and rename it to the real file.
*/
protected void saveWorkspace() {
synchronized (this) {
File tempFile = new File(_workspaceDir, "workspace.temp.json");
@ -265,6 +304,10 @@ public class ProjectManager {
}
}
/**
* A utility class to prioritize projects for saving, depending on how long ago
* they have been changed but have not been saved.
*/
static protected class SaveRecord {
final Project project;
final long overdue;
@ -275,29 +318,37 @@ public class ProjectManager {
}
}
static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour
static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs
protected void saveProjects(boolean allModified) {
List<SaveRecord> records = new ArrayList<SaveRecord>();
Date now = new Date();
boolean gc = false;
synchronized (this) {
for (long id : _projectsMetadata.keySet()) {
ProjectMetadata metadata = _projectsMetadata.get(id);
Project project = _projects.get(id);
if (project != null) {
boolean hasUnsavedChanges = metadata.getModified().getTime() > project.lastSave.getTime();
boolean hasUnsavedChanges =
metadata.getModified().getTime() > project.lastSave.getTime();
if (hasUnsavedChanges) {
long msecsOverdue = now.getTime() - project.lastSave.getTime();
records.add(new SaveRecord(project, msecsOverdue));
} else if (now.getTime() - project.lastSave.getTime() > 1000 * 60 * 60) {
} else if (now.getTime() - project.lastSave.getTime() > s_projectFlushDelay) {
/*
* It's been 1 hour since the project was last saved and it hasn't
* been modified. We can safely remove it from the cache to save some memory.
* It's been a while since the project was last saved and it hasn't been
* modified. We can safely remove it from the cache to save some memory.
*/
_projects.remove(id);
gc = true;
}
}
}
@ -323,7 +374,8 @@ public class ProjectManager {
for (int i = 0;
i < records.size() &&
(allModified || (new Date().getTime() - now.getTime() < 30000)); i++) {
(allModified || (new Date().getTime() - now.getTime() < s_quickSaveTimeout));
i++) {
try {
records.get(i).project.save();
@ -332,6 +384,10 @@ public class ProjectManager {
}
}
}
if (gc) {
System.gc();
}
}
public void deleteProject(Project project) {

24
src/main/java/com/metaweb/gridworks/ProjectMetadata.java
View File

@ -57,8 +57,8 @@ public class ProjectMetadata implements Jsonizable {
writer.endObject();
}
public void save(File dir) throws Exception {
File tempFile = new File(dir, "metadata.temp.json");
public void save(File projectDir) throws Exception {
File tempFile = new File(projectDir, "metadata.temp.json");
try {
saveToFile(tempFile);
} catch (Exception e) {
@ -68,8 +68,8 @@ public class ProjectMetadata implements Jsonizable {
return;
}
File file = new File(dir, "metadata.json");
File oldFile = new File(dir, "metadata.old.json");
File file = new File(projectDir, "metadata.json");
File oldFile = new File(projectDir, "metadata.old.json");
if (file.exists()) {
file.renameTo(oldFile);
@ -81,8 +81,8 @@ public class ProjectMetadata implements Jsonizable {
}
}
protected void saveToFile(File file) throws Exception {
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
protected void saveToFile(File metadataFile) throws Exception {
Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile));
try {
Properties options = new Properties();
options.setProperty("mode", "save");
@ -95,27 +95,27 @@ public class ProjectMetadata implements Jsonizable {
}
}
static public ProjectMetadata load(File dir) {
static public ProjectMetadata load(File projectDir) {
try {
return loadFromFile(new File(dir, "metadata.json"));
return loadFromFile(new File(projectDir, "metadata.json"));
} catch (Exception e) {
}
try {
return loadFromFile(new File(dir, "metadata.temp.json"));
return loadFromFile(new File(projectDir, "metadata.temp.json"));
} catch (Exception e) {
}
try {
return loadFromFile(new File(dir, "metadata.old.json"));
return loadFromFile(new File(projectDir, "metadata.old.json"));
} catch (Exception e) {
}
return null;
}
static protected ProjectMetadata loadFromFile(File file) throws Exception {
FileReader reader = new FileReader(file);
static protected ProjectMetadata loadFromFile(File metadataFile) throws Exception {
FileReader reader = new FileReader(metadataFile);
try {
JSONTokener tokener = new JSONTokener(reader);
JSONObject obj = (JSONObject) tokener.nextValue();