s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Periodically clean up stale importing jobs to free up disk space.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2240 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2011-09-15 23:52:05 +00:00
parent 0693205430
commit 02c58e2c56
9 changed files with 139 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
extensions/gdata/module/scripts/index/importing-controller.js
View File

@ -176,7 +176,7 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
this._parsingPanelElmts.startOverButton.click(function() {
// explicitly cancel the import job
$.post("/command/core/cancel-importing-job?" + $.param({ "jobID": self._jobID }));
Refine.CreateProjectUI.cancelImportinJob(self._jobID);
delete self._doc;
delete self._jobID;
@ -279,7 +279,8 @@ Refine.GDataImportingController.prototype._updatePreview = function() {
});
} else {
self._parsingPanelElmts.progressPanel.hide();
alert('Errors:\n' + Refine.CreateProjectUI.composeErrorMessage(job));
alert('Errors:\n' +
(result.message) ? result.message : Refine.CreateProjectUI.composeErrorMessage(job));
}
},
"json"
@ -338,38 +339,43 @@ Refine.GDataImportingController.prototype._createProject = function() {
{
"options" : JSON.stringify(options)
},
function() {},
function(o) {
if (o.status == 'error') {
alert(o.message);
} else {
var start = new Date();
var timerID = window.setInterval(
function() {
self._createProjectUI.pollImportJob(
start,
self._jobID,
timerID,
function(job) {
return "projectID" in job.config;
},
function(jobID, job) {
window.clearInterval(timerID);
Refine.CreateProjectUI.cancelImportinJob(jobID);
document.location = "project?project=" + job.config.projectID;
},
function(job) {
alert(Refine.CreateProjectUI.composeErrorMessage(job));
}
);
},
1000
);
self._createProjectUI.showImportProgressPanel("Creating project ...", function() {
// stop the timed polling
window.clearInterval(timerID);
// explicitly cancel the import job
$.post("/command/core/cancel-importing-job?" + $.param({ "jobID": jobID }));
self._createProjectUI.showSourceSelectionPanel();
});
}
},
"json"
);
var start = new Date();
var timerID = window.setInterval(
function() {
self._createProjectUI.pollImportJob(
start,
self._jobID,
timerID,
function(job) {
return "projectID" in job.config;
},
function(jobID, job) {
window.clearInterval(timerID);
document.location = "project?project=" + job.config.projectID;
},
function(job) {
alert(Refine.CreateProjectUI.composeErrorMessage(job));
}
);
},
1000
);
this._createProjectUI.showImportProgressPanel("Creating project ...", function() {
// stop the timed polling
window.clearInterval(timerID);
// explicitly cancel the import job
$.post("/command/core/cancel-importing-job?" + $.param({ "jobID": jobID }));
self._createProjectUI.showSourceSelectionPanel();
});
};

10
extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java
View File

@ -258,7 +258,8 @@ public class GDataImportingController implements ImportingController {
HttpUtilities.respond(response, "error", "No such import job");
return;
}
job.updating = true;
try {
// This is for setting progress during the parsing process.
job.config = new JSONObject();
@ -306,6 +307,9 @@ public class GDataImportingController implements ImportingController {
} catch (JSONException e) {
throw new ServletException(e);
} finally {
job.touch();
job.updating = false;
}
}
@ -325,6 +329,7 @@ public class GDataImportingController implements ImportingController {
return;
}
job.updating = true;
try {
final JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(
request.getParameter("options"));
@ -364,6 +369,9 @@ public class GDataImportingController implements ImportingController {
JSONUtilities.safePut(job.config, "state", "created-project");
JSONUtilities.safePut(job.config, "projectID", project.id);
}
job.touch();
job.updating = false;
}
}
}.start();

3
main/src/com/google/refine/commands/importing/CancelImportingJobCommand.java
View File

@ -55,6 +55,9 @@ public class CancelImportingJobCommand extends Command {
HttpUtilities.respond(response, "error", "No such import job");
} else {
job.canceled = true;
ImportingManager.disposeJob(jobID);
HttpUtilities.respond(response, "ok", "Job canceled");
}
}

22
main/src/com/google/refine/importing/DefaultImportingController.java
View File

@ -107,7 +107,8 @@ public class DefaultImportingController implements ImportingController {
HttpUtilities.respond(response, "error", "No such import job");
return;
}
job.updating = true;
try {
JSONObject config = job.getOrCreateDefaultConfig();
if (!("new".equals(config.getString("state")))) {
@ -119,6 +120,9 @@ public class DefaultImportingController implements ImportingController {
request, response, parameters, job, config);
} catch (JSONException e) {
throw new ServletException(e);
} finally {
job.touch();
job.updating = false;
}
}
@ -131,7 +135,8 @@ public class DefaultImportingController implements ImportingController {
HttpUtilities.respond(response, "error", "No such import job");
return;
}
job.updating = true;
try {
JSONObject config = job.getOrCreateDefaultConfig();
if (!("ready".equals(config.getString("state")))) {
@ -147,6 +152,9 @@ public class DefaultImportingController implements ImportingController {
replyWithJobData(request, response, job);
} catch (JSONException e) {
throw new ServletException(e);
} finally {
job.touch();
job.updating = false;
}
}
@ -159,7 +167,8 @@ public class DefaultImportingController implements ImportingController {
HttpUtilities.respond(response, "error", "No such import job");
return;
}
job.updating = true;
try {
JSONObject config = job.getOrCreateDefaultConfig();
if (!("ready".equals(config.getString("state")))) {
@ -199,6 +208,9 @@ public class DefaultImportingController implements ImportingController {
}
} catch (JSONException e) {
throw new ServletException(e);
} finally {
job.touch();
job.updating = false;
}
}
@ -236,7 +248,9 @@ public class DefaultImportingController implements ImportingController {
HttpUtilities.respond(response, "error", "No such import job");
return;
}
job.updating = true;
job.touch();
try {
JSONObject config = job.getOrCreateDefaultConfig();
if (!("ready".equals(config.getString("state")))) {

4
main/src/com/google/refine/importing/ImportingJob.java
View File

@ -52,11 +52,13 @@ public class ImportingJob implements Jsonizable {
final public long id;
final public File dir; // Temporary directory where the data about this job is stored
public long lastTouched;
public JSONObject config = null;
public Project project;
public ProjectMetadata metadata;
public long lastTouched;
public boolean updating;
public boolean canceled;
public ImportingJob(long id, File dir) {

41
main/src/com/google/refine/importing/ImportingManager.java
View File

@ -42,10 +42,14 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import org.apache.commons.io.FileUtils;
import org.json.JSONException;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.RefineServlet;
@ -74,6 +78,8 @@ public class ImportingManager {
}
}
final static Logger logger = LoggerFactory.getLogger("importing");
static private RefineServlet servlet;
static private File importDir;
final static private Map<Long, ImportingJob> jobs = new HashMap<Long, ImportingJob>();
@ -96,8 +102,30 @@ public class ImportingManager {
// Mapping from controller name to controller
final static public Map<String, ImportingController> controllers = new HashMap<String, ImportingController>();
// timer for periodically deleting stale importing jobs
static private Timer _timer;
final static private long s_timerPeriod = 1000 * 60 * 10; // 10 minutes
final static private long s_stalePeriod = 1000 * 60 * 60; // 60 minutes
static private class CleaningTimerTask extends TimerTask {
@Override
public void run() {
try {
cleanUpStaleJobs();
} finally {
_timer.schedule(new CleaningTimerTask(), s_timerPeriod);
// we don't use scheduleAtFixedRate because that might result in
// bunched up events when the computer is put in sleep mode
}
}
}
static public void initialize(RefineServlet servlet) {
ImportingManager.servlet = servlet;
_timer = new Timer("autosave");
_timer.schedule(new CleaningTimerTask(), s_timerPeriod);
}
static public void registerFormat(String format, String label) {
@ -254,4 +282,17 @@ public class ImportingManager {
return mimeTypeFormat;
}
}
static private void cleanUpStaleJobs() {
long now = System.currentTimeMillis();
for (Long id : new HashSet<Long>(jobs.keySet())) {
ImportingJob job = jobs.get(id);
if (job != null && !job.updating && now - job.lastTouched > s_stalePeriod) {
job.dispose();
jobs.remove(id);
logger.info("Disposed " + id);
}
}
}
}

2
main/src/com/google/refine/importing/ImportingUtilities.java
View File

@ -923,6 +923,8 @@ public class ImportingUtilities {
JSONUtilities.safePut(job.config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
}
job.touch();
job.updating = false;
}
}

4
main/webapp/modules/core/scripts/index/create-project-ui.js
View File

@ -251,3 +251,7 @@ Refine.CreateProjectUI.composeErrorMessage = function(job) {
$.each(job.config.errors, function() { messages.push(this.message) });
return messages.join('\n');
};
Refine.CreateProjectUI.cancelImportinJob = function(jobID) {
$.post("/command/core/cancel-importing-job?" + $.param({ "jobID": jobID }));
};

22
main/webapp/modules/core/scripts/index/default-importing-controller/controller.js
View File

@ -50,6 +50,10 @@ Refine.DefaultImportingController.sources = [];
Refine.DefaultImportingController.parserUIs = {};
Refine.DefaultImportingController.prototype._startOver = function() {
if (this._jobID) {
Refine.CreateProjectUI.cancelImportinJob(this._jobID);
}
this._disposeFileSelectionPanel();
this._disposeFileSelectionPanel();
@ -117,7 +121,7 @@ Refine.DefaultImportingController.prototype.startImportJob = function(form, prog
window.clearInterval(timerID);
// explicitly cancel the import job
$.post("/command/core/cancel-importing-job?" + $.param({ "jobID": jobID }));
Refine.CreateProjectUI.cancelImportinJob(jobID);
self._createProjectUI.showSourceSelectionPanel();
});
@ -206,7 +210,13 @@ Refine.DefaultImportingController.prototype.updateFormatAndOptions = function(op
"format" : this._format,
"options" : JSON.stringify(options)
},
callback,
function(o) {
if (o.status == 'error') {
alert(o.message);
} else {
callback(o);
}
},
"json"
);
};
@ -265,7 +275,12 @@ Refine.DefaultImportingController.prototype._createProject = function() {
"format" : this._format,
"options" : JSON.stringify(options)
},
function() {
function(o) {
if (o.status == 'error') {
alert(o.message);
return;
}
var start = new Date();
var timerID = window.setInterval(
function() {
@ -277,6 +292,7 @@ Refine.DefaultImportingController.prototype._createProject = function() {
return "projectID" in job.config;
},
function(jobID, job) {
Refine.CreateProjectUI.cancelImportinJob(jobID);
document.location = "project?project=" + job.config.projectID;
},
function(job) {