Periodically clean up stale importing jobs to free up disk space.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2240 7d457c2a-affb-35e4-300a-418c747d4874

extensions/gdata/module/scripts/index/importing-controller.js

@@ -176,7 +176,7 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
   
   this._parsingPanelElmts.startOverButton.click(function() {
     // explicitly cancel the import job
-    $.post("/command/core/cancel-importing-job?" + $.param({ "jobID": self._jobID }));
+    Refine.CreateProjectUI.cancelImportinJob(self._jobID);
     
     delete self._doc;
     delete self._jobID;
@@ -279,7 +279,8 @@ Refine.GDataImportingController.prototype._updatePreview = function() {
         });
       } else {
         self._parsingPanelElmts.progressPanel.hide();
-        alert('Errors:\n' + Refine.CreateProjectUI.composeErrorMessage(job));
+        alert('Errors:\n' + 
+          (result.message) ? result.message : Refine.CreateProjectUI.composeErrorMessage(job));
       }
     },
     "json"
@@ -338,10 +339,10 @@ Refine.GDataImportingController.prototype._createProject = function() {
     {
       "options" : JSON.stringify(options)
     },
-    function() {},
+    function(o) {
-    "json"
+      if (o.status == 'error') {
-  );
+        alert(o.message);
-  
+      } else {
         var start = new Date();
         var timerID = window.setInterval(
           function() {
@@ -354,6 +355,7 @@ Refine.GDataImportingController.prototype._createProject = function() {
                 },
                 function(jobID, job) {
                   window.clearInterval(timerID);
+                  Refine.CreateProjectUI.cancelImportinJob(jobID);
                   document.location = "project?project=" + job.config.projectID;
                 },
                 function(job) {
@@ -363,7 +365,7 @@ Refine.GDataImportingController.prototype._createProject = function() {
           },
           1000
         );
-  this._createProjectUI.showImportProgressPanel("Creating project ...", function() {
+        self._createProjectUI.showImportProgressPanel("Creating project ...", function() {
           // stop the timed polling
           window.clearInterval(timerID);
 
@@ -372,4 +374,8 @@ Refine.GDataImportingController.prototype._createProject = function() {
 
           self._createProjectUI.showSourceSelectionPanel();
         });
+      }
+    },
+    "json"
+  );
 };

extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java

@@ -259,6 +259,7 @@ public class GDataImportingController implements ImportingController {
             return;
         }
         
+        job.updating = true;
         try {
             // This is for setting progress during the parsing process.
             job.config = new JSONObject();
@@ -306,6 +307,9 @@ public class GDataImportingController implements ImportingController {
 
         } catch (JSONException e) {
             throw new ServletException(e);
+        } finally {
+            job.touch();
+            job.updating = false;
         }
     }
     
@@ -325,6 +329,7 @@ public class GDataImportingController implements ImportingController {
             return;
         }
         
+        job.updating = true;
         try {
             final JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(
                 request.getParameter("options"));
@@ -364,6 +369,9 @@ public class GDataImportingController implements ImportingController {
                             JSONUtilities.safePut(job.config, "state", "created-project");
                             JSONUtilities.safePut(job.config, "projectID", project.id);
                         }
+                        
+                        job.touch();
+                        job.updating = false;
                     }
                 }
             }.start();

main/src/com/google/refine/commands/importing/CancelImportingJobCommand.java

@@ -55,6 +55,9 @@ public class CancelImportingJobCommand extends Command {
             HttpUtilities.respond(response, "error", "No such import job");
         } else {
             job.canceled = true;
+            
+            ImportingManager.disposeJob(jobID);
+            
             HttpUtilities.respond(response, "ok", "Job canceled");
         }
     }

main/src/com/google/refine/importing/DefaultImportingController.java

@@ -108,6 +108,7 @@ public class DefaultImportingController implements ImportingController {
             return;
         }
         
+        job.updating = true;
         try {
             JSONObject config = job.getOrCreateDefaultConfig();
             if (!("new".equals(config.getString("state")))) {
@@ -119,6 +120,9 @@ public class DefaultImportingController implements ImportingController {
                 request, response, parameters, job, config);
         } catch (JSONException e) {
             throw new ServletException(e);
+        } finally {
+            job.touch();
+            job.updating = false;
         }
     }
     
@@ -132,6 +136,7 @@ public class DefaultImportingController implements ImportingController {
             return;
         }
         
+        job.updating = true;
         try {
             JSONObject config = job.getOrCreateDefaultConfig();
             if (!("ready".equals(config.getString("state")))) {
@@ -147,6 +152,9 @@ public class DefaultImportingController implements ImportingController {
             replyWithJobData(request, response, job);
         } catch (JSONException e) {
             throw new ServletException(e);
+        } finally {
+            job.touch();
+            job.updating = false;
         }
     }
     
@@ -160,6 +168,7 @@ public class DefaultImportingController implements ImportingController {
             return;
         }
         
+        job.updating = true;
         try {
             JSONObject config = job.getOrCreateDefaultConfig();
             if (!("ready".equals(config.getString("state")))) {
@@ -199,6 +208,9 @@ public class DefaultImportingController implements ImportingController {
             }
         } catch (JSONException e) {
             throw new ServletException(e);
+        } finally {
+            job.touch();
+            job.updating = false;
         }
     }
     
@@ -237,6 +249,8 @@ public class DefaultImportingController implements ImportingController {
             return;
         }
         
+        job.updating = true;
+        job.touch();
         try {
             JSONObject config = job.getOrCreateDefaultConfig();
             if (!("ready".equals(config.getString("state")))) {

main/src/com/google/refine/importing/ImportingJob.java

@@ -52,11 +52,13 @@ public class ImportingJob implements Jsonizable {
     final public long id;
     final public File dir; // Temporary directory where the data about this job is stored
     
-    public long lastTouched;
     public JSONObject config = null;
     
     public Project project;
     public ProjectMetadata metadata;
+    
+    public long lastTouched;
+    public boolean updating;
     public boolean canceled;
     
     public ImportingJob(long id, File dir) {

main/src/com/google/refine/importing/ImportingManager.java

@@ -42,10 +42,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
 
 import org.apache.commons.io.FileUtils;
 import org.json.JSONException;
 import org.json.JSONWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.refine.RefineServlet;
 
@@ -74,6 +78,8 @@ public class ImportingManager {
         }
     }
     
+    final static Logger logger = LoggerFactory.getLogger("importing");
+    
     static private RefineServlet servlet;
     static private File importDir;
     final static private Map<Long, ImportingJob> jobs = new HashMap<Long, ImportingJob>();
@@ -96,8 +102,30 @@ public class ImportingManager {
     // Mapping from controller name to controller
     final static public Map<String, ImportingController> controllers = new HashMap<String, ImportingController>();
     
+    // timer for periodically deleting stale importing jobs
+    static private Timer _timer;
+
+    final static private long s_timerPeriod = 1000 * 60 * 10; // 10 minutes
+    final static private long s_stalePeriod = 1000 * 60 * 60; // 60 minutes
+    
+    static private class CleaningTimerTask extends TimerTask {
+        @Override
+        public void run() {
+            try {
+                cleanUpStaleJobs();
+            } finally {
+                _timer.schedule(new CleaningTimerTask(), s_timerPeriod);
+                // we don't use scheduleAtFixedRate because that might result in
+                // bunched up events when the computer is put in sleep mode
+            }
+        }
+    }
+    
     static public void initialize(RefineServlet servlet) {
         ImportingManager.servlet = servlet;
+        
+        _timer = new Timer("autosave");
+        _timer.schedule(new CleaningTimerTask(), s_timerPeriod);
     }
     
     static public void registerFormat(String format, String label) {
@@ -254,4 +282,17 @@ public class ImportingManager {
             return mimeTypeFormat;
         }
     }
+    
+    static private void cleanUpStaleJobs() {
+        long now = System.currentTimeMillis();
+        for (Long id : new HashSet<Long>(jobs.keySet())) {
+            ImportingJob job = jobs.get(id);
+            if (job != null && !job.updating && now - job.lastTouched > s_stalePeriod) {
+                job.dispose();
+                jobs.remove(id);
+                
+                logger.info("Disposed " + id);
+            }
+        }
+    }
 }

main/src/com/google/refine/importing/ImportingUtilities.java

@@ -923,6 +923,8 @@ public class ImportingUtilities {
                 JSONUtilities.safePut(job.config, "errors",
                     DefaultImportingController.convertErrorsToJsonArray(exceptions));
             }
+            job.touch();
+            job.updating = false;
         }
     }
     

main/webapp/modules/core/scripts/index/create-project-ui.js

@@ -251,3 +251,7 @@ Refine.CreateProjectUI.composeErrorMessage = function(job) {
   $.each(job.config.errors, function() { messages.push(this.message) });
   return messages.join('\n');
 };
+
+Refine.CreateProjectUI.cancelImportinJob = function(jobID) {
+  $.post("/command/core/cancel-importing-job?" + $.param({ "jobID": jobID }));
+};

main/webapp/modules/core/scripts/index/default-importing-controller/controller.js

@@ -50,6 +50,10 @@ Refine.DefaultImportingController.sources = [];
 Refine.DefaultImportingController.parserUIs = {};
 
 Refine.DefaultImportingController.prototype._startOver = function() {
+  if (this._jobID) {
+    Refine.CreateProjectUI.cancelImportinJob(this._jobID);
+  }
+  
   this._disposeFileSelectionPanel();
   this._disposeFileSelectionPanel();
 
@@ -117,7 +121,7 @@ Refine.DefaultImportingController.prototype.startImportJob = function(form, prog
           window.clearInterval(timerID);
 
           // explicitly cancel the import job
-          $.post("/command/core/cancel-importing-job?" + $.param({ "jobID": jobID }));
+          Refine.CreateProjectUI.cancelImportinJob(jobID);
 
           self._createProjectUI.showSourceSelectionPanel();
         });
@@ -206,7 +210,13 @@ Refine.DefaultImportingController.prototype.updateFormatAndOptions = function(op
       "format" : this._format,
       "options" : JSON.stringify(options)
     },
-    callback,
+    function(o) {
+      if (o.status == 'error') {
+        alert(o.message);
+      } else {
+        callback(o);
+      }
+    },
     "json"
   );
 };
@@ -265,7 +275,12 @@ Refine.DefaultImportingController.prototype._createProject = function() {
         "format" : this._format,
         "options" : JSON.stringify(options)
       },
-      function() {
+      function(o) {
+        if (o.status == 'error') {
+          alert(o.message);
+          return;
+        }
+        
         var start = new Date();
         var timerID = window.setInterval(
           function() {
@@ -277,6 +292,7 @@ Refine.DefaultImportingController.prototype._createProject = function() {
                   return "projectID" in job.config;
                 },
                 function(jobID, job) {
+                  Refine.CreateProjectUI.cancelImportinJob(jobID);
                   document.location = "project?project=" + job.config.projectID;
                 },
                 function(job) {