Made CreateProjectCommand scriptable again, so it can be called from client libraries.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2216 7d457c2a-affb-35e4-300a-418c747d4874

main/src/com/google/refine/commands/project/CreateProjectCommand.java

@@ -34,20 +34,27 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.commands.project;
 
 import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Properties;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.json.JSONArray;
+import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.refine.ProjectManager;
-import com.google.refine.ProjectMetadata;
 import com.google.refine.commands.Command;
 import com.google.refine.commands.HttpUtilities;
-import com.google.refine.model.Project;
+import com.google.refine.importing.ImportingJob;
+import com.google.refine.importing.ImportingManager;
+import com.google.refine.importing.ImportingUtilities;
+import com.google.refine.importing.ImportingManager.Format;
+import com.google.refine.util.JSONUtilities;
 import com.google.refine.util.ParsingUtilities;
 
 public class CreateProjectCommand extends Command {
@@ -60,39 +67,112 @@ public class CreateProjectCommand extends Command {
 
         ProjectManager.singleton.setBusy(true);
         try {
+            Properties parameters = ParsingUtilities.parseUrlParameters(request);
+            ImportingJob job = ImportingManager.createJob();
+            JSONObject config = job.getOrCreateDefaultConfig();
+            ImportingUtilities.loadDataAndPrepareJob(
+                    request, response, parameters, job, config);
             
-            /*
+            String format = parameters.getProperty("format");
-             * The uploaded file is in the POST body as a "file part". If
-             * we call request.getParameter() then the POST body will get
-             * read and we won't have a chance to parse the body ourselves.
-             * This is why we have to parse the URL for parameters ourselves.
-             * Don't call request.getParameter() before calling internalImport().
-             */
-            Properties options = ParsingUtilities.parseUrlParameters(request);
             
-            Project project = new Project();
+            // If a format is specified, it might still be wrong, so we need
-            ProjectMetadata pm = new ProjectMetadata();
+            // to check if we have a parser for it. If not, null it out.
+            if (format != null && !format.isEmpty()) {
+                Format formatRecord = ImportingManager.formatToRecord.get(format);
+                if (formatRecord == null || formatRecord.parser == null) {
+                    format = null;
+                }
+            }
             
-            //internalImport(request, project, pm, options);
+            // If we don't have a format specified, try to guess it.
+            if (format == null || format.isEmpty()) {
+                // Use legacy parameters to guess the format.
+                if ("false".equals(parameters.getProperty("split-into-columns"))) {
+                    format = "text/line-based";
+                } else if (",".equals(parameters.getProperty("separator")) ||
+                           "\\t".equals(parameters.getProperty("separator"))) {
+                    format = "text/line-based/*sv";
+                } else {
+                    JSONArray rankedFormats = JSONUtilities.getArray(config, "rankedFormats");
+                    if (rankedFormats != null && rankedFormats.length() > 0) {
+                        format = rankedFormats.getString(0);
+                    }
+                }
                 
-            /*
+                if (format == null || format.isEmpty()) {
-             * The import process above populates options with parameters
+                    // If we have failed in guessing, default to something simple.
-             * in the POST body. That's why we're constructing the project
+                    format = "text/line-based";
-             * metadata object after calling internalImport().
+                }
-             */
+            }
-            pm.setName(options.getProperty("project-name"));
-            pm.setPassword(options.getProperty("project-password"));
-            pm.setEncoding(options.getProperty("encoding"));
-            pm.setEncodingConfidence(options.getProperty("encoding_confidence"));
-            ProjectManager.singleton.registerProject(project, pm);
             
-            project.update();
+            JSONObject optionObj = null;
+            String optionsString = request.getParameter("options");
+            if (optionsString != null && !optionsString.isEmpty()) {
+                optionObj = ParsingUtilities.evaluateJsonStringToObject(optionsString);
+            } else {
+                Format formatRecord = ImportingManager.formatToRecord.get(format);
+                optionObj = formatRecord.parser.createParserUIInitializationData(
+                    job, ImportingUtilities.getSelectedFileRecords(job), format);
+            }
+            adjustLegacyOptions(format, parameters, optionObj);
             
-            HttpUtilities.redirect(response, "/project?project=" + project.id);
+            String projectName = parameters.getProperty("project-name");
+            if (projectName != null && !projectName.isEmpty()) {
+                JSONUtilities.safePut(optionObj, "projectName", projectName);
+            }
+            
+            List<Exception> exceptions = new LinkedList<Exception>();
+            
+            long projectId = ImportingUtilities.createProject(job, format, optionObj, exceptions, true);
+
+            HttpUtilities.redirect(response, "/project?project=" + projectId);
         } catch (Exception e) {
             respondWithErrorPage(request, response, "Failed to import file", e);
         } finally {
             ProjectManager.singleton.setBusy(false);
         }
     }
+
+    static private void adjustLegacyOptions(String format, Properties parameters, JSONObject optionObj) {
+        if (",".equals(parameters.getProperty("separator"))) {
+            JSONUtilities.safePut(optionObj, "separator", ",");
+        } else if ("\\t".equals(parameters.getProperty("separator"))) {
+            JSONUtilities.safePut(optionObj, "separator", "\t");
+        }
+        
+        adjustLegacyIntegerOption(format, parameters, optionObj, "ignore", "ignoreLines");
+        adjustLegacyIntegerOption(format, parameters, optionObj, "header-lines", "headerLines");
+        adjustLegacyIntegerOption(format, parameters, optionObj, "skip", "skipDataLines");
+        adjustLegacyIntegerOption(format, parameters, optionObj, "limit", "limit");
+        
+        adjustLegacyBooleanOption(format, parameters, optionObj, "guess-value-type", "guessCellValueTypes", false);
+        adjustLegacyBooleanOption(format, parameters, optionObj, "ignore-quotes", "processQuotes", true);
+    }
+
+    static private void adjustLegacyIntegerOption(
+        String format, Properties parameters, JSONObject optionObj, String legacyName, String newName) {
+        
+        String s = parameters.getProperty(legacyName);
+        if (s != null && !s.isEmpty()) {
+            try {
+                JSONUtilities.safePut(optionObj, newName, Integer.parseInt(s));
+            } catch (NumberFormatException e) {
+                // Ignore
+            }
+        }
+    }
+    
+    static private void adjustLegacyBooleanOption(
+        String format,
+        Properties parameters,
+        JSONObject optionObj,
+        String legacyName,
+        String newName,
+        boolean invert) {
+        
+        String s = parameters.getProperty(legacyName);
+        if (s != null && !s.isEmpty()) {
+            JSONUtilities.safePut(optionObj, newName, Boolean.parseBoolean(s));
+        }
+    }
 }

main/src/com/google/refine/importing/DefaultImportingController.java

@@ -107,7 +107,7 @@ public class DefaultImportingController implements ImportingController {
         }
 
         try {
-            final JSONObject config = getConfig(job);
+            JSONObject config = job.getOrCreateDefaultConfig();
             if (!("new".equals(config.getString("state")))) {
                 HttpUtilities.respond(response, "error", "Job already started; cannot load more data");
                 return;
@@ -131,7 +131,7 @@ public class DefaultImportingController implements ImportingController {
         }
     
         try {
-            JSONObject config = getConfig(job);
+            JSONObject config = job.getOrCreateDefaultConfig();
             if (!("ready".equals(config.getString("state")))) {
                 HttpUtilities.respond(response, "error", "Job not ready");
                 return;
@@ -159,7 +159,7 @@ public class DefaultImportingController implements ImportingController {
         }
     
         try {
-            JSONObject config = getConfig(job);
+            JSONObject config = job.getOrCreateDefaultConfig();
             if (!("ready".equals(config.getString("state")))) {
                 HttpUtilities.respond(response, "error", "Job not ready");
                 return;
@@ -215,7 +215,7 @@ public class DefaultImportingController implements ImportingController {
         }
     
         try {
-            JSONObject config = getConfig(job);
+            JSONObject config = job.getOrCreateDefaultConfig();
             if (!("ready".equals(config.getString("state")))) {
                 HttpUtilities.respond(response, "error", "Job not ready");
                 return;
@@ -227,7 +227,7 @@ public class DefaultImportingController implements ImportingController {
             
             List<Exception> exceptions = new LinkedList<Exception>();
             
-            ImportingUtilities.createProject(job, format, optionObj, exceptions);
+            ImportingUtilities.createProject(job, format, optionObj, exceptions, false);
             
             HttpUtilities.respond(response, "ok", "done");
         } catch (JSONException e) {
@@ -235,15 +235,6 @@ public class DefaultImportingController implements ImportingController {
         }
     }
 
-    private JSONObject getConfig(ImportingJob job) {
-        if (job.config == null) {
-            job.config = new JSONObject();
-            JSONUtilities.safePut(job.config, "state", "new");
-            JSONUtilities.safePut(job.config, "hasData", false);
-        }
-        return job.config;
-    }
-    
     private void replyWithJobData(HttpServletRequest request, HttpServletResponse response, ImportingJob job)
         throws ServletException, IOException {
         

main/src/com/google/refine/importing/ImportingJob.java

@@ -45,6 +45,7 @@ import org.json.JSONWriter;
 import com.google.refine.Jsonizable;
 import com.google.refine.ProjectMetadata;
 import com.google.refine.model.Project;
+import com.google.refine.util.JSONUtilities;
 
 
 public class ImportingJob implements Jsonizable {
@@ -65,6 +66,15 @@ public class ImportingJob implements Jsonizable {
         dir.mkdirs();
     }
     
+    public JSONObject getOrCreateDefaultConfig() {
+        if (config == null) {
+            config = new JSONObject();
+            JSONUtilities.safePut(config, "state", "new");
+            JSONUtilities.safePut(config, "hasData", false);
+        }
+        return config;
+    }
+    
     public void touch() {
         lastTouched = System.currentTimeMillis();
     }

main/src/com/google/refine/importing/ImportingUtilities.java

@@ -861,7 +861,8 @@ public class ImportingUtilities {
             final ImportingJob job,
             final String format,
             final JSONObject optionObj,
-            final List<Exception> exceptions) {
+            final List<Exception> exceptions,
+            boolean synchronous) {
         final Format record = ImportingManager.formatToRecord.get(format);
         if (record == null || record.parser == null) {
             // TODO: what to do?
@@ -871,9 +872,29 @@ public class ImportingUtilities {
         JSONUtilities.safePut(job.config, "state", "creating-project");
         
         final Project project = new Project();
+        if (synchronous) {
+            createProjectSynchronously(
+                job, format, optionObj, exceptions, record, project);
+        } else {
             new Thread() {
                 @Override
                 public void run() {
+                    createProjectSynchronously(
+                        job, format, optionObj, exceptions, record, project);
+                }
+            }.start();
+        }
+        return project.id;
+    }
+    
+    static private void createProjectSynchronously(
+        final ImportingJob job,
+        final String format,
+        final JSONObject optionObj,
+        final List<Exception> exceptions,
+        final Format record,
+        final Project project
+    ) {
         ProjectMetadata pm = new ProjectMetadata();
         pm.setName(JSONUtilities.getString(optionObj, "projectName", "Untitled"));
         pm.setEncoding(JSONUtilities.getString(optionObj, "encoding", "UTF-8"));
@@ -898,10 +919,6 @@ public class ImportingUtilities {
             JSONUtilities.safePut(job.config, "state", "created-project");
         }
     }
-        }.start();
-        
-        return project.id;
-    }
     
     static public void setCreatingProjectProgress(ImportingJob job, String message, int percent) {
         JSONObject progress = JSONUtilities.getObject(job.config, "progress");