diff --git a/main/src/com/google/refine/commands/project/CreateProjectCommand.java b/main/src/com/google/refine/commands/project/CreateProjectCommand.java index 05aeba61e..3324c25aa 100644 --- a/main/src/com/google/refine/commands/project/CreateProjectCommand.java +++ b/main/src/com/google/refine/commands/project/CreateProjectCommand.java @@ -34,20 +34,27 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.commands.project; import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import java.util.Properties; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.json.JSONArray; +import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.refine.ProjectManager; -import com.google.refine.ProjectMetadata; import com.google.refine.commands.Command; import com.google.refine.commands.HttpUtilities; -import com.google.refine.model.Project; +import com.google.refine.importing.ImportingJob; +import com.google.refine.importing.ImportingManager; +import com.google.refine.importing.ImportingUtilities; +import com.google.refine.importing.ImportingManager.Format; +import com.google.refine.util.JSONUtilities; import com.google.refine.util.ParsingUtilities; public class CreateProjectCommand extends Command { @@ -60,39 +67,112 @@ public class CreateProjectCommand extends Command { ProjectManager.singleton.setBusy(true); try { + Properties parameters = ParsingUtilities.parseUrlParameters(request); + ImportingJob job = ImportingManager.createJob(); + JSONObject config = job.getOrCreateDefaultConfig(); + ImportingUtilities.loadDataAndPrepareJob( + request, response, parameters, job, config); + + String format = parameters.getProperty("format"); + + // If a format is specified, it might still be wrong, so we need + // to check if we have a parser for it. If not, null it out. + if (format != null && !format.isEmpty()) { + Format formatRecord = ImportingManager.formatToRecord.get(format); + if (formatRecord == null || formatRecord.parser == null) { + format = null; + } + } + + // If we don't have a format specified, try to guess it. + if (format == null || format.isEmpty()) { + // Use legacy parameters to guess the format. + if ("false".equals(parameters.getProperty("split-into-columns"))) { + format = "text/line-based"; + } else if (",".equals(parameters.getProperty("separator")) || + "\\t".equals(parameters.getProperty("separator"))) { + format = "text/line-based/*sv"; + } else { + JSONArray rankedFormats = JSONUtilities.getArray(config, "rankedFormats"); + if (rankedFormats != null && rankedFormats.length() > 0) { + format = rankedFormats.getString(0); + } + } + + if (format == null || format.isEmpty()) { + // If we have failed in guessing, default to something simple. + format = "text/line-based"; + } + } + + JSONObject optionObj = null; + String optionsString = request.getParameter("options"); + if (optionsString != null && !optionsString.isEmpty()) { + optionObj = ParsingUtilities.evaluateJsonStringToObject(optionsString); + } else { + Format formatRecord = ImportingManager.formatToRecord.get(format); + optionObj = formatRecord.parser.createParserUIInitializationData( + job, ImportingUtilities.getSelectedFileRecords(job), format); + } + adjustLegacyOptions(format, parameters, optionObj); + + String projectName = parameters.getProperty("project-name"); + if (projectName != null && !projectName.isEmpty()) { + JSONUtilities.safePut(optionObj, "projectName", projectName); + } + + List exceptions = new LinkedList(); + + long projectId = ImportingUtilities.createProject(job, format, optionObj, exceptions, true); - /* - * The uploaded file is in the POST body as a "file part". If - * we call request.getParameter() then the POST body will get - * read and we won't have a chance to parse the body ourselves. - * This is why we have to parse the URL for parameters ourselves. - * Don't call request.getParameter() before calling internalImport(). - */ - Properties options = ParsingUtilities.parseUrlParameters(request); - - Project project = new Project(); - ProjectMetadata pm = new ProjectMetadata(); - - //internalImport(request, project, pm, options); - - /* - * The import process above populates options with parameters - * in the POST body. That's why we're constructing the project - * metadata object after calling internalImport(). - */ - pm.setName(options.getProperty("project-name")); - pm.setPassword(options.getProperty("project-password")); - pm.setEncoding(options.getProperty("encoding")); - pm.setEncodingConfidence(options.getProperty("encoding_confidence")); - ProjectManager.singleton.registerProject(project, pm); - - project.update(); - - HttpUtilities.redirect(response, "/project?project=" + project.id); + HttpUtilities.redirect(response, "/project?project=" + projectId); } catch (Exception e) { respondWithErrorPage(request, response, "Failed to import file", e); } finally { ProjectManager.singleton.setBusy(false); } } + + static private void adjustLegacyOptions(String format, Properties parameters, JSONObject optionObj) { + if (",".equals(parameters.getProperty("separator"))) { + JSONUtilities.safePut(optionObj, "separator", ","); + } else if ("\\t".equals(parameters.getProperty("separator"))) { + JSONUtilities.safePut(optionObj, "separator", "\t"); + } + + adjustLegacyIntegerOption(format, parameters, optionObj, "ignore", "ignoreLines"); + adjustLegacyIntegerOption(format, parameters, optionObj, "header-lines", "headerLines"); + adjustLegacyIntegerOption(format, parameters, optionObj, "skip", "skipDataLines"); + adjustLegacyIntegerOption(format, parameters, optionObj, "limit", "limit"); + + adjustLegacyBooleanOption(format, parameters, optionObj, "guess-value-type", "guessCellValueTypes", false); + adjustLegacyBooleanOption(format, parameters, optionObj, "ignore-quotes", "processQuotes", true); + } + + static private void adjustLegacyIntegerOption( + String format, Properties parameters, JSONObject optionObj, String legacyName, String newName) { + + String s = parameters.getProperty(legacyName); + if (s != null && !s.isEmpty()) { + try { + JSONUtilities.safePut(optionObj, newName, Integer.parseInt(s)); + } catch (NumberFormatException e) { + // Ignore + } + } + } + + static private void adjustLegacyBooleanOption( + String format, + Properties parameters, + JSONObject optionObj, + String legacyName, + String newName, + boolean invert) { + + String s = parameters.getProperty(legacyName); + if (s != null && !s.isEmpty()) { + JSONUtilities.safePut(optionObj, newName, Boolean.parseBoolean(s)); + } + } } diff --git a/main/src/com/google/refine/importing/DefaultImportingController.java b/main/src/com/google/refine/importing/DefaultImportingController.java index 7a0f6614c..3d54d6c9c 100644 --- a/main/src/com/google/refine/importing/DefaultImportingController.java +++ b/main/src/com/google/refine/importing/DefaultImportingController.java @@ -107,7 +107,7 @@ public class DefaultImportingController implements ImportingController { } try { - final JSONObject config = getConfig(job); + JSONObject config = job.getOrCreateDefaultConfig(); if (!("new".equals(config.getString("state")))) { HttpUtilities.respond(response, "error", "Job already started; cannot load more data"); return; @@ -131,7 +131,7 @@ public class DefaultImportingController implements ImportingController { } try { - JSONObject config = getConfig(job); + JSONObject config = job.getOrCreateDefaultConfig(); if (!("ready".equals(config.getString("state")))) { HttpUtilities.respond(response, "error", "Job not ready"); return; @@ -159,7 +159,7 @@ public class DefaultImportingController implements ImportingController { } try { - JSONObject config = getConfig(job); + JSONObject config = job.getOrCreateDefaultConfig(); if (!("ready".equals(config.getString("state")))) { HttpUtilities.respond(response, "error", "Job not ready"); return; @@ -215,7 +215,7 @@ public class DefaultImportingController implements ImportingController { } try { - JSONObject config = getConfig(job); + JSONObject config = job.getOrCreateDefaultConfig(); if (!("ready".equals(config.getString("state")))) { HttpUtilities.respond(response, "error", "Job not ready"); return; @@ -227,7 +227,7 @@ public class DefaultImportingController implements ImportingController { List exceptions = new LinkedList(); - ImportingUtilities.createProject(job, format, optionObj, exceptions); + ImportingUtilities.createProject(job, format, optionObj, exceptions, false); HttpUtilities.respond(response, "ok", "done"); } catch (JSONException e) { @@ -235,15 +235,6 @@ public class DefaultImportingController implements ImportingController { } } - private JSONObject getConfig(ImportingJob job) { - if (job.config == null) { - job.config = new JSONObject(); - JSONUtilities.safePut(job.config, "state", "new"); - JSONUtilities.safePut(job.config, "hasData", false); - } - return job.config; - } - private void replyWithJobData(HttpServletRequest request, HttpServletResponse response, ImportingJob job) throws ServletException, IOException { diff --git a/main/src/com/google/refine/importing/ImportingJob.java b/main/src/com/google/refine/importing/ImportingJob.java index 8ab30c57a..e576cdc8b 100644 --- a/main/src/com/google/refine/importing/ImportingJob.java +++ b/main/src/com/google/refine/importing/ImportingJob.java @@ -45,6 +45,7 @@ import org.json.JSONWriter; import com.google.refine.Jsonizable; import com.google.refine.ProjectMetadata; import com.google.refine.model.Project; +import com.google.refine.util.JSONUtilities; public class ImportingJob implements Jsonizable { @@ -65,6 +66,15 @@ public class ImportingJob implements Jsonizable { dir.mkdirs(); } + public JSONObject getOrCreateDefaultConfig() { + if (config == null) { + config = new JSONObject(); + JSONUtilities.safePut(config, "state", "new"); + JSONUtilities.safePut(config, "hasData", false); + } + return config; + } + public void touch() { lastTouched = System.currentTimeMillis(); } diff --git a/main/src/com/google/refine/importing/ImportingUtilities.java b/main/src/com/google/refine/importing/ImportingUtilities.java index 21e0c01e4..a6de54a36 100644 --- a/main/src/com/google/refine/importing/ImportingUtilities.java +++ b/main/src/com/google/refine/importing/ImportingUtilities.java @@ -861,7 +861,8 @@ public class ImportingUtilities { final ImportingJob job, final String format, final JSONObject optionObj, - final List exceptions) { + final List exceptions, + boolean synchronous) { final Format record = ImportingManager.formatToRecord.get(format); if (record == null || record.parser == null) { // TODO: what to do? @@ -871,38 +872,54 @@ public class ImportingUtilities { JSONUtilities.safePut(job.config, "state", "creating-project"); final Project project = new Project(); - new Thread() { - @Override - public void run() { - ProjectMetadata pm = new ProjectMetadata(); - pm.setName(JSONUtilities.getString(optionObj, "projectName", "Untitled")); - pm.setEncoding(JSONUtilities.getString(optionObj, "encoding", "UTF-8")); - - record.parser.parse( - project, - pm, - job, - getSelectedFileRecords(job), - format, - -1, - optionObj, - exceptions - ); - - if (!job.canceled) { - project.update(); // update all internal models, indexes, caches, etc. - - ProjectManager.singleton.registerProject(project, pm); - - JSONUtilities.safePut(job.config, "projectID", project.id); - JSONUtilities.safePut(job.config, "state", "created-project"); + if (synchronous) { + createProjectSynchronously( + job, format, optionObj, exceptions, record, project); + } else { + new Thread() { + @Override + public void run() { + createProjectSynchronously( + job, format, optionObj, exceptions, record, project); } - } - }.start(); - + }.start(); + } return project.id; } + static private void createProjectSynchronously( + final ImportingJob job, + final String format, + final JSONObject optionObj, + final List exceptions, + final Format record, + final Project project + ) { + ProjectMetadata pm = new ProjectMetadata(); + pm.setName(JSONUtilities.getString(optionObj, "projectName", "Untitled")); + pm.setEncoding(JSONUtilities.getString(optionObj, "encoding", "UTF-8")); + + record.parser.parse( + project, + pm, + job, + getSelectedFileRecords(job), + format, + -1, + optionObj, + exceptions + ); + + if (!job.canceled) { + project.update(); // update all internal models, indexes, caches, etc. + + ProjectManager.singleton.registerProject(project, pm); + + JSONUtilities.safePut(job.config, "projectID", project.id); + JSONUtilities.safePut(job.config, "state", "created-project"); + } + } + static public void setCreatingProjectProgress(ImportingJob job, String message, int percent) { JSONObject progress = JSONUtilities.getObject(job.config, "progress"); if (progress == null) {