From 1e5f89e84cb7c88cc157ae246c0bf7b9da3dc501 Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Thu, 25 Jul 2013 15:41:08 -0400 Subject: [PATCH] Centralize handling of import job config object & synchronize to allow multiple accessors --- .../refine/extension/gdata/GDataImporter.java | 8 +- .../gdata/GDataImportingController.java | 13 +-- .../project/CreateProjectCommand.java | 2 +- .../refine/importers/ImporterUtilities.java | 6 +- .../importing/DefaultImportingController.java | 2 +- .../google/refine/importing/ImportingJob.java | 104 ++++++++++++++++-- .../refine/importing/ImportingUtilities.java | 57 +++------- 7 files changed, 117 insertions(+), 75 deletions(-) diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java index d843cd461..d4a038956 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java @@ -172,13 +172,7 @@ public class GDataImporter { } static private void setProgress(ImportingJob job, String fileSource, int percent) { - JSONObject progress = JSONUtilities.getObject(job.config, "progress"); - if (progress == null) { - progress = new JSONObject(); - JSONUtilities.safePut(job.config, "progress", progress); - } - JSONUtilities.safePut(progress, "message", "Reading " + fileSource); - JSONUtilities.safePut(progress, "percent", percent); + job.setProgress(percent, "Reading " + fileSource); } static private class WorksheetBatchRowReader implements TableDataReader { diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java index 0ee9b4c7d..c81807936 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java @@ -300,9 +300,6 @@ public class GDataImportingController implements ImportingController { job.updating = true; try { - // This is for setting progress during the parsing process. - job.config = new JSONObject(); - JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject( request.getParameter("options")); @@ -371,7 +368,7 @@ public class GDataImportingController implements ImportingController { final List exceptions = new LinkedList(); - JSONUtilities.safePut(job.config, "state", "creating-project"); + job.setState("creating-project"); final Project project = new Project(); new Thread() { @@ -393,16 +390,14 @@ public class GDataImportingController implements ImportingController { if (!job.canceled) { if (exceptions.size() > 0) { - JSONUtilities.safePut(job.config, "errors", - DefaultImportingController.convertErrorsToJsonArray(exceptions)); - JSONUtilities.safePut(job.config, "state", "error"); + job.setError(exceptions); } else { project.update(); // update all internal models, indexes, caches, etc. ProjectManager.singleton.registerProject(project, pm); - JSONUtilities.safePut(job.config, "state", "created-project"); - JSONUtilities.safePut(job.config, "projectID", project.id); + job.setState("created-project"); + job.setProjectID(project.id); } job.touch(); diff --git a/main/src/com/google/refine/commands/project/CreateProjectCommand.java b/main/src/com/google/refine/commands/project/CreateProjectCommand.java index 3324c25aa..52a8e47d3 100644 --- a/main/src/com/google/refine/commands/project/CreateProjectCommand.java +++ b/main/src/com/google/refine/commands/project/CreateProjectCommand.java @@ -112,7 +112,7 @@ public class CreateProjectCommand extends Command { } else { Format formatRecord = ImportingManager.formatToRecord.get(format); optionObj = formatRecord.parser.createParserUIInitializationData( - job, ImportingUtilities.getSelectedFileRecords(job), format); + job, job.getSelectedFileRecords(), format); } adjustLegacyOptions(format, parameters, optionObj); diff --git a/main/src/com/google/refine/importers/ImporterUtilities.java b/main/src/com/google/refine/importers/ImporterUtilities.java index fe2ea3001..7d543a3fb 100644 --- a/main/src/com/google/refine/importers/ImporterUtilities.java +++ b/main/src/com/google/refine/importers/ImporterUtilities.java @@ -209,10 +209,8 @@ public class ImporterUtilities { long totalBytesRead = 0; void setProgress(String fileSource, long bytesRead) { - ImportingUtilities.setCreatingProjectProgress( - job, - "Reading " + fileSource, - totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2)); + job.setProgress(totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2), + "Reading " + fileSource); } @Override diff --git a/main/src/com/google/refine/importing/DefaultImportingController.java b/main/src/com/google/refine/importing/DefaultImportingController.java index 4d79dde99..40029be91 100644 --- a/main/src/com/google/refine/importing/DefaultImportingController.java +++ b/main/src/com/google/refine/importing/DefaultImportingController.java @@ -228,7 +228,7 @@ public class DefaultImportingController implements ImportingController { Format formatRecord = ImportingManager.formatToRecord.get(format); if (formatRecord != null && formatRecord.parser != null) { JSONObject options = formatRecord.parser.createParserUIInitializationData( - job, ImportingUtilities.getSelectedFileRecords(job), format); + job, job.getSelectedFileRecords(), format); JSONObject result = new JSONObject(); JSONUtilities.safePut(result, "status", "ok"); JSONUtilities.safePut(result, "options", options); diff --git a/main/src/com/google/refine/importing/ImportingJob.java b/main/src/com/google/refine/importing/ImportingJob.java index e2b426a73..bfcb14582 100644 --- a/main/src/com/google/refine/importing/ImportingJob.java +++ b/main/src/com/google/refine/importing/ImportingJob.java @@ -35,9 +35,12 @@ package com.google.refine.importing; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; import org.apache.commons.io.FileUtils; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; @@ -52,7 +55,7 @@ public class ImportingJob implements Jsonizable { final public long id; final public File dir; // Temporary directory where the data about this job is stored - public JSONObject config = null; + private JSONObject config; public Project project; public ProjectMetadata metadata; @@ -61,22 +64,101 @@ public class ImportingJob implements Jsonizable { public boolean updating; public boolean canceled; + final private Object lock = new Object(); + public ImportingJob(long id, File dir) { this.id = id; this.dir = dir; + + JSONObject cfg = new JSONObject(); + JSONUtilities.safePut(cfg, "state", "new"); + JSONUtilities.safePut(cfg, "hasData", false); + this.config = cfg; dir.mkdirs(); } + public JSONObject getOrCreateDefaultConfig() { - if (config == null) { - config = new JSONObject(); - JSONUtilities.safePut(config, "state", "new"); - JSONUtilities.safePut(config, "hasData", false); - } return config; } + public void setState(String state) { + synchronized(config) { + JSONUtilities.safePut(config, "state", state); + } + } + + public void setError(List exceptions) { + synchronized(config) { + JSONUtilities.safePut(config, "errors", + DefaultImportingController.convertErrorsToJsonArray(exceptions)); + setState("error"); + } + } + + public void setProjectID(long id2) { + synchronized (config) { + JSONUtilities.safePut(config, "projectID", project.id); + } + } + + public void setProgress(int percent, String message) { + synchronized (config) { + JSONObject progress = JSONUtilities.getObject(config, "progress"); + if (progress == null) { + progress = new JSONObject(); + JSONUtilities.safePut(config, "progress", progress); + } + JSONUtilities.safePut(progress, "message", message); + JSONUtilities.safePut(progress, "percent", percent); + JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000); + JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000); + } + } + + public void setFileSelection(JSONArray fileSelectionArray) { + synchronized (config) { + JSONUtilities.safePut(config, "fileSelection", fileSelectionArray); + } + } + + public void setRankedFormats(JSONArray rankedFormats) { + synchronized (config) { + JSONUtilities.safePut(config, "rankedFormats", rankedFormats); + } + } + + + public JSONObject getRetrievalRecord() { + synchronized(config) { + return JSONUtilities.getObject(config,"retrievalRecord"); + } + } + + + public List getSelectedFileRecords() { + List results = new ArrayList(); + + JSONObject retrievalRecord = JSONUtilities.getObject(config,"retrievalRecord"); + if (retrievalRecord != null) { + JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files"); + if (fileRecordArray != null) { + JSONArray fileSelectionArray = JSONUtilities.getArray(config,"fileSelection"); + if (fileSelectionArray != null) { + for (int i = 0; i < fileSelectionArray.length(); i++) { + int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1); + if (index >= 0 && index < fileRecordArray.length()) { + results.add(JSONUtilities.getObjectElement(fileRecordArray, index)); + } + } + } + } + } + return results; + } + + public void touch() { lastTouched = System.currentTimeMillis(); } @@ -111,8 +193,12 @@ public class ImportingJob implements Jsonizable { @Override public void write(JSONWriter writer, Properties options) throws JSONException { - writer.object(); - writer.key("config"); writer.value(config); - writer.endObject(); + + synchronized(lock) { + writer.object(); + writer.key("config"); writer.value(config); + writer.endObject(); + } } + } diff --git a/main/src/com/google/refine/importing/ImportingUtilities.java b/main/src/com/google/refine/importing/ImportingUtilities.java index 703023a16..5fd2a1cda 100644 --- a/main/src/com/google/refine/importing/ImportingUtilities.java +++ b/main/src/com/google/refine/importing/ImportingUtilities.java @@ -147,14 +147,14 @@ public class ImportingUtilities { } static public void updateJobWithNewFileSelection(ImportingJob job, JSONArray fileSelectionArray) { - JSONUtilities.safePut(job.config, "fileSelection", fileSelectionArray); + job.setFileSelection(fileSelectionArray); String bestFormat = ImportingUtilities.getCommonFormatForSelectedFiles(job, fileSelectionArray); bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat); JSONArray rankedFormats = new JSONArray(); - JSONUtilities.safePut(job.config, "rankedFormats", rankedFormats); ImportingUtilities.rankFormats(job, bestFormat, rankedFormats); + job.setRankedFormats(rankedFormats); } static public void retrieveContentFromPostRequest( @@ -210,6 +210,7 @@ public class ImportingUtilities { } }); + @SuppressWarnings("rawtypes") List tempFiles = upload.parseRequest(request); progress.setProgress("Uploading data ...", -1); @@ -747,7 +748,7 @@ public class ImportingUtilities { } static public String getCommonFormatForSelectedFiles(ImportingJob job, JSONArray fileSelectionIndexes) { - JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord"); + JSONObject retrievalRecord = job.getRetrievalRecord(); final Map formatToCount = new HashMap(); List formats = new ArrayList(); @@ -780,7 +781,7 @@ public class ImportingUtilities { } static String guessBetterFormat(ImportingJob job, String bestFormat) { - JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord"); + JSONObject retrievalRecord = job.getRetrievalRecord(); return retrievalRecord != null ? guessBetterFormat(job, retrievalRecord, bestFormat) : bestFormat; } @@ -879,27 +880,7 @@ public class ImportingUtilities { JSONUtilities.append(rankedFormats, format); } } - - static public List getSelectedFileRecords(ImportingJob job) { - List results = new ArrayList(); - - JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord"); - if (retrievalRecord != null) { - JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files"); - if (fileRecordArray != null) { - JSONArray fileSelectionArray = JSONUtilities.getArray(job.config, "fileSelection"); - if (fileSelectionArray != null) { - for (int i = 0; i < fileSelectionArray.length(); i++) { - int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1); - if (index >= 0 && index < fileRecordArray.length()) { - results.add(JSONUtilities.getObjectElement(fileRecordArray, index)); - } - } - } - } - } - return results; - } + static public void previewParse(ImportingJob job, String format, JSONObject optionObj, List exceptions) { Format record = ImportingManager.formatToRecord.get(format); @@ -914,7 +895,7 @@ public class ImportingUtilities { job.project, job.metadata, job, - getSelectedFileRecords(job), + job.getSelectedFileRecords(), format, 100, optionObj, @@ -936,7 +917,7 @@ public class ImportingUtilities { return -1; } - JSONUtilities.safePut(job.config, "state", "creating-project"); + job.setState("creating-project"); final Project project = new Project(); if (synchronous) { @@ -975,7 +956,7 @@ public class ImportingUtilities { project, pm, job, - getSelectedFileRecords(job), + job.getSelectedFileRecords(), format, -1, optionObj, @@ -988,27 +969,15 @@ public class ImportingUtilities { ProjectManager.singleton.registerProject(project, pm); - JSONUtilities.safePut(job.config, "projectID", project.id); - JSONUtilities.safePut(job.config, "state", "created-project"); + job.setProjectID(project.id); + job.setState("created-project"); } else { - JSONUtilities.safePut(job.config, "state", "error"); - JSONUtilities.safePut(job.config, "errors", - DefaultImportingController.convertErrorsToJsonArray(exceptions)); + job.setError(exceptions); } job.touch(); job.updating = false; } } - static public void setCreatingProjectProgress(ImportingJob job, String message, int percent) { - JSONObject progress = JSONUtilities.getObject(job.config, "progress"); - if (progress == null) { - progress = new JSONObject(); - JSONUtilities.safePut(job.config, "progress", progress); - } - JSONUtilities.safePut(progress, "message", message); - JSONUtilities.safePut(progress, "percent", percent); - JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000); - JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000); - } + }