s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Centralize handling of import job config object & synchronize to allow

Browse Source 
multiple accessors
This commit is contained in:
Tom Morris 2013-07-25 15:41:08 -04:00
parent dc206e1889
commit 1e5f89e84c
7 changed files with 117 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java
View File

@ -172,13 +172,7 @@ public class GDataImporter {
} }
static private void setProgress(ImportingJob job, String fileSource, int percent) { static private void setProgress(ImportingJob job, String fileSource, int percent) {
JSONObject progress = JSONUtilities.getObject(job.config, "progress"); job.setProgress(percent, "Reading " + fileSource);
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(job.config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", "Reading " + fileSource);
JSONUtilities.safePut(progress, "percent", percent);
} }
static private class WorksheetBatchRowReader implements TableDataReader { static private class WorksheetBatchRowReader implements TableDataReader {

13
extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java
View File

@ -300,9 +300,6 @@ public class GDataImportingController implements ImportingController {
job.updating = true; job.updating = true;
try { try {
// This is for setting progress during the parsing process.
job.config = new JSONObject();
JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject( JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(
request.getParameter("options")); request.getParameter("options"));
@ -371,7 +368,7 @@ public class GDataImportingController implements ImportingController {
final List<Exception> exceptions = new LinkedList<Exception>(); final List<Exception> exceptions = new LinkedList<Exception>();
JSONUtilities.safePut(job.config, "state", "creating-project"); job.setState("creating-project");
final Project project = new Project(); final Project project = new Project();
new Thread() { new Thread() {
@ -393,16 +390,14 @@ public class GDataImportingController implements ImportingController {
if (!job.canceled) { if (!job.canceled) {
if (exceptions.size() > 0) { if (exceptions.size() > 0) {
JSONUtilities.safePut(job.config, "errors", job.setError(exceptions);
DefaultImportingController.convertErrorsToJsonArray(exceptions));
JSONUtilities.safePut(job.config, "state", "error");
} else { } else {
project.update(); // update all internal models, indexes, caches, etc. project.update(); // update all internal models, indexes, caches, etc.
ProjectManager.singleton.registerProject(project, pm); ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "state", "created-project"); job.setState("created-project");
JSONUtilities.safePut(job.config, "projectID", project.id); job.setProjectID(project.id);
} }
job.touch(); job.touch();

2
main/src/com/google/refine/commands/project/CreateProjectCommand.java
View File

@ -112,7 +112,7 @@ public class CreateProjectCommand extends Command {
} else { } else {
Format formatRecord = ImportingManager.formatToRecord.get(format); Format formatRecord = ImportingManager.formatToRecord.get(format);
optionObj = formatRecord.parser.createParserUIInitializationData( optionObj = formatRecord.parser.createParserUIInitializationData(
job, ImportingUtilities.getSelectedFileRecords(job), format); job, job.getSelectedFileRecords(), format);
} }
adjustLegacyOptions(format, parameters, optionObj); adjustLegacyOptions(format, parameters, optionObj);

6
main/src/com/google/refine/importers/ImporterUtilities.java
View File

@ -209,10 +209,8 @@ public class ImporterUtilities {
long totalBytesRead = 0; long totalBytesRead = 0;
void setProgress(String fileSource, long bytesRead) { void setProgress(String fileSource, long bytesRead) {
ImportingUtilities.setCreatingProjectProgress( job.setProgress(totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2),
job, "Reading " + fileSource);
"Reading " + fileSource,
totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2));
} }
@Override @Override

2
main/src/com/google/refine/importing/DefaultImportingController.java
View File

@ -228,7 +228,7 @@ public class DefaultImportingController implements ImportingController {
Format formatRecord = ImportingManager.formatToRecord.get(format); Format formatRecord = ImportingManager.formatToRecord.get(format);
if (formatRecord != null && formatRecord.parser != null) { if (formatRecord != null && formatRecord.parser != null) {
JSONObject options = formatRecord.parser.createParserUIInitializationData( JSONObject options = formatRecord.parser.createParserUIInitializationData(
job, ImportingUtilities.getSelectedFileRecords(job), format); job, job.getSelectedFileRecords(), format);
JSONObject result = new JSONObject(); JSONObject result = new JSONObject();
JSONUtilities.safePut(result, "status", "ok"); JSONUtilities.safePut(result, "status", "ok");
JSONUtilities.safePut(result, "options", options); JSONUtilities.safePut(result, "options", options);

104
main/src/com/google/refine/importing/ImportingJob.java
View File

@ -35,9 +35,12 @@ package com.google.refine.importing;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties; import java.util.Properties;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
@ -52,7 +55,7 @@ public class ImportingJob implements Jsonizable {
final public long id; final public long id;
final public File dir; // Temporary directory where the data about this job is stored final public File dir; // Temporary directory where the data about this job is stored
public JSONObject config = null; private JSONObject config;
public Project project; public Project project;
public ProjectMetadata metadata; public ProjectMetadata metadata;
@ -61,22 +64,101 @@ public class ImportingJob implements Jsonizable {
public boolean updating; public boolean updating;
public boolean canceled; public boolean canceled;
final private Object lock = new Object();
public ImportingJob(long id, File dir) { public ImportingJob(long id, File dir) {
this.id = id; this.id = id;
this.dir = dir; this.dir = dir;
JSONObject cfg = new JSONObject();
JSONUtilities.safePut(cfg, "state", "new");
JSONUtilities.safePut(cfg, "hasData", false);
this.config = cfg;
dir.mkdirs(); dir.mkdirs();
} }
public JSONObject getOrCreateDefaultConfig() { public JSONObject getOrCreateDefaultConfig() {
if (config == null) {
config = new JSONObject();
JSONUtilities.safePut(config, "state", "new");
JSONUtilities.safePut(config, "hasData", false);
}
return config; return config;
} }
public void setState(String state) {
synchronized(config) {
JSONUtilities.safePut(config, "state", state);
}
}
public void setError(List<Exception> exceptions) {
synchronized(config) {
JSONUtilities.safePut(config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
setState("error");
}
}
public void setProjectID(long id2) {
synchronized (config) {
JSONUtilities.safePut(config, "projectID", project.id);
}
}
public void setProgress(int percent, String message) {
synchronized (config) {
JSONObject progress = JSONUtilities.getObject(config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", message);
JSONUtilities.safePut(progress, "percent", percent);
JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000);
JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000);
}
}
public void setFileSelection(JSONArray fileSelectionArray) {
synchronized (config) {
JSONUtilities.safePut(config, "fileSelection", fileSelectionArray);
}
}
public void setRankedFormats(JSONArray rankedFormats) {
synchronized (config) {
JSONUtilities.safePut(config, "rankedFormats", rankedFormats);
}
}
public JSONObject getRetrievalRecord() {
synchronized(config) {
return JSONUtilities.getObject(config,"retrievalRecord");
}
}
public List<JSONObject> getSelectedFileRecords() {
List<JSONObject> results = new ArrayList<JSONObject>();
JSONObject retrievalRecord = JSONUtilities.getObject(config,"retrievalRecord");
if (retrievalRecord != null) {
JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files");
if (fileRecordArray != null) {
JSONArray fileSelectionArray = JSONUtilities.getArray(config,"fileSelection");
if (fileSelectionArray != null) {
for (int i = 0; i < fileSelectionArray.length(); i++) {
int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1);
if (index >= 0 && index < fileRecordArray.length()) {
results.add(JSONUtilities.getObjectElement(fileRecordArray, index));
}
}
}
}
}
return results;
}
public void touch() { public void touch() {
lastTouched = System.currentTimeMillis(); lastTouched = System.currentTimeMillis();
} }
@ -111,8 +193,12 @@ public class ImportingJob implements Jsonizable {
@Override @Override
public void write(JSONWriter writer, Properties options) public void write(JSONWriter writer, Properties options)
throws JSONException { throws JSONException {
writer.object();
writer.key("config"); writer.value(config); synchronized(lock) {
writer.endObject(); writer.object();
writer.key("config"); writer.value(config);
writer.endObject();
}
} }
} }

57
main/src/com/google/refine/importing/ImportingUtilities.java
View File

@ -147,14 +147,14 @@ public class ImportingUtilities {
} }
static public void updateJobWithNewFileSelection(ImportingJob job, JSONArray fileSelectionArray) { static public void updateJobWithNewFileSelection(ImportingJob job, JSONArray fileSelectionArray) {
JSONUtilities.safePut(job.config, "fileSelection", fileSelectionArray); job.setFileSelection(fileSelectionArray);
String bestFormat = ImportingUtilities.getCommonFormatForSelectedFiles(job, fileSelectionArray); String bestFormat = ImportingUtilities.getCommonFormatForSelectedFiles(job, fileSelectionArray);
bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat); bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat);
JSONArray rankedFormats = new JSONArray(); JSONArray rankedFormats = new JSONArray();
JSONUtilities.safePut(job.config, "rankedFormats", rankedFormats);
ImportingUtilities.rankFormats(job, bestFormat, rankedFormats); ImportingUtilities.rankFormats(job, bestFormat, rankedFormats);
job.setRankedFormats(rankedFormats);
} }
static public void retrieveContentFromPostRequest( static public void retrieveContentFromPostRequest(
@ -210,6 +210,7 @@ public class ImportingUtilities {
} }
}); });
@SuppressWarnings("rawtypes")
List tempFiles = upload.parseRequest(request); List tempFiles = upload.parseRequest(request);
progress.setProgress("Uploading data ...", -1); progress.setProgress("Uploading data ...", -1);
@ -747,7 +748,7 @@ public class ImportingUtilities {
} }
static public String getCommonFormatForSelectedFiles(ImportingJob job, JSONArray fileSelectionIndexes) { static public String getCommonFormatForSelectedFiles(ImportingJob job, JSONArray fileSelectionIndexes) {
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord"); JSONObject retrievalRecord = job.getRetrievalRecord();
final Map<String, Integer> formatToCount = new HashMap<String, Integer>(); final Map<String, Integer> formatToCount = new HashMap<String, Integer>();
List<String> formats = new ArrayList<String>(); List<String> formats = new ArrayList<String>();
@ -780,7 +781,7 @@ public class ImportingUtilities {
} }
static String guessBetterFormat(ImportingJob job, String bestFormat) { static String guessBetterFormat(ImportingJob job, String bestFormat) {
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord"); JSONObject retrievalRecord = job.getRetrievalRecord();
return retrievalRecord != null ? guessBetterFormat(job, retrievalRecord, bestFormat) : bestFormat; return retrievalRecord != null ? guessBetterFormat(job, retrievalRecord, bestFormat) : bestFormat;
} }
@ -879,27 +880,7 @@ public class ImportingUtilities {
JSONUtilities.append(rankedFormats, format); JSONUtilities.append(rankedFormats, format);
} }
} }
static public List<JSONObject> getSelectedFileRecords(ImportingJob job) {
List<JSONObject> results = new ArrayList<JSONObject>();
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
if (retrievalRecord != null) {
JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files");
if (fileRecordArray != null) {
JSONArray fileSelectionArray = JSONUtilities.getArray(job.config, "fileSelection");
if (fileSelectionArray != null) {
for (int i = 0; i < fileSelectionArray.length(); i++) {
int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1);
if (index >= 0 && index < fileRecordArray.length()) {
results.add(JSONUtilities.getObjectElement(fileRecordArray, index));
}
}
}
}
}
return results;
}
static public void previewParse(ImportingJob job, String format, JSONObject optionObj, List<Exception> exceptions) { static public void previewParse(ImportingJob job, String format, JSONObject optionObj, List<Exception> exceptions) {
Format record = ImportingManager.formatToRecord.get(format); Format record = ImportingManager.formatToRecord.get(format);
@ -914,7 +895,7 @@ public class ImportingUtilities {
job.project, job.project,
job.metadata, job.metadata,
job, job,
getSelectedFileRecords(job), job.getSelectedFileRecords(),
format, format,
100, 100,
optionObj, optionObj,
@ -936,7 +917,7 @@ public class ImportingUtilities {
return -1; return -1;
} }
JSONUtilities.safePut(job.config, "state", "creating-project"); job.setState("creating-project");
final Project project = new Project(); final Project project = new Project();
if (synchronous) { if (synchronous) {
@ -975,7 +956,7 @@ public class ImportingUtilities {
project, project,
pm, pm,
job, job,
getSelectedFileRecords(job), job.getSelectedFileRecords(),
format, format,
-1, -1,
optionObj, optionObj,
@ -988,27 +969,15 @@ public class ImportingUtilities {
ProjectManager.singleton.registerProject(project, pm); ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "projectID", project.id); job.setProjectID(project.id);
JSONUtilities.safePut(job.config, "state", "created-project"); job.setState("created-project");
} else { } else {
JSONUtilities.safePut(job.config, "state", "error"); job.setError(exceptions);
JSONUtilities.safePut(job.config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
} }
job.touch(); job.touch();
job.updating = false; job.updating = false;
} }
} }
static public void setCreatingProjectProgress(ImportingJob job, String message, int percent) {
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(job.config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", message);
JSONUtilities.safePut(progress, "percent", percent);
JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000);
JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000);
}
} }