s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Centralize handling of import job config object & synchronize to allow

Browse Source 
multiple accessors
This commit is contained in:
Tom Morris 2013-07-25 15:41:08 -04:00
parent dc206e1889
commit 1e5f89e84c
7 changed files with 117 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java
View File

@ -172,13 +172,7 @@ public class GDataImporter {
}
static private void setProgress(ImportingJob job, String fileSource, int percent) {
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(job.config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", "Reading " + fileSource);
JSONUtilities.safePut(progress, "percent", percent);
job.setProgress(percent, "Reading " + fileSource);
}
static private class WorksheetBatchRowReader implements TableDataReader {

13
extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java
View File

@ -300,9 +300,6 @@ public class GDataImportingController implements ImportingController {
job.updating = true;
try {
// This is for setting progress during the parsing process.
job.config = new JSONObject();
JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(
request.getParameter("options"));
@ -371,7 +368,7 @@ public class GDataImportingController implements ImportingController {
final List<Exception> exceptions = new LinkedList<Exception>();
JSONUtilities.safePut(job.config, "state", "creating-project");
job.setState("creating-project");
final Project project = new Project();
new Thread() {
@ -393,16 +390,14 @@ public class GDataImportingController implements ImportingController {
if (!job.canceled) {
if (exceptions.size() > 0) {
JSONUtilities.safePut(job.config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
JSONUtilities.safePut(job.config, "state", "error");
job.setError(exceptions);
} else {
project.update(); // update all internal models, indexes, caches, etc.
ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "state", "created-project");
JSONUtilities.safePut(job.config, "projectID", project.id);
job.setState("created-project");
job.setProjectID(project.id);
}
job.touch();

2
main/src/com/google/refine/commands/project/CreateProjectCommand.java
View File

@ -112,7 +112,7 @@ public class CreateProjectCommand extends Command {
} else {
Format formatRecord = ImportingManager.formatToRecord.get(format);
optionObj = formatRecord.parser.createParserUIInitializationData(
job, ImportingUtilities.getSelectedFileRecords(job), format);
job, job.getSelectedFileRecords(), format);
}
adjustLegacyOptions(format, parameters, optionObj);

6
main/src/com/google/refine/importers/ImporterUtilities.java
View File

@ -209,10 +209,8 @@ public class ImporterUtilities {
long totalBytesRead = 0;
void setProgress(String fileSource, long bytesRead) {
ImportingUtilities.setCreatingProjectProgress(
job,
"Reading " + fileSource,
totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2));
job.setProgress(totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2),
"Reading " + fileSource);
}
@Override

2
main/src/com/google/refine/importing/DefaultImportingController.java
View File

@ -228,7 +228,7 @@ public class DefaultImportingController implements ImportingController {
Format formatRecord = ImportingManager.formatToRecord.get(format);
if (formatRecord != null && formatRecord.parser != null) {
JSONObject options = formatRecord.parser.createParserUIInitializationData(
job, ImportingUtilities.getSelectedFileRecords(job), format);
job, job.getSelectedFileRecords(), format);
JSONObject result = new JSONObject();
JSONUtilities.safePut(result, "status", "ok");
JSONUtilities.safePut(result, "options", options);

104
main/src/com/google/refine/importing/ImportingJob.java
View File

@ -35,9 +35,12 @@ package com.google.refine.importing;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
@ -52,7 +55,7 @@ public class ImportingJob implements Jsonizable {
final public long id;
final public File dir; // Temporary directory where the data about this job is stored
public JSONObject config = null;
private JSONObject config;
public Project project;
public ProjectMetadata metadata;
@ -61,22 +64,101 @@ public class ImportingJob implements Jsonizable {
public boolean updating;
public boolean canceled;
final private Object lock = new Object();
public ImportingJob(long id, File dir) {
this.id = id;
this.dir = dir;
JSONObject cfg = new JSONObject();
JSONUtilities.safePut(cfg, "state", "new");
JSONUtilities.safePut(cfg, "hasData", false);
this.config = cfg;
dir.mkdirs();
}
public JSONObject getOrCreateDefaultConfig() {
if (config == null) {
config = new JSONObject();
JSONUtilities.safePut(config, "state", "new");
JSONUtilities.safePut(config, "hasData", false);
}
return config;
}
public void setState(String state) {
synchronized(config) {
JSONUtilities.safePut(config, "state", state);
}
}
public void setError(List<Exception> exceptions) {
synchronized(config) {
JSONUtilities.safePut(config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
setState("error");
}
}
public void setProjectID(long id2) {
synchronized (config) {
JSONUtilities.safePut(config, "projectID", project.id);
}
}
public void setProgress(int percent, String message) {
synchronized (config) {
JSONObject progress = JSONUtilities.getObject(config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", message);
JSONUtilities.safePut(progress, "percent", percent);
JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000);
JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000);
}
}
public void setFileSelection(JSONArray fileSelectionArray) {
synchronized (config) {
JSONUtilities.safePut(config, "fileSelection", fileSelectionArray);
}
}
public void setRankedFormats(JSONArray rankedFormats) {
synchronized (config) {
JSONUtilities.safePut(config, "rankedFormats", rankedFormats);
}
}
public JSONObject getRetrievalRecord() {
synchronized(config) {
return JSONUtilities.getObject(config,"retrievalRecord");
}
}
public List<JSONObject> getSelectedFileRecords() {
List<JSONObject> results = new ArrayList<JSONObject>();
JSONObject retrievalRecord = JSONUtilities.getObject(config,"retrievalRecord");
if (retrievalRecord != null) {
JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files");
if (fileRecordArray != null) {
JSONArray fileSelectionArray = JSONUtilities.getArray(config,"fileSelection");
if (fileSelectionArray != null) {
for (int i = 0; i < fileSelectionArray.length(); i++) {
int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1);
if (index >= 0 && index < fileRecordArray.length()) {
results.add(JSONUtilities.getObjectElement(fileRecordArray, index));
}
}
}
}
}
return results;
}
public void touch() {
lastTouched = System.currentTimeMillis();
}
@ -111,8 +193,12 @@ public class ImportingJob implements Jsonizable {
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("config"); writer.value(config);
writer.endObject();
synchronized(lock) {
writer.object();
writer.key("config"); writer.value(config);
writer.endObject();
}
}
}

57
main/src/com/google/refine/importing/ImportingUtilities.java
View File

@ -147,14 +147,14 @@ public class ImportingUtilities {
}
static public void updateJobWithNewFileSelection(ImportingJob job, JSONArray fileSelectionArray) {
JSONUtilities.safePut(job.config, "fileSelection", fileSelectionArray);
job.setFileSelection(fileSelectionArray);
String bestFormat = ImportingUtilities.getCommonFormatForSelectedFiles(job, fileSelectionArray);
bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat);
JSONArray rankedFormats = new JSONArray();
JSONUtilities.safePut(job.config, "rankedFormats", rankedFormats);
ImportingUtilities.rankFormats(job, bestFormat, rankedFormats);
job.setRankedFormats(rankedFormats);
}
static public void retrieveContentFromPostRequest(
@ -210,6 +210,7 @@ public class ImportingUtilities {
}
});
@SuppressWarnings("rawtypes")
List tempFiles = upload.parseRequest(request);
progress.setProgress("Uploading data ...", -1);
@ -747,7 +748,7 @@ public class ImportingUtilities {
}
static public String getCommonFormatForSelectedFiles(ImportingJob job, JSONArray fileSelectionIndexes) {
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
JSONObject retrievalRecord = job.getRetrievalRecord();
final Map<String, Integer> formatToCount = new HashMap<String, Integer>();
List<String> formats = new ArrayList<String>();
@ -780,7 +781,7 @@ public class ImportingUtilities {
}
static String guessBetterFormat(ImportingJob job, String bestFormat) {
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
JSONObject retrievalRecord = job.getRetrievalRecord();
return retrievalRecord != null ? guessBetterFormat(job, retrievalRecord, bestFormat) : bestFormat;
}
@ -879,27 +880,7 @@ public class ImportingUtilities {
JSONUtilities.append(rankedFormats, format);
}
}
static public List<JSONObject> getSelectedFileRecords(ImportingJob job) {
List<JSONObject> results = new ArrayList<JSONObject>();
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
if (retrievalRecord != null) {
JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files");
if (fileRecordArray != null) {
JSONArray fileSelectionArray = JSONUtilities.getArray(job.config, "fileSelection");
if (fileSelectionArray != null) {
for (int i = 0; i < fileSelectionArray.length(); i++) {
int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1);
if (index >= 0 && index < fileRecordArray.length()) {
results.add(JSONUtilities.getObjectElement(fileRecordArray, index));
}
}
}
}
}
return results;
}
static public void previewParse(ImportingJob job, String format, JSONObject optionObj, List<Exception> exceptions) {
Format record = ImportingManager.formatToRecord.get(format);
@ -914,7 +895,7 @@ public class ImportingUtilities {
job.project,
job.metadata,
job,
getSelectedFileRecords(job),
job.getSelectedFileRecords(),
format,
100,
optionObj,
@ -936,7 +917,7 @@ public class ImportingUtilities {
return -1;
}
JSONUtilities.safePut(job.config, "state", "creating-project");
job.setState("creating-project");
final Project project = new Project();
if (synchronous) {
@ -975,7 +956,7 @@ public class ImportingUtilities {
project,
pm,
job,
getSelectedFileRecords(job),
job.getSelectedFileRecords(),
format,
-1,
optionObj,
@ -988,27 +969,15 @@ public class ImportingUtilities {
ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "projectID", project.id);
JSONUtilities.safePut(job.config, "state", "created-project");
job.setProjectID(project.id);
job.setState("created-project");
} else {
JSONUtilities.safePut(job.config, "state", "error");
JSONUtilities.safePut(job.config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
job.setError(exceptions);
}
job.touch();
job.updating = false;
}
}
static public void setCreatingProjectProgress(ImportingJob job, String message, int percent) {
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
if (progress == null) {
progress = new JSONObject();
JSONUtilities.safePut(job.config, "progress", progress);
}
JSONUtilities.safePut(progress, "message", message);
JSONUtilities.safePut(progress, "percent", percent);
JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000);
JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000);
}
}