Centralize handling of import job config object & synchronize to allow
multiple accessors
This commit is contained in:
parent
dc206e1889
commit
1e5f89e84c
@ -172,13 +172,7 @@ public class GDataImporter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static private void setProgress(ImportingJob job, String fileSource, int percent) {
|
static private void setProgress(ImportingJob job, String fileSource, int percent) {
|
||||||
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
|
job.setProgress(percent, "Reading " + fileSource);
|
||||||
if (progress == null) {
|
|
||||||
progress = new JSONObject();
|
|
||||||
JSONUtilities.safePut(job.config, "progress", progress);
|
|
||||||
}
|
|
||||||
JSONUtilities.safePut(progress, "message", "Reading " + fileSource);
|
|
||||||
JSONUtilities.safePut(progress, "percent", percent);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static private class WorksheetBatchRowReader implements TableDataReader {
|
static private class WorksheetBatchRowReader implements TableDataReader {
|
||||||
|
@ -300,9 +300,6 @@ public class GDataImportingController implements ImportingController {
|
|||||||
|
|
||||||
job.updating = true;
|
job.updating = true;
|
||||||
try {
|
try {
|
||||||
// This is for setting progress during the parsing process.
|
|
||||||
job.config = new JSONObject();
|
|
||||||
|
|
||||||
JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(
|
JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(
|
||||||
request.getParameter("options"));
|
request.getParameter("options"));
|
||||||
|
|
||||||
@ -371,7 +368,7 @@ public class GDataImportingController implements ImportingController {
|
|||||||
|
|
||||||
final List<Exception> exceptions = new LinkedList<Exception>();
|
final List<Exception> exceptions = new LinkedList<Exception>();
|
||||||
|
|
||||||
JSONUtilities.safePut(job.config, "state", "creating-project");
|
job.setState("creating-project");
|
||||||
|
|
||||||
final Project project = new Project();
|
final Project project = new Project();
|
||||||
new Thread() {
|
new Thread() {
|
||||||
@ -393,16 +390,14 @@ public class GDataImportingController implements ImportingController {
|
|||||||
|
|
||||||
if (!job.canceled) {
|
if (!job.canceled) {
|
||||||
if (exceptions.size() > 0) {
|
if (exceptions.size() > 0) {
|
||||||
JSONUtilities.safePut(job.config, "errors",
|
job.setError(exceptions);
|
||||||
DefaultImportingController.convertErrorsToJsonArray(exceptions));
|
|
||||||
JSONUtilities.safePut(job.config, "state", "error");
|
|
||||||
} else {
|
} else {
|
||||||
project.update(); // update all internal models, indexes, caches, etc.
|
project.update(); // update all internal models, indexes, caches, etc.
|
||||||
|
|
||||||
ProjectManager.singleton.registerProject(project, pm);
|
ProjectManager.singleton.registerProject(project, pm);
|
||||||
|
|
||||||
JSONUtilities.safePut(job.config, "state", "created-project");
|
job.setState("created-project");
|
||||||
JSONUtilities.safePut(job.config, "projectID", project.id);
|
job.setProjectID(project.id);
|
||||||
}
|
}
|
||||||
|
|
||||||
job.touch();
|
job.touch();
|
||||||
|
@ -112,7 +112,7 @@ public class CreateProjectCommand extends Command {
|
|||||||
} else {
|
} else {
|
||||||
Format formatRecord = ImportingManager.formatToRecord.get(format);
|
Format formatRecord = ImportingManager.formatToRecord.get(format);
|
||||||
optionObj = formatRecord.parser.createParserUIInitializationData(
|
optionObj = formatRecord.parser.createParserUIInitializationData(
|
||||||
job, ImportingUtilities.getSelectedFileRecords(job), format);
|
job, job.getSelectedFileRecords(), format);
|
||||||
}
|
}
|
||||||
adjustLegacyOptions(format, parameters, optionObj);
|
adjustLegacyOptions(format, parameters, optionObj);
|
||||||
|
|
||||||
|
@ -209,10 +209,8 @@ public class ImporterUtilities {
|
|||||||
long totalBytesRead = 0;
|
long totalBytesRead = 0;
|
||||||
|
|
||||||
void setProgress(String fileSource, long bytesRead) {
|
void setProgress(String fileSource, long bytesRead) {
|
||||||
ImportingUtilities.setCreatingProjectProgress(
|
job.setProgress(totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2),
|
||||||
job,
|
"Reading " + fileSource);
|
||||||
"Reading " + fileSource,
|
|
||||||
totalSize2 == 0 ? -1 : (int) (100 * (totalBytesRead + bytesRead) / totalSize2));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -228,7 +228,7 @@ public class DefaultImportingController implements ImportingController {
|
|||||||
Format formatRecord = ImportingManager.formatToRecord.get(format);
|
Format formatRecord = ImportingManager.formatToRecord.get(format);
|
||||||
if (formatRecord != null && formatRecord.parser != null) {
|
if (formatRecord != null && formatRecord.parser != null) {
|
||||||
JSONObject options = formatRecord.parser.createParserUIInitializationData(
|
JSONObject options = formatRecord.parser.createParserUIInitializationData(
|
||||||
job, ImportingUtilities.getSelectedFileRecords(job), format);
|
job, job.getSelectedFileRecords(), format);
|
||||||
JSONObject result = new JSONObject();
|
JSONObject result = new JSONObject();
|
||||||
JSONUtilities.safePut(result, "status", "ok");
|
JSONUtilities.safePut(result, "status", "ok");
|
||||||
JSONUtilities.safePut(result, "options", options);
|
JSONUtilities.safePut(result, "options", options);
|
||||||
|
@ -35,9 +35,12 @@ package com.google.refine.importing;
|
|||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.json.JSONArray;
|
||||||
import org.json.JSONException;
|
import org.json.JSONException;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.json.JSONWriter;
|
import org.json.JSONWriter;
|
||||||
@ -52,7 +55,7 @@ public class ImportingJob implements Jsonizable {
|
|||||||
final public long id;
|
final public long id;
|
||||||
final public File dir; // Temporary directory where the data about this job is stored
|
final public File dir; // Temporary directory where the data about this job is stored
|
||||||
|
|
||||||
public JSONObject config = null;
|
private JSONObject config;
|
||||||
|
|
||||||
public Project project;
|
public Project project;
|
||||||
public ProjectMetadata metadata;
|
public ProjectMetadata metadata;
|
||||||
@ -61,22 +64,101 @@ public class ImportingJob implements Jsonizable {
|
|||||||
public boolean updating;
|
public boolean updating;
|
||||||
public boolean canceled;
|
public boolean canceled;
|
||||||
|
|
||||||
|
final private Object lock = new Object();
|
||||||
|
|
||||||
public ImportingJob(long id, File dir) {
|
public ImportingJob(long id, File dir) {
|
||||||
this.id = id;
|
this.id = id;
|
||||||
this.dir = dir;
|
this.dir = dir;
|
||||||
|
|
||||||
|
JSONObject cfg = new JSONObject();
|
||||||
|
JSONUtilities.safePut(cfg, "state", "new");
|
||||||
|
JSONUtilities.safePut(cfg, "hasData", false);
|
||||||
|
this.config = cfg;
|
||||||
|
|
||||||
dir.mkdirs();
|
dir.mkdirs();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public JSONObject getOrCreateDefaultConfig() {
|
public JSONObject getOrCreateDefaultConfig() {
|
||||||
if (config == null) {
|
|
||||||
config = new JSONObject();
|
|
||||||
JSONUtilities.safePut(config, "state", "new");
|
|
||||||
JSONUtilities.safePut(config, "hasData", false);
|
|
||||||
}
|
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setState(String state) {
|
||||||
|
synchronized(config) {
|
||||||
|
JSONUtilities.safePut(config, "state", state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setError(List<Exception> exceptions) {
|
||||||
|
synchronized(config) {
|
||||||
|
JSONUtilities.safePut(config, "errors",
|
||||||
|
DefaultImportingController.convertErrorsToJsonArray(exceptions));
|
||||||
|
setState("error");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProjectID(long id2) {
|
||||||
|
synchronized (config) {
|
||||||
|
JSONUtilities.safePut(config, "projectID", project.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProgress(int percent, String message) {
|
||||||
|
synchronized (config) {
|
||||||
|
JSONObject progress = JSONUtilities.getObject(config, "progress");
|
||||||
|
if (progress == null) {
|
||||||
|
progress = new JSONObject();
|
||||||
|
JSONUtilities.safePut(config, "progress", progress);
|
||||||
|
}
|
||||||
|
JSONUtilities.safePut(progress, "message", message);
|
||||||
|
JSONUtilities.safePut(progress, "percent", percent);
|
||||||
|
JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000);
|
||||||
|
JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFileSelection(JSONArray fileSelectionArray) {
|
||||||
|
synchronized (config) {
|
||||||
|
JSONUtilities.safePut(config, "fileSelection", fileSelectionArray);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRankedFormats(JSONArray rankedFormats) {
|
||||||
|
synchronized (config) {
|
||||||
|
JSONUtilities.safePut(config, "rankedFormats", rankedFormats);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public JSONObject getRetrievalRecord() {
|
||||||
|
synchronized(config) {
|
||||||
|
return JSONUtilities.getObject(config,"retrievalRecord");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public List<JSONObject> getSelectedFileRecords() {
|
||||||
|
List<JSONObject> results = new ArrayList<JSONObject>();
|
||||||
|
|
||||||
|
JSONObject retrievalRecord = JSONUtilities.getObject(config,"retrievalRecord");
|
||||||
|
if (retrievalRecord != null) {
|
||||||
|
JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files");
|
||||||
|
if (fileRecordArray != null) {
|
||||||
|
JSONArray fileSelectionArray = JSONUtilities.getArray(config,"fileSelection");
|
||||||
|
if (fileSelectionArray != null) {
|
||||||
|
for (int i = 0; i < fileSelectionArray.length(); i++) {
|
||||||
|
int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1);
|
||||||
|
if (index >= 0 && index < fileRecordArray.length()) {
|
||||||
|
results.add(JSONUtilities.getObjectElement(fileRecordArray, index));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
public void touch() {
|
public void touch() {
|
||||||
lastTouched = System.currentTimeMillis();
|
lastTouched = System.currentTimeMillis();
|
||||||
}
|
}
|
||||||
@ -111,8 +193,12 @@ public class ImportingJob implements Jsonizable {
|
|||||||
@Override
|
@Override
|
||||||
public void write(JSONWriter writer, Properties options)
|
public void write(JSONWriter writer, Properties options)
|
||||||
throws JSONException {
|
throws JSONException {
|
||||||
writer.object();
|
|
||||||
writer.key("config"); writer.value(config);
|
synchronized(lock) {
|
||||||
writer.endObject();
|
writer.object();
|
||||||
|
writer.key("config"); writer.value(config);
|
||||||
|
writer.endObject();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -147,14 +147,14 @@ public class ImportingUtilities {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static public void updateJobWithNewFileSelection(ImportingJob job, JSONArray fileSelectionArray) {
|
static public void updateJobWithNewFileSelection(ImportingJob job, JSONArray fileSelectionArray) {
|
||||||
JSONUtilities.safePut(job.config, "fileSelection", fileSelectionArray);
|
job.setFileSelection(fileSelectionArray);
|
||||||
|
|
||||||
String bestFormat = ImportingUtilities.getCommonFormatForSelectedFiles(job, fileSelectionArray);
|
String bestFormat = ImportingUtilities.getCommonFormatForSelectedFiles(job, fileSelectionArray);
|
||||||
bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat);
|
bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat);
|
||||||
|
|
||||||
JSONArray rankedFormats = new JSONArray();
|
JSONArray rankedFormats = new JSONArray();
|
||||||
JSONUtilities.safePut(job.config, "rankedFormats", rankedFormats);
|
|
||||||
ImportingUtilities.rankFormats(job, bestFormat, rankedFormats);
|
ImportingUtilities.rankFormats(job, bestFormat, rankedFormats);
|
||||||
|
job.setRankedFormats(rankedFormats);
|
||||||
}
|
}
|
||||||
|
|
||||||
static public void retrieveContentFromPostRequest(
|
static public void retrieveContentFromPostRequest(
|
||||||
@ -210,6 +210,7 @@ public class ImportingUtilities {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@SuppressWarnings("rawtypes")
|
||||||
List tempFiles = upload.parseRequest(request);
|
List tempFiles = upload.parseRequest(request);
|
||||||
|
|
||||||
progress.setProgress("Uploading data ...", -1);
|
progress.setProgress("Uploading data ...", -1);
|
||||||
@ -747,7 +748,7 @@ public class ImportingUtilities {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static public String getCommonFormatForSelectedFiles(ImportingJob job, JSONArray fileSelectionIndexes) {
|
static public String getCommonFormatForSelectedFiles(ImportingJob job, JSONArray fileSelectionIndexes) {
|
||||||
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
|
JSONObject retrievalRecord = job.getRetrievalRecord();
|
||||||
|
|
||||||
final Map<String, Integer> formatToCount = new HashMap<String, Integer>();
|
final Map<String, Integer> formatToCount = new HashMap<String, Integer>();
|
||||||
List<String> formats = new ArrayList<String>();
|
List<String> formats = new ArrayList<String>();
|
||||||
@ -780,7 +781,7 @@ public class ImportingUtilities {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static String guessBetterFormat(ImportingJob job, String bestFormat) {
|
static String guessBetterFormat(ImportingJob job, String bestFormat) {
|
||||||
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
|
JSONObject retrievalRecord = job.getRetrievalRecord();
|
||||||
return retrievalRecord != null ? guessBetterFormat(job, retrievalRecord, bestFormat) : bestFormat;
|
return retrievalRecord != null ? guessBetterFormat(job, retrievalRecord, bestFormat) : bestFormat;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -880,26 +881,6 @@ public class ImportingUtilities {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static public List<JSONObject> getSelectedFileRecords(ImportingJob job) {
|
|
||||||
List<JSONObject> results = new ArrayList<JSONObject>();
|
|
||||||
|
|
||||||
JSONObject retrievalRecord = JSONUtilities.getObject(job.config, "retrievalRecord");
|
|
||||||
if (retrievalRecord != null) {
|
|
||||||
JSONArray fileRecordArray = JSONUtilities.getArray(retrievalRecord, "files");
|
|
||||||
if (fileRecordArray != null) {
|
|
||||||
JSONArray fileSelectionArray = JSONUtilities.getArray(job.config, "fileSelection");
|
|
||||||
if (fileSelectionArray != null) {
|
|
||||||
for (int i = 0; i < fileSelectionArray.length(); i++) {
|
|
||||||
int index = JSONUtilities.getIntElement(fileSelectionArray, i, -1);
|
|
||||||
if (index >= 0 && index < fileRecordArray.length()) {
|
|
||||||
results.add(JSONUtilities.getObjectElement(fileRecordArray, index));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
static public void previewParse(ImportingJob job, String format, JSONObject optionObj, List<Exception> exceptions) {
|
static public void previewParse(ImportingJob job, String format, JSONObject optionObj, List<Exception> exceptions) {
|
||||||
Format record = ImportingManager.formatToRecord.get(format);
|
Format record = ImportingManager.formatToRecord.get(format);
|
||||||
@ -914,7 +895,7 @@ public class ImportingUtilities {
|
|||||||
job.project,
|
job.project,
|
||||||
job.metadata,
|
job.metadata,
|
||||||
job,
|
job,
|
||||||
getSelectedFileRecords(job),
|
job.getSelectedFileRecords(),
|
||||||
format,
|
format,
|
||||||
100,
|
100,
|
||||||
optionObj,
|
optionObj,
|
||||||
@ -936,7 +917,7 @@ public class ImportingUtilities {
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
JSONUtilities.safePut(job.config, "state", "creating-project");
|
job.setState("creating-project");
|
||||||
|
|
||||||
final Project project = new Project();
|
final Project project = new Project();
|
||||||
if (synchronous) {
|
if (synchronous) {
|
||||||
@ -975,7 +956,7 @@ public class ImportingUtilities {
|
|||||||
project,
|
project,
|
||||||
pm,
|
pm,
|
||||||
job,
|
job,
|
||||||
getSelectedFileRecords(job),
|
job.getSelectedFileRecords(),
|
||||||
format,
|
format,
|
||||||
-1,
|
-1,
|
||||||
optionObj,
|
optionObj,
|
||||||
@ -988,27 +969,15 @@ public class ImportingUtilities {
|
|||||||
|
|
||||||
ProjectManager.singleton.registerProject(project, pm);
|
ProjectManager.singleton.registerProject(project, pm);
|
||||||
|
|
||||||
JSONUtilities.safePut(job.config, "projectID", project.id);
|
job.setProjectID(project.id);
|
||||||
JSONUtilities.safePut(job.config, "state", "created-project");
|
job.setState("created-project");
|
||||||
} else {
|
} else {
|
||||||
JSONUtilities.safePut(job.config, "state", "error");
|
job.setError(exceptions);
|
||||||
JSONUtilities.safePut(job.config, "errors",
|
|
||||||
DefaultImportingController.convertErrorsToJsonArray(exceptions));
|
|
||||||
}
|
}
|
||||||
job.touch();
|
job.touch();
|
||||||
job.updating = false;
|
job.updating = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static public void setCreatingProjectProgress(ImportingJob job, String message, int percent) {
|
|
||||||
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
|
|
||||||
if (progress == null) {
|
|
||||||
progress = new JSONObject();
|
|
||||||
JSONUtilities.safePut(job.config, "progress", progress);
|
|
||||||
}
|
|
||||||
JSONUtilities.safePut(progress, "message", message);
|
|
||||||
JSONUtilities.safePut(progress, "percent", percent);
|
|
||||||
JSONUtilities.safePut(progress, "memory", Runtime.getRuntime().totalMemory() / 1000000);
|
|
||||||
JSONUtilities.safePut(progress, "maxmemory", Runtime.getRuntime().maxMemory() / 1000000);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user