s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added support for importing from fusion tables.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2239 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2011-09-15 21:40:40 +00:00
parent ebede9b424
commit 0693205430
12 changed files with 485 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html Normal file
View File

@ -0,0 +1,33 @@
<div bind="wizardHeader" class="gdata-importing-wizard-header"><div class="grid-layout layout-tightest layout-full"><table><tr>
<td width="1%"><button bind="startOverButton" class="button">&laquo; Start Over</button></td>
<td width="98%">Configure Parsing Options</td>
<td style="text-align: right;">Project&nbsp;name</td>
<td width="1%"><input class="inline" type="text" size="30" bind="projectNameInput" /></td>
<td width="1%"><button bind="createProjectButton" class="button button-primary">Create Project &raquo;</button></td>
</tr></table></div></div>
<div bind="dataPanel" class="gdata-importing-parsing-data-panel"></div>
<div bind="progressPanel" class="gdata-importing-progress-data-panel">
<img src="images/large-spinner.gif" /> Updating preview ...
</div>
<div bind="controlPanel" class="gdata-importing-parsing-control-panel"><div class="grid-layout layout-normal"><table>
<tr>
<td>Options</td>
<td><button class="button" bind="previewButton">Update&nbsp;Preview</button></td>
</tr>
<tr>
<td><div class="grid-layout layout-tightest"><table>
<tr><td width="1%"><input type="checkbox" bind="skipCheckbox" /></td><td>Discard initial</td>
<td><input bind="skipInput" type="text" class="lightweight" size="2" value="0" /> row(s) of data</td></tr>
<tr><td width="1%"><input type="checkbox" bind="limitCheckbox" /></td><td>Load at most</td>
<td><input bind="limitInput" type="text" class="lightweight" size="2" value="0" /> row(s) of data</td></tr>
<tr><td width="1%"><input type="checkbox" bind="storeBlankRowsCheckbox" /></td>
<td colspan="2">Store blank rows</td></tr>
<tr><td width="1%"><input type="checkbox" bind="storeBlankCellsAsNullsCheckbox" /></td>
<td colspan="2">Store blank cells as nulls</td></tr>
</table></div></td>
</tr>
</table></div></div>

14
extensions/gdata/module/scripts/index/gdata-source-ui.js
View File

@ -137,17 +137,15 @@ Refine.GDataSourceUI.prototype._renderDocuments = function(o) {
td = tr.insertCell(tr.cells.length);
$('<span>')
.addClass('gdata-doc-authors')
.text(doc.authors.join(', '))
.text((doc.authors) ? doc.authors.join(', ') : '<unknown>')
.appendTo(td);
td = tr.insertCell(tr.cells.length);
if (doc.updated) {
$('<span>')
.addClass('gdata-doc-date')
.text(formatRelativeDate(doc.updated))
.attr('title', doc.updated)
.appendTo(td);
}
$('<span>')
.addClass('gdata-doc-date')
.text((doc.updated) ? formatRelativeDate(doc.updated) : '<unknown>')
.attr('title', (doc.updated) ? doc.updated : '<unknown>')
.appendTo(td);
};
var docs = o.documents;

91
extensions/gdata/module/scripts/index/importing-controller.js
View File

@ -84,7 +84,6 @@ Refine.GDataImportingController.prototype.getOptions = function() {
var options = {
docUrl: this._doc.docSelfLink,
docType: this._doc.type,
sheetUrl: this._sheetUrl
};
var parseIntDefault = function(s, def) {
@ -99,22 +98,25 @@ Refine.GDataImportingController.prototype.getOptions = function() {
return def;
};
this._parsingPanelElmts.sheetRecordContainer.find('input').each(function() {
if (this.checked) {
options.sheetUrl = this.getAttribute('sheetUrl');
}
});
if (this._doc.type != 'table') {
this._parsingPanelElmts.sheetRecordContainer.find('input').each(function() {
if (this.checked) {
options.sheetUrl = this.getAttribute('sheetUrl');
}
});
if (this._parsingPanelElmts.ignoreCheckbox[0].checked) {
options.ignoreLines = parseIntDefault(this._parsingPanelElmts.ignoreInput[0].value, -1);
} else {
options.ignoreLines = -1;
}
if (this._parsingPanelElmts.headerLinesCheckbox[0].checked) {
options.headerLines = parseIntDefault(this._parsingPanelElmts.headerLinesInput[0].value, 0);
} else {
options.headerLines = 0;
if (this._parsingPanelElmts.ignoreCheckbox[0].checked) {
options.ignoreLines = parseIntDefault(this._parsingPanelElmts.ignoreInput[0].value, -1);
} else {
options.ignoreLines = -1;
}
if (this._parsingPanelElmts.headerLinesCheckbox[0].checked) {
options.headerLines = parseIntDefault(this._parsingPanelElmts.headerLinesInput[0].value, 0);
} else {
options.headerLines = 0;
}
}
if (this._parsingPanelElmts.skipCheckbox[0].checked) {
options.skipDataLines = parseIntDefault(this._parsingPanelElmts.skipInput[0].value, 0);
} else {
@ -135,7 +137,10 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
var self = this;
this._parsingPanel.unbind().empty().html(
DOM.loadHTML("gdata", "scripts/index/gdata-parsing-panel.html"));
DOM.loadHTML("gdata",
this._doc.type == 'table' ?
'scripts/index/gdata-fusion-tables-parsing-panel.html' :
'scripts/index/gdata-parsing-panel.html'));
this._parsingPanelElmts = DOM.bind(this._parsingPanel);
if (this._parsingPanelResizer) {
@ -184,30 +189,33 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
this._parsingPanelElmts.projectNameInput[0].value = this._doc.title;
var sheetTable = this._parsingPanelElmts.sheetRecordContainer[0];
$.each(this._options.worksheets, function(i, v) {
var tr = sheetTable.insertRow(sheetTable.rows.length);
var td0 = $(tr.insertCell(0)).attr('width', '1%');
var checkbox = $('<input>')
.attr('type', 'radio')
.attr('name', 'gdata-importing-parsing-worksheet')
.attr('sheetUrl', this.link)
.appendTo(td0);
if (i === 0) {
checkbox.attr('checked', 'true');
}
$(tr.insertCell(1)).text(this.name);
$(tr.insertCell(2)).text(this.rows + ' rows');
});
if (this._doc.type != 'table') {
var sheetTable = this._parsingPanelElmts.sheetRecordContainer[0];
$.each(this._options.worksheets, function(i, v) {
var tr = sheetTable.insertRow(sheetTable.rows.length);
var td0 = $(tr.insertCell(0)).attr('width', '1%');
var checkbox = $('<input>')
.attr('type', 'radio')
.attr('name', 'gdata-importing-parsing-worksheet')
.attr('sheetUrl', this.link)
.appendTo(td0);
if (i === 0) {
checkbox.attr('checked', 'true');
}
$(tr.insertCell(1)).text(this.name);
$(tr.insertCell(2)).text(this.rows + ' rows');
});
if (this._options.ignoreLines > 0) {
this._parsingPanelElmts.ignoreCheckbox.attr("checked", "checked");
this._parsingPanelElmts.ignoreInput[0].value = this._options.ignoreLines.toString();
}
if (this._options.headerLines > 0) {
this._parsingPanelElmts.headerLinesCheckbox.attr("checked", "checked");
this._parsingPanelElmts.headerLinesInput[0].value = this._options.headerLines.toString();
if (this._options.ignoreLines > 0) {
this._parsingPanelElmts.ignoreCheckbox.attr("checked", "checked");
this._parsingPanelElmts.ignoreInput[0].value = this._options.ignoreLines.toString();
}
if (this._options.headerLines > 0) {
this._parsingPanelElmts.headerLinesCheckbox.attr("checked", "checked");
this._parsingPanelElmts.headerLinesInput[0].value = this._options.headerLines.toString();
}
}
if (this._options.limit > 0) {
this._parsingPanelElmts.limitCheckbox.attr("checked", "checked");
this._parsingPanelElmts.limitInput[0].value = this._options.limit.toString();
@ -262,7 +270,7 @@ Refine.GDataImportingController.prototype._updatePreview = function() {
"options" : JSON.stringify(this.getOptions())
},
function(result) {
if (result.code == "ok") {
if (result.status == "ok") {
self._getPreviewData(function(projectData) {
self._parsingPanelElmts.progressPanel.hide();
self._parsingPanelElmts.dataPanel.show();
@ -271,7 +279,7 @@ Refine.GDataImportingController.prototype._updatePreview = function() {
});
} else {
self._parsingPanelElmts.progressPanel.hide();
alert('Errors:\n' + result.errors.join('\n'));
alert('Errors:\n' + Refine.CreateProjectUI.composeErrorMessage(job));
}
},
"json"
@ -345,10 +353,11 @@ Refine.GDataImportingController.prototype._createProject = function() {
return "projectID" in job.config;
},
function(jobID, job) {
window.clearInterval(timerID);
document.location = "project?project=" + job.config.projectID;
},
function(job) {
alert(job.config.error + '\n' + job.config.errorDetails);
alert(Refine.CreateProjectUI.composeErrorMessage(job));
}
);
},

2
extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java
View File

@ -33,7 +33,7 @@ public class AuthorizeCommand extends Command {
String requestUrl = AuthSubUtil.getRequestUrl(
authorizedUrl.toExternalForm(), // execution continues at authorized on redirect
"https://docs.google.com/feeds https://spreadsheets.google.com/feeds",
"https://spreadsheets.google.com/feeds https://www.google.com/fusiontables/api/query",
false,
true);
response.sendRedirect(requestUrl);

64
extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java
View File

@ -28,9 +28,23 @@
*/
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.regex.MatchResult;
import java.util.regex.Pattern;
import com.google.gdata.client.GoogleService;
import com.google.gdata.client.Service.GDataRequest;
import com.google.gdata.client.Service.GDataRequest.RequestType;
import com.google.gdata.client.docs.DocsService;
import com.google.gdata.client.spreadsheet.FeedURLFactory;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.util.ContentType;
import com.google.gdata.util.ServiceException;
/**
* @author Tom Morris <tfmorris@gmail.com>
@ -60,4 +74,54 @@ abstract public class GDataExtension {
service.setAuthSubToken(token);
return service;
}
static public GoogleService getFusionTablesGoogleService(String token) {
GoogleService service = new GoogleService("fusiontables", SERVICE_APP_NAME);
service.setAuthSubToken(token);
return service;
}
final static private String FUSION_TABLES_SERVICE_URL =
"https://www.google.com/fusiontables/api/query";
final static private Pattern CSV_VALUE_PATTERN =
Pattern.compile("([^,\\r\\n\"]*|\"(([^\"]*\"\")*[^\"]*)\")(,|\\r?\\n)");
static public List<List<String>> runFusionTablesSelect(GoogleService service, String selectQuery)
throws IOException, ServiceException {
URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" +
URLEncoder.encode(selectQuery, "UTF-8"));
GDataRequest request = service.getRequestFactory().getRequest(
RequestType.QUERY, url, ContentType.TEXT_PLAIN);
request.execute();
List<List<String>> rows = new ArrayList<List<String>>();
List<String> row = null;
Scanner scanner = new Scanner(request.getResponseStream(), "UTF-8");
while (scanner.hasNextLine()) {
scanner.findWithinHorizon(CSV_VALUE_PATTERN, 0);
MatchResult match = scanner.match();
String quotedString = match.group(2);
String decoded = quotedString == null ? match.group(1) : quotedString.replaceAll("\"\"", "\"");
if (row == null) {
row = new ArrayList<String>();
}
row.add(decoded);
if (!match.group(4).equals(",")) {
if (row != null) {
rows.add(row);
row = null;
}
}
}
if (row != null) {
rows.add(row);
}
return rows;
}
}

233
extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java
View File

@ -36,7 +36,7 @@ import java.util.List;
import org.json.JSONObject;
import com.google.gdata.client.docs.DocsService;
import com.google.gdata.client.GoogleService;
import com.google.gdata.client.spreadsheet.CellQuery;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.data.spreadsheet.Cell;
@ -75,19 +75,19 @@ public class GDataImporter {
SpreadsheetService service = GDataExtension.getSpreadsheetService(token);
parse(
service,
job.project,
job.metadata,
project,
metadata,
job,
limit,
options,
exceptions
);
} else if ("table".equals(docType)) {
DocsService service = GDataExtension.getDocsService(token);
GoogleService service = GDataExtension.getFusionTablesGoogleService(token);
parse(
service,
job.project,
job.metadata,
project,
metadata,
job,
limit,
options,
@ -165,21 +165,6 @@ public class GDataImporter {
}
}
static public void parse(
DocsService service,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
JSONObject options,
List<Exception> exceptions) {
String docUrlString = JSONUtilities.getString(options, "docUrl", null);
if (docUrlString != null) {
// TODO[dfhuynh]
}
}
static private void setProgress(ImportingJob job, String fileSource, int percent) {
JSONObject progress = JSONUtilities.getObject(job.config, "progress");
if (progress == null) {
@ -287,4 +272,210 @@ public class GDataImporter {
return rowsOfCells;
}
}
static public void parse(
GoogleService service,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
JSONObject options,
List<Exception> exceptions) {
String docUrlString = JSONUtilities.getString(options, "docUrl", null);
if (docUrlString == null) {
return;
}
int equal = docUrlString.lastIndexOf('=');
if (equal < 0) {
return;
}
String id = docUrlString.substring(equal + 1);
try {
List<FTColumnData> columns = new ArrayList<GDataImporter.FTColumnData>();
List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, "DESCRIBE " + id);
if (rows.size() > 1) {
for (int i = 1; i < rows.size(); i++) {
List<String> row = rows.get(i);
if (row.size() >= 2) {
FTColumnData cd = new FTColumnData();
cd.name = row.get(1);
cd.type = FTColumnType.STRING;
if (row.size() > 2) {
String type = row.get(2).toLowerCase();
if (type.equals("number")) {
cd.type = FTColumnType.NUMBER;
} else if (type.equals("datetime")) {
cd.type = FTColumnType.DATETIME;
} else if (type.equals("location")) {
cd.type = FTColumnType.LOCATION;
}
}
columns.add(cd);
}
}
setProgress(job, docUrlString, -1);
// Force these options for the next call because each fusion table
// is strictly structured with a single line of headers.
JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore
JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
TabularImportingParserBase.readTable(
project,
metadata,
job,
new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100),
docUrlString,
limit,
options,
exceptions
);
setProgress(job, docUrlString, 100);
}
} catch (IOException e) {
e.printStackTrace();
exceptions.add(e);
} catch (ServiceException e) {
e.printStackTrace();
exceptions.add(e);
}
}
static private enum FTColumnType {
STRING,
NUMBER,
DATETIME,
LOCATION
}
final static private class FTColumnData {
String name;
FTColumnType type;
}
static private class FusionTableBatchRowReader implements TableDataReader {
final ImportingJob job;
final String fileSource;
final GoogleService service;
final List<FTColumnData> columns;
final int batchSize;
final String baseQuery;
int nextRow = 0; // 0-based
int batchRowStart = 0; // 0-based
boolean end = false;
List<List<Object>> rowsOfCells = null;
boolean usedHeaders = false;
public FusionTableBatchRowReader(ImportingJob job, String fileSource,
GoogleService service, String tableId, List<FTColumnData> columns,
int batchSize) {
this.job = job;
this.fileSource = fileSource;
this.service = service;
this.columns = columns;
this.batchSize = batchSize;
StringBuffer sb = new StringBuffer();
sb.append("SELECT ");
boolean first = true;
for (FTColumnData cd : columns) {
if (first) {
first = false;
} else {
sb.append(",");
}
sb.append("'");
sb.append(cd.name);
sb.append("'");
}
sb.append(" FROM ");
sb.append(tableId);
baseQuery = sb.toString();
}
@Override
public List<Object> getNextRowOfCells() throws IOException {
if (!usedHeaders) {
List<Object> row = new ArrayList<Object>(columns.size());
for (FTColumnData cd : columns) {
row.add(cd.name);
}
usedHeaders = true;
return row;
}
if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) {
int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size());
try {
rowsOfCells = getRowsOfCells(newBatchRowStart);
batchRowStart = newBatchRowStart;
setProgress(job, fileSource, -1 /* batchRowStart * 100 / totalRows */);
} catch (ServiceException e) {
throw new IOException(e);
}
}
if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) {
return rowsOfCells.get(nextRow++ - batchRowStart);
} else {
return null;
}
}
private List<List<Object>> getRowsOfCells(int startRow) throws IOException, ServiceException {
List<List<Object>> rowsOfCells = new ArrayList<List<Object>>(batchSize);
String query = baseQuery + " OFFSET " + startRow + " LIMIT " + batchSize;
List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, query);
if (rows.size() > 1) {
for (int i = 1; i < rows.size(); i++) {
List<String> row = rows.get(i);
List<Object> rowOfCells = new ArrayList<Object>(row.size());
for (int j = 0; j < row.size() && j < columns.size(); j++) {
String text = row.get(j);
if (text.isEmpty()) {
rowOfCells.add(null);
} else {
FTColumnData cd = columns.get(j);
if (cd.type == FTColumnType.NUMBER) {
try {
rowOfCells.add(Long.parseLong(text));
continue;
} catch (NumberFormatException e) {
// ignore
}
try {
double d = Double.parseDouble(text);
if (!Double.isInfinite(d) && !Double.isNaN(d)) {
rowOfCells.add(d);
continue;
}
} catch (NumberFormatException e) {
// ignore
}
}
rowOfCells.add(text);
}
}
rowsOfCells.add(rowOfCells);
}
}
end = rows.size() < batchSize + 1;
return rowsOfCells;
}
}
}

108
extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java
View File

@ -49,17 +49,13 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.gdata.client.Query;
import com.google.gdata.client.GoogleService;
import com.google.gdata.client.docs.DocsService;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.data.Category;
import com.google.gdata.data.DateTime;
import com.google.gdata.data.Person;
import com.google.gdata.data.docs.DocumentListEntry;
import com.google.gdata.data.docs.DocumentListFeed;
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
import com.google.gdata.data.spreadsheet.SpreadsheetFeed;
import com.google.gdata.data.spreadsheet.TableEntry;
import com.google.gdata.data.spreadsheet.WorksheetEntry;
import com.google.gdata.util.ServiceException;
@ -67,6 +63,7 @@ import com.google.refine.ProjectManager;
import com.google.refine.ProjectMetadata;
import com.google.refine.RefineServlet;
import com.google.refine.commands.HttpUtilities;
import com.google.refine.importing.DefaultImportingController;
import com.google.refine.importing.ImportingController;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingManager;
@ -125,11 +122,10 @@ public class GDataImportingController implements ImportingController {
writer.array();
try {
DocsService service = GDataExtension.getDocsService(token);
listSpreadsheets(service, writer);
listDocumentsOfType(service, writer, "http://schemas.google.com/docs/2007#table");
listSpreadsheets(GDataExtension.getDocsService(token), writer);
listFusionTables(GDataExtension.getFusionTablesGoogleService(token), writer);
} catch (ServiceException e) {
// TODO: just ignore?
e.printStackTrace();
}
writer.endArray();
@ -169,36 +165,27 @@ public class GDataImportingController implements ImportingController {
}
}
private void listDocumentsOfType(DocsService service, JSONWriter writer, String type)
throws IOException, ServiceException, JSONException {
URL feedUrl = new URL("https://docs.google.com/feeds/default/private/full");
private void listFusionTables(GoogleService service, JSONWriter writer)
throws IOException, ServiceException, JSONException {
Query query = new Query(feedUrl);
query.addCategoryFilter(
new Query.CategoryFilter(
new Category("http://schemas.google.com/g/2005#kind", type)));
DocumentListFeed feed = service.query(query, DocumentListFeed.class);
for (DocumentListEntry entry : feed.getEntries()) {
writer.object();
writer.key("docId"); writer.value(entry.getId());
writer.key("docLink"); writer.value(entry.getHtmlLink().getHref());
writer.key("docSelfLink"); writer.value(entry.getSelfLink().getHref());
writer.key("title"); writer.value(entry.getTitle().getPlainText());
writer.key("type"); writer.value(entry.getType());
DateTime updated = entry.getUpdated();
if (updated != null) {
writer.key("updated"); writer.value(updated.toStringRfc822());
List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, "SHOW TABLES");
if (rows.size() > 1) { // excluding headers
for (int i = 1; i < rows.size(); i++) {
List<String> row = rows.get(i);
if (row.size() >= 2) {
String id = row.get(0);
String name = row.get(1);
String link = "https://www.google.com/fusiontables/DataSource?dsrcid=" + id;
writer.object();
writer.key("docId"); writer.value(id);
writer.key("docLink"); writer.value(link);
writer.key("docSelfLink"); writer.value(link);
writer.key("title"); writer.value(name);
writer.key("type"); writer.value("table");
writer.endObject();
}
}
writer.key("authors"); writer.array();
for (Person person : entry.getAuthors()) {
writer.value(person.getName());
}
writer.endArray();
writer.endObject();
}
}
@ -221,16 +208,17 @@ public class GDataImportingController implements ImportingController {
JSONUtilities.safePut(result, "status", "ok");
JSONUtilities.safePut(result, "options", options);
JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore
JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
JSONUtilities.safePut(options, "skipDataLines", 0); // number of initial data lines to skip
JSONUtilities.safePut(options, "storeBlankRows", true);
JSONUtilities.safePut(options, "storeBlankCellsAsNulls", true);
JSONArray worksheets = new JSONArray();
JSONUtilities.safePut(options, "worksheets", worksheets);
if ("spreadsheet".equals(type)) {
JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore
JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
JSONArray worksheets = new JSONArray();
JSONUtilities.safePut(options, "worksheets", worksheets);
SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token);
SpreadsheetEntry spreadsheetEntry = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
for (WorksheetEntry worksheetEntry : spreadsheetEntry.getWorksheets()) {
@ -242,15 +230,7 @@ public class GDataImportingController implements ImportingController {
JSONUtilities.append(worksheets, worksheetO);
}
} else if ("table".equals(type)) {
DocsService docsService = GDataExtension.getDocsService(token);
TableEntry tableEntry = docsService.getEntry(url, TableEntry.class);
JSONObject worksheetO = new JSONObject();
JSONUtilities.safePut(worksheetO, "name", tableEntry.getTitle().getPlainText());
JSONUtilities.safePut(worksheetO, "rows", -1);
JSONUtilities.safePut(worksheetO, "link", tableEntry.getSelfLink().getHref());
JSONUtilities.append(worksheets, worksheetO);
// No metadata for a fusion table.
}
/* TODO: else */
@ -307,15 +287,13 @@ public class GDataImportingController implements ImportingController {
if (exceptions.size() == 0) {
job.project.update(); // update all internal models, indexes, caches, etc.
writer.key("code"); writer.value("ok");
writer.key("status"); writer.value("ok");
} else {
writer.key("code"); writer.value("error");
writer.key("status"); writer.value("error");
writer.key("errors");
writer.array();
for (Exception e : exceptions) {
writer.value(e.getLocalizedMessage());
}
DefaultImportingController.writeErrors(writer, exceptions);
writer.endArray();
}
writer.endObject();
@ -374,12 +352,18 @@ public class GDataImportingController implements ImportingController {
);
if (!job.canceled) {
project.update(); // update all internal models, indexes, caches, etc.
ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "projectID", project.id);
JSONUtilities.safePut(job.config, "state", "created-project");
if (exceptions.size() > 0) {
JSONUtilities.safePut(job.config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
JSONUtilities.safePut(job.config, "state", "error");
} else {
project.update(); // update all internal models, indexes, caches, etc.
ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "state", "created-project");
JSONUtilities.safePut(job.config, "projectID", project.id);
}
}
}
}.start();

54
main/src/com/google/refine/importing/DefaultImportingController.java
View File

@ -34,6 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.importing;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.Writer;
import java.util.LinkedList;
import java.util.List;
@ -173,7 +175,28 @@ public class DefaultImportingController implements ImportingController {
ImportingUtilities.previewParse(job, format, optionObj, exceptions);
HttpUtilities.respond(response, "ok", "done");
Writer w = response.getWriter();
JSONWriter writer = new JSONWriter(w);
try {
writer.object();
if (exceptions.size() == 0) {
job.project.update(); // update all internal models, indexes, caches, etc.
writer.key("status"); writer.value("ok");
} else {
writer.key("status"); writer.value("error");
writer.key("errors");
writer.array();
writeErrors(writer, exceptions);
writer.endArray();
}
writer.endObject();
} catch (JSONException e) {
throw new ServletException(e);
} finally {
w.flush();
w.close();
}
} catch (JSONException e) {
throw new ServletException(e);
}
@ -252,4 +275,33 @@ public class DefaultImportingController implements ImportingController {
w.close();
}
}
static public void writeErrors(JSONWriter writer, List<Exception> exceptions) throws JSONException {
for (Exception e : exceptions) {
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
writer.object();
writer.key("message");
writer.value(e.getLocalizedMessage());
writer.key("stack");
writer.value(sw.toString());
writer.endObject();
}
}
static public JSONArray convertErrorsToJsonArray(List<Exception> exceptions) {
JSONArray a = new JSONArray();
for (Exception e : exceptions) {
StringWriter sw = new StringWriter();
e.printStackTrace(new PrintWriter(sw));
JSONObject o = new JSONObject();
JSONUtilities.safePut(o, "message", e.getLocalizedMessage());
JSONUtilities.safePut(o, "stack", sw.toString());
JSONUtilities.append(a, o);
}
return a;
}
}

18
main/src/com/google/refine/importing/ImportingUtilities.java
View File

@ -911,12 +911,18 @@ public class ImportingUtilities {
);
if (!job.canceled) {
project.update(); // update all internal models, indexes, caches, etc.
ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "projectID", project.id);
JSONUtilities.safePut(job.config, "state", "created-project");
if (exceptions.size() == 0) {
project.update(); // update all internal models, indexes, caches, etc.
ProjectManager.singleton.registerProject(project, pm);
JSONUtilities.safePut(job.config, "projectID", project.id);
JSONUtilities.safePut(job.config, "state", "created-project");
} else {
JSONUtilities.safePut(job.config, "state", "error");
JSONUtilities.safePut(job.config, "errors",
DefaultImportingController.convertErrorsToJsonArray(exceptions));
}
}
}

6
main/webapp/modules/core/scripts/index/create-project-ui.js
View File

@ -245,3 +245,9 @@ Refine.CreateProjectUI.prototype.showImportJobError = function(message, stack) {
self.showSourceSelectionPanel();
});
};
Refine.CreateProjectUI.composeErrorMessage = function(job) {
var messages = [];
$.each(job.config.errors, function() { messages.push(this.message) });
return messages.join('\n');
};

2
main/webapp/modules/core/scripts/index/default-importing-controller/controller.js
View File

@ -280,7 +280,7 @@ Refine.DefaultImportingController.prototype._createProject = function() {
document.location = "project?project=" + job.config.projectID;
},
function(job) {
alert(job.config.error + '\n' + job.config.errorDetails);
alert('Errors:\n' + Refine.CreateProjectUI.composeErrorMessage(job));
self._onImportJobReady();
}
);

2
main/webapp/modules/core/styles/index/default-importing-file-selection-panel.less
View File

@ -48,7 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.default-importing-file-selection-file-panel > table {
border-collapse: collapse;
width: 100%;
}
}
.default-importing-file-selection-file-panel > table > tbody > tr > td,
.default-importing-file-selection-file-panel > table > tbody > tr > th {
padding: @padding_tight @padding_normal;