diff --git a/extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html b/extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html new file mode 100644 index 000000000..7be80b23a --- /dev/null +++ b/extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html @@ -0,0 +1,33 @@ +
+ + + + + +
Configure Parsing OptionsProject name
+ +
+ +
+ Updating preview ... +
+ +
+ + + + + + + +
Options
+ + + + + + + + + +
Discard initial row(s) of data
Load at most row(s) of data
Store blank rows
Store blank cells as nulls
\ No newline at end of file diff --git a/extensions/gdata/module/scripts/index/gdata-source-ui.js b/extensions/gdata/module/scripts/index/gdata-source-ui.js index e21715ddd..b8cf3e1f4 100644 --- a/extensions/gdata/module/scripts/index/gdata-source-ui.js +++ b/extensions/gdata/module/scripts/index/gdata-source-ui.js @@ -137,17 +137,15 @@ Refine.GDataSourceUI.prototype._renderDocuments = function(o) { td = tr.insertCell(tr.cells.length); $('') .addClass('gdata-doc-authors') - .text(doc.authors.join(', ')) + .text((doc.authors) ? doc.authors.join(', ') : '') .appendTo(td); td = tr.insertCell(tr.cells.length); - if (doc.updated) { - $('') - .addClass('gdata-doc-date') - .text(formatRelativeDate(doc.updated)) - .attr('title', doc.updated) - .appendTo(td); - } + $('') + .addClass('gdata-doc-date') + .text((doc.updated) ? formatRelativeDate(doc.updated) : '') + .attr('title', (doc.updated) ? doc.updated : '') + .appendTo(td); }; var docs = o.documents; diff --git a/extensions/gdata/module/scripts/index/importing-controller.js b/extensions/gdata/module/scripts/index/importing-controller.js index 6bf4faeef..67831ae67 100644 --- a/extensions/gdata/module/scripts/index/importing-controller.js +++ b/extensions/gdata/module/scripts/index/importing-controller.js @@ -84,7 +84,6 @@ Refine.GDataImportingController.prototype.getOptions = function() { var options = { docUrl: this._doc.docSelfLink, docType: this._doc.type, - sheetUrl: this._sheetUrl }; var parseIntDefault = function(s, def) { @@ -99,22 +98,25 @@ Refine.GDataImportingController.prototype.getOptions = function() { return def; }; - this._parsingPanelElmts.sheetRecordContainer.find('input').each(function() { - if (this.checked) { - options.sheetUrl = this.getAttribute('sheetUrl'); - } - }); + if (this._doc.type != 'table') { + this._parsingPanelElmts.sheetRecordContainer.find('input').each(function() { + if (this.checked) { + options.sheetUrl = this.getAttribute('sheetUrl'); + } + }); - if (this._parsingPanelElmts.ignoreCheckbox[0].checked) { - options.ignoreLines = parseIntDefault(this._parsingPanelElmts.ignoreInput[0].value, -1); - } else { - options.ignoreLines = -1; - } - if (this._parsingPanelElmts.headerLinesCheckbox[0].checked) { - options.headerLines = parseIntDefault(this._parsingPanelElmts.headerLinesInput[0].value, 0); - } else { - options.headerLines = 0; + if (this._parsingPanelElmts.ignoreCheckbox[0].checked) { + options.ignoreLines = parseIntDefault(this._parsingPanelElmts.ignoreInput[0].value, -1); + } else { + options.ignoreLines = -1; + } + if (this._parsingPanelElmts.headerLinesCheckbox[0].checked) { + options.headerLines = parseIntDefault(this._parsingPanelElmts.headerLinesInput[0].value, 0); + } else { + options.headerLines = 0; + } } + if (this._parsingPanelElmts.skipCheckbox[0].checked) { options.skipDataLines = parseIntDefault(this._parsingPanelElmts.skipInput[0].value, 0); } else { @@ -135,7 +137,10 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() { var self = this; this._parsingPanel.unbind().empty().html( - DOM.loadHTML("gdata", "scripts/index/gdata-parsing-panel.html")); + DOM.loadHTML("gdata", + this._doc.type == 'table' ? + 'scripts/index/gdata-fusion-tables-parsing-panel.html' : + 'scripts/index/gdata-parsing-panel.html')); this._parsingPanelElmts = DOM.bind(this._parsingPanel); if (this._parsingPanelResizer) { @@ -184,30 +189,33 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() { this._parsingPanelElmts.projectNameInput[0].value = this._doc.title; - var sheetTable = this._parsingPanelElmts.sheetRecordContainer[0]; - $.each(this._options.worksheets, function(i, v) { - var tr = sheetTable.insertRow(sheetTable.rows.length); - var td0 = $(tr.insertCell(0)).attr('width', '1%'); - var checkbox = $('') - .attr('type', 'radio') - .attr('name', 'gdata-importing-parsing-worksheet') - .attr('sheetUrl', this.link) - .appendTo(td0); - if (i === 0) { - checkbox.attr('checked', 'true'); - } - $(tr.insertCell(1)).text(this.name); - $(tr.insertCell(2)).text(this.rows + ' rows'); - }); + if (this._doc.type != 'table') { + var sheetTable = this._parsingPanelElmts.sheetRecordContainer[0]; + $.each(this._options.worksheets, function(i, v) { + var tr = sheetTable.insertRow(sheetTable.rows.length); + var td0 = $(tr.insertCell(0)).attr('width', '1%'); + var checkbox = $('') + .attr('type', 'radio') + .attr('name', 'gdata-importing-parsing-worksheet') + .attr('sheetUrl', this.link) + .appendTo(td0); + if (i === 0) { + checkbox.attr('checked', 'true'); + } + $(tr.insertCell(1)).text(this.name); + $(tr.insertCell(2)).text(this.rows + ' rows'); + }); - if (this._options.ignoreLines > 0) { - this._parsingPanelElmts.ignoreCheckbox.attr("checked", "checked"); - this._parsingPanelElmts.ignoreInput[0].value = this._options.ignoreLines.toString(); - } - if (this._options.headerLines > 0) { - this._parsingPanelElmts.headerLinesCheckbox.attr("checked", "checked"); - this._parsingPanelElmts.headerLinesInput[0].value = this._options.headerLines.toString(); + if (this._options.ignoreLines > 0) { + this._parsingPanelElmts.ignoreCheckbox.attr("checked", "checked"); + this._parsingPanelElmts.ignoreInput[0].value = this._options.ignoreLines.toString(); + } + if (this._options.headerLines > 0) { + this._parsingPanelElmts.headerLinesCheckbox.attr("checked", "checked"); + this._parsingPanelElmts.headerLinesInput[0].value = this._options.headerLines.toString(); + } } + if (this._options.limit > 0) { this._parsingPanelElmts.limitCheckbox.attr("checked", "checked"); this._parsingPanelElmts.limitInput[0].value = this._options.limit.toString(); @@ -262,7 +270,7 @@ Refine.GDataImportingController.prototype._updatePreview = function() { "options" : JSON.stringify(this.getOptions()) }, function(result) { - if (result.code == "ok") { + if (result.status == "ok") { self._getPreviewData(function(projectData) { self._parsingPanelElmts.progressPanel.hide(); self._parsingPanelElmts.dataPanel.show(); @@ -271,7 +279,7 @@ Refine.GDataImportingController.prototype._updatePreview = function() { }); } else { self._parsingPanelElmts.progressPanel.hide(); - alert('Errors:\n' + result.errors.join('\n')); + alert('Errors:\n' + Refine.CreateProjectUI.composeErrorMessage(job)); } }, "json" @@ -345,10 +353,11 @@ Refine.GDataImportingController.prototype._createProject = function() { return "projectID" in job.config; }, function(jobID, job) { + window.clearInterval(timerID); document.location = "project?project=" + job.config.projectID; }, function(job) { - alert(job.config.error + '\n' + job.config.errorDetails); + alert(Refine.CreateProjectUI.composeErrorMessage(job)); } ); }, diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java b/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java index 7d3fd2336..7d0ee72b5 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java @@ -33,7 +33,7 @@ public class AuthorizeCommand extends Command { String requestUrl = AuthSubUtil.getRequestUrl( authorizedUrl.toExternalForm(), // execution continues at authorized on redirect - "https://docs.google.com/feeds https://spreadsheets.google.com/feeds", + "https://spreadsheets.google.com/feeds https://www.google.com/fusiontables/api/query", false, true); response.sendRedirect(requestUrl); diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java index 02693e1d5..71b5f3949 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java @@ -28,9 +28,23 @@ */ package com.google.refine.extension.gdata; +import java.io.IOException; +import java.net.URL; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; +import java.util.regex.MatchResult; +import java.util.regex.Pattern; + +import com.google.gdata.client.GoogleService; +import com.google.gdata.client.Service.GDataRequest; +import com.google.gdata.client.Service.GDataRequest.RequestType; import com.google.gdata.client.docs.DocsService; import com.google.gdata.client.spreadsheet.FeedURLFactory; import com.google.gdata.client.spreadsheet.SpreadsheetService; +import com.google.gdata.util.ContentType; +import com.google.gdata.util.ServiceException; /** * @author Tom Morris @@ -60,4 +74,54 @@ abstract public class GDataExtension { service.setAuthSubToken(token); return service; } + + static public GoogleService getFusionTablesGoogleService(String token) { + GoogleService service = new GoogleService("fusiontables", SERVICE_APP_NAME); + service.setAuthSubToken(token); + return service; + } + + final static private String FUSION_TABLES_SERVICE_URL = + "https://www.google.com/fusiontables/api/query"; + + final static private Pattern CSV_VALUE_PATTERN = + Pattern.compile("([^,\\r\\n\"]*|\"(([^\"]*\"\")*[^\"]*)\")(,|\\r?\\n)"); + + static public List> runFusionTablesSelect(GoogleService service, String selectQuery) + throws IOException, ServiceException { + + URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" + + URLEncoder.encode(selectQuery, "UTF-8")); + GDataRequest request = service.getRequestFactory().getRequest( + RequestType.QUERY, url, ContentType.TEXT_PLAIN); + + request.execute(); + + List> rows = new ArrayList>(); + List row = null; + + Scanner scanner = new Scanner(request.getResponseStream(), "UTF-8"); + while (scanner.hasNextLine()) { + scanner.findWithinHorizon(CSV_VALUE_PATTERN, 0); + MatchResult match = scanner.match(); + String quotedString = match.group(2); + String decoded = quotedString == null ? match.group(1) : quotedString.replaceAll("\"\"", "\""); + + if (row == null) { + row = new ArrayList(); + } + row.add(decoded); + + if (!match.group(4).equals(",")) { + if (row != null) { + rows.add(row); + row = null; + } + } + } + if (row != null) { + rows.add(row); + } + return rows; + } } diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java index f2836b3c4..9ee7c84e1 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java @@ -36,7 +36,7 @@ import java.util.List; import org.json.JSONObject; -import com.google.gdata.client.docs.DocsService; +import com.google.gdata.client.GoogleService; import com.google.gdata.client.spreadsheet.CellQuery; import com.google.gdata.client.spreadsheet.SpreadsheetService; import com.google.gdata.data.spreadsheet.Cell; @@ -75,19 +75,19 @@ public class GDataImporter { SpreadsheetService service = GDataExtension.getSpreadsheetService(token); parse( service, - job.project, - job.metadata, + project, + metadata, job, limit, options, exceptions ); } else if ("table".equals(docType)) { - DocsService service = GDataExtension.getDocsService(token); + GoogleService service = GDataExtension.getFusionTablesGoogleService(token); parse( service, - job.project, - job.metadata, + project, + metadata, job, limit, options, @@ -165,21 +165,6 @@ public class GDataImporter { } } - static public void parse( - DocsService service, - Project project, - ProjectMetadata metadata, - final ImportingJob job, - int limit, - JSONObject options, - List exceptions) { - - String docUrlString = JSONUtilities.getString(options, "docUrl", null); - if (docUrlString != null) { - // TODO[dfhuynh] - } - } - static private void setProgress(ImportingJob job, String fileSource, int percent) { JSONObject progress = JSONUtilities.getObject(job.config, "progress"); if (progress == null) { @@ -287,4 +272,210 @@ public class GDataImporter { return rowsOfCells; } } + + static public void parse( + GoogleService service, + Project project, + ProjectMetadata metadata, + final ImportingJob job, + int limit, + JSONObject options, + List exceptions) { + + String docUrlString = JSONUtilities.getString(options, "docUrl", null); + if (docUrlString == null) { + return; + } + int equal = docUrlString.lastIndexOf('='); + if (equal < 0) { + return; + } + + String id = docUrlString.substring(equal + 1); + + try { + List columns = new ArrayList(); + List> rows = GDataExtension.runFusionTablesSelect(service, "DESCRIBE " + id); + if (rows.size() > 1) { + for (int i = 1; i < rows.size(); i++) { + List row = rows.get(i); + if (row.size() >= 2) { + FTColumnData cd = new FTColumnData(); + cd.name = row.get(1); + cd.type = FTColumnType.STRING; + + if (row.size() > 2) { + String type = row.get(2).toLowerCase(); + if (type.equals("number")) { + cd.type = FTColumnType.NUMBER; + } else if (type.equals("datetime")) { + cd.type = FTColumnType.DATETIME; + } else if (type.equals("location")) { + cd.type = FTColumnType.LOCATION; + } + } + columns.add(cd); + } + } + + setProgress(job, docUrlString, -1); + + // Force these options for the next call because each fusion table + // is strictly structured with a single line of headers. + JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore + JSONUtilities.safePut(options, "headerLines", 1); // number of header lines + + TabularImportingParserBase.readTable( + project, + metadata, + job, + new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100), + docUrlString, + limit, + options, + exceptions + ); + setProgress(job, docUrlString, 100); + } + } catch (IOException e) { + e.printStackTrace(); + exceptions.add(e); + } catch (ServiceException e) { + e.printStackTrace(); + exceptions.add(e); + } + } + + static private enum FTColumnType { + STRING, + NUMBER, + DATETIME, + LOCATION + } + + final static private class FTColumnData { + String name; + FTColumnType type; + } + + static private class FusionTableBatchRowReader implements TableDataReader { + final ImportingJob job; + final String fileSource; + + final GoogleService service; + final List columns; + final int batchSize; + + final String baseQuery; + + int nextRow = 0; // 0-based + int batchRowStart = 0; // 0-based + boolean end = false; + List> rowsOfCells = null; + boolean usedHeaders = false; + + public FusionTableBatchRowReader(ImportingJob job, String fileSource, + GoogleService service, String tableId, List columns, + int batchSize) { + this.job = job; + this.fileSource = fileSource; + this.service = service; + this.columns = columns; + this.batchSize = batchSize; + + StringBuffer sb = new StringBuffer(); + sb.append("SELECT "); + + boolean first = true; + for (FTColumnData cd : columns) { + if (first) { + first = false; + } else { + sb.append(","); + } + sb.append("'"); + sb.append(cd.name); + sb.append("'"); + } + sb.append(" FROM "); + sb.append(tableId); + + baseQuery = sb.toString(); + } + + @Override + public List getNextRowOfCells() throws IOException { + if (!usedHeaders) { + List row = new ArrayList(columns.size()); + for (FTColumnData cd : columns) { + row.add(cd.name); + } + usedHeaders = true; + return row; + } + + if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) { + int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size()); + try { + rowsOfCells = getRowsOfCells(newBatchRowStart); + batchRowStart = newBatchRowStart; + + setProgress(job, fileSource, -1 /* batchRowStart * 100 / totalRows */); + } catch (ServiceException e) { + throw new IOException(e); + } + } + + if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) { + return rowsOfCells.get(nextRow++ - batchRowStart); + } else { + return null; + } + } + + + private List> getRowsOfCells(int startRow) throws IOException, ServiceException { + List> rowsOfCells = new ArrayList>(batchSize); + + String query = baseQuery + " OFFSET " + startRow + " LIMIT " + batchSize; + + List> rows = GDataExtension.runFusionTablesSelect(service, query); + if (rows.size() > 1) { + for (int i = 1; i < rows.size(); i++) { + List row = rows.get(i); + List rowOfCells = new ArrayList(row.size()); + for (int j = 0; j < row.size() && j < columns.size(); j++) { + String text = row.get(j); + if (text.isEmpty()) { + rowOfCells.add(null); + } else { + FTColumnData cd = columns.get(j); + if (cd.type == FTColumnType.NUMBER) { + try { + rowOfCells.add(Long.parseLong(text)); + continue; + } catch (NumberFormatException e) { + // ignore + } + try { + double d = Double.parseDouble(text); + if (!Double.isInfinite(d) && !Double.isNaN(d)) { + rowOfCells.add(d); + continue; + } + } catch (NumberFormatException e) { + // ignore + } + } + + rowOfCells.add(text); + } + } + rowsOfCells.add(rowOfCells); + } + } + end = rows.size() < batchSize + 1; + return rowsOfCells; + } + } } \ No newline at end of file diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java index 5e5aec952..29c27b483 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java @@ -49,17 +49,13 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; -import com.google.gdata.client.Query; +import com.google.gdata.client.GoogleService; import com.google.gdata.client.docs.DocsService; import com.google.gdata.client.spreadsheet.SpreadsheetService; -import com.google.gdata.data.Category; import com.google.gdata.data.DateTime; import com.google.gdata.data.Person; -import com.google.gdata.data.docs.DocumentListEntry; -import com.google.gdata.data.docs.DocumentListFeed; import com.google.gdata.data.spreadsheet.SpreadsheetEntry; import com.google.gdata.data.spreadsheet.SpreadsheetFeed; -import com.google.gdata.data.spreadsheet.TableEntry; import com.google.gdata.data.spreadsheet.WorksheetEntry; import com.google.gdata.util.ServiceException; @@ -67,6 +63,7 @@ import com.google.refine.ProjectManager; import com.google.refine.ProjectMetadata; import com.google.refine.RefineServlet; import com.google.refine.commands.HttpUtilities; +import com.google.refine.importing.DefaultImportingController; import com.google.refine.importing.ImportingController; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingManager; @@ -125,11 +122,10 @@ public class GDataImportingController implements ImportingController { writer.array(); try { - DocsService service = GDataExtension.getDocsService(token); - listSpreadsheets(service, writer); - listDocumentsOfType(service, writer, "http://schemas.google.com/docs/2007#table"); + listSpreadsheets(GDataExtension.getDocsService(token), writer); + listFusionTables(GDataExtension.getFusionTablesGoogleService(token), writer); } catch (ServiceException e) { - // TODO: just ignore? + e.printStackTrace(); } writer.endArray(); @@ -169,36 +165,27 @@ public class GDataImportingController implements ImportingController { } } - private void listDocumentsOfType(DocsService service, JSONWriter writer, String type) - throws IOException, ServiceException, JSONException { - URL feedUrl = new URL("https://docs.google.com/feeds/default/private/full"); + private void listFusionTables(GoogleService service, JSONWriter writer) + throws IOException, ServiceException, JSONException { - Query query = new Query(feedUrl); - query.addCategoryFilter( - new Query.CategoryFilter( - new Category("http://schemas.google.com/g/2005#kind", type))); - - DocumentListFeed feed = service.query(query, DocumentListFeed.class); - for (DocumentListEntry entry : feed.getEntries()) { - writer.object(); - writer.key("docId"); writer.value(entry.getId()); - writer.key("docLink"); writer.value(entry.getHtmlLink().getHref()); - writer.key("docSelfLink"); writer.value(entry.getSelfLink().getHref()); - writer.key("title"); writer.value(entry.getTitle().getPlainText()); - writer.key("type"); writer.value(entry.getType()); - - DateTime updated = entry.getUpdated(); - if (updated != null) { - writer.key("updated"); writer.value(updated.toStringRfc822()); + List> rows = GDataExtension.runFusionTablesSelect(service, "SHOW TABLES"); + if (rows.size() > 1) { // excluding headers + for (int i = 1; i < rows.size(); i++) { + List row = rows.get(i); + if (row.size() >= 2) { + String id = row.get(0); + String name = row.get(1); + String link = "https://www.google.com/fusiontables/DataSource?dsrcid=" + id; + + writer.object(); + writer.key("docId"); writer.value(id); + writer.key("docLink"); writer.value(link); + writer.key("docSelfLink"); writer.value(link); + writer.key("title"); writer.value(name); + writer.key("type"); writer.value("table"); + writer.endObject(); + } } - - writer.key("authors"); writer.array(); - for (Person person : entry.getAuthors()) { - writer.value(person.getName()); - } - writer.endArray(); - - writer.endObject(); } } @@ -221,16 +208,17 @@ public class GDataImportingController implements ImportingController { JSONUtilities.safePut(result, "status", "ok"); JSONUtilities.safePut(result, "options", options); - JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore - JSONUtilities.safePut(options, "headerLines", 1); // number of header lines JSONUtilities.safePut(options, "skipDataLines", 0); // number of initial data lines to skip JSONUtilities.safePut(options, "storeBlankRows", true); JSONUtilities.safePut(options, "storeBlankCellsAsNulls", true); - JSONArray worksheets = new JSONArray(); - JSONUtilities.safePut(options, "worksheets", worksheets); - if ("spreadsheet".equals(type)) { + JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore + JSONUtilities.safePut(options, "headerLines", 1); // number of header lines + + JSONArray worksheets = new JSONArray(); + JSONUtilities.safePut(options, "worksheets", worksheets); + SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token); SpreadsheetEntry spreadsheetEntry = spreadsheetService.getEntry(url, SpreadsheetEntry.class); for (WorksheetEntry worksheetEntry : spreadsheetEntry.getWorksheets()) { @@ -242,15 +230,7 @@ public class GDataImportingController implements ImportingController { JSONUtilities.append(worksheets, worksheetO); } } else if ("table".equals(type)) { - DocsService docsService = GDataExtension.getDocsService(token); - TableEntry tableEntry = docsService.getEntry(url, TableEntry.class); - - JSONObject worksheetO = new JSONObject(); - JSONUtilities.safePut(worksheetO, "name", tableEntry.getTitle().getPlainText()); - JSONUtilities.safePut(worksheetO, "rows", -1); - JSONUtilities.safePut(worksheetO, "link", tableEntry.getSelfLink().getHref()); - - JSONUtilities.append(worksheets, worksheetO); + // No metadata for a fusion table. } /* TODO: else */ @@ -307,15 +287,13 @@ public class GDataImportingController implements ImportingController { if (exceptions.size() == 0) { job.project.update(); // update all internal models, indexes, caches, etc. - writer.key("code"); writer.value("ok"); + writer.key("status"); writer.value("ok"); } else { - writer.key("code"); writer.value("error"); + writer.key("status"); writer.value("error"); writer.key("errors"); writer.array(); - for (Exception e : exceptions) { - writer.value(e.getLocalizedMessage()); - } + DefaultImportingController.writeErrors(writer, exceptions); writer.endArray(); } writer.endObject(); @@ -374,12 +352,18 @@ public class GDataImportingController implements ImportingController { ); if (!job.canceled) { - project.update(); // update all internal models, indexes, caches, etc. - - ProjectManager.singleton.registerProject(project, pm); - - JSONUtilities.safePut(job.config, "projectID", project.id); - JSONUtilities.safePut(job.config, "state", "created-project"); + if (exceptions.size() > 0) { + JSONUtilities.safePut(job.config, "errors", + DefaultImportingController.convertErrorsToJsonArray(exceptions)); + JSONUtilities.safePut(job.config, "state", "error"); + } else { + project.update(); // update all internal models, indexes, caches, etc. + + ProjectManager.singleton.registerProject(project, pm); + + JSONUtilities.safePut(job.config, "state", "created-project"); + JSONUtilities.safePut(job.config, "projectID", project.id); + } } } }.start(); diff --git a/main/src/com/google/refine/importing/DefaultImportingController.java b/main/src/com/google/refine/importing/DefaultImportingController.java index 3d54d6c9c..00abfec55 100644 --- a/main/src/com/google/refine/importing/DefaultImportingController.java +++ b/main/src/com/google/refine/importing/DefaultImportingController.java @@ -34,6 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.importing; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.io.Writer; import java.util.LinkedList; import java.util.List; @@ -173,7 +175,28 @@ public class DefaultImportingController implements ImportingController { ImportingUtilities.previewParse(job, format, optionObj, exceptions); - HttpUtilities.respond(response, "ok", "done"); + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + try { + writer.object(); + if (exceptions.size() == 0) { + job.project.update(); // update all internal models, indexes, caches, etc. + + writer.key("status"); writer.value("ok"); + } else { + writer.key("status"); writer.value("error"); + writer.key("errors"); + writer.array(); + writeErrors(writer, exceptions); + writer.endArray(); + } + writer.endObject(); + } catch (JSONException e) { + throw new ServletException(e); + } finally { + w.flush(); + w.close(); + } } catch (JSONException e) { throw new ServletException(e); } @@ -252,4 +275,33 @@ public class DefaultImportingController implements ImportingController { w.close(); } } + + static public void writeErrors(JSONWriter writer, List exceptions) throws JSONException { + for (Exception e : exceptions) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + + writer.object(); + writer.key("message"); + writer.value(e.getLocalizedMessage()); + writer.key("stack"); + writer.value(sw.toString()); + writer.endObject(); + } + } + + static public JSONArray convertErrorsToJsonArray(List exceptions) { + JSONArray a = new JSONArray(); + for (Exception e : exceptions) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + + JSONObject o = new JSONObject(); + JSONUtilities.safePut(o, "message", e.getLocalizedMessage()); + JSONUtilities.safePut(o, "stack", sw.toString()); + JSONUtilities.append(a, o); + } + return a; + } + } diff --git a/main/src/com/google/refine/importing/ImportingUtilities.java b/main/src/com/google/refine/importing/ImportingUtilities.java index a6de54a36..1ac30f41b 100644 --- a/main/src/com/google/refine/importing/ImportingUtilities.java +++ b/main/src/com/google/refine/importing/ImportingUtilities.java @@ -911,12 +911,18 @@ public class ImportingUtilities { ); if (!job.canceled) { - project.update(); // update all internal models, indexes, caches, etc. - - ProjectManager.singleton.registerProject(project, pm); - - JSONUtilities.safePut(job.config, "projectID", project.id); - JSONUtilities.safePut(job.config, "state", "created-project"); + if (exceptions.size() == 0) { + project.update(); // update all internal models, indexes, caches, etc. + + ProjectManager.singleton.registerProject(project, pm); + + JSONUtilities.safePut(job.config, "projectID", project.id); + JSONUtilities.safePut(job.config, "state", "created-project"); + } else { + JSONUtilities.safePut(job.config, "state", "error"); + JSONUtilities.safePut(job.config, "errors", + DefaultImportingController.convertErrorsToJsonArray(exceptions)); + } } } diff --git a/main/webapp/modules/core/scripts/index/create-project-ui.js b/main/webapp/modules/core/scripts/index/create-project-ui.js index 9f9027ba5..bd4c4b2ef 100644 --- a/main/webapp/modules/core/scripts/index/create-project-ui.js +++ b/main/webapp/modules/core/scripts/index/create-project-ui.js @@ -245,3 +245,9 @@ Refine.CreateProjectUI.prototype.showImportJobError = function(message, stack) { self.showSourceSelectionPanel(); }); }; + +Refine.CreateProjectUI.composeErrorMessage = function(job) { + var messages = []; + $.each(job.config.errors, function() { messages.push(this.message) }); + return messages.join('\n'); +}; diff --git a/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js b/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js index b4f5908b3..1814131bc 100644 --- a/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js +++ b/main/webapp/modules/core/scripts/index/default-importing-controller/controller.js @@ -280,7 +280,7 @@ Refine.DefaultImportingController.prototype._createProject = function() { document.location = "project?project=" + job.config.projectID; }, function(job) { - alert(job.config.error + '\n' + job.config.errorDetails); + alert('Errors:\n' + Refine.CreateProjectUI.composeErrorMessage(job)); self._onImportJobReady(); } ); diff --git a/main/webapp/modules/core/styles/index/default-importing-file-selection-panel.less b/main/webapp/modules/core/styles/index/default-importing-file-selection-panel.less index 4fcdeb239..90a32366a 100644 --- a/main/webapp/modules/core/styles/index/default-importing-file-selection-panel.less +++ b/main/webapp/modules/core/styles/index/default-importing-file-selection-panel.less @@ -48,7 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .default-importing-file-selection-file-panel > table { border-collapse: collapse; width: 100%; -} + } .default-importing-file-selection-file-panel > table > tbody > tr > td, .default-importing-file-selection-file-panel > table > tbody > tr > th { padding: @padding_tight @padding_normal;