Added support for importing from fusion tables.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2239 7d457c2a-affb-35e4-300a-418c747d4874

extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html

@@ -0,0 +1,33 @@
+<div bind="wizardHeader" class="gdata-importing-wizard-header"><div class="grid-layout layout-tightest layout-full"><table><tr>
+  <td width="1%"><button bind="startOverButton" class="button">&laquo; Start Over</button></td>
+  <td width="98%">Configure Parsing Options</td>
+  <td style="text-align: right;">Project&nbsp;name</td>
+  <td width="1%"><input class="inline" type="text" size="30" bind="projectNameInput" /></td>
+  <td width="1%"><button bind="createProjectButton" class="button button-primary">Create Project &raquo;</button></td>
+</tr></table></div></div>
+
+<div bind="dataPanel" class="gdata-importing-parsing-data-panel"></div>
+
+<div bind="progressPanel" class="gdata-importing-progress-data-panel">
+    <img src="images/large-spinner.gif" /> Updating preview ...
+</div>
+
+<div bind="controlPanel" class="gdata-importing-parsing-control-panel"><div class="grid-layout layout-normal"><table>
+  <tr>
+    <td>Options</td>
+    <td><button class="button" bind="previewButton">Update&nbsp;Preview</button></td>
+  </tr>
+  <tr>
+    <td><div class="grid-layout layout-tightest"><table>
+      <tr><td width="1%"><input type="checkbox" bind="skipCheckbox" /></td><td>Discard initial</td>
+        <td><input bind="skipInput" type="text" class="lightweight" size="2" value="0" /> row(s) of data</td></tr>
+      <tr><td width="1%"><input type="checkbox" bind="limitCheckbox" /></td><td>Load at most</td>
+        <td><input bind="limitInput" type="text" class="lightweight" size="2" value="0" /> row(s) of data</td></tr>
+        
+      <tr><td width="1%"><input type="checkbox" bind="storeBlankRowsCheckbox" /></td>
+        <td colspan="2">Store blank rows</td></tr>
+      <tr><td width="1%"><input type="checkbox" bind="storeBlankCellsAsNullsCheckbox" /></td>
+        <td colspan="2">Store blank cells as nulls</td></tr>
+    </table></div></td>
+  </tr>
+</table></div></div>

extensions/gdata/module/scripts/index/gdata-source-ui.js

@@ -137,17 +137,15 @@ Refine.GDataSourceUI.prototype._renderDocuments = function(o) {
     td = tr.insertCell(tr.cells.length);
     $('<span>')
     .addClass('gdata-doc-authors')
-    .text(doc.authors.join(', '))
+    .text((doc.authors) ? doc.authors.join(', ') : '<unknown>')
     .appendTo(td);
     
     td = tr.insertCell(tr.cells.length);
-    if (doc.updated) {
     $('<span>')
     .addClass('gdata-doc-date')
-      .text(formatRelativeDate(doc.updated))
+    .text((doc.updated) ? formatRelativeDate(doc.updated) : '<unknown>')
-      .attr('title', doc.updated)
+    .attr('title', (doc.updated) ? doc.updated : '<unknown>')
     .appendTo(td);
-    }
   };
   
   var docs = o.documents;

extensions/gdata/module/scripts/index/importing-controller.js

@@ -84,7 +84,6 @@ Refine.GDataImportingController.prototype.getOptions = function() {
   var options = {
     docUrl: this._doc.docSelfLink,
     docType: this._doc.type,
-    sheetUrl: this._sheetUrl
   };
 
   var parseIntDefault = function(s, def) {
@@ -99,6 +98,7 @@ Refine.GDataImportingController.prototype.getOptions = function() {
     return def;
   };
 
+  if (this._doc.type != 'table') {
     this._parsingPanelElmts.sheetRecordContainer.find('input').each(function() {
       if (this.checked) {
         options.sheetUrl = this.getAttribute('sheetUrl');
@@ -115,6 +115,8 @@ Refine.GDataImportingController.prototype.getOptions = function() {
     } else {
       options.headerLines = 0;
     }
+  }
+  
   if (this._parsingPanelElmts.skipCheckbox[0].checked) {
     options.skipDataLines = parseIntDefault(this._parsingPanelElmts.skipInput[0].value, 0);
   } else {
@@ -135,7 +137,10 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
   var self = this;
   
   this._parsingPanel.unbind().empty().html(
-      DOM.loadHTML("gdata", "scripts/index/gdata-parsing-panel.html"));
+      DOM.loadHTML("gdata",
+        this._doc.type == 'table' ?
+          'scripts/index/gdata-fusion-tables-parsing-panel.html' :
+          'scripts/index/gdata-parsing-panel.html'));
   this._parsingPanelElmts = DOM.bind(this._parsingPanel);
   
   if (this._parsingPanelResizer) {
@@ -184,6 +189,7 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
   
   this._parsingPanelElmts.projectNameInput[0].value = this._doc.title;
 
+  if (this._doc.type != 'table') {
     var sheetTable = this._parsingPanelElmts.sheetRecordContainer[0];
     $.each(this._options.worksheets, function(i, v) {
       var tr = sheetTable.insertRow(sheetTable.rows.length);
@@ -208,6 +214,8 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
       this._parsingPanelElmts.headerLinesCheckbox.attr("checked", "checked");
       this._parsingPanelElmts.headerLinesInput[0].value = this._options.headerLines.toString();
     }
+  }
+  
   if (this._options.limit > 0) {
     this._parsingPanelElmts.limitCheckbox.attr("checked", "checked");
     this._parsingPanelElmts.limitInput[0].value = this._options.limit.toString();
@@ -262,7 +270,7 @@ Refine.GDataImportingController.prototype._updatePreview = function() {
       "options" : JSON.stringify(this.getOptions())
     },
     function(result) {
-      if (result.code == "ok") {
+      if (result.status == "ok") {
         self._getPreviewData(function(projectData) {
           self._parsingPanelElmts.progressPanel.hide();
           self._parsingPanelElmts.dataPanel.show();
@@ -271,7 +279,7 @@ Refine.GDataImportingController.prototype._updatePreview = function() {
         });
       } else {
         self._parsingPanelElmts.progressPanel.hide();
-        alert('Errors:\n' + result.errors.join('\n'));
+        alert('Errors:\n' + Refine.CreateProjectUI.composeErrorMessage(job));
       }
     },
     "json"
@@ -345,10 +353,11 @@ Refine.GDataImportingController.prototype._createProject = function() {
             return "projectID" in job.config;
           },
           function(jobID, job) {
+            window.clearInterval(timerID);
             document.location = "project?project=" + job.config.projectID;
           },
           function(job) {
-            alert(job.config.error + '\n' + job.config.errorDetails);
+            alert(Refine.CreateProjectUI.composeErrorMessage(job));
           }
       );
     },

extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java

@@ -33,7 +33,7 @@ public class AuthorizeCommand extends Command {
         
         String requestUrl = AuthSubUtil.getRequestUrl(
             authorizedUrl.toExternalForm(), // execution continues at authorized on redirect
-            "https://docs.google.com/feeds https://spreadsheets.google.com/feeds",
+            "https://spreadsheets.google.com/feeds https://www.google.com/fusiontables/api/query",
             false,
             true);
         response.sendRedirect(requestUrl);

extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java

@@ -28,9 +28,23 @@
  */
 package com.google.refine.extension.gdata;
 
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Scanner;
+import java.util.regex.MatchResult;
+import java.util.regex.Pattern;
+
+import com.google.gdata.client.GoogleService;
+import com.google.gdata.client.Service.GDataRequest;
+import com.google.gdata.client.Service.GDataRequest.RequestType;
 import com.google.gdata.client.docs.DocsService;
 import com.google.gdata.client.spreadsheet.FeedURLFactory;
 import com.google.gdata.client.spreadsheet.SpreadsheetService;
+import com.google.gdata.util.ContentType;
+import com.google.gdata.util.ServiceException;
 
 /**
  * @author Tom Morris <tfmorris@gmail.com>
@@ -60,4 +74,54 @@ abstract public class GDataExtension {
         service.setAuthSubToken(token);
         return service;
     }
+    
+    static public GoogleService getFusionTablesGoogleService(String token) {
+        GoogleService service = new GoogleService("fusiontables", SERVICE_APP_NAME);
+        service.setAuthSubToken(token);
+        return service;
+    }
+
+    final static private String FUSION_TABLES_SERVICE_URL =
+        "https://www.google.com/fusiontables/api/query";
+
+    final static private Pattern CSV_VALUE_PATTERN =
+        Pattern.compile("([^,\\r\\n\"]*|\"(([^\"]*\"\")*[^\"]*)\")(,|\\r?\\n)");
+    
+    static public List<List<String>> runFusionTablesSelect(GoogleService service, String selectQuery)
+            throws IOException, ServiceException {
+        
+        URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" +
+                URLEncoder.encode(selectQuery, "UTF-8"));
+        GDataRequest request = service.getRequestFactory().getRequest(
+                RequestType.QUERY, url, ContentType.TEXT_PLAIN);
+        
+        request.execute();
+        
+        List<List<String>> rows = new ArrayList<List<String>>();
+        List<String> row = null;
+        
+        Scanner scanner = new Scanner(request.getResponseStream(), "UTF-8");
+        while (scanner.hasNextLine()) {
+            scanner.findWithinHorizon(CSV_VALUE_PATTERN, 0);
+            MatchResult match = scanner.match();
+            String quotedString = match.group(2);
+            String decoded = quotedString == null ? match.group(1) : quotedString.replaceAll("\"\"", "\"");
+            
+            if (row == null) {
+                row = new ArrayList<String>();
+            }
+            row.add(decoded);
+            
+            if (!match.group(4).equals(",")) {
+                if (row != null) {
+                    rows.add(row);
+                    row = null;
+                }
+            }
+        }
+        if (row != null) {
+            rows.add(row);
+        }
+        return rows;
+    }
 }

extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java

@@ -36,7 +36,7 @@ import java.util.List;
 
 import org.json.JSONObject;
 
-import com.google.gdata.client.docs.DocsService;
+import com.google.gdata.client.GoogleService;
 import com.google.gdata.client.spreadsheet.CellQuery;
 import com.google.gdata.client.spreadsheet.SpreadsheetService;
 import com.google.gdata.data.spreadsheet.Cell;
@@ -75,19 +75,19 @@ public class GDataImporter {
             SpreadsheetService service = GDataExtension.getSpreadsheetService(token);
             parse(
                 service,
-                job.project,
+                project,
-                job.metadata,
+                metadata,
                 job,
                 limit,
                 options,
                 exceptions
             );
         } else if ("table".equals(docType)) {
-            DocsService service = GDataExtension.getDocsService(token);
+            GoogleService service = GDataExtension.getFusionTablesGoogleService(token);
             parse(
                 service,
-                job.project,
+                project,
-                job.metadata,
+                metadata,
                 job,
                 limit,
                 options,
@@ -165,21 +165,6 @@ public class GDataImporter {
         }
     }
     
-    static public void parse(
-        DocsService service,
-        Project project,
-        ProjectMetadata metadata,
-        final ImportingJob job,
-        int limit,
-        JSONObject options,
-        List<Exception> exceptions) {
-        
-        String docUrlString = JSONUtilities.getString(options, "docUrl", null);
-        if (docUrlString != null) {
-            // TODO[dfhuynh]
-        }
-    }
-    
     static private void setProgress(ImportingJob job, String fileSource, int percent) {
         JSONObject progress = JSONUtilities.getObject(job.config, "progress");
         if (progress == null) {
@@ -287,4 +272,210 @@ public class GDataImporter {
             return rowsOfCells;
         }
     }
+    
+    static public void parse(
+            GoogleService service,
+            Project project,
+            ProjectMetadata metadata,
+            final ImportingJob job,
+            int limit,
+            JSONObject options,
+            List<Exception> exceptions) {
+        
+        String docUrlString = JSONUtilities.getString(options, "docUrl", null);
+        if (docUrlString == null) {
+            return;
+        }
+        int equal = docUrlString.lastIndexOf('=');
+        if (equal < 0) {
+            return;
+        }
+        
+        String id = docUrlString.substring(equal + 1);
+        
+        try {
+            List<FTColumnData> columns = new ArrayList<GDataImporter.FTColumnData>();
+            List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, "DESCRIBE " + id);
+            if (rows.size() > 1) {
+                for (int i = 1; i < rows.size(); i++) {
+                    List<String> row = rows.get(i);
+                    if (row.size() >= 2) {
+                        FTColumnData cd = new FTColumnData();
+                        cd.name = row.get(1);
+                        cd.type = FTColumnType.STRING;
+                        
+                        if (row.size() > 2) {
+                            String type = row.get(2).toLowerCase();
+                            if (type.equals("number")) {
+                                cd.type = FTColumnType.NUMBER;
+                            } else if (type.equals("datetime")) {
+                                cd.type = FTColumnType.DATETIME;
+                            } else if (type.equals("location")) {
+                                cd.type = FTColumnType.LOCATION;
+                            }
+                        }
+                        columns.add(cd);
+                    }
+                }
+                
+                setProgress(job, docUrlString, -1);
+                
+                // Force these options for the next call because each fusion table
+                // is strictly structured with a single line of headers.
+                JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore
+                JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
+                
+                TabularImportingParserBase.readTable(
+                    project,
+                    metadata,
+                    job,
+                    new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100),
+                    docUrlString,
+                    limit,
+                    options,
+                    exceptions
+                );
+                setProgress(job, docUrlString, 100);
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+            exceptions.add(e);
+        } catch (ServiceException e) {
+            e.printStackTrace();
+            exceptions.add(e);
+        }
+    }
+    
+    static private enum FTColumnType {
+        STRING,
+        NUMBER,
+        DATETIME,
+        LOCATION
+    }
+    
+    final static private class FTColumnData {
+        String name;
+        FTColumnType type;
+    }
+    
+    static private class FusionTableBatchRowReader implements TableDataReader {
+        final ImportingJob job;
+        final String fileSource;
+        
+        final GoogleService service;
+        final List<FTColumnData> columns;
+        final int batchSize;
+        
+        final String baseQuery;
+        
+        int nextRow = 0; // 0-based
+        int batchRowStart = 0; // 0-based
+        boolean end = false;
+        List<List<Object>> rowsOfCells = null;
+        boolean usedHeaders = false;
+        
+        public FusionTableBatchRowReader(ImportingJob job, String fileSource,
+                GoogleService service, String tableId, List<FTColumnData> columns,
+                int batchSize) {
+            this.job = job;
+            this.fileSource = fileSource;
+            this.service = service;
+            this.columns = columns;
+            this.batchSize = batchSize;
+            
+            StringBuffer sb = new StringBuffer();
+            sb.append("SELECT ");
+            
+            boolean first = true;
+            for (FTColumnData cd : columns) {
+                if (first) {
+                    first = false;
+                } else {
+                    sb.append(",");
+                }
+                sb.append("'");
+                sb.append(cd.name);
+                sb.append("'");
+            }
+            sb.append(" FROM ");
+            sb.append(tableId);
+            
+            baseQuery = sb.toString();
+        }
+        
+        @Override
+        public List<Object> getNextRowOfCells() throws IOException {
+            if (!usedHeaders) {
+                List<Object> row = new ArrayList<Object>(columns.size());
+                for (FTColumnData cd : columns) {
+                    row.add(cd.name);
+                }
+                usedHeaders = true;
+                return row;
+            }
+            
+            if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) {
+                int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size());
+                try {
+                    rowsOfCells = getRowsOfCells(newBatchRowStart);
+                    batchRowStart = newBatchRowStart;
+                    
+                    setProgress(job, fileSource, -1 /* batchRowStart * 100 / totalRows */);
+                } catch (ServiceException e) {
+                    throw new IOException(e);
+                }
+            }
+            
+            if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) {
+                return rowsOfCells.get(nextRow++ - batchRowStart);
+            } else {
+                return null;
+            }
+        }
+        
+        
+        private List<List<Object>> getRowsOfCells(int startRow) throws IOException, ServiceException {
+            List<List<Object>> rowsOfCells = new ArrayList<List<Object>>(batchSize);
+            
+            String query = baseQuery + " OFFSET " + startRow + " LIMIT " + batchSize;
+            
+            List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, query);
+            if (rows.size() > 1) {
+                for (int i = 1; i < rows.size(); i++) {
+                    List<String> row = rows.get(i);
+                    List<Object> rowOfCells = new ArrayList<Object>(row.size());
+                    for (int j = 0; j < row.size() && j < columns.size(); j++) {
+                        String text = row.get(j);
+                        if (text.isEmpty()) {
+                            rowOfCells.add(null);
+                        } else {
+                            FTColumnData cd = columns.get(j);
+                            if (cd.type == FTColumnType.NUMBER) {
+                                try {
+                                    rowOfCells.add(Long.parseLong(text));
+                                    continue;
+                                } catch (NumberFormatException e) {
+                                    // ignore
+                                }
+                                try {
+                                    double d = Double.parseDouble(text);
+                                    if (!Double.isInfinite(d) && !Double.isNaN(d)) {
+                                        rowOfCells.add(d);
+                                        continue;
+                                    }
+                                } catch (NumberFormatException e) {
+                                    // ignore
+                                }
+                            }
+                            
+                            rowOfCells.add(text);
+                        }
+                    }
+                    rowsOfCells.add(rowOfCells);
+                }
+            }
+            end = rows.size() < batchSize + 1;
+            return rowsOfCells;
+        }
+    }
 }

extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java

@@ -49,17 +49,13 @@ import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.JSONWriter;
 
-import com.google.gdata.client.Query;
+import com.google.gdata.client.GoogleService;
 import com.google.gdata.client.docs.DocsService;
 import com.google.gdata.client.spreadsheet.SpreadsheetService;
-import com.google.gdata.data.Category;
 import com.google.gdata.data.DateTime;
 import com.google.gdata.data.Person;
-import com.google.gdata.data.docs.DocumentListEntry;
-import com.google.gdata.data.docs.DocumentListFeed;
 import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
 import com.google.gdata.data.spreadsheet.SpreadsheetFeed;
-import com.google.gdata.data.spreadsheet.TableEntry;
 import com.google.gdata.data.spreadsheet.WorksheetEntry;
 import com.google.gdata.util.ServiceException;
 
@@ -67,6 +63,7 @@ import com.google.refine.ProjectManager;
 import com.google.refine.ProjectMetadata;
 import com.google.refine.RefineServlet;
 import com.google.refine.commands.HttpUtilities;
+import com.google.refine.importing.DefaultImportingController;
 import com.google.refine.importing.ImportingController;
 import com.google.refine.importing.ImportingJob;
 import com.google.refine.importing.ImportingManager;
@@ -125,11 +122,10 @@ public class GDataImportingController implements ImportingController {
             writer.array();
             
             try {
-                DocsService service = GDataExtension.getDocsService(token);
+                listSpreadsheets(GDataExtension.getDocsService(token), writer);
-                listSpreadsheets(service, writer);
+                listFusionTables(GDataExtension.getFusionTablesGoogleService(token), writer);
-                listDocumentsOfType(service, writer, "http://schemas.google.com/docs/2007#table");
             } catch (ServiceException e) {
-                // TODO: just ignore?
+                e.printStackTrace();
             }
             
             writer.endArray();
@@ -169,38 +165,29 @@ public class GDataImportingController implements ImportingController {
         }
     }
     
-    private void listDocumentsOfType(DocsService service, JSONWriter writer, String type)
+    private void listFusionTables(GoogleService service, JSONWriter writer)
         throws IOException, ServiceException, JSONException {
-        URL feedUrl = new URL("https://docs.google.com/feeds/default/private/full");
         
-        Query query = new Query(feedUrl);
+        List<List<String>> rows = GDataExtension.runFusionTablesSelect(service, "SHOW TABLES");
-        query.addCategoryFilter(
+        if (rows.size() > 1) { // excluding headers
-            new Query.CategoryFilter(
+            for (int i = 1; i < rows.size(); i++) {
-                new Category("http://schemas.google.com/g/2005#kind", type)));
+                List<String> row = rows.get(i);
+                if (row.size() >= 2) {
+                    String id = row.get(0);
+                    String name = row.get(1);
+                    String link = "https://www.google.com/fusiontables/DataSource?dsrcid=" + id;
                     
-        DocumentListFeed feed = service.query(query, DocumentListFeed.class);
-        for (DocumentListEntry entry : feed.getEntries()) {
                     writer.object();
-            writer.key("docId"); writer.value(entry.getId());
+                    writer.key("docId"); writer.value(id);
-            writer.key("docLink"); writer.value(entry.getHtmlLink().getHref());
+                    writer.key("docLink"); writer.value(link);
-            writer.key("docSelfLink"); writer.value(entry.getSelfLink().getHref());
+                    writer.key("docSelfLink"); writer.value(link);
-            writer.key("title"); writer.value(entry.getTitle().getPlainText());
+                    writer.key("title"); writer.value(name);
-            writer.key("type"); writer.value(entry.getType());
+                    writer.key("type"); writer.value("table");
-            
-            DateTime updated = entry.getUpdated();
-            if (updated != null) {
-                writer.key("updated"); writer.value(updated.toStringRfc822());
-            }
-            
-            writer.key("authors"); writer.array();
-            for (Person person : entry.getAuthors()) {
-                writer.value(person.getName());
-            }
-            writer.endArray();
-            
                     writer.endObject();
                 }
             }
+        }
+    }
     
     private void doInitializeParserUI(
         HttpServletRequest request, HttpServletResponse response, Properties parameters)
@@ -221,16 +208,17 @@ public class GDataImportingController implements ImportingController {
             JSONUtilities.safePut(result, "status", "ok");
             JSONUtilities.safePut(result, "options", options);
             
-            JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore
-            JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
             JSONUtilities.safePut(options, "skipDataLines", 0); // number of initial data lines to skip
             JSONUtilities.safePut(options, "storeBlankRows", true);
             JSONUtilities.safePut(options, "storeBlankCellsAsNulls", true);
             
+            if ("spreadsheet".equals(type)) {
+                JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore
+                JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
+                
                 JSONArray worksheets = new JSONArray();
                 JSONUtilities.safePut(options, "worksheets", worksheets);
                 
-            if ("spreadsheet".equals(type)) {
                 SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token);
                 SpreadsheetEntry spreadsheetEntry = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
                 for (WorksheetEntry worksheetEntry : spreadsheetEntry.getWorksheets()) {
@@ -242,15 +230,7 @@ public class GDataImportingController implements ImportingController {
                     JSONUtilities.append(worksheets, worksheetO);
                 }
             } else if ("table".equals(type)) {
-                DocsService docsService = GDataExtension.getDocsService(token);
+                // No metadata for a fusion table.
-                TableEntry tableEntry = docsService.getEntry(url, TableEntry.class);
-                
-                JSONObject worksheetO = new JSONObject();
-                JSONUtilities.safePut(worksheetO, "name", tableEntry.getTitle().getPlainText());
-                JSONUtilities.safePut(worksheetO, "rows", -1);
-                JSONUtilities.safePut(worksheetO, "link", tableEntry.getSelfLink().getHref());
-                
-                JSONUtilities.append(worksheets, worksheetO);
             }
             /* TODO: else */
             
@@ -307,15 +287,13 @@ public class GDataImportingController implements ImportingController {
                 if (exceptions.size() == 0) {
                     job.project.update(); // update all internal models, indexes, caches, etc.
                     
-                    writer.key("code"); writer.value("ok");
+                    writer.key("status"); writer.value("ok");
                 } else {
-                    writer.key("code"); writer.value("error");
+                    writer.key("status"); writer.value("error");
                     
                     writer.key("errors");
                     writer.array();
-                    for (Exception e : exceptions) {
+                    DefaultImportingController.writeErrors(writer, exceptions);
-                        writer.value(e.getLocalizedMessage());
-                    }
                     writer.endArray();
                 }
                 writer.endObject();
@@ -374,12 +352,18 @@ public class GDataImportingController implements ImportingController {
                     );
                     
                     if (!job.canceled) {
+                        if (exceptions.size() > 0) {
+                            JSONUtilities.safePut(job.config, "errors",
+                                DefaultImportingController.convertErrorsToJsonArray(exceptions));
+                            JSONUtilities.safePut(job.config, "state", "error");
+                        } else {
                             project.update(); // update all internal models, indexes, caches, etc.
                             
                             ProjectManager.singleton.registerProject(project, pm);
                             
-                        JSONUtilities.safePut(job.config, "projectID", project.id);
                             JSONUtilities.safePut(job.config, "state", "created-project");
+                            JSONUtilities.safePut(job.config, "projectID", project.id);
+                        }
                     }
                 }
             }.start();

main/src/com/google/refine/importing/DefaultImportingController.java

@@ -34,6 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.importing;
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.io.Writer;
 import java.util.LinkedList;
 import java.util.List;
@@ -173,7 +175,28 @@ public class DefaultImportingController implements ImportingController {
             
             ImportingUtilities.previewParse(job, format, optionObj, exceptions);
             
-            HttpUtilities.respond(response, "ok", "done");
+            Writer w = response.getWriter();
+            JSONWriter writer = new JSONWriter(w);
+            try {
+                writer.object();
+                if (exceptions.size() == 0) {
+                    job.project.update(); // update all internal models, indexes, caches, etc.
+                    
+                    writer.key("status"); writer.value("ok");
+                } else {
+                    writer.key("status"); writer.value("error");
+                    writer.key("errors");
+                    writer.array();
+                    writeErrors(writer, exceptions);
+                    writer.endArray();
+                }
+                writer.endObject();
+            } catch (JSONException e) {
+                throw new ServletException(e);
+            } finally {
+                w.flush();
+                w.close();
+            }
         } catch (JSONException e) {
             throw new ServletException(e);
         }
@@ -252,4 +275,33 @@ public class DefaultImportingController implements ImportingController {
             w.close();
         }
     }
+    
+    static public void writeErrors(JSONWriter writer, List<Exception> exceptions) throws JSONException {
+        for (Exception e : exceptions) {
+            StringWriter sw = new StringWriter();
+            e.printStackTrace(new PrintWriter(sw));
+            
+            writer.object();
+            writer.key("message");
+            writer.value(e.getLocalizedMessage());
+            writer.key("stack");
+            writer.value(sw.toString());
+            writer.endObject();
+        }
+    }
+    
+    static public JSONArray convertErrorsToJsonArray(List<Exception> exceptions) {
+        JSONArray a = new JSONArray();
+        for (Exception e : exceptions) {
+            StringWriter sw = new StringWriter();
+            e.printStackTrace(new PrintWriter(sw));
+            
+            JSONObject o = new JSONObject();
+            JSONUtilities.safePut(o, "message", e.getLocalizedMessage());
+            JSONUtilities.safePut(o, "stack", sw.toString());
+            JSONUtilities.append(a, o);
+        }
+        return a;
+    }
+
 }

main/src/com/google/refine/importing/ImportingUtilities.java

@@ -911,12 +911,18 @@ public class ImportingUtilities {
         );
         
         if (!job.canceled) {
+            if (exceptions.size() == 0) {
                 project.update(); // update all internal models, indexes, caches, etc.
                 
                 ProjectManager.singleton.registerProject(project, pm);
                 
                 JSONUtilities.safePut(job.config, "projectID", project.id);
                 JSONUtilities.safePut(job.config, "state", "created-project");
+            } else {
+                JSONUtilities.safePut(job.config, "state", "error");
+                JSONUtilities.safePut(job.config, "errors",
+                    DefaultImportingController.convertErrorsToJsonArray(exceptions));
+            }
         }
     }
     

main/webapp/modules/core/scripts/index/create-project-ui.js

@@ -245,3 +245,9 @@ Refine.CreateProjectUI.prototype.showImportJobError = function(message, stack) {
     self.showSourceSelectionPanel();
   });
 };
+
+Refine.CreateProjectUI.composeErrorMessage = function(job) {
+  var messages = [];
+  $.each(job.config.errors, function() { messages.push(this.message) });
+  return messages.join('\n');
+};

main/webapp/modules/core/scripts/index/default-importing-controller/controller.js

@@ -280,7 +280,7 @@ Refine.DefaultImportingController.prototype._createProject = function() {
                   document.location = "project?project=" + job.config.projectID;
                 },
                 function(job) {
-                  alert(job.config.error + '\n' + job.config.errorDetails);
+                  alert('Errors:\n' + Refine.CreateProjectUI.composeErrorMessage(job));
                   self._onImportJobReady();
                 }
             );

main/webapp/modules/core/styles/index/default-importing-file-selection-panel.less

@@ -48,7 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .default-importing-file-selection-file-panel > table {
   border-collapse: collapse;
   width: 100%;
-}
+  }
 .default-importing-file-selection-file-panel > table > tbody > tr > td,
 .default-importing-file-selection-file-panel > table > tbody > tr > th {
   padding: @padding_tight @padding_normal;