Support uploading directly to a new Google Fusion table.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2244 7d457c2a-affb-35e4-300a-418c747d4874

extensions/gdata/module/scripts/project/exporters.js

@@ -31,12 +31,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
  */
 
-CustomTabularExporterDialog.uploadTargets.push({
+(function() {
-  id: 'gdata/google-spreadsheet',
+  var handleUpload = function(options, exportAllRows, onDone, prompt) {
-  label: 'A new Google spreadsheet',
-  handler: function(options, exportAllRows, onDone) {
     var doUpload = function() {
-      var name = window.prompt('Enter name of new spreadsheet', theProject.metadata.name);
+      var name = window.prompt(prompt, theProject.metadata.name);
       if (name) {
         var dismiss = DialogSystem.showBusy('Uploading...');
         $.post(
@@ -78,5 +76,20 @@ CustomTabularExporterDialog.uploadTargets.push({
     };
 
     authenticate();
-  }
+  };
-});
+  
+  CustomTabularExporterDialog.uploadTargets.push({
+    id: 'gdata/google-spreadsheet',
+    label: 'A new Google spreadsheet',
+    handler: function(options, exportAllRows, onDone) {
+      handleUpload(options, exportAllRows, onDone, 'Enter a name for the new Google spreadsheet');
+    }
+  });
+  CustomTabularExporterDialog.uploadTargets.push({
+    id: 'gdata/fusion-table',
+    label: 'A new Google Fusion table',
+    handler: function(options, exportAllRows, onDone) {
+      handleUpload(options, exportAllRows, onDone, 'Enter a name for the new Google Fusion table');
+    }
+  });
+})();

extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java

@@ -29,6 +29,7 @@
 package com.google.refine.extension.gdata;
 
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.net.URL;
 import java.net.URLEncoder;
 import java.util.ArrayList;
@@ -90,13 +91,36 @@ abstract public class GDataExtension {
     static public List<List<String>> runFusionTablesSelect(GoogleService service, String selectQuery)
             throws IOException, ServiceException {
         
-        URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" +
+        GDataRequest request = createFusionTablesRequest(service, RequestType.QUERY, selectQuery);
-                URLEncoder.encode(selectQuery, "UTF-8"));
-        GDataRequest request = service.getRequestFactory().getRequest(
-                RequestType.QUERY, url, ContentType.TEXT_PLAIN);
-        
         request.execute();
+        return parseFusionTablesResults(request);
+    }
     
+    static public GDataRequest createFusionTablesRequest(
+            GoogleService service, RequestType requestType, String query)
+            throws IOException, ServiceException {
+        URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" +
+                URLEncoder.encode(query, "UTF-8"));
+        return service.getRequestFactory().getRequest(
+                requestType, url, ContentType.TEXT_PLAIN);
+    }
+    
+    static public GDataRequest createFusionTablesPostRequest(
+            GoogleService service, RequestType requestType, String query)
+            throws IOException, ServiceException {
+        URL url = new URL(FUSION_TABLES_SERVICE_URL);
+        GDataRequest request = service.getRequestFactory().getRequest(
+            requestType, url, new ContentType("application/x-www-form-urlencoded"));
+        
+        OutputStreamWriter writer =
+            new OutputStreamWriter(request.getRequestStream());
+        writer.append("sql=" + URLEncoder.encode(query, "UTF-8"));
+        writer.flush();
+        
+        return request;
+    }
+    
+    static public List<List<String>> parseFusionTablesResults(GDataRequest request) throws IOException {
         List<List<String>> rows = new ArrayList<List<String>>();
         List<String> row = null;
         

extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java

@@ -33,6 +33,8 @@ import java.io.IOException;
 import java.io.Writer;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
 
@@ -45,6 +47,9 @@ import org.json.JSONWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.gdata.client.GoogleService;
+import com.google.gdata.client.Service.GDataRequest;
+import com.google.gdata.client.Service.GDataRequest.RequestType;
 import com.google.gdata.client.docs.DocsService;
 import com.google.gdata.client.spreadsheet.CellQuery;
 import com.google.gdata.client.spreadsheet.SpreadsheetService;
@@ -96,13 +101,21 @@ public class UploadCommand extends Command {
             JSONWriter writer = new JSONWriter(w);
             try {
                 writer.object();
-                String url = upload(project, engine, params, token, name);
+                
+                List<Exception> exceptions = new LinkedList<Exception>();
+                String url = upload(project, engine, params, token, name, exceptions);
                 if (url != null) {
                     writer.key("status"); writer.value("ok");
                     writer.key("url"); writer.value(url);
-                } else {
+                } else if (exceptions.size() == 0) {
                     writer.key("status"); writer.value("error");
                     writer.key("message"); writer.value("No such format");
+                } else {
+                    for (Exception e : exceptions) {
+                        logger.warn(e.getLocalizedMessage(), e);
+                    }
+                    writer.key("status"); writer.value("error");
+                    writer.key("message"); writer.value(exceptions.get(0).getLocalizedMessage());
                 }
             } catch (Exception e) {
                 e.printStackTrace();
@@ -122,56 +135,67 @@ public class UploadCommand extends Command {
     
     static private String upload(
             Project project, Engine engine, Properties params,
-            String token, String name) throws Exception {
+            String token, String name, List<Exception> exceptions) {
         String format = params.getProperty("format");
         if ("gdata/google-spreadsheet".equals(format)) {
-            return uploadSpreadsheet(project, engine, params, token, name);
+            return uploadSpreadsheet(project, engine, params, token, name, exceptions);
+        } else if ("gdata/fusion-table".equals(format)) {
+            return uploadFusionTable(project, engine, params, token, name, exceptions);
         }
         return null;
     }
     
     static private String uploadSpreadsheet(
             final Project project, final Engine engine, final Properties params,
-            String token, String name)
+            String token, String name, List<Exception> exceptions) {
-            throws MalformedURLException, IOException, ServiceException {
+        
         DocsService docsService = GDataExtension.getDocsService(token);
         final SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token);
         
-        SpreadsheetEntry spreadsheetEntry = new SpreadsheetEntry();
+        try {
-        spreadsheetEntry.setTitle(new PlainTextConstruct(name));
+            SpreadsheetEntry spreadsheetEntry = new SpreadsheetEntry();
+            spreadsheetEntry.setTitle(new PlainTextConstruct(name));
             
-        final SpreadsheetEntry spreadsheetEntry2 = docsService.insert(
+            final SpreadsheetEntry spreadsheetEntry2 = docsService.insert(
-            new URL("https://docs.google.com/feeds/default/private/full/"), spreadsheetEntry);
+                new URL("https://docs.google.com/feeds/default/private/full/"), spreadsheetEntry);
             
-        int[] size = CustomizableTabularExporterUtilities.countColumnsRows(
+            int[] size = CustomizableTabularExporterUtilities.countColumnsRows(
-                project, engine, params);
+                    project, engine, params);
             
-        URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl();
+            URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl();
-        WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]); 
+            WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]); 
-        worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data"));
+            worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data"));
             
-        final WorksheetEntry worksheetEntry2 =
+            final WorksheetEntry worksheetEntry2 =
-            spreadsheetService.insert(worksheetFeedUrl, worksheetEntry);
+                spreadsheetService.insert(worksheetFeedUrl, worksheetEntry);
             
-        spreadsheetEntry2.getDefaultWorksheet().delete();
+            spreadsheetEntry2.getDefaultWorksheet().delete();
             
-        new Thread() {
+            new Thread() {
-            @Override
+                @Override
-            public void run() {
+                public void run() {
-                spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V1);
+                    spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V1);
-                try {
+                    try {
-                    uploadToCellFeed(
+                        uploadToCellFeed(
-                        project, engine, params,
+                            project, engine, params,
-                        spreadsheetService,
+                            spreadsheetService,
-                        spreadsheetEntry2,
+                            spreadsheetEntry2,
-                        worksheetEntry2);
+                            worksheetEntry2);
-                } catch (Exception e) {
+                    } catch (Exception e) {
-                    logger.error("Error uploading data to Google Spreadsheets", e);
+                        logger.error("Error uploading data to Google Spreadsheets", e);
+                    }
                 }
-            }
+            }.start();
-        }.start();
             
-        return spreadsheetEntry2.getDocumentLink().getHref();
+            return spreadsheetEntry2.getDocumentLink().getHref();
+        } catch (MalformedURLException e) {
+            exceptions.add(e);
+        } catch (IOException e) {
+            exceptions.add(e);
+        } catch (ServiceException e) {
+            exceptions.add(e);
+        }
+        return null;
     }
     
     static private void uploadToCellFeed(
@@ -267,4 +291,146 @@ public class UploadCommand extends Command {
         CustomizableTabularExporterUtilities.exportRows(
                 project, engine, params, serializer);
     }
+    
+    static private String uploadFusionTable(
+            Project project, final Engine engine, final Properties params,
+            String token, String name, List<Exception> exceptions) {
+        GoogleService service = GDataExtension.getFusionTablesGoogleService(token);
+        FusionTableSerializer serializer = new FusionTableSerializer(service, name, exceptions);
+        
+        CustomizableTabularExporterUtilities.exportRows(
+                project, engine, params, serializer);
+        
+        return serializer.tableId == null || exceptions.size() > 0 ? null :
+            "https://www.google.com/fusiontables/DataSource?dsrcid=" + serializer.tableId;
+    }
+    
+    final static private class FusionTableSerializer implements TabularSerializer {
+        GoogleService service;
+        String tableName;
+        List<Exception> exceptions;
+        
+        String tableId;
+        List<String> columnNames;
+        StringBuffer sbBatch;
+        int rows;
+        
+        FusionTableSerializer(GoogleService service, String tableName, List<Exception> exceptions) {
+            this.service = service;
+            this.tableName = tableName;
+            this.exceptions = exceptions;
+        }
+        
+        @Override
+        public void startFile(JSONObject options) {
+        }
+
+        @Override
+        public void endFile() {
+            if (sbBatch != null) {
+                sendBatch();
+            }
+        }
+
+        @Override
+        public void addRow(List<CellData> cells, boolean isHeader) {
+            if (isHeader) {
+                columnNames = new ArrayList<String>(cells.size());
+                
+                StringBuffer sb = new StringBuffer();
+                sb.append("CREATE TABLE '");
+                sb.append(tableName);
+                sb.append("' (");
+                boolean first = true;
+                for (CellData cellData : cells) {
+                    columnNames.add(cellData.text);
+                    
+                    if (first) {
+                        first = false;
+                    } else {
+                        sb.append(',');
+                    }
+                    sb.append("'");
+                    sb.append(cellData.text);
+                    sb.append("': STRING");
+                }
+                sb.append(")");
+                
+                try {
+                    String createQuery = sb.toString();
+                    
+                    GDataRequest createTableRequest = GDataExtension.createFusionTablesPostRequest(
+                            service, RequestType.INSERT, createQuery);
+                    createTableRequest.execute();
+                    
+                    List<List<String>> createTableResults =
+                        GDataExtension.parseFusionTablesResults(createTableRequest);
+                    if (createTableResults != null && createTableResults.size() == 2) {
+                        tableId = createTableResults.get(1).get(0);
+                    }
+                } catch (Exception e) {
+                    exceptions.add(e);
+                }
+            } else if (tableId != null) {
+                if (sbBatch == null) {
+                    sbBatch = new StringBuffer();
+                }
+                formulateInsert(cells, sbBatch);
+                
+                rows++;
+                if (rows % 20 == 0) {
+                    sendBatch();
+                }
+            }
+        }
+        
+        void sendBatch() {
+            try {
+                GDataRequest createTableRequest = GDataExtension.createFusionTablesPostRequest(
+                        service, RequestType.INSERT, sbBatch.toString());
+                createTableRequest.execute();
+            } catch (IOException e) {
+                exceptions.add(e);
+            } catch (ServiceException e) {
+                exceptions.add(e);
+            } finally {
+                sbBatch = null;
+            }
+        }
+        
+        void formulateInsert(List<CellData> cells, StringBuffer sb) {
+            StringBuffer sbColumnNames = new StringBuffer();
+            StringBuffer sbValues = new StringBuffer();
+            boolean first = true;
+            for (int i = 0; i < cells.size() && i < columnNames.size(); i++) {
+                CellData cellData = cells.get(i);
+                if (first) {
+                    first = false;
+                } else {
+                    sbColumnNames.append(',');
+                    sbValues.append(',');
+                }
+                sbColumnNames.append("'");
+                sbColumnNames.append(columnNames.get(i));
+                sbColumnNames.append("'");
+                
+                sbValues.append("'");
+                if (cellData != null && cellData.text != null) {
+                    sbValues.append(cellData.text.replaceAll("'", "\\'"));
+                }
+                sbValues.append("'");
+            }
+            
+            if (sb.length() > 0) {
+                sb.append(';');
+            }
+            sb.append("INSERT INTO ");
+            sb.append(tableId);
+            sb.append("(");
+            sb.append(sbColumnNames.toString());
+            sb.append(") values (");
+            sb.append(sbValues.toString());
+            sb.append(")");
+        }
+    }
 }

main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html

@@ -139,15 +139,15 @@
       </table></div></div>
       
       
-      <div bind="uploadTabBody" id="custom-tabular-exporter-tabs-upload" style="display: none;"><div class="grid-layout grid-layout-for-ui layout-loose layout-full"><table bind="uploadTargetTable">
+      <div bind="uploadTabBody" id="custom-tabular-exporter-tabs-upload" style="display: none;"><div class="grid-layout grid-layout-for-ui layout-loose layout-full"><table>
         <tr>
-          <th colspan="2">Upload to</th>
+          <th>Upload to</th>
         </tr>
         <tr>
-          <td colspan="2"><div class="grid-layout grid-layout-for-text layout-tightest layout-full"><table><tr>
+          <td><div class="grid-layout grid-layout-for-text layout-tightest"><table bind="uploadTargetTable"></table></div></td>
-            <td width="100%"> </td>
+        </tr>
-            <td width="1%"><button class="button button-primary" bind="uploadButton">Upload</button></td>
+        <tr>
-          </tr></table></div></td>
+          <td><button class="button button-primary" bind="uploadButton">Upload</button></td>
         </tr>
       </table></div></div>
       

main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.js

@@ -131,7 +131,7 @@ CustomTabularExporterDialog.prototype._createDialog = function(options) {
     var table = this._elmts.uploadTargetTable[0];
     for (var i = 0; i < CustomTabularExporterDialog.uploadTargets.length; i++) {
       var target = CustomTabularExporterDialog.uploadTargets[i];
-      var tr = table.insertRow(table.rows.length - 1);
+      var tr = table.insertRow(table.rows.length);
       
       var td0 = $(tr.insertCell(0))
         .attr('width', '1');