diff --git a/extensions/gdata/module/scripts/project/exporters.js b/extensions/gdata/module/scripts/project/exporters.js index abf8c3a07..de715f5c3 100644 --- a/extensions/gdata/module/scripts/project/exporters.js +++ b/extensions/gdata/module/scripts/project/exporters.js @@ -31,12 +31,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -CustomTabularExporterDialog.uploadTargets.push({ - id: 'gdata/google-spreadsheet', - label: 'A new Google spreadsheet', - handler: function(options, exportAllRows, onDone) { +(function() { + var handleUpload = function(options, exportAllRows, onDone, prompt) { var doUpload = function() { - var name = window.prompt('Enter name of new spreadsheet', theProject.metadata.name); + var name = window.prompt(prompt, theProject.metadata.name); if (name) { var dismiss = DialogSystem.showBusy('Uploading...'); $.post( @@ -50,7 +48,7 @@ CustomTabularExporterDialog.uploadTargets.push({ }, function(o) { dismiss(); - + if (o.url) { window.open(o.url, '_blank'); } @@ -60,14 +58,14 @@ CustomTabularExporterDialog.uploadTargets.push({ ); } }; - + var messageListener = function(evt) { window.removeEventListener("message", messageListener, false); if ($.cookie('authsub_token')) { doUpload(); } }; - + var authenticate = function() { window.addEventListener("message", messageListener, false); window.open( @@ -76,7 +74,22 @@ CustomTabularExporterDialog.uploadTargets.push({ "resizable=1,width=600,height=450" ); }; - + authenticate(); - } -}); + }; + + CustomTabularExporterDialog.uploadTargets.push({ + id: 'gdata/google-spreadsheet', + label: 'A new Google spreadsheet', + handler: function(options, exportAllRows, onDone) { + handleUpload(options, exportAllRows, onDone, 'Enter a name for the new Google spreadsheet'); + } + }); + CustomTabularExporterDialog.uploadTargets.push({ + id: 'gdata/fusion-table', + label: 'A new Google Fusion table', + handler: function(options, exportAllRows, onDone) { + handleUpload(options, exportAllRows, onDone, 'Enter a name for the new Google Fusion table'); + } + }); +})(); diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java index 71b5f3949..ad8c3af8b 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java @@ -29,6 +29,7 @@ package com.google.refine.extension.gdata; import java.io.IOException; +import java.io.OutputStreamWriter; import java.net.URL; import java.net.URLEncoder; import java.util.ArrayList; @@ -90,13 +91,36 @@ abstract public class GDataExtension { static public List> runFusionTablesSelect(GoogleService service, String selectQuery) throws IOException, ServiceException { - URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" + - URLEncoder.encode(selectQuery, "UTF-8")); - GDataRequest request = service.getRequestFactory().getRequest( - RequestType.QUERY, url, ContentType.TEXT_PLAIN); - + GDataRequest request = createFusionTablesRequest(service, RequestType.QUERY, selectQuery); request.execute(); + return parseFusionTablesResults(request); + } + + static public GDataRequest createFusionTablesRequest( + GoogleService service, RequestType requestType, String query) + throws IOException, ServiceException { + URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" + + URLEncoder.encode(query, "UTF-8")); + return service.getRequestFactory().getRequest( + requestType, url, ContentType.TEXT_PLAIN); + } + + static public GDataRequest createFusionTablesPostRequest( + GoogleService service, RequestType requestType, String query) + throws IOException, ServiceException { + URL url = new URL(FUSION_TABLES_SERVICE_URL); + GDataRequest request = service.getRequestFactory().getRequest( + requestType, url, new ContentType("application/x-www-form-urlencoded")); + OutputStreamWriter writer = + new OutputStreamWriter(request.getRequestStream()); + writer.append("sql=" + URLEncoder.encode(query, "UTF-8")); + writer.flush(); + + return request; + } + + static public List> parseFusionTablesResults(GDataRequest request) throws IOException { List> rows = new ArrayList>(); List row = null; diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java b/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java index 8a614bdb8..ebd603f0c 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java @@ -33,6 +33,8 @@ import java.io.IOException; import java.io.Writer; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.Properties; @@ -45,6 +47,9 @@ import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.gdata.client.GoogleService; +import com.google.gdata.client.Service.GDataRequest; +import com.google.gdata.client.Service.GDataRequest.RequestType; import com.google.gdata.client.docs.DocsService; import com.google.gdata.client.spreadsheet.CellQuery; import com.google.gdata.client.spreadsheet.SpreadsheetService; @@ -96,13 +101,21 @@ public class UploadCommand extends Command { JSONWriter writer = new JSONWriter(w); try { writer.object(); - String url = upload(project, engine, params, token, name); + + List exceptions = new LinkedList(); + String url = upload(project, engine, params, token, name, exceptions); if (url != null) { writer.key("status"); writer.value("ok"); writer.key("url"); writer.value(url); - } else { + } else if (exceptions.size() == 0) { writer.key("status"); writer.value("error"); writer.key("message"); writer.value("No such format"); + } else { + for (Exception e : exceptions) { + logger.warn(e.getLocalizedMessage(), e); + } + writer.key("status"); writer.value("error"); + writer.key("message"); writer.value(exceptions.get(0).getLocalizedMessage()); } } catch (Exception e) { e.printStackTrace(); @@ -122,56 +135,67 @@ public class UploadCommand extends Command { static private String upload( Project project, Engine engine, Properties params, - String token, String name) throws Exception { + String token, String name, List exceptions) { String format = params.getProperty("format"); if ("gdata/google-spreadsheet".equals(format)) { - return uploadSpreadsheet(project, engine, params, token, name); + return uploadSpreadsheet(project, engine, params, token, name, exceptions); + } else if ("gdata/fusion-table".equals(format)) { + return uploadFusionTable(project, engine, params, token, name, exceptions); } return null; } static private String uploadSpreadsheet( final Project project, final Engine engine, final Properties params, - String token, String name) - throws MalformedURLException, IOException, ServiceException { + String token, String name, List exceptions) { + DocsService docsService = GDataExtension.getDocsService(token); final SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token); - SpreadsheetEntry spreadsheetEntry = new SpreadsheetEntry(); - spreadsheetEntry.setTitle(new PlainTextConstruct(name)); - - final SpreadsheetEntry spreadsheetEntry2 = docsService.insert( - new URL("https://docs.google.com/feeds/default/private/full/"), spreadsheetEntry); - - int[] size = CustomizableTabularExporterUtilities.countColumnsRows( - project, engine, params); - - URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl(); - WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]); - worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data")); - - final WorksheetEntry worksheetEntry2 = - spreadsheetService.insert(worksheetFeedUrl, worksheetEntry); - - spreadsheetEntry2.getDefaultWorksheet().delete(); - - new Thread() { - @Override - public void run() { - spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V1); - try { - uploadToCellFeed( - project, engine, params, - spreadsheetService, - spreadsheetEntry2, - worksheetEntry2); - } catch (Exception e) { - logger.error("Error uploading data to Google Spreadsheets", e); + try { + SpreadsheetEntry spreadsheetEntry = new SpreadsheetEntry(); + spreadsheetEntry.setTitle(new PlainTextConstruct(name)); + + final SpreadsheetEntry spreadsheetEntry2 = docsService.insert( + new URL("https://docs.google.com/feeds/default/private/full/"), spreadsheetEntry); + + int[] size = CustomizableTabularExporterUtilities.countColumnsRows( + project, engine, params); + + URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl(); + WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]); + worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data")); + + final WorksheetEntry worksheetEntry2 = + spreadsheetService.insert(worksheetFeedUrl, worksheetEntry); + + spreadsheetEntry2.getDefaultWorksheet().delete(); + + new Thread() { + @Override + public void run() { + spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V1); + try { + uploadToCellFeed( + project, engine, params, + spreadsheetService, + spreadsheetEntry2, + worksheetEntry2); + } catch (Exception e) { + logger.error("Error uploading data to Google Spreadsheets", e); + } } - } - }.start(); - - return spreadsheetEntry2.getDocumentLink().getHref(); + }.start(); + + return spreadsheetEntry2.getDocumentLink().getHref(); + } catch (MalformedURLException e) { + exceptions.add(e); + } catch (IOException e) { + exceptions.add(e); + } catch (ServiceException e) { + exceptions.add(e); + } + return null; } static private void uploadToCellFeed( @@ -267,4 +291,146 @@ public class UploadCommand extends Command { CustomizableTabularExporterUtilities.exportRows( project, engine, params, serializer); } + + static private String uploadFusionTable( + Project project, final Engine engine, final Properties params, + String token, String name, List exceptions) { + GoogleService service = GDataExtension.getFusionTablesGoogleService(token); + FusionTableSerializer serializer = new FusionTableSerializer(service, name, exceptions); + + CustomizableTabularExporterUtilities.exportRows( + project, engine, params, serializer); + + return serializer.tableId == null || exceptions.size() > 0 ? null : + "https://www.google.com/fusiontables/DataSource?dsrcid=" + serializer.tableId; + } + + final static private class FusionTableSerializer implements TabularSerializer { + GoogleService service; + String tableName; + List exceptions; + + String tableId; + List columnNames; + StringBuffer sbBatch; + int rows; + + FusionTableSerializer(GoogleService service, String tableName, List exceptions) { + this.service = service; + this.tableName = tableName; + this.exceptions = exceptions; + } + + @Override + public void startFile(JSONObject options) { + } + + @Override + public void endFile() { + if (sbBatch != null) { + sendBatch(); + } + } + + @Override + public void addRow(List cells, boolean isHeader) { + if (isHeader) { + columnNames = new ArrayList(cells.size()); + + StringBuffer sb = new StringBuffer(); + sb.append("CREATE TABLE '"); + sb.append(tableName); + sb.append("' ("); + boolean first = true; + for (CellData cellData : cells) { + columnNames.add(cellData.text); + + if (first) { + first = false; + } else { + sb.append(','); + } + sb.append("'"); + sb.append(cellData.text); + sb.append("': STRING"); + } + sb.append(")"); + + try { + String createQuery = sb.toString(); + + GDataRequest createTableRequest = GDataExtension.createFusionTablesPostRequest( + service, RequestType.INSERT, createQuery); + createTableRequest.execute(); + + List> createTableResults = + GDataExtension.parseFusionTablesResults(createTableRequest); + if (createTableResults != null && createTableResults.size() == 2) { + tableId = createTableResults.get(1).get(0); + } + } catch (Exception e) { + exceptions.add(e); + } + } else if (tableId != null) { + if (sbBatch == null) { + sbBatch = new StringBuffer(); + } + formulateInsert(cells, sbBatch); + + rows++; + if (rows % 20 == 0) { + sendBatch(); + } + } + } + + void sendBatch() { + try { + GDataRequest createTableRequest = GDataExtension.createFusionTablesPostRequest( + service, RequestType.INSERT, sbBatch.toString()); + createTableRequest.execute(); + } catch (IOException e) { + exceptions.add(e); + } catch (ServiceException e) { + exceptions.add(e); + } finally { + sbBatch = null; + } + } + + void formulateInsert(List cells, StringBuffer sb) { + StringBuffer sbColumnNames = new StringBuffer(); + StringBuffer sbValues = new StringBuffer(); + boolean first = true; + for (int i = 0; i < cells.size() && i < columnNames.size(); i++) { + CellData cellData = cells.get(i); + if (first) { + first = false; + } else { + sbColumnNames.append(','); + sbValues.append(','); + } + sbColumnNames.append("'"); + sbColumnNames.append(columnNames.get(i)); + sbColumnNames.append("'"); + + sbValues.append("'"); + if (cellData != null && cellData.text != null) { + sbValues.append(cellData.text.replaceAll("'", "\\'")); + } + sbValues.append("'"); + } + + if (sb.length() > 0) { + sb.append(';'); + } + sb.append("INSERT INTO "); + sb.append(tableId); + sb.append("("); + sb.append(sbColumnNames.toString()); + sb.append(") values ("); + sb.append(sbValues.toString()); + sb.append(")"); + } + } } diff --git a/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html b/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html index 98fd7aa51..03c107dec 100644 --- a/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html +++ b/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html @@ -139,15 +139,15 @@ - diff --git a/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.js b/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.js index d880fa22e..56b385c08 100644 --- a/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.js +++ b/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.js @@ -131,7 +131,7 @@ CustomTabularExporterDialog.prototype._createDialog = function(options) { var table = this._elmts.uploadTargetTable[0]; for (var i = 0; i < CustomTabularExporterDialog.uploadTargets.length; i++) { var target = CustomTabularExporterDialog.uploadTargets[i]; - var tr = table.insertRow(table.rows.length - 1); + var tr = table.insertRow(table.rows.length); var td0 = $(tr.insertCell(0)) .attr('width', '1');