diff --git a/extensions/gdata/module/MOD-INF/controller.js b/extensions/gdata/module/MOD-INF/controller.js index fba51f6f2..06ae8d700 100644 --- a/extensions/gdata/module/MOD-INF/controller.js +++ b/extensions/gdata/module/MOD-INF/controller.js @@ -47,6 +47,7 @@ function init() { RS.registerCommand(module, "authorize", Packages.com.google.refine.extension.gdata.AuthorizeCommand()); RS.registerCommand(module, "authorize2", Packages.com.google.refine.extension.gdata.AuthorizeCommand2()); RS.registerCommand(module, "deauthorize", Packages.com.google.refine.extension.gdata.DeAuthorizeCommand()); + RS.registerCommand(module, "upload", Packages.com.google.refine.extension.gdata.UploadCommand()); // Register importer and exporter var IM = Packages.com.google.refine.importing.ImportingManager; @@ -80,6 +81,15 @@ function init() { "styles/importing-controller.less" ] ); + + // Script files to inject into /project page + ClientSideResourceManager.addPaths( + "project/scripts", + module, + [ + "scripts/project/exporters.js" + ] + ); } /* diff --git a/extensions/gdata/module/scripts/project/exporters.js b/extensions/gdata/module/scripts/project/exporters.js new file mode 100644 index 000000000..abf8c3a07 --- /dev/null +++ b/extensions/gdata/module/scripts/project/exporters.js @@ -0,0 +1,82 @@ +/* + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + */ + +CustomTabularExporterDialog.uploadTargets.push({ + id: 'gdata/google-spreadsheet', + label: 'A new Google spreadsheet', + handler: function(options, exportAllRows, onDone) { + var doUpload = function() { + var name = window.prompt('Enter name of new spreadsheet', theProject.metadata.name); + if (name) { + var dismiss = DialogSystem.showBusy('Uploading...'); + $.post( + "/command/gdata/upload", + { + "project" : theProject.id, + "engine" : exportAllRows ? '' : JSON.stringify(ui.browsingEngine.getJSON()), + "name" : name, + "format" : options.format, + "options" : JSON.stringify(options) + }, + function(o) { + dismiss(); + + if (o.url) { + window.open(o.url, '_blank'); + } + onDone(); + }, + "json" + ); + } + }; + + var messageListener = function(evt) { + window.removeEventListener("message", messageListener, false); + if ($.cookie('authsub_token')) { + doUpload(); + } + }; + + var authenticate = function() { + window.addEventListener("message", messageListener, false); + window.open( + "/command/gdata/authorize", + "google-refine-gdata-signin", + "resizable=1,width=600,height=450" + ); + }; + + authenticate(); + } +}); diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java b/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java index 7d0ee72b5..05d47655d 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand.java @@ -1,4 +1,31 @@ -package com.google.refine.extension.gdata; +/* + * Copyright (c) 2011, Thomas F. Morris + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */package com.google.refine.extension.gdata; import java.io.IOException; import java.net.URL; @@ -33,7 +60,7 @@ public class AuthorizeCommand extends Command { String requestUrl = AuthSubUtil.getRequestUrl( authorizedUrl.toExternalForm(), // execution continues at authorized on redirect - "https://spreadsheets.google.com/feeds https://www.google.com/fusiontables/api/query", + "https://docs.google.com/feeds https://spreadsheets.google.com/feeds https://www.google.com/fusiontables/api/query", false, true); response.sendRedirect(requestUrl); diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand2.java b/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand2.java index 0c5690520..be1693bb7 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand2.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/AuthorizeCommand2.java @@ -1,3 +1,31 @@ +/* + * Copyright (c) 2011, Thomas F. Morris + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ package com.google.refine.extension.gdata; import java.io.IOException; diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/DeAuthorizeCommand.java b/extensions/gdata/src/com/google/refine/extension/gdata/DeAuthorizeCommand.java index 0bddc5a4e..91dd7b631 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/DeAuthorizeCommand.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/DeAuthorizeCommand.java @@ -1,3 +1,31 @@ +/* + * Copyright (c) 2011, Thomas F. Morris + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ package com.google.refine.extension.gdata; import java.io.IOException; diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java b/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java new file mode 100644 index 000000000..8a614bdb8 --- /dev/null +++ b/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2010, Thomas F. Morris + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of Google nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.refine.extension.gdata; + +import java.io.IOException; +import java.io.Writer; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gdata.client.docs.DocsService; +import com.google.gdata.client.spreadsheet.CellQuery; +import com.google.gdata.client.spreadsheet.SpreadsheetService; +import com.google.gdata.data.Link; +import com.google.gdata.data.PlainTextConstruct; +import com.google.gdata.data.batch.BatchOperationType; +import com.google.gdata.data.batch.BatchStatus; +import com.google.gdata.data.batch.BatchUtils; +import com.google.gdata.data.docs.SpreadsheetEntry; +import com.google.gdata.data.spreadsheet.Cell; +import com.google.gdata.data.spreadsheet.CellEntry; +import com.google.gdata.data.spreadsheet.CellFeed; +import com.google.gdata.data.spreadsheet.WorksheetEntry; +import com.google.gdata.util.ServiceException; + +import com.google.refine.ProjectManager; +import com.google.refine.browsing.Engine; +import com.google.refine.commands.Command; +import com.google.refine.commands.HttpUtilities; +import com.google.refine.commands.project.ExportRowsCommand; +import com.google.refine.exporters.CustomizableTabularExporterUtilities; +import com.google.refine.exporters.TabularSerializer; +import com.google.refine.model.Project; + +public class UploadCommand extends Command { + static final Logger logger = LoggerFactory.getLogger("gdata_upload"); + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + String token = TokenCookie.getToken(request); + if (token == null) { + HttpUtilities.respond(response, "error", "Not authorized"); + return; + } + + ProjectManager.singleton.setBusy(true); + try { + Project project = getProject(request); + Engine engine = getEngine(request, project); + Properties params = ExportRowsCommand.getRequestParameters(request); + String name = params.getProperty("name"); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + Writer w = response.getWriter(); + JSONWriter writer = new JSONWriter(w); + try { + writer.object(); + String url = upload(project, engine, params, token, name); + if (url != null) { + writer.key("status"); writer.value("ok"); + writer.key("url"); writer.value(url); + } else { + writer.key("status"); writer.value("error"); + writer.key("message"); writer.value("No such format"); + } + } catch (Exception e) { + e.printStackTrace(); + writer.key("status"); writer.value("error"); + writer.key("message"); writer.value(e.getMessage()); + } finally { + writer.endObject(); + w.flush(); + w.close(); + } + } catch (Exception e) { + throw new ServletException(e); + } finally { + ProjectManager.singleton.setBusy(false); + } + } + + static private String upload( + Project project, Engine engine, Properties params, + String token, String name) throws Exception { + String format = params.getProperty("format"); + if ("gdata/google-spreadsheet".equals(format)) { + return uploadSpreadsheet(project, engine, params, token, name); + } + return null; + } + + static private String uploadSpreadsheet( + final Project project, final Engine engine, final Properties params, + String token, String name) + throws MalformedURLException, IOException, ServiceException { + DocsService docsService = GDataExtension.getDocsService(token); + final SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token); + + SpreadsheetEntry spreadsheetEntry = new SpreadsheetEntry(); + spreadsheetEntry.setTitle(new PlainTextConstruct(name)); + + final SpreadsheetEntry spreadsheetEntry2 = docsService.insert( + new URL("https://docs.google.com/feeds/default/private/full/"), spreadsheetEntry); + + int[] size = CustomizableTabularExporterUtilities.countColumnsRows( + project, engine, params); + + URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl(); + WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]); + worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data")); + + final WorksheetEntry worksheetEntry2 = + spreadsheetService.insert(worksheetFeedUrl, worksheetEntry); + + spreadsheetEntry2.getDefaultWorksheet().delete(); + + new Thread() { + @Override + public void run() { + spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V1); + try { + uploadToCellFeed( + project, engine, params, + spreadsheetService, + spreadsheetEntry2, + worksheetEntry2); + } catch (Exception e) { + logger.error("Error uploading data to Google Spreadsheets", e); + } + } + }.start(); + + return spreadsheetEntry2.getDocumentLink().getHref(); + } + + static private void uploadToCellFeed( + Project project, + Engine engine, + Properties params, + final SpreadsheetService service, + final SpreadsheetEntry spreadsheetEntry, + final WorksheetEntry worksheetEntry) + throws IOException, ServiceException { + + final URL cellFeedUrl = worksheetEntry.getCellFeedUrl(); + final CellEntry[][] cellEntries = + new CellEntry[worksheetEntry.getRowCount()][worksheetEntry.getColCount()]; + { + CellQuery cellQuery = new CellQuery(cellFeedUrl); + cellQuery.setReturnEmpty(true); + + CellFeed fetchingCellFeed = service.getFeed(cellQuery, CellFeed.class); + for (CellEntry cellEntry : fetchingCellFeed.getEntries()) { + Cell cell = cellEntry.getCell(); + cellEntries[cell.getRow() - 1][cell.getCol() - 1] = cellEntry; + } + } + + TabularSerializer serializer = new TabularSerializer() { + CellFeed cellFeed = service.getFeed(cellFeedUrl, CellFeed.class); + CellFeed batchRequest = null; + int row = 0; + + @Override + public void startFile(JSONObject options) { + } + + @Override + public void endFile() { + if (batchRequest != null) { + sendBatch(); + } + } + + private void sendBatch() { + try { + Link batchLink = cellFeed.getLink(Link.Rel.FEED_BATCH, Link.Type.ATOM); + CellFeed batchResponse = service.batch(new URL(batchLink.getHref()), batchRequest); + + for (CellEntry entry : batchResponse.getEntries()) { + String batchId = BatchUtils.getBatchId(entry); + if (!BatchUtils.isSuccess(entry)) { + BatchStatus status = BatchUtils.getBatchStatus(entry); + logger.warn( + String.format( + "Error: %s failed (%s) %s\n", + batchId, status.getReason(), status.getContent())); + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + batchRequest = null; + } + + @Override + public void addRow(List cells, boolean isHeader) { + if (batchRequest == null) { + batchRequest = new CellFeed(); + } + for (int c = 0; c < cells.size(); c++) { + CellData cellData = cells.get(c); + if (cellData != null && cellData.text != null) { + String cellId = String.format("R%sC%s", row + 1, c + 1); + + CellEntry cellEntry = cellEntries[row][c]; + cellEntry.changeInputValueLocal(cellData.text); + if (cellData.link != null) { + cellEntry.addHtmlLink(cellData.link, null, cellData.text); + } + cellEntry.setId(cellId); + BatchUtils.setBatchId(cellEntry, cellId); + BatchUtils.setBatchOperationType(cellEntry, BatchOperationType.UPDATE); + + batchRequest.getEntries().add(cellEntry); + } + } + row++; + if (row % 20 == 0) { + sendBatch(); + } + } + }; + + CustomizableTabularExporterUtilities.exportRows( + project, engine, params, serializer); + } +} diff --git a/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java b/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java index 584dda642..9c71bc5c3 100644 --- a/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java +++ b/main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java @@ -171,6 +171,34 @@ abstract public class CustomizableTabularExporterUtilities { filteredRows.accept(project, visitor); } + static public int[] countColumnsRows( + final Project project, + final Engine engine, + Properties params) { + RowCountingTabularSerializer serializer = new RowCountingTabularSerializer(); + exportRows(project, engine, params, serializer); + return new int[] { serializer.columns, serializer.rows }; + } + + static private class RowCountingTabularSerializer implements TabularSerializer { + int columns; + int rows; + + @Override + public void startFile(JSONObject options) { + } + + @Override + public void endFile() { + } + + @Override + public void addRow(List cells, boolean isHeader) { + columns = Math.max(columns, cells.size()); + rows++; + } + } + private enum ReconOutputMode { ENTITY_NAME, ENTITY_ID, diff --git a/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html b/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html index 8c57c53e2..98fd7aa51 100644 --- a/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html +++ b/main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html @@ -3,49 +3,12 @@
- -
@@ -109,22 +72,86 @@
Select and Order Columns to Export
- + -
+ + +
- + - + + +
Output column headersOutput column headers Output blank rowsOutput blank rowsIgnore facets and filters and export all rows
+ + + + + +