diff --git a/.classpath b/.classpath index 84016272a..b55cf5463 100644 --- a/.classpath +++ b/.classpath @@ -38,16 +38,7 @@ - - - - - - - - - @@ -63,14 +54,6 @@ - - - - - - - - @@ -82,7 +65,6 @@ - @@ -119,5 +101,17 @@ + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore index ccafd80f9..4f246f10d 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,6 @@ broker/core/data/ broker/core/test-output/ tmp/ /test-output +test-out/ /bin open-refine.log diff --git a/extensions/gdata/module/MOD-INF/controller.js b/extensions/gdata/module/MOD-INF/controller.js index 428941de3..a5ef68568 100644 --- a/extensions/gdata/module/MOD-INF/controller.js +++ b/extensions/gdata/module/MOD-INF/controller.js @@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. var html = "text/html"; var encoding = "UTF-8"; -var version = "0.2"; +var version = "0.3"; var ClientSideResourceManager = Packages.com.google.refine.ClientSideResourceManager; /* @@ -61,10 +61,6 @@ function init() { new Packages.com.google.refine.extension.gdata.GDataImportingController() ); - -//Packages.com.google.refine.exporters.ExporterRegistry.registerExporter( -//"gdata-exporter", new Packages.com.google.refine.extension.gdata.GDataExporter()); - // Script files to inject into /index page ClientSideResourceManager.addPaths( "index/scripts", @@ -102,7 +98,7 @@ function process(path, request, response) { // Analyze path and handle this request yourself. if (path == "authorize") { var context = {}; - context.authorizationUrl = Packages.com.google.refine.extension.gdata.GDataExtension.getAuthorizationUrl(module, request); + context.authorizationUrl = Packages.com.google.refine.extension.gdata.GoogleAPIExtension.getAuthorizationUrl(module, request); send(request, response, "authorize.vt", context); } else if (path == "authorized") { @@ -111,9 +107,10 @@ function process(path, request, response) { context.callback = request.getParameter("cb"); (function() { - var token = Packages.com.google.refine.extension.gdata.GDataExtension.getTokenFromCode(module,request); - if (token) { - Packages.com.google.refine.extension.gdata.TokenCookie.setToken(request, response, token); + var tokenAndExpiresInSeconds = Packages.com.google.refine.extension.gdata.GoogleAPIExtension.getTokenFromCode(module,request); + if (tokenAndExpiresInSeconds) { + var tokenInfo = tokenAndExpiresInSeconds.split(","); + Packages.com.google.refine.extension.gdata.TokenCookie.setToken(request, response, tokenInfo[0], tokenInfo[1]); return; } Packages.com.google.refine.extension.gdata.TokenCookie.deleteToken(request, response); diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-base-1.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-base-1.0.jar deleted file mode 100644 index 2f65d50ae..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-base-1.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-client-1.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-client-1.0.jar deleted file mode 100644 index 99d073cab..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-client-1.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-client-meta-1.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-client-meta-1.0.jar deleted file mode 100644 index 362bca10d..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-client-meta-1.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-core-1.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-core-1.0.jar deleted file mode 100644 index 8c493c674..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-core-1.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-docs-3.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-docs-3.0.jar deleted file mode 100644 index 9f28745e6..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-docs-3.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-docs-meta-3.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-docs-meta-3.0.jar deleted file mode 100644 index f5edda213..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-docs-meta-3.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-media-1.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-media-1.0.jar deleted file mode 100644 index 69a545ae4..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-media-1.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-3.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-3.0.jar deleted file mode 100644 index e4ddb4ac6..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-3.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-meta-3.0.jar b/extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-meta-3.0.jar deleted file mode 100644 index e46c83e1a..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-meta-3.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-client-1.20.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-client-1.20.0.jar deleted file mode 100644 index 778aa0192..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/google-api-client-1.20.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-client-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-client-1.23.0.jar new file mode 100644 index 000000000..3df306b2c Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-api-client-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.20.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.20.0.jar deleted file mode 100644 index d3389c132..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.20.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.23.0.jar new file mode 100644 index 000000000..95bd0431c Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v2-rev168-1.20.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v2-rev168-1.20.0.jar deleted file mode 100644 index 79ff52bb3..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v2-rev168-1.20.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v3-rev101-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v3-rev101-1.23.0.jar new file mode 100644 index 000000000..a20f60fd0 Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v3-rev101-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev21-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev21-1.23.0.jar new file mode 100644 index 000000000..c215965ff Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev21-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev3-1.20.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev3-1.20.0.jar deleted file mode 100644 index 5ef00235d..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev3-1.20.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-api-services-sheets-v4-rev502-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-api-services-sheets-v4-rev502-1.23.0.jar new file mode 100644 index 000000000..87b3b0f43 Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-api-services-sheets-v4-rev502-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-auth-library-oauth2-http-0.9.0.jar b/extensions/gdata/module/MOD-INF/lib/google-auth-library-oauth2-http-0.9.0.jar new file mode 100644 index 000000000..b14bd5f10 Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-auth-library-oauth2-http-0.9.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-http-client-1.20.0.jar b/extensions/gdata/module/MOD-INF/lib/google-http-client-1.20.0.jar deleted file mode 100644 index 82887fb3e..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/google-http-client-1.20.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-http-client-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-http-client-1.23.0.jar new file mode 100644 index 000000000..3e6667d29 Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-http-client-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-http-client-jackson-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-http-client-jackson-1.23.0.jar new file mode 100644 index 000000000..61d1a3a75 Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-http-client-jackson-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar b/extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar deleted file mode 100644 index 674aea920..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar b/extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar deleted file mode 100644 index 417dc5152..000000000 Binary files a/extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar and /dev/null differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.23.0.jar new file mode 100644 index 000000000..be88de28a Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-oauth-client-jetty-1.23.0.jar b/extensions/gdata/module/MOD-INF/lib/google-oauth-client-jetty-1.23.0.jar new file mode 100644 index 000000000..f0d2ee31c Binary files /dev/null and b/extensions/gdata/module/MOD-INF/lib/google-oauth-client-jetty-1.23.0.jar differ diff --git a/extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar b/extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.23.0.jar similarity index 82% rename from extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar rename to extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.23.0.jar index d444049ca..487e3c62d 100644 Binary files a/extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar and b/extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.23.0.jar differ diff --git a/extensions/gdata/module/langs/translation-en.json b/extensions/gdata/module/langs/translation-en.json index 089eaeac7..150432d77 100644 --- a/extensions/gdata/module/langs/translation-en.json +++ b/extensions/gdata/module/langs/translation-en.json @@ -44,6 +44,7 @@ "gdata-exporter": { "uploading": "Uploading...", "upload-error": "Upload error: ", + "upload-success": "Project was uploaded successfully to Google Drive with Id ", "new-spreadsheet": "A new Google spreadsheet", "enter-spreadsheet": "Enter a name for the new Google spreadsheet", "new-fusion": "A new Google Fusion table", diff --git a/extensions/gdata/module/scripts/index/gdata-source-ui.js b/extensions/gdata/module/scripts/index/gdata-source-ui.js index 597725b03..9a430877e 100644 --- a/extensions/gdata/module/scripts/index/gdata-source-ui.js +++ b/extensions/gdata/module/scripts/index/gdata-source-ui.js @@ -83,7 +83,13 @@ Refine.GDataSourceUI.prototype.attachUI = function(body) { } else { doc.type = 'table'; } - self._controller.startImportingDocument(doc); + + if (GdataExtension.isAuthorized()) { + self._controller.startImportingDocument(doc); + } else { + var fn = self._controller.startImportingDocument; + GdataExtension.showAuthorizationDialog(fn.bind(self._controller, doc)); + } } }); diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java b/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java index 16b35562c..c7f219f78 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java @@ -121,8 +121,8 @@ public class FusionTableHandler { static public Fusiontables getFusionTablesService(String token) { Credential credential = new GoogleCredential().setAccessToken(token); Fusiontables fusiontables = new Fusiontables.Builder( - GDataExtension.HTTP_TRANSPORT, GDataExtension.JSON_FACTORY, credential) - .setApplicationName(GDataExtension.SERVICE_APP_NAME) + GoogleAPIExtension.HTTP_TRANSPORT, GoogleAPIExtension.JSON_FACTORY, credential) + .setApplicationName(GoogleAPIExtension.SERVICE_APP_NAME) .build();; return fusiontables; diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java deleted file mode 100644 index 62aa91d32..000000000 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (c) 2010, Thomas F. Morris - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.gdata; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Arrays; - -import javax.servlet.http.HttpServletRequest; - -import com.google.api.client.auth.oauth2.AuthorizationCodeResponseUrl; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeRequestUrl; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeTokenRequest; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.drive.Drive; -import com.google.gdata.client.docs.DocsService; -import com.google.gdata.client.spreadsheet.SpreadsheetService; - -import com.google.refine.util.ParsingUtilities; - -import edu.mit.simile.butterfly.ButterflyModule; - -/** - * @author Tom Morris - * @copyright 2010 Thomas F. Morris - * @license New BSD http://www.opensource.org/licenses/bsd-license.php - */ -abstract public class GDataExtension { - static final String SERVICE_APP_NAME = "OpenRefine-GData-Extension"; - static final String CLIENT_ID = "647865400439.apps.googleusercontent.com"; - static final String CLIENT_SECRET = "0mW9OJji1yrgJk5AjJc5Pn6I"; // not really that secret, but the protocol accounts for that - - /** Global instance of the HTTP transport. */ - static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); - - /** Global instance of the JSON factory. */ - static final JsonFactory JSON_FACTORY = new JacksonFactory(); - - static public String getAuthorizationUrl(ButterflyModule module, HttpServletRequest request) - throws MalformedURLException { - String authorizedUrl = makeRedirectUrl(module, request); - - // New Oauth2 - GoogleAuthorizationCodeRequestUrl url = new GoogleAuthorizationCodeRequestUrl( - CLIENT_ID, - authorizedUrl, // execution continues at authorized on redirect - Arrays.asList("https://www.googleapis.com/auth/fusiontables", - "https://www.googleapis.com/auth/drive", // create new spreadsheets - "https://spreadsheets.google.com/feeds")); - - return url.toString(); - - } - - private static String makeRedirectUrl(ButterflyModule module, HttpServletRequest request) - throws MalformedURLException { - StringBuffer sb = new StringBuffer(module.getMountPoint().getMountPoint()); - sb.append("authorized?winname="); - sb.append(ParsingUtilities.encode(request.getParameter("winname"))); - sb.append("&cb="); - sb.append(ParsingUtilities.encode(request.getParameter("cb"))); - - URL thisUrl = new URL(request.getRequestURL().toString()); - URL authorizedUrl = new URL(thisUrl, sb.toString()); - return authorizedUrl.toExternalForm(); - } - - static public String getTokenFromCode(ButterflyModule module, HttpServletRequest request) - throws MalformedURLException { - String redirectUrl = makeRedirectUrl(module, request); - StringBuffer fullUrlBuf = request.getRequestURL(); - if (request.getQueryString() != null) { - fullUrlBuf.append('?').append(request.getQueryString()); - } - AuthorizationCodeResponseUrl authResponse = - new AuthorizationCodeResponseUrl(fullUrlBuf.toString()); - // check for user-denied error - if (authResponse.getError() != null) { - // authorization denied... - } else { - // request access token using authResponse.getCode()... - String code = authResponse.getCode(); - try { - GoogleTokenResponse response = new GoogleAuthorizationCodeTokenRequest(HTTP_TRANSPORT, - JSON_FACTORY, CLIENT_ID, CLIENT_SECRET, code, redirectUrl).execute(); - String token = response.getAccessToken(); - return token; - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - return null; - } - - - static public DocsService getDocsService(String token) { - DocsService service = new DocsService(SERVICE_APP_NAME); - if (token != null) { - service.setAuthSubToken(token); - } - return service; - } - - static public SpreadsheetService getSpreadsheetService(String token) { - SpreadsheetService service = new SpreadsheetService(SERVICE_APP_NAME); - if (token != null) { - service.setAuthSubToken(token); - } - return service; - } - - static public Drive getDriveService(String token) { - GoogleCredential credential = new GoogleCredential().setAccessToken(token); - return new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential) - .setApplicationName(SERVICE_APP_NAME).build(); - } - - static boolean isSpreadsheetURL(String url) { - // e.g. http://spreadsheets.google.com/ccc?key=tI36b9Fxk1lFBS83iR_3XQA&hl=en - // TODO: The following should work, but the GData implementation is too limited -// try { -// FeedURLFactory.getSpreadsheetKeyFromUrl(url); -// return true; -// } catch (IllegalArgumentException e) { -// return false; -// } - try { - return url.contains("spreadsheet") && getSpreadsheetID(new URL(url)) != null; - } catch (MalformedURLException e) { - return false; - } - } - - static String getSpreadsheetID(URL url) { - return getParamValue(url,"key"); - } - - static private String getParamValue(URL url, String key) { - String query = url.getQuery(); - if (query != null) { - String[] parts = query.split("&"); - for (String part : parts) { - if (part.startsWith(key+"=")) { - int offset = key.length()+1; - String tableId = part.substring(offset); - return tableId; - } - } - } - return null; - } - -} diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java index a4d58c2a6..3e2e1d9fd 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java @@ -1,49 +1,19 @@ -/* - * Copyright (c) 2010, Thomas F. Morris - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ package com.google.refine.extension.gdata; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import com.google.gdata.client.spreadsheet.CellQuery; -import com.google.gdata.client.spreadsheet.SpreadsheetService; -import com.google.gdata.data.spreadsheet.Cell; -import com.google.gdata.data.spreadsheet.CellEntry; -import com.google.gdata.data.spreadsheet.CellFeed; -import com.google.gdata.data.spreadsheet.SpreadsheetEntry; -import com.google.gdata.data.spreadsheet.WorksheetEntry; -import com.google.gdata.util.ServiceException; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.model.Sheet; +import com.google.api.services.sheets.v4.model.Spreadsheet; +import com.google.api.services.sheets.v4.model.ValueRange; import com.google.refine.importers.TabularImportingParserBase; import com.google.refine.importers.TabularImportingParserBase.TableDataReader; @@ -52,14 +22,9 @@ import com.google.refine.model.Project; import com.google.refine.model.medadata.ProjectMetadata; import com.google.refine.util.JSONUtilities; -/** - * OpenRefine parser for Google Spreadsheets. - * - * @author Tom Morris - * @copyright 2010 Thomas F. Morris - * @license New BSD http://www.opensource.org/licenses/bsd-license.php - */ public class GDataImporter { + static final Logger logger = LoggerFactory.getLogger("GDataImporter"); + static public void parse( String token, Project project, @@ -67,11 +32,11 @@ public class GDataImporter { final ImportingJob job, int limit, JSONObject options, - List exceptions) { + List exceptions) throws IOException { String docType = JSONUtilities.getString(options, "docType", null); if ("spreadsheet".equals(docType)) { - SpreadsheetService service = GDataExtension.getSpreadsheetService(token); + Sheets service = GoogleAPIExtension.getSheetsService(token); parse( service, project, @@ -94,7 +59,7 @@ public class GDataImporter { } static public void parse( - SpreadsheetService service, + Sheets service, Project project, ProjectMetadata metadata, final ImportingJob job, @@ -104,6 +69,10 @@ public class GDataImporter { String docUrlString = JSONUtilities.getString(options, "docUrl", null); String worksheetUrlString = JSONUtilities.getString(options, "sheetUrl", null); + + // the index of the worksheet + int worksheetIndex = JSONUtilities.getInt(options, "worksheetIndex", 0); + if (docUrlString != null && worksheetUrlString != null) { try { parseOneWorkSheet( @@ -112,7 +81,7 @@ public class GDataImporter { metadata, job, new URL(docUrlString), - new URL(worksheetUrlString), + worksheetIndex, limit, options, exceptions); @@ -124,35 +93,35 @@ public class GDataImporter { } static public void parseOneWorkSheet( - SpreadsheetService service, + Sheets service, Project project, ProjectMetadata metadata, final ImportingJob job, URL docURL, - URL worksheetURL, + int worksheetIndex, int limit, JSONObject options, List exceptions) { try { - WorksheetEntry worksheetEntry = service.getEntry(worksheetURL, WorksheetEntry.class); + String spreadsheetId = GoogleAPIExtension.extractSpreadSheetId(docURL.toString()); + + Spreadsheet response = service.spreadsheets().get(spreadsheetId) + .setIncludeGridData(true) + .execute(); + Sheet worksheetEntry = response.getSheets().get(worksheetIndex); + String spreadsheetName = docURL.toExternalForm(); - try { - SpreadsheetEntry spreadsheetEntry = service.getEntry(docURL, SpreadsheetEntry.class); - spreadsheetName = spreadsheetEntry.getTitle().getPlainText(); - } catch (ServiceException e) { // RedirectRequiredException among others - // fall back to just using the URL (better for traceability anyway?) - } String fileSource = spreadsheetName + " # " + - worksheetEntry.getTitle().getPlainText(); + worksheetEntry.getProperties().getTitle(); setProgress(job, fileSource, 0); TabularImportingParserBase.readTable( project, metadata, job, - new WorksheetBatchRowReader(job, fileSource, service, worksheetEntry, 20), + new WorksheetBatchRowReader(job, fileSource, service, spreadsheetId, worksheetEntry), fileSource, limit, options, @@ -160,10 +129,7 @@ public class GDataImporter { ); setProgress(job, fileSource, 100); } catch (IOException e) { - e.printStackTrace(); - exceptions.add(e); - } catch (ServiceException e) { - e.printStackTrace(); + logger.error(ExceptionUtils.getStackTrace(e)); exceptions.add(e); } } @@ -176,98 +142,61 @@ public class GDataImporter { final ImportingJob job; final String fileSource; - final SpreadsheetService service; - final WorksheetEntry worksheet; - final int batchSize; + final Sheets service; + final String spreadsheetId; + final Sheet worksheet; - final int totalRows; - - int nextRow = 0; // 0-based - int batchRowStart = 0; // 0-based - List> rowsOfCells = null; + private int indexRow = 0; + private List> rowsOfCells = null; public WorksheetBatchRowReader(ImportingJob job, String fileSource, - SpreadsheetService service, WorksheetEntry worksheet, - int batchSize) { + Sheets service, String spreadsheetId, Sheet worksheet) { this.job = job; this.fileSource = fileSource; this.service = service; + this.spreadsheetId = spreadsheetId; this.worksheet = worksheet; - this.batchSize = batchSize; - - this.totalRows = worksheet.getRowCount(); } @Override public List getNextRowOfCells() throws IOException { - if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && nextRow < totalRows)) { - int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size()); - try { - rowsOfCells = getRowsOfCells( - service, - worksheet, - newBatchRowStart + 1, // convert to 1-based - batchSize); - - batchRowStart = newBatchRowStart; - - setProgress(job, fileSource, batchRowStart * 100 / totalRows); - } catch (ServiceException e) { - throw new IOException(e); - } + if (rowsOfCells == null) { + rowsOfCells = getRowsOfCells( + service, + spreadsheetId, + worksheet); } - if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) { - return rowsOfCells.get(nextRow++ - batchRowStart); + if (rowsOfCells == null) { + return null; + } + + if (rowsOfCells.size() > 0) { + setProgress(job, fileSource, 100 * indexRow / rowsOfCells.size()); + } else { + setProgress(job, fileSource, 100); + } + + if (indexRow < rowsOfCells.size()) { + return rowsOfCells.get(indexRow++); } else { return null; } } - List> getRowsOfCells( - SpreadsheetService service, - WorksheetEntry worksheet, - int startRow, // 1-based - int rowCount - ) throws IOException, ServiceException { - URL cellFeedUrl = worksheet.getCellFeedUrl(); + Sheets service, + String spreadsheetId, + Sheet worksheet + ) throws IOException { + String range = worksheet.getProperties().getTitle(); + ValueRange result = service.spreadsheets().values().get(spreadsheetId, range).execute(); - int minRow = startRow; - int maxRow = Math.min(worksheet.getRowCount(), startRow + rowCount - 1); - int cols = worksheet.getColCount(); - int rows = worksheet.getRowCount(); + rowsOfCells = result.getValues(); - CellQuery cellQuery = new CellQuery(cellFeedUrl); - cellQuery.setMinimumRow(minRow); - cellQuery.setMaximumRow(maxRow); - cellQuery.setMaximumCol(cols); - cellQuery.setMaxResults(rows * cols); - cellQuery.setReturnEmpty(false); - - CellFeed cellFeed = service.query(cellQuery, CellFeed.class); - List cellEntries = cellFeed.getEntries(); - - List> rowsOfCells = new ArrayList>(rowCount); - for (CellEntry cellEntry : cellEntries) { - Cell cell = cellEntry.getCell(); - if (cell != null) { - int row = cell.getRow() - startRow; - int col = cell.getCol() - 1; - - while (row >= rowsOfCells.size()) { - rowsOfCells.add(new ArrayList()); - } - List rowOfCells = rowsOfCells.get(row); - - while (col >= rowOfCells.size()) { - rowOfCells.add(null); - } - rowOfCells.set(col, cell.getValue()); - } - } return rowsOfCells; } + } } diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java index 3e9e47da1..8813977fa 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java @@ -1,41 +1,7 @@ -/* - -Copyright 2011, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - package com.google.refine.extension.gdata; import java.io.IOException; import java.io.Writer; -import java.net.URL; import java.util.LinkedList; import java.util.List; import java.util.Properties; @@ -44,25 +10,24 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.api.services.drive.Drive; +import com.google.api.services.drive.model.File; +import com.google.api.services.drive.model.FileList; +import com.google.api.services.drive.model.User; import com.google.api.services.fusiontables.Fusiontables; import com.google.api.services.fusiontables.model.Table; import com.google.api.services.fusiontables.model.TableList; -import com.google.gdata.client.docs.DocsService; -import com.google.gdata.client.spreadsheet.FeedURLFactory; -import com.google.gdata.client.spreadsheet.SpreadsheetService; -import com.google.gdata.data.DateTime; -import com.google.gdata.data.Person; -import com.google.gdata.data.spreadsheet.SpreadsheetEntry; -import com.google.gdata.data.spreadsheet.SpreadsheetFeed; -import com.google.gdata.data.spreadsheet.WorksheetEntry; -import com.google.gdata.data.spreadsheet.WorksheetFeed; -import com.google.gdata.util.AuthenticationException; -import com.google.gdata.util.ServiceException; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.model.Sheet; +import com.google.api.services.sheets.v4.model.Spreadsheet; import com.google.refine.ProjectManager; import com.google.refine.RefineServlet; @@ -77,7 +42,7 @@ import com.google.refine.util.JSONUtilities; import com.google.refine.util.ParsingUtilities; public class GDataImportingController implements ImportingController { - + private static final Logger logger = LoggerFactory.getLogger("GDataImportingController"); protected RefineServlet servlet; @Override @@ -128,13 +93,11 @@ public class GDataImportingController implements ImportingController { writer.array(); try { - listSpreadsheets(GDataExtension.getDocsService(token), writer); + listSpreadsheets(GoogleAPIExtension.getDriveService(token), writer); listFusionTables(FusionTableHandler.getFusionTablesService(token), writer); - } catch (AuthenticationException e) { - TokenCookie.deleteToken(request, response); - } catch (ServiceException e) { - e.printStackTrace(); - } finally { + } catch (Exception e) { + logger.error("doListDocuments exception:" + ExceptionUtils.getStackTrace(e)); + } finally { writer.endArray(); writer.endObject(); } @@ -146,26 +109,30 @@ public class GDataImportingController implements ImportingController { } } - private void listSpreadsheets(DocsService service, JSONWriter writer) - throws IOException, ServiceException, JSONException { - URL metafeedUrl = new URL("https://spreadsheets.google.com/feeds/spreadsheets/private/full"); - SpreadsheetFeed feed = service.getFeed(metafeedUrl, SpreadsheetFeed.class); - for (SpreadsheetEntry entry : feed.getEntries()) { + private void listSpreadsheets(Drive drive, JSONWriter writer) + throws IOException, JSONException { + com.google.api.services.drive.Drive.Files.List files = drive.files().list(); + files.setQ("mimeType = 'application/vnd.google-apps.spreadsheet'"); + files.setFields("nextPageToken, files(id, name, webViewLink, owners, modifiedTime)"); + FileList fileList = files.execute(); + + for (File entry : fileList.getFiles()) { writer.object(); writer.key("docId"); writer.value(entry.getId()); - writer.key("docLink"); writer.value(entry.getHtmlLink().getHref()); - writer.key("docSelfLink"); writer.value(entry.getSelfLink().getHref()); - writer.key("title"); writer.value(entry.getTitle().getPlainText()); + writer.key("docLink"); writer.value(entry.getWebViewLink()); + writer.key("docSelfLink"); writer.value(entry.getWebViewLink()); + writer.key("title"); writer.value(entry.getName()); + writer.key("type"); writer.value("spreadsheet"); - DateTime updated = entry.getUpdated(); + com.google.api.client.util.DateTime updated = entry.getModifiedTime(); if (updated != null) { - writer.key("updated"); writer.value(updated.toStringRfc822()); + writer.key("updated"); writer.value(updated.toString()); } writer.key("authors"); writer.array(); - for (Person person : entry.getAuthors()) { - writer.value(person.getName()); + for (User user : entry.getOwners()) { + writer.value(user.getDisplayName()); } writer.endArray(); @@ -174,10 +141,14 @@ public class GDataImportingController implements ImportingController { } private void listFusionTables(Fusiontables service, JSONWriter writer) - throws IOException, ServiceException, JSONException { + throws IOException, JSONException { Fusiontables.Table.List listTables = service.table().list(); TableList tablelist = listTables.execute(); + + if (tablelist == null || tablelist.getItems() == null) + return; + for (Table table : tablelist.getItems()) { String id = table.getTableId(); String name = table.getName(); @@ -197,95 +168,56 @@ public class GDataImportingController implements ImportingController { private void doInitializeParserUI( HttpServletRequest request, HttpServletResponse response, Properties parameters) throws ServletException, IOException { - String token = TokenCookie.getToken(request); - String type = parameters.getProperty("docType"); String urlString = parameters.getProperty("docUrl"); + JSONObject result = new JSONObject(); + JSONObject options = new JSONObject(); - URL url = new URL(urlString); - try { - JSONObject result = new JSONObject(); - JSONObject options = new JSONObject(); - JSONUtilities.safePut(result, "status", "ok"); - JSONUtilities.safePut(result, "options", options); + JSONUtilities.safePut(result, "status", "ok"); + JSONUtilities.safePut(result, "options", options); + JSONUtilities.safePut(options, "skipDataLines", 0); // number of initial data lines to skip + JSONUtilities.safePut(options, "storeBlankRows", true); + JSONUtilities.safePut(options, "storeBlankCellsAsNulls", true); + + if ("spreadsheet".equals(type)) { + JSONArray worksheets = new JSONArray(); + // extract spreadSheetId from URL + String spreadSheetId = GoogleAPIExtension.extractSpreadSheetId(urlString); - JSONUtilities.safePut(options, "skipDataLines", 0); // number of initial data lines to skip - JSONUtilities.safePut(options, "storeBlankRows", true); - JSONUtilities.safePut(options, "storeBlankCellsAsNulls", true); + JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore + JSONUtilities.safePut(options, "headerLines", 1); // number of header lines + JSONUtilities.safePut(options, "worksheets", worksheets); - if ("spreadsheet".equals(type)) { - JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore - JSONUtilities.safePut(options, "headerLines", 1); // number of header lines + List worksheetEntries = + getWorksheetEntriesForDoc(token, spreadSheetId); + for (Sheet sheet : worksheetEntries) { + JSONObject worksheetO = new JSONObject(); + JSONUtilities.safePut(worksheetO, "name", sheet.getProperties().getTitle()); + JSONUtilities.safePut(worksheetO, "rows", sheet.getProperties().getGridProperties().getRowCount()); + JSONUtilities.safePut(worksheetO, "link", + "https://sheets.googleapis.com/v4/spreadsheets/" + spreadSheetId + "/values/" + sheet.getProperties().getTitle()); - JSONArray worksheets = new JSONArray(); - JSONUtilities.safePut(options, "worksheets", worksheets); - - List worksheetEntries = - reallyTryToGetWorksheetEntriesForDoc(url, token); - for (WorksheetEntry worksheetEntry : worksheetEntries) { - JSONObject worksheetO = new JSONObject(); - JSONUtilities.safePut(worksheetO, "name", worksheetEntry.getTitle().getPlainText()); - JSONUtilities.safePut(worksheetO, "rows", worksheetEntry.getRowCount()); - JSONUtilities.safePut(worksheetO, "link", worksheetEntry.getSelfLink().getHref()); - - JSONUtilities.append(worksheets, worksheetO); - } - } else if ("table".equals(type)) { - // No metadata for a fusion table. + JSONUtilities.append(worksheets, worksheetO); } - /* TODO: else */ + } else if ("table".equals(type)) { + // No metadata for a fusion table. + } - HttpUtilities.respond(response, result.toString()); - } catch (ServiceException e) { - e.printStackTrace(); - HttpUtilities.respond(response, "error", "Internal error: " + e.getLocalizedMessage()); - } + HttpUtilities.respond(response, result.toString()); } - - private List reallyTryToGetWorksheetEntriesForDoc(URL docUrl, String token) throws IOException, ServiceException { - try { - return getWorksheetEntriesForDoc(docUrl, token); - } catch (ServiceException e) { - /* - * TODO: figure out if we can rewire the URL somehow. This code below - * doesn't work but maybe we need to try something similar to it. - * - String urlString = docUrl.toString(); - if (urlString.startsWith("https://docs.google.com/spreadsheet/ccc?key=") || - urlString.startsWith("http://docs.google.com/spreadsheet/ccc?key=")) { - - String urlString2 = "https://spreadsheets.google.com/spreadsheet/ccc?key=" + - urlString.substring(urlString.indexOf("?key=") + 5); - - return getWorksheetEntriesForDoc(new URL(urlString2), token); - } - */ - throw e; - } - } - - private List getWorksheetEntriesForDoc(URL docUrl, String token) throws IOException, ServiceException { - if (token != null) { - try { - SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token); - SpreadsheetEntry spreadsheetEntry = spreadsheetService.getEntry(docUrl, SpreadsheetEntry.class); - return spreadsheetEntry.getWorksheets(); - } catch (ServiceException e) { - // Ignore and fall through, pretending that we're not logged in. - } - } - return getWorksheetEntriesForDoc(docUrl); - } - - private List getWorksheetEntriesForDoc(URL docUrl) throws IOException, ServiceException { - SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(null); - String visibility = "public"; - FeedURLFactory factory = FeedURLFactory.getDefault(); - String key = GDataExtension.getSpreadsheetID(docUrl); - docUrl = factory.getWorksheetFeedUrl(key, visibility, "values"); - WorksheetFeed feed = spreadsheetService.getFeed(docUrl, WorksheetFeed.class); - return feed.getEntries(); + + private List getWorksheetEntriesForDoc(String token, String spreadsheetId) throws IOException { + Sheets sheetsService = GoogleAPIExtension.getSheetsService(token); + + boolean includeGridData = true; + + Sheets.Spreadsheets.Get request = sheetsService.spreadsheets().get(spreadsheetId); + request.setIncludeGridData(includeGridData); + + Spreadsheet response = request.execute(); + + return response.getSheets(); } private void doParsePreview( @@ -381,15 +313,19 @@ public class GDataImportingController implements ImportingController { pm.setName(JSONUtilities.getString(optionObj, "projectName", "Untitled")); pm.setEncoding(JSONUtilities.getString(optionObj, "encoding", "UTF-8")); - GDataImporter.parse( - token, - project, - pm, - job, - -1, - optionObj, - exceptions - ); + try { + GDataImporter.parse( + token, + project, + pm, + job, + -1, + optionObj, + exceptions + ); + } catch (IOException e) { + logger.error(ExceptionUtils.getStackTrace(e)); + } if (!job.canceled) { if (exceptions.size() > 0) { diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java b/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java new file mode 100644 index 000000000..7b6c0b059 --- /dev/null +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java @@ -0,0 +1,205 @@ +package com.google.refine.extension.gdata; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Arrays; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.servlet.http.HttpServletRequest; + +import com.google.api.client.auth.oauth2.AuthorizationCodeResponseUrl; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeRequestUrl; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeTokenRequest; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.services.drive.Drive; +import com.google.api.services.drive.DriveScopes; +import com.google.api.services.fusiontables.FusiontablesScopes; +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.SheetsScopes; + +import com.google.refine.util.ParsingUtilities; + +import edu.mit.simile.butterfly.ButterflyModule; + +abstract public class GoogleAPIExtension { + protected static final String SERVICE_APP_NAME = "OpenRefine-Google-Service"; + private static final String CLIENT_ID = "455686949425-d237cmorii0ge8if7it5r1qijce6caf0.apps.googleusercontent.com"; + private static final String CLIENT_SECRET = "wm5qVtjp3VDfuAx2P2qm6GJb"; + + /** Global instance of the HTTP transport. */ + protected static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); + + /** Global instance of the JSON factory. */ + protected static final JsonFactory JSON_FACTORY = new JacksonFactory(); + + private static final String[] SCOPES = {DriveScopes.DRIVE, SheetsScopes.SPREADSHEETS, FusiontablesScopes.FUSIONTABLES}; + + static public String getAuthorizationUrl(ButterflyModule module, HttpServletRequest request) + throws MalformedURLException { + String authorizedUrl = makeRedirectUrl(module, request); + + + GoogleAuthorizationCodeRequestUrl url = new GoogleAuthorizationCodeRequestUrl( + CLIENT_ID, + authorizedUrl, // execution continues at authorized on redirect + Arrays.asList(SCOPES)); + + return url.toString(); + + } + + private static String makeRedirectUrl(ButterflyModule module, HttpServletRequest request) + throws MalformedURLException { + StringBuffer sb = new StringBuffer(module.getMountPoint().getMountPoint()); + sb.append("authorized?winname="); + sb.append(ParsingUtilities.encode(request.getParameter("winname"))); + sb.append("&cb="); + sb.append(ParsingUtilities.encode(request.getParameter("cb"))); + + URL thisUrl = new URL(request.getRequestURL().toString()); + URL authorizedUrl = new URL(thisUrl, sb.toString()); + + return authorizedUrl.toExternalForm(); + } + + static public String getTokenFromCode(ButterflyModule module, HttpServletRequest request) + throws IOException { + String redirectUrl = makeRedirectUrl(module, request); + StringBuffer fullUrlBuf = request.getRequestURL(); + if (request.getQueryString() != null) { + fullUrlBuf.append('?').append(request.getQueryString()); + } + AuthorizationCodeResponseUrl authResponse = + new AuthorizationCodeResponseUrl(fullUrlBuf.toString()); + // check for user-denied error + if (authResponse.getError() != null) { + // authorization denied... + } else { + // request access token using authResponse.getCode()... + String code = authResponse.getCode(); + GoogleTokenResponse response = new GoogleAuthorizationCodeTokenRequest(HTTP_TRANSPORT, + JSON_FACTORY, CLIENT_ID, CLIENT_SECRET, code, redirectUrl).execute(); + String tokenAndExpiresInSeconds = response.getAccessToken() + "," + response.getExpiresInSeconds(); + return tokenAndExpiresInSeconds; + } + return null; + } + + static public Drive getDriveService(String token) { + GoogleCredential credential = new GoogleCredential().setAccessToken(token); + + return new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).setHttpRequestInitializer(new HttpRequestInitializer() { + @Override + public void initialize(HttpRequest httpRequest) throws IOException { + credential.initialize(httpRequest); + httpRequest.setConnectTimeout(3 * 60000); // 3 minutes connect timeout + httpRequest.setReadTimeout(3 * 60000); // 3 minutes read timeout + } + }) + .setApplicationName(SERVICE_APP_NAME).build(); + } + + static boolean isSpreadsheetURL(String url) { + try { + return url.contains("spreadsheet") && getSpreadsheetID(new URL(url)) != null; + } catch (MalformedURLException e) { + return false; + } + } + + static String getSpreadsheetID(URL url) { + return getParamValue(url,"key"); + } + + static private String getParamValue(URL url, String key) { + String query = url.getQuery(); + if (query != null) { + String[] parts = query.split("&"); + for (String part : parts) { + if (part.startsWith(key+"=")) { + int offset = key.length()+1; + String tableId = part.substring(offset); + return tableId; + } + } + } + return null; + } + + /** + * Build and return an authorized Sheets API client service. + * @return an authorized Sheets API client service + * @throws IOException + */ + public static Sheets getSheetsService(String token) throws IOException { + return new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, + new GoogleCredential().setAccessToken(token)) + .setApplicationName(SERVICE_APP_NAME) + .build(); + } + + public static String extractSpreadSheetId(String url) + throws IllegalArgumentException { + URL urlAsUrl; + + Matcher matcher = Pattern.compile("(?<=\\/d\\/).*(?=\\/.*)").matcher(url); + if (matcher.find()) { + return matcher.group(0); + } + + try { + urlAsUrl = new URL(url); + String query = urlAsUrl.getQuery(); + if ( query != null ) { + + String[] parts = query.split("&"); + + int offset = -1; + int numParts = 0; + String keyOrId = ""; + + for (String part : parts) { + if (part.startsWith("id=")) { + offset = ("id=").length(); + keyOrId = part.substring(offset); + numParts = 4; + break; + } else if (part.startsWith("key=")) { + offset = ("key=").length(); + keyOrId = part.substring(offset); + if (!keyOrId.isEmpty()) { + return keyOrId; + } + numParts = 2; + break; + } + } + + if (offset > -1) { + String[] dottedParts = keyOrId.split("\\."); + if (dottedParts.length == numParts) { + return dottedParts[0] + "." + dottedParts[1]; + } + } + } + } catch ( MalformedURLException e ) { + // This is not a URL, maybe it is just an id + String[] dottedParts = url.split("\\."); + + if (dottedParts.length == 4 || dottedParts.length == 2) { + return dottedParts[0] + "." + dottedParts[1]; + } + } + + throw new IllegalArgumentException("Uknown URL format."); + } +} diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/SpreadsheetSerializer.java b/extensions/gdata/src/com/google/refine/extension/gdata/SpreadsheetSerializer.java new file mode 100644 index 000000000..145a92a2d --- /dev/null +++ b/extensions/gdata/src/com/google/refine/extension/gdata/SpreadsheetSerializer.java @@ -0,0 +1,124 @@ +package com.google.refine.extension.gdata; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.api.services.sheets.v4.Sheets; +import com.google.api.services.sheets.v4.model.AppendCellsRequest; +import com.google.api.services.sheets.v4.model.BatchUpdateSpreadsheetRequest; +import com.google.api.services.sheets.v4.model.BatchUpdateSpreadsheetResponse; +import com.google.api.services.sheets.v4.model.ExtendedValue; +import com.google.api.services.sheets.v4.model.Request; +import com.google.api.services.sheets.v4.model.RowData; + +import com.google.refine.exporters.TabularSerializer; + +final class SpreadsheetSerializer implements TabularSerializer { + static final Logger logger = LoggerFactory.getLogger("SpreadsheetSerializer"); + + private static final int BATCH_SIZE = 1000; + + private Sheets service; + private String spreadsheetId; + private List exceptions; + + // A list of updates to apply to the spreadsheet. + private List requests = new ArrayList<>(); + + private Request batchRequest = null; + private int row = 0; + + private List rows; + + SpreadsheetSerializer(Sheets service, String spreadsheetId, List exceptions) { + this.service = service; + this.spreadsheetId = spreadsheetId; + this.exceptions = exceptions; + } + + @Override + public void startFile(JSONObject options) { + } + + + @Override + public void endFile() { + if (batchRequest != null) { + sendBatch(rows); + } + + BatchUpdateSpreadsheetRequest requestBody = new BatchUpdateSpreadsheetRequest(); + requestBody.setIncludeSpreadsheetInResponse(false); + requestBody.setRequests(requests); + + Sheets.Spreadsheets.BatchUpdate request; + try { + logger.debug("spreadsheetId: " + spreadsheetId); + logger.debug("requestBody:" + requestBody.toString()); + request = service.spreadsheets().batchUpdate(spreadsheetId, requestBody); + + BatchUpdateSpreadsheetResponse response = request.execute(); + logger.debug("response:" + response.toPrettyString()); + } catch (IOException e) { + exceptions.add(e); + } + } + + @Override + public void addRow(List cells, boolean isHeader) { + if (batchRequest == null) { + batchRequest = new Request(); + rows = new ArrayList(BATCH_SIZE); + } + List cellDatas = new ArrayList<>(); + RowData rowData = new RowData(); + + for (int c = 0; c < cells.size(); c++) { + CellData cellData = cells.get(c); + if (cellData != null && cellData.text != null) { + cellDatas.add(cellData2sheetCellData(cellData)); + } + } + + rowData.setValues(cellDatas); + rows.add(rowData); + row++; + + if (row % BATCH_SIZE == 0) { + sendBatch(rows); + } + } + + private com.google.api.services.sheets.v4.model.CellData cellData2sheetCellData(CellData cellData) { + com.google.api.services.sheets.v4.model.CellData sheetCellData = new com.google.api.services.sheets.v4.model.CellData(); + + ExtendedValue ev = new ExtendedValue(); + ev.setStringValue(cellData.value.toString()); + + sheetCellData.setUserEnteredValue(ev); + + return sheetCellData; + } + + private void sendBatch(List rows) { + AppendCellsRequest acr = new AppendCellsRequest(); + acr.setFields("*"); + acr.setSheetId(0); + acr.setRows(rows); + batchRequest.setAppendCells(acr); + + requests.add(batchRequest); + } + + public String getUrl() throws UnsupportedEncodingException { + String urlString= "https://docs.google.com/spreadsheets/d/" + spreadsheetId + "/edit#gid=0"; + return URLEncoder.encode(urlString, "UTF-8"); + } +} diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/TokenCookie.java b/extensions/gdata/src/com/google/refine/extension/gdata/TokenCookie.java index 17cd18de7..7233b5d85 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/TokenCookie.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/TokenCookie.java @@ -7,9 +7,7 @@ import javax.servlet.http.HttpServletResponse; import com.google.refine.util.CookiesUtilities; public class TokenCookie { - private static final String COOKIE_NAME = "oauth2_token"; - private static int MAX_AGE = 30 * 24 * 60 * 60; // 30 days public static String getToken(HttpServletRequest request) { Cookie cookie = CookiesUtilities.getCookie(request, COOKIE_NAME); @@ -17,8 +15,9 @@ public class TokenCookie { } public static void setToken(HttpServletRequest request, - HttpServletResponse response, String token) { - CookiesUtilities.setCookie(request, response, COOKIE_NAME, token, MAX_AGE); + HttpServletResponse response, String token, String expiresInSeconds) { + CookiesUtilities.setCookie(request, response, COOKIE_NAME, token, + Integer.parseInt(expiresInSeconds)); } public static void deleteToken(HttpServletRequest request, diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java b/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java index e81def7ea..8a32541cc 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java @@ -1,37 +1,10 @@ -/* - * Copyright (c) 2010,2011,2015 Thomas F. Morris - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - package com.google.refine.extension.gdata; +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.Writer; -import java.net.MalformedURLException; import java.net.URL; import java.util.LinkedList; import java.util.List; @@ -41,27 +14,17 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.json.JSONObject; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.codec.binary.Base64; +import com.google.api.client.http.FileContent; import com.google.api.services.drive.Drive; import com.google.api.services.drive.model.File; -import com.google.gdata.client.spreadsheet.CellQuery; -import com.google.gdata.client.spreadsheet.SpreadsheetService; -import com.google.gdata.data.Link; -import com.google.gdata.data.PlainTextConstruct; -import com.google.gdata.data.batch.BatchOperationType; -import com.google.gdata.data.batch.BatchStatus; -import com.google.gdata.data.batch.BatchUtils; -import com.google.gdata.data.spreadsheet.Cell; -import com.google.gdata.data.spreadsheet.CellEntry; -import com.google.gdata.data.spreadsheet.CellFeed; -import com.google.gdata.data.spreadsheet.SpreadsheetEntry; -import com.google.gdata.data.spreadsheet.SpreadsheetFeed; -import com.google.gdata.data.spreadsheet.WorksheetEntry; -import com.google.gdata.util.ServiceException; +import com.google.api.services.drive.model.File.ContentHints; +import com.google.api.services.drive.model.File.ContentHints.Thumbnail; import com.google.refine.ProjectManager; import com.google.refine.browsing.Engine; @@ -69,13 +32,14 @@ import com.google.refine.commands.Command; import com.google.refine.commands.HttpUtilities; import com.google.refine.commands.project.ExportRowsCommand; import com.google.refine.exporters.CustomizableTabularExporterUtilities; -import com.google.refine.exporters.TabularSerializer; +import com.google.refine.io.FileProjectManager; import com.google.refine.model.Project; public class UploadCommand extends Command { static final Logger logger = LoggerFactory.getLogger("gdata_upload"); - private static final String SPREADSHEET_FEED = "https://spreadsheets.google.com/feeds/spreadsheets/private/full"; + private static final String METADATA_DESCRIPTION = "OpenRefine project dump"; + private static final String METADATA_ICONLINK = "https://raw.githubusercontent.com/OpenRefine/OpenRefine/master/main/webapp/modules/core/images/logo-openrefine-550.png"; @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { @@ -131,7 +95,7 @@ public class UploadCommand extends Command { ProjectManager.singleton.setBusy(false); } } - + static private String upload( Project project, Engine engine, Properties params, String token, String name, List exceptions) { @@ -140,172 +104,96 @@ public class UploadCommand extends Command { return uploadSpreadsheet(project, engine, params, token, name, exceptions); } else if ("gdata/fusion-table".equals(format)) { return uploadFusionTable(project, engine, params, token, name, exceptions); + } else if (("raw/openrefine-project").equals(format)) { + return uploadOpenRefineProject(project, token, name, exceptions); } return null; } + private static byte[] getImageFromUrl(String urlText) throws IOException { + URL url = new URL(urlText); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + try (InputStream inputStream = url.openStream()) { + int n = 0; + byte [] buffer = new byte[ 1024 ]; + while (-1 != (n = inputStream.read(buffer))) { + output.write(buffer, 0, n); + } + } + + return output.toByteArray(); + } + + private static String uploadOpenRefineProject(Project project, String token, + String name, List exceptions) { + FileOutputStream fos = null; + + try { + java.io.File filePath = java.io.File.createTempFile(name, ".tgz"); + filePath.deleteOnExit(); + + fos = new FileOutputStream(filePath); + FileProjectManager.gzipTarToOutputStream(project, fos); + + File fileMetadata = new File(); + String asB64 = Base64.encodeBase64URLSafeString(getImageFromUrl(METADATA_ICONLINK)); + + Thumbnail tn = new Thumbnail(); + tn.setMimeType("image/x-icon").setImage(asB64); + ContentHints contentHints = new ContentHints(); + contentHints.setThumbnail(tn); + + fileMetadata.setName(name) + .setDescription(METADATA_DESCRIPTION) + .setContentHints(contentHints); + FileContent projectContent = new FileContent("application/zip", filePath); + File file = GoogleAPIExtension.getDriveService(token) + .files().create(fileMetadata, projectContent) + .setFields("id") + .execute(); + logger.info("File ID: " + file.getId()); + + return file.getId(); + } catch (IOException e) { + logger.error(ExceptionUtils.getStackTrace(e)); + exceptions.add(e); + } + + return null; + } + static private String uploadSpreadsheet( final Project project, final Engine engine, final Properties params, String token, String name, List exceptions) { - Drive driveService = GDataExtension.getDriveService(token); - final SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token); + Drive driveService = GoogleAPIExtension.getDriveService(token); try { File body = new File(); - body.setTitle(name); + body.setName(name); body.setDescription("Spreadsheet uploaded from OpenRefine project: " + name); body.setMimeType("application/vnd.google-apps.spreadsheet"); - File file = driveService.files().insert(body).execute(); - String fileID = file.getId(); + File file = driveService.files().create(body).execute(); + String spreadsheetId = file.getId(); - // Iterate through all spreadsheets to find one with our ID - SpreadsheetEntry spreadsheetEntry2 = null; - SpreadsheetFeed feed = spreadsheetService.getFeed(new URL(SPREADSHEET_FEED), SpreadsheetFeed.class); - List spreadsheets = feed.getEntries(); - for (com.google.gdata.data.spreadsheet.SpreadsheetEntry spreadsheet : spreadsheets) { - if (spreadsheet.getId().endsWith(fileID)) { - spreadsheetEntry2 = spreadsheet; - } - } + SpreadsheetSerializer serializer = new SpreadsheetSerializer( + GoogleAPIExtension.getSheetsService(token), + spreadsheetId, + exceptions); - // Bail if we didn't find our spreadsheet (shouldn't happen) - if (spreadsheetEntry2 == null) { - logger.error("Failed to find match for ID: " + fileID); - return null; - } - - int[] size = CustomizableTabularExporterUtilities.countColumnsRows( - project, engine, params); + CustomizableTabularExporterUtilities.exportRows( + project, engine, params, serializer); - URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl(); - WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]); - worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data")); - - final WorksheetEntry worksheetEntry2 = - spreadsheetService.insert(worksheetFeedUrl, worksheetEntry); - - spreadsheetEntry2.getDefaultWorksheet().delete(); - - final SpreadsheetEntry spreadsheetEntry3 = spreadsheetEntry2; - new Thread() { - @Override - public void run() { - spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V3); - try { - uploadToCellFeed( - project, engine, params, - spreadsheetService, - spreadsheetEntry3, - worksheetEntry2); - } catch (Exception e) { - logger.error("Error uploading data to Google Spreadsheets", e); - } - } - }.start(); - - return spreadsheetEntry2.getSpreadsheetLink().getHref(); - } catch (IOException | ServiceException e) { + return serializer.getUrl(); + } catch (IOException e) { + logger.error(ExceptionUtils.getStackTrace(e)); exceptions.add(e); } return null; } - static private void uploadToCellFeed( - Project project, - Engine engine, - Properties params, - final SpreadsheetService service, - final SpreadsheetEntry spreadsheetEntry, - final WorksheetEntry worksheetEntry) - throws IOException, ServiceException { - - final URL cellFeedUrl = worksheetEntry.getCellFeedUrl(); - final CellEntry[][] cellEntries = - new CellEntry[worksheetEntry.getRowCount()][worksheetEntry.getColCount()]; - { - CellQuery cellQuery = new CellQuery(cellFeedUrl); - cellQuery.setReturnEmpty(true); - - CellFeed fetchingCellFeed = service.getFeed(cellQuery, CellFeed.class); - for (CellEntry cellEntry : fetchingCellFeed.getEntries()) { - Cell cell = cellEntry.getCell(); - cellEntries[cell.getRow() - 1][cell.getCol() - 1] = cellEntry; - } - } - - TabularSerializer serializer = new TabularSerializer() { - CellFeed cellFeed = service.getFeed(cellFeedUrl, CellFeed.class); - CellFeed batchRequest = null; - int row = 0; - - @Override - public void startFile(JSONObject options) { - } - - @Override - public void endFile() { - if (batchRequest != null) { - sendBatch(); - } - } - - private void sendBatch() { - try { - Link batchLink = cellFeed.getLink(Link.Rel.FEED_BATCH, Link.Type.ATOM); - CellFeed batchResponse = service.batch(new URL(batchLink.getHref()), batchRequest); - - for (CellEntry entry : batchResponse.getEntries()) { - String batchId = BatchUtils.getBatchId(entry); - if (!BatchUtils.isSuccess(entry)) { - BatchStatus status = BatchUtils.getBatchStatus(entry); - logger.warn( - String.format( - "Error: %s failed (%s) %s\n", - batchId, status.getReason(), status.getContent())); - break; - } - } - } catch (Exception e) { - e.printStackTrace(); - } - batchRequest = null; - } - - @Override - public void addRow(List cells, boolean isHeader) { - if (batchRequest == null) { - batchRequest = new CellFeed(); - } - for (int c = 0; c < cells.size(); c++) { - CellData cellData = cells.get(c); - if (cellData != null && cellData.text != null) { - String cellId = String.format("R%sC%s", row + 1, c + 1); - - CellEntry cellEntry = cellEntries[row][c]; - cellEntry.changeInputValueLocal(cellData.text); - if (cellData.link != null) { - cellEntry.addHtmlLink(cellData.link, null, cellData.text); - } - cellEntry.setId(cellId); - BatchUtils.setBatchId(cellEntry, cellId); - BatchUtils.setBatchOperationType(cellEntry, BatchOperationType.UPDATE); - - batchRequest.getEntries().add(cellEntry); - } - } - row++; - if (row % 20 == 0) { - sendBatch(); - } - } - }; - - CustomizableTabularExporterUtilities.exportRows( - project, engine, params, serializer); - } - static private String uploadFusionTable( Project project, final Engine engine, final Properties params, String token, String name, List exceptions) { diff --git a/main/src/com/google/refine/commands/project/ExportProjectCommand.java b/main/src/com/google/refine/commands/project/ExportProjectCommand.java index 203227f1e..3252f6a2a 100644 --- a/main/src/com/google/refine/commands/project/ExportProjectCommand.java +++ b/main/src/com/google/refine/commands/project/ExportProjectCommand.java @@ -35,16 +35,14 @@ package com.google.refine.commands.project; import java.io.IOException; import java.io.OutputStream; -import java.util.zip.GZIPOutputStream; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.tools.tar.TarOutputStream; - import com.google.refine.ProjectManager; import com.google.refine.commands.Command; +import com.google.refine.io.FileProjectManager; import com.google.refine.model.Project; public class ExportProjectCommand extends Command { @@ -61,7 +59,7 @@ public class ExportProjectCommand extends Command { OutputStream os = response.getOutputStream(); try { - gzipTarToOutputStream(project, os); + FileProjectManager.gzipTarToOutputStream(project, os); } finally { os.close(); } @@ -69,24 +67,4 @@ public class ExportProjectCommand extends Command { respondException(response, e); } } - - protected void gzipTarToOutputStream(Project project, OutputStream os) throws IOException { - GZIPOutputStream gos = new GZIPOutputStream(os); - try { - tarToOutputStream(project, gos); - } finally { - gos.close(); - } - } - - protected void tarToOutputStream(Project project, OutputStream os) throws IOException { - TarOutputStream tos = new TarOutputStream(os); - try { - ProjectManager.singleton.exportProject(project.id, tos); - } finally { - tos.close(); - } - } - - } diff --git a/main/src/com/google/refine/io/FileProjectManager.java b/main/src/com/google/refine/io/FileProjectManager.java index eb5c54662..14df3e784 100644 --- a/main/src/com/google/refine/io/FileProjectManager.java +++ b/main/src/com/google/refine/io/FileProjectManager.java @@ -44,6 +44,7 @@ import java.io.OutputStream; import java.util.HashMap; import java.util.Properties; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import org.apache.tools.tar.TarEntry; import org.apache.tools.tar.TarInputStream; @@ -472,4 +473,15 @@ public class FileProjectManager extends ProjectManager { public HistoryEntryManager getHistoryEntryManager(){ return new FileHistoryEntryManager(); } + + public static void gzipTarToOutputStream(Project project, OutputStream os) throws IOException { + GZIPOutputStream gos = new GZIPOutputStream(os); + TarOutputStream tos = new TarOutputStream(gos); + try { + ProjectManager.singleton.exportProject(project.id, tos); + } finally { + tos.close(); + gos.close(); + } + } } \ No newline at end of file diff --git a/main/webapp/modules/core/langs/translation-en.json b/main/webapp/modules/core/langs/translation-en.json index 4e9ea8cd4..5fe0605a6 100644 --- a/main/webapp/modules/core/langs/translation-en.json +++ b/main/webapp/modules/core/langs/translation-en.json @@ -273,7 +273,10 @@ "char-enc": "Character encoding", "line-sep": "Line separator", "upload-to": "Upload to", - "json-text": "The following JSON text encodes the options you have set in the other tabs. You can copy it out and save it for later, and paste it back in and click Apply to re-use the same options." + "json-text": "The following JSON text encodes the options you have set in the other tabs. You can copy it out and save it for later, and paste it back in and click Apply to re-use the same options.", + "choose-export-destination": "Please choose the destination for project export", + "export-to-local": "Export to local", + "export-to-google-drive": "Export to Google Drive" }, "core-facets": { "remove-facet": "Remove this facet", diff --git a/main/webapp/modules/core/scripts/dialogs/export-project-dialog.html b/main/webapp/modules/core/scripts/dialogs/export-project-dialog.html new file mode 100644 index 000000000..9bd2a46c8 --- /dev/null +++ b/main/webapp/modules/core/scripts/dialogs/export-project-dialog.html @@ -0,0 +1,20 @@ +
+
+
+ +
+ + + + + +
+ + + + +
+ + +
+
\ No newline at end of file diff --git a/main/webapp/modules/core/scripts/project/exporters.js b/main/webapp/modules/core/scripts/project/exporters.js index 13eba106c..fb0c664fc 100644 --- a/main/webapp/modules/core/scripts/project/exporters.js +++ b/main/webapp/modules/core/scripts/project/exporters.js @@ -169,23 +169,85 @@ ExporterManager.prepareExportRowsForm = function(format, includeEngine, ext) { ExporterManager.handlers.exportProject = function() { var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-'); - var form = document.createElement("form"); - $(form) - .css("display", "none") - .attr("method", "post") - .attr("action", "command/core/export-project/" + name + ".openrefine.tar.gz") - .attr("target", "refine-export"); - $('') - .attr("name", "project") - .attr("value", theProject.id) - .appendTo(form); + // dialog + var dialog = $(DOM.loadHTML("core", "scripts/dialogs/export-project-dialog.html")); + var _elmts = DOM.bind(dialog); + + _elmts.dialogHeader.html($.i18n._('core-dialogs')["choose-export-destination"]); + _elmts.toLocalRadio.html($.i18n._('core-dialogs')["export-to-local"]); + _elmts.toGoogleDriveRadio.html($.i18n._('core-dialogs')["export-to-google-drive"]); + _elmts.exportButton.html($.i18n._('core-buttons')["export"]); + _elmts.cancelButton.html($.i18n._('core-buttons')["cancel"]); + + _elmts.exportButton.click(function() { + if ($("input[name='export-destination']")[0].checked) { + exportToLocal(name); + } else { + exportToGoogleDrive(name); + } + + DialogSystem.dismissAll(); + }); + + _elmts.cancelButton.click(function() { DialogSystem.dismissAll(); }); + + DialogSystem.showDialog(dialog); + + // save to google drive + var doExportToGoogleDrive = function() { + var name = window.prompt(prompt, theProject.metadata.name); + if (name) { + var dismiss = DialogSystem.showBusy($.i18n._('gdata-exporter')["uploading"]); + $.post( + "command/gdata/upload", + { + "project" : theProject.id, + "name" : name, + "format" : "raw/openrefine-project" + }, + function(o) { + dismiss(); - document.body.appendChild(form); + if (o.url) { + alert($.i18n._('gdata-exporter')["upload-success"] + o.url); + } else { + alert($.i18n._('gdata-exporter')["upload-error"] + o.message) + } + onDone(); + }, + "json" + ); + } + }; - window.open("about:blank", "refine-export"); - form.submit(); - - document.body.removeChild(form); + function exportToGoogleDrive(name) { + if (GdataExtension.isAuthorized()) { + doExportToGoogleDrive(); + } else { + GdataExtension.showAuthorizationDialog(doExportToGoogleDrive); + } + } + + // save to local + function exportToLocal(name) { + var form = document.createElement("form"); + $(form) + .css("display", "none") + .attr("method", "post") + .attr("action", "command/core/export-project/" + name + ".openrefine.tar.gz") + .attr("target", "refine-export"); + $('') + .attr("name", "project") + .attr("value", theProject.id) + .appendTo(form); + + document.body.appendChild(form); + + window.open("about:blank", "refine-export"); + form.submit(); + + document.body.removeChild(form); + } }; ExporterManager.handlers.projectDataPackage = function() {