Merge remote-tracking branch 'origin/master'
This commit is contained in:
Weblate
commit
6911c33fe4
30
.classpath
30
.classpath
@ -38,16 +38,7 @@
|
||||
<classpathentry exported="true" kind="lib" path="server/lib/slf4j-log4j12-1.5.6.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="broker/appengine/WEB-INF/lib/slf4j-jdk14-1.5.6.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="broker/core/module/MOD-INF/lib/bdb-je-4.0.103.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-core-1.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-3.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jsoup-1.4.1.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-docs-3.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-docs-meta-3.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-media-1.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-base-1.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-client-1.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-client-meta-1.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/gdata-spreadsheet-meta-3.0.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="extensions/gdata/module/MOD-INF/lib/mail.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/odfdom-java-0.8.7.jar" sourcepath="main/webapp/WEB-INF/lib-src/odfdom-java-0.8.7-sources.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/resolver.jar"/>
|
||||
@ -63,14 +54,6 @@
|
||||
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/fluent-hc-4.2.5.jar"/>
|
||||
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/httpmime-4.2.5.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/commons-logging-1.1.1.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-1.20.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.20.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v2-rev168-1.20.0.jar" sourcepath="C:/Users/tfmorris/.sourceattacher/google-api-services-drive-v2-rev168-1.20.0-sources.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev3-1.20.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-1.20.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/transaction-api-1.1.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-3.13-20150929.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-3.13-20150929.jar"/>
|
||||
@ -82,7 +65,6 @@
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-2.9.1.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-databind-2.9.1.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/json-20160810.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/commons-beanutils-1.9.3.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/commons-collections-3.2.2.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/commons-digester-1.8.1.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/commons-lang3-3.6.jar"/>
|
||||
@ -119,5 +101,17 @@
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/testng-6.9.10.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/datapackage-java-1.0-SNAPSHOT.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/tableschema-java-1.0-SNAPSHOT.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-fusiontables-v2-rev21-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-client-servlet-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-jackson-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-drive-v3-rev101-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-api-services-sheets-v4-rev502-1.23.0.jar"/>
|
||||
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/commons-beanutils-1.9.3.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-auth-library-oauth2-http-0.9.0.jar"/>
|
||||
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-jetty-1.23.0.jar"/>
|
||||
<classpathentry kind="output" path="main/webapp/WEB-INF/classes"/>
|
||||
</classpath>
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -23,5 +23,6 @@ broker/core/data/
|
||||
broker/core/test-output/
|
||||
tmp/
|
||||
/test-output
|
||||
test-out/
|
||||
/bin
|
||||
open-refine.log
|
||||
|
||||
@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
var html = "text/html";
|
||||
var encoding = "UTF-8";
|
||||
var version = "0.2";
|
||||
var version = "0.3";
|
||||
var ClientSideResourceManager = Packages.com.google.refine.ClientSideResourceManager;
|
||||
|
||||
/*
|
||||
@ -61,10 +61,6 @@ function init() {
|
||||
new Packages.com.google.refine.extension.gdata.GDataImportingController()
|
||||
);
|
||||
|
||||
|
||||
//Packages.com.google.refine.exporters.ExporterRegistry.registerExporter(
|
||||
//"gdata-exporter", new Packages.com.google.refine.extension.gdata.GDataExporter());
|
||||
|
||||
// Script files to inject into /index page
|
||||
ClientSideResourceManager.addPaths(
|
||||
"index/scripts",
|
||||
@ -102,7 +98,7 @@ function process(path, request, response) {
|
||||
// Analyze path and handle this request yourself.
|
||||
if (path == "authorize") {
|
||||
var context = {};
|
||||
context.authorizationUrl = Packages.com.google.refine.extension.gdata.GDataExtension.getAuthorizationUrl(module, request);
|
||||
context.authorizationUrl = Packages.com.google.refine.extension.gdata.GoogleAPIExtension.getAuthorizationUrl(module, request);
|
||||
|
||||
send(request, response, "authorize.vt", context);
|
||||
} else if (path == "authorized") {
|
||||
@ -111,9 +107,10 @@ function process(path, request, response) {
|
||||
context.callback = request.getParameter("cb");
|
||||
|
||||
(function() {
|
||||
var token = Packages.com.google.refine.extension.gdata.GDataExtension.getTokenFromCode(module,request);
|
||||
if (token) {
|
||||
Packages.com.google.refine.extension.gdata.TokenCookie.setToken(request, response, token);
|
||||
var tokenAndExpiresInSeconds = Packages.com.google.refine.extension.gdata.GoogleAPIExtension.getTokenFromCode(module,request);
|
||||
if (tokenAndExpiresInSeconds) {
|
||||
var tokenInfo = tokenAndExpiresInSeconds.split(",");
|
||||
Packages.com.google.refine.extension.gdata.TokenCookie.setToken(request, response, tokenInfo[0], tokenInfo[1]);
|
||||
return;
|
||||
}
|
||||
Packages.com.google.refine.extension.gdata.TokenCookie.deleteToken(request, response);
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
extensions/gdata/module/MOD-INF/lib/google-api-client-1.23.0.jar
Normal file
BIN
extensions/gdata/module/MOD-INF/lib/google-api-client-1.23.0.jar
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -44,6 +44,7 @@
|
||||
"gdata-exporter": {
|
||||
"uploading": "Uploading...",
|
||||
"upload-error": "Upload error: ",
|
||||
"upload-success": "Project was uploaded successfully to Google Drive with Id ",
|
||||
"new-spreadsheet": "A new Google spreadsheet",
|
||||
"enter-spreadsheet": "Enter a name for the new Google spreadsheet",
|
||||
"new-fusion": "A new Google Fusion table",
|
||||
|
||||
@ -83,7 +83,13 @@ Refine.GDataSourceUI.prototype.attachUI = function(body) {
|
||||
} else {
|
||||
doc.type = 'table';
|
||||
}
|
||||
|
||||
if (GdataExtension.isAuthorized()) {
|
||||
self._controller.startImportingDocument(doc);
|
||||
} else {
|
||||
var fn = self._controller.startImportingDocument;
|
||||
GdataExtension.showAuthorizationDialog(fn.bind(self._controller, doc));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@ -121,8 +121,8 @@ public class FusionTableHandler {
|
||||
static public Fusiontables getFusionTablesService(String token) {
|
||||
Credential credential = new GoogleCredential().setAccessToken(token);
|
||||
Fusiontables fusiontables = new Fusiontables.Builder(
|
||||
GDataExtension.HTTP_TRANSPORT, GDataExtension.JSON_FACTORY, credential)
|
||||
.setApplicationName(GDataExtension.SERVICE_APP_NAME)
|
||||
GoogleAPIExtension.HTTP_TRANSPORT, GoogleAPIExtension.JSON_FACTORY, credential)
|
||||
.setApplicationName(GoogleAPIExtension.SERVICE_APP_NAME)
|
||||
.build();;
|
||||
|
||||
return fusiontables;
|
||||
|
||||
@ -1,186 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2010, Thomas F. Morris
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
* - Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* - Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* Neither the name of Google nor the names of its contributors may be used to
|
||||
* endorse or promote products derived from this software without specific
|
||||
* prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
||||
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
package com.google.refine.extension.gdata;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import com.google.api.client.auth.oauth2.AuthorizationCodeResponseUrl;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeRequestUrl;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeTokenRequest;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.http.javanet.NetHttpTransport;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.client.json.jackson2.JacksonFactory;
|
||||
import com.google.api.services.drive.Drive;
|
||||
import com.google.gdata.client.docs.DocsService;
|
||||
import com.google.gdata.client.spreadsheet.SpreadsheetService;
|
||||
|
||||
import com.google.refine.util.ParsingUtilities;
|
||||
|
||||
import edu.mit.simile.butterfly.ButterflyModule;
|
||||
|
||||
/**
|
||||
* @author Tom Morris <tfmorris@gmail.com>
|
||||
* @copyright 2010 Thomas F. Morris
|
||||
* @license New BSD http://www.opensource.org/licenses/bsd-license.php
|
||||
*/
|
||||
abstract public class GDataExtension {
|
||||
static final String SERVICE_APP_NAME = "OpenRefine-GData-Extension";
|
||||
static final String CLIENT_ID = "647865400439.apps.googleusercontent.com";
|
||||
static final String CLIENT_SECRET = "0mW9OJji1yrgJk5AjJc5Pn6I"; // not really that secret, but the protocol accounts for that
|
||||
|
||||
/** Global instance of the HTTP transport. */
|
||||
static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
|
||||
|
||||
/** Global instance of the JSON factory. */
|
||||
static final JsonFactory JSON_FACTORY = new JacksonFactory();
|
||||
|
||||
static public String getAuthorizationUrl(ButterflyModule module, HttpServletRequest request)
|
||||
throws MalformedURLException {
|
||||
String authorizedUrl = makeRedirectUrl(module, request);
|
||||
|
||||
// New Oauth2
|
||||
GoogleAuthorizationCodeRequestUrl url = new GoogleAuthorizationCodeRequestUrl(
|
||||
CLIENT_ID,
|
||||
authorizedUrl, // execution continues at authorized on redirect
|
||||
Arrays.asList("https://www.googleapis.com/auth/fusiontables",
|
||||
"https://www.googleapis.com/auth/drive", // create new spreadsheets
|
||||
"https://spreadsheets.google.com/feeds"));
|
||||
|
||||
return url.toString();
|
||||
|
||||
}
|
||||
|
||||
private static String makeRedirectUrl(ButterflyModule module, HttpServletRequest request)
|
||||
throws MalformedURLException {
|
||||
StringBuffer sb = new StringBuffer(module.getMountPoint().getMountPoint());
|
||||
sb.append("authorized?winname=");
|
||||
sb.append(ParsingUtilities.encode(request.getParameter("winname")));
|
||||
sb.append("&cb=");
|
||||
sb.append(ParsingUtilities.encode(request.getParameter("cb")));
|
||||
|
||||
URL thisUrl = new URL(request.getRequestURL().toString());
|
||||
URL authorizedUrl = new URL(thisUrl, sb.toString());
|
||||
return authorizedUrl.toExternalForm();
|
||||
}
|
||||
|
||||
static public String getTokenFromCode(ButterflyModule module, HttpServletRequest request)
|
||||
throws MalformedURLException {
|
||||
String redirectUrl = makeRedirectUrl(module, request);
|
||||
StringBuffer fullUrlBuf = request.getRequestURL();
|
||||
if (request.getQueryString() != null) {
|
||||
fullUrlBuf.append('?').append(request.getQueryString());
|
||||
}
|
||||
AuthorizationCodeResponseUrl authResponse =
|
||||
new AuthorizationCodeResponseUrl(fullUrlBuf.toString());
|
||||
// check for user-denied error
|
||||
if (authResponse.getError() != null) {
|
||||
// authorization denied...
|
||||
} else {
|
||||
// request access token using authResponse.getCode()...
|
||||
String code = authResponse.getCode();
|
||||
try {
|
||||
GoogleTokenResponse response = new GoogleAuthorizationCodeTokenRequest(HTTP_TRANSPORT,
|
||||
JSON_FACTORY, CLIENT_ID, CLIENT_SECRET, code, redirectUrl).execute();
|
||||
String token = response.getAccessToken();
|
||||
return token;
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
static public DocsService getDocsService(String token) {
|
||||
DocsService service = new DocsService(SERVICE_APP_NAME);
|
||||
if (token != null) {
|
||||
service.setAuthSubToken(token);
|
||||
}
|
||||
return service;
|
||||
}
|
||||
|
||||
static public SpreadsheetService getSpreadsheetService(String token) {
|
||||
SpreadsheetService service = new SpreadsheetService(SERVICE_APP_NAME);
|
||||
if (token != null) {
|
||||
service.setAuthSubToken(token);
|
||||
}
|
||||
return service;
|
||||
}
|
||||
|
||||
static public Drive getDriveService(String token) {
|
||||
GoogleCredential credential = new GoogleCredential().setAccessToken(token);
|
||||
return new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential)
|
||||
.setApplicationName(SERVICE_APP_NAME).build();
|
||||
}
|
||||
|
||||
static boolean isSpreadsheetURL(String url) {
|
||||
// e.g. http://spreadsheets.google.com/ccc?key=tI36b9Fxk1lFBS83iR_3XQA&hl=en
|
||||
// TODO: The following should work, but the GData implementation is too limited
|
||||
// try {
|
||||
// FeedURLFactory.getSpreadsheetKeyFromUrl(url);
|
||||
// return true;
|
||||
// } catch (IllegalArgumentException e) {
|
||||
// return false;
|
||||
// }
|
||||
try {
|
||||
return url.contains("spreadsheet") && getSpreadsheetID(new URL(url)) != null;
|
||||
} catch (MalformedURLException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static String getSpreadsheetID(URL url) {
|
||||
return getParamValue(url,"key");
|
||||
}
|
||||
|
||||
static private String getParamValue(URL url, String key) {
|
||||
String query = url.getQuery();
|
||||
if (query != null) {
|
||||
String[] parts = query.split("&");
|
||||
for (String part : parts) {
|
||||
if (part.startsWith(key+"=")) {
|
||||
int offset = key.length()+1;
|
||||
String tableId = part.substring(offset);
|
||||
return tableId;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,49 +1,19 @@
|
||||
/*
|
||||
* Copyright (c) 2010, Thomas F. Morris
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
* - Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* - Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* Neither the name of Google nor the names of its contributors may be used to
|
||||
* endorse or promote products derived from this software without specific
|
||||
* prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
||||
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
package com.google.refine.extension.gdata;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.gdata.client.spreadsheet.CellQuery;
|
||||
import com.google.gdata.client.spreadsheet.SpreadsheetService;
|
||||
import com.google.gdata.data.spreadsheet.Cell;
|
||||
import com.google.gdata.data.spreadsheet.CellEntry;
|
||||
import com.google.gdata.data.spreadsheet.CellFeed;
|
||||
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
|
||||
import com.google.gdata.data.spreadsheet.WorksheetEntry;
|
||||
import com.google.gdata.util.ServiceException;
|
||||
import com.google.api.services.sheets.v4.Sheets;
|
||||
import com.google.api.services.sheets.v4.model.Sheet;
|
||||
import com.google.api.services.sheets.v4.model.Spreadsheet;
|
||||
import com.google.api.services.sheets.v4.model.ValueRange;
|
||||
|
||||
import com.google.refine.importers.TabularImportingParserBase;
|
||||
import com.google.refine.importers.TabularImportingParserBase.TableDataReader;
|
||||
@ -52,14 +22,9 @@ import com.google.refine.model.Project;
|
||||
import com.google.refine.model.medadata.ProjectMetadata;
|
||||
import com.google.refine.util.JSONUtilities;
|
||||
|
||||
/**
|
||||
* OpenRefine parser for Google Spreadsheets.
|
||||
*
|
||||
* @author Tom Morris <tfmorris@gmail.com>
|
||||
* @copyright 2010 Thomas F. Morris
|
||||
* @license New BSD http://www.opensource.org/licenses/bsd-license.php
|
||||
*/
|
||||
public class GDataImporter {
|
||||
static final Logger logger = LoggerFactory.getLogger("GDataImporter");
|
||||
|
||||
static public void parse(
|
||||
String token,
|
||||
Project project,
|
||||
@ -67,11 +32,11 @@ public class GDataImporter {
|
||||
final ImportingJob job,
|
||||
int limit,
|
||||
JSONObject options,
|
||||
List<Exception> exceptions) {
|
||||
List<Exception> exceptions) throws IOException {
|
||||
|
||||
String docType = JSONUtilities.getString(options, "docType", null);
|
||||
if ("spreadsheet".equals(docType)) {
|
||||
SpreadsheetService service = GDataExtension.getSpreadsheetService(token);
|
||||
Sheets service = GoogleAPIExtension.getSheetsService(token);
|
||||
parse(
|
||||
service,
|
||||
project,
|
||||
@ -94,7 +59,7 @@ public class GDataImporter {
|
||||
}
|
||||
|
||||
static public void parse(
|
||||
SpreadsheetService service,
|
||||
Sheets service,
|
||||
Project project,
|
||||
ProjectMetadata metadata,
|
||||
final ImportingJob job,
|
||||
@ -104,6 +69,10 @@ public class GDataImporter {
|
||||
|
||||
String docUrlString = JSONUtilities.getString(options, "docUrl", null);
|
||||
String worksheetUrlString = JSONUtilities.getString(options, "sheetUrl", null);
|
||||
|
||||
// the index of the worksheet
|
||||
int worksheetIndex = JSONUtilities.getInt(options, "worksheetIndex", 0);
|
||||
|
||||
if (docUrlString != null && worksheetUrlString != null) {
|
||||
try {
|
||||
parseOneWorkSheet(
|
||||
@ -112,7 +81,7 @@ public class GDataImporter {
|
||||
metadata,
|
||||
job,
|
||||
new URL(docUrlString),
|
||||
new URL(worksheetUrlString),
|
||||
worksheetIndex,
|
||||
limit,
|
||||
options,
|
||||
exceptions);
|
||||
@ -124,35 +93,35 @@ public class GDataImporter {
|
||||
}
|
||||
|
||||
static public void parseOneWorkSheet(
|
||||
SpreadsheetService service,
|
||||
Sheets service,
|
||||
Project project,
|
||||
ProjectMetadata metadata,
|
||||
final ImportingJob job,
|
||||
URL docURL,
|
||||
URL worksheetURL,
|
||||
int worksheetIndex,
|
||||
int limit,
|
||||
JSONObject options,
|
||||
List<Exception> exceptions) {
|
||||
|
||||
try {
|
||||
WorksheetEntry worksheetEntry = service.getEntry(worksheetURL, WorksheetEntry.class);
|
||||
String spreadsheetId = GoogleAPIExtension.extractSpreadSheetId(docURL.toString());
|
||||
|
||||
Spreadsheet response = service.spreadsheets().get(spreadsheetId)
|
||||
.setIncludeGridData(true)
|
||||
.execute();
|
||||
Sheet worksheetEntry = response.getSheets().get(worksheetIndex);
|
||||
|
||||
String spreadsheetName = docURL.toExternalForm();
|
||||
try {
|
||||
SpreadsheetEntry spreadsheetEntry = service.getEntry(docURL, SpreadsheetEntry.class);
|
||||
spreadsheetName = spreadsheetEntry.getTitle().getPlainText();
|
||||
} catch (ServiceException e) { // RedirectRequiredException among others
|
||||
// fall back to just using the URL (better for traceability anyway?)
|
||||
}
|
||||
|
||||
String fileSource = spreadsheetName + " # " +
|
||||
worksheetEntry.getTitle().getPlainText();
|
||||
worksheetEntry.getProperties().getTitle();
|
||||
|
||||
setProgress(job, fileSource, 0);
|
||||
TabularImportingParserBase.readTable(
|
||||
project,
|
||||
metadata,
|
||||
job,
|
||||
new WorksheetBatchRowReader(job, fileSource, service, worksheetEntry, 20),
|
||||
new WorksheetBatchRowReader(job, fileSource, service, spreadsheetId, worksheetEntry),
|
||||
fileSource,
|
||||
limit,
|
||||
options,
|
||||
@ -160,10 +129,7 @@ public class GDataImporter {
|
||||
);
|
||||
setProgress(job, fileSource, 100);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
exceptions.add(e);
|
||||
} catch (ServiceException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(ExceptionUtils.getStackTrace(e));
|
||||
exceptions.add(e);
|
||||
}
|
||||
}
|
||||
@ -176,98 +142,61 @@ public class GDataImporter {
|
||||
final ImportingJob job;
|
||||
final String fileSource;
|
||||
|
||||
final SpreadsheetService service;
|
||||
final WorksheetEntry worksheet;
|
||||
final int batchSize;
|
||||
final Sheets service;
|
||||
final String spreadsheetId;
|
||||
final Sheet worksheet;
|
||||
|
||||
final int totalRows;
|
||||
|
||||
int nextRow = 0; // 0-based
|
||||
int batchRowStart = 0; // 0-based
|
||||
List<List<Object>> rowsOfCells = null;
|
||||
private int indexRow = 0;
|
||||
private List<List<Object>> rowsOfCells = null;
|
||||
|
||||
public WorksheetBatchRowReader(ImportingJob job, String fileSource,
|
||||
SpreadsheetService service, WorksheetEntry worksheet,
|
||||
int batchSize) {
|
||||
Sheets service, String spreadsheetId, Sheet worksheet) {
|
||||
this.job = job;
|
||||
this.fileSource = fileSource;
|
||||
this.service = service;
|
||||
this.spreadsheetId = spreadsheetId;
|
||||
this.worksheet = worksheet;
|
||||
this.batchSize = batchSize;
|
||||
|
||||
this.totalRows = worksheet.getRowCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Object> getNextRowOfCells() throws IOException {
|
||||
if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && nextRow < totalRows)) {
|
||||
int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size());
|
||||
try {
|
||||
if (rowsOfCells == null) {
|
||||
rowsOfCells = getRowsOfCells(
|
||||
service,
|
||||
worksheet,
|
||||
newBatchRowStart + 1, // convert to 1-based
|
||||
batchSize);
|
||||
|
||||
batchRowStart = newBatchRowStart;
|
||||
|
||||
setProgress(job, fileSource, batchRowStart * 100 / totalRows);
|
||||
} catch (ServiceException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
spreadsheetId,
|
||||
worksheet);
|
||||
}
|
||||
|
||||
if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) {
|
||||
return rowsOfCells.get(nextRow++ - batchRowStart);
|
||||
if (rowsOfCells == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (rowsOfCells.size() > 0) {
|
||||
setProgress(job, fileSource, 100 * indexRow / rowsOfCells.size());
|
||||
} else {
|
||||
setProgress(job, fileSource, 100);
|
||||
}
|
||||
|
||||
if (indexRow < rowsOfCells.size()) {
|
||||
return rowsOfCells.get(indexRow++);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
List<List<Object>> getRowsOfCells(
|
||||
SpreadsheetService service,
|
||||
WorksheetEntry worksheet,
|
||||
int startRow, // 1-based
|
||||
int rowCount
|
||||
) throws IOException, ServiceException {
|
||||
URL cellFeedUrl = worksheet.getCellFeedUrl();
|
||||
Sheets service,
|
||||
String spreadsheetId,
|
||||
Sheet worksheet
|
||||
) throws IOException {
|
||||
String range = worksheet.getProperties().getTitle();
|
||||
ValueRange result = service.spreadsheets().values().get(spreadsheetId, range).execute();
|
||||
|
||||
int minRow = startRow;
|
||||
int maxRow = Math.min(worksheet.getRowCount(), startRow + rowCount - 1);
|
||||
int cols = worksheet.getColCount();
|
||||
int rows = worksheet.getRowCount();
|
||||
rowsOfCells = result.getValues();
|
||||
|
||||
CellQuery cellQuery = new CellQuery(cellFeedUrl);
|
||||
cellQuery.setMinimumRow(minRow);
|
||||
cellQuery.setMaximumRow(maxRow);
|
||||
cellQuery.setMaximumCol(cols);
|
||||
cellQuery.setMaxResults(rows * cols);
|
||||
cellQuery.setReturnEmpty(false);
|
||||
|
||||
CellFeed cellFeed = service.query(cellQuery, CellFeed.class);
|
||||
List<CellEntry> cellEntries = cellFeed.getEntries();
|
||||
|
||||
List<List<Object>> rowsOfCells = new ArrayList<List<Object>>(rowCount);
|
||||
for (CellEntry cellEntry : cellEntries) {
|
||||
Cell cell = cellEntry.getCell();
|
||||
if (cell != null) {
|
||||
int row = cell.getRow() - startRow;
|
||||
int col = cell.getCol() - 1;
|
||||
|
||||
while (row >= rowsOfCells.size()) {
|
||||
rowsOfCells.add(new ArrayList<Object>());
|
||||
}
|
||||
List<Object> rowOfCells = rowsOfCells.get(row);
|
||||
|
||||
while (col >= rowOfCells.size()) {
|
||||
rowOfCells.add(null);
|
||||
}
|
||||
rowOfCells.set(col, cell.getValue());
|
||||
}
|
||||
}
|
||||
return rowsOfCells;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,41 +1,7 @@
|
||||
/*
|
||||
|
||||
Copyright 2011, Google Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
package com.google.refine.extension.gdata;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.net.URL;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
@ -44,25 +10,24 @@ import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.api.services.drive.Drive;
|
||||
import com.google.api.services.drive.model.File;
|
||||
import com.google.api.services.drive.model.FileList;
|
||||
import com.google.api.services.drive.model.User;
|
||||
import com.google.api.services.fusiontables.Fusiontables;
|
||||
import com.google.api.services.fusiontables.model.Table;
|
||||
import com.google.api.services.fusiontables.model.TableList;
|
||||
import com.google.gdata.client.docs.DocsService;
|
||||
import com.google.gdata.client.spreadsheet.FeedURLFactory;
|
||||
import com.google.gdata.client.spreadsheet.SpreadsheetService;
|
||||
import com.google.gdata.data.DateTime;
|
||||
import com.google.gdata.data.Person;
|
||||
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
|
||||
import com.google.gdata.data.spreadsheet.SpreadsheetFeed;
|
||||
import com.google.gdata.data.spreadsheet.WorksheetEntry;
|
||||
import com.google.gdata.data.spreadsheet.WorksheetFeed;
|
||||
import com.google.gdata.util.AuthenticationException;
|
||||
import com.google.gdata.util.ServiceException;
|
||||
import com.google.api.services.sheets.v4.Sheets;
|
||||
import com.google.api.services.sheets.v4.model.Sheet;
|
||||
import com.google.api.services.sheets.v4.model.Spreadsheet;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.RefineServlet;
|
||||
@ -77,7 +42,7 @@ import com.google.refine.util.JSONUtilities;
|
||||
import com.google.refine.util.ParsingUtilities;
|
||||
|
||||
public class GDataImportingController implements ImportingController {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger("GDataImportingController");
|
||||
protected RefineServlet servlet;
|
||||
|
||||
@Override
|
||||
@ -128,12 +93,10 @@ public class GDataImportingController implements ImportingController {
|
||||
writer.array();
|
||||
|
||||
try {
|
||||
listSpreadsheets(GDataExtension.getDocsService(token), writer);
|
||||
listSpreadsheets(GoogleAPIExtension.getDriveService(token), writer);
|
||||
listFusionTables(FusionTableHandler.getFusionTablesService(token), writer);
|
||||
} catch (AuthenticationException e) {
|
||||
TokenCookie.deleteToken(request, response);
|
||||
} catch (ServiceException e) {
|
||||
e.printStackTrace();
|
||||
} catch (Exception e) {
|
||||
logger.error("doListDocuments exception:" + ExceptionUtils.getStackTrace(e));
|
||||
} finally {
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
@ -146,26 +109,30 @@ public class GDataImportingController implements ImportingController {
|
||||
}
|
||||
}
|
||||
|
||||
private void listSpreadsheets(DocsService service, JSONWriter writer)
|
||||
throws IOException, ServiceException, JSONException {
|
||||
URL metafeedUrl = new URL("https://spreadsheets.google.com/feeds/spreadsheets/private/full");
|
||||
SpreadsheetFeed feed = service.getFeed(metafeedUrl, SpreadsheetFeed.class);
|
||||
for (SpreadsheetEntry entry : feed.getEntries()) {
|
||||
private void listSpreadsheets(Drive drive, JSONWriter writer)
|
||||
throws IOException, JSONException {
|
||||
com.google.api.services.drive.Drive.Files.List files = drive.files().list();
|
||||
files.setQ("mimeType = 'application/vnd.google-apps.spreadsheet'");
|
||||
files.setFields("nextPageToken, files(id, name, webViewLink, owners, modifiedTime)");
|
||||
FileList fileList = files.execute();
|
||||
|
||||
for (File entry : fileList.getFiles()) {
|
||||
writer.object();
|
||||
writer.key("docId"); writer.value(entry.getId());
|
||||
writer.key("docLink"); writer.value(entry.getHtmlLink().getHref());
|
||||
writer.key("docSelfLink"); writer.value(entry.getSelfLink().getHref());
|
||||
writer.key("title"); writer.value(entry.getTitle().getPlainText());
|
||||
writer.key("docLink"); writer.value(entry.getWebViewLink());
|
||||
writer.key("docSelfLink"); writer.value(entry.getWebViewLink());
|
||||
writer.key("title"); writer.value(entry.getName());
|
||||
|
||||
writer.key("type"); writer.value("spreadsheet");
|
||||
|
||||
DateTime updated = entry.getUpdated();
|
||||
com.google.api.client.util.DateTime updated = entry.getModifiedTime();
|
||||
if (updated != null) {
|
||||
writer.key("updated"); writer.value(updated.toStringRfc822());
|
||||
writer.key("updated"); writer.value(updated.toString());
|
||||
}
|
||||
|
||||
writer.key("authors"); writer.array();
|
||||
for (Person person : entry.getAuthors()) {
|
||||
writer.value(person.getName());
|
||||
for (User user : entry.getOwners()) {
|
||||
writer.value(user.getDisplayName());
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
@ -174,10 +141,14 @@ public class GDataImportingController implements ImportingController {
|
||||
}
|
||||
|
||||
private void listFusionTables(Fusiontables service, JSONWriter writer)
|
||||
throws IOException, ServiceException, JSONException {
|
||||
throws IOException, JSONException {
|
||||
|
||||
Fusiontables.Table.List listTables = service.table().list();
|
||||
TableList tablelist = listTables.execute();
|
||||
|
||||
if (tablelist == null || tablelist.getItems() == null)
|
||||
return;
|
||||
|
||||
for (Table table : tablelist.getItems()) {
|
||||
String id = table.getTableId();
|
||||
String name = table.getName();
|
||||
@ -197,95 +168,56 @@ public class GDataImportingController implements ImportingController {
|
||||
private void doInitializeParserUI(
|
||||
HttpServletRequest request, HttpServletResponse response, Properties parameters)
|
||||
throws ServletException, IOException {
|
||||
|
||||
String token = TokenCookie.getToken(request);
|
||||
|
||||
String type = parameters.getProperty("docType");
|
||||
String urlString = parameters.getProperty("docUrl");
|
||||
|
||||
URL url = new URL(urlString);
|
||||
try {
|
||||
JSONObject result = new JSONObject();
|
||||
JSONObject options = new JSONObject();
|
||||
|
||||
JSONUtilities.safePut(result, "status", "ok");
|
||||
JSONUtilities.safePut(result, "options", options);
|
||||
|
||||
JSONUtilities.safePut(options, "skipDataLines", 0); // number of initial data lines to skip
|
||||
JSONUtilities.safePut(options, "storeBlankRows", true);
|
||||
JSONUtilities.safePut(options, "storeBlankCellsAsNulls", true);
|
||||
|
||||
if ("spreadsheet".equals(type)) {
|
||||
JSONArray worksheets = new JSONArray();
|
||||
// extract spreadSheetId from URL
|
||||
String spreadSheetId = GoogleAPIExtension.extractSpreadSheetId(urlString);
|
||||
|
||||
JSONUtilities.safePut(options, "ignoreLines", -1); // number of blank lines at the beginning to ignore
|
||||
JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
|
||||
|
||||
JSONArray worksheets = new JSONArray();
|
||||
JSONUtilities.safePut(options, "worksheets", worksheets);
|
||||
|
||||
List<WorksheetEntry> worksheetEntries =
|
||||
reallyTryToGetWorksheetEntriesForDoc(url, token);
|
||||
for (WorksheetEntry worksheetEntry : worksheetEntries) {
|
||||
List<Sheet> worksheetEntries =
|
||||
getWorksheetEntriesForDoc(token, spreadSheetId);
|
||||
for (Sheet sheet : worksheetEntries) {
|
||||
JSONObject worksheetO = new JSONObject();
|
||||
JSONUtilities.safePut(worksheetO, "name", worksheetEntry.getTitle().getPlainText());
|
||||
JSONUtilities.safePut(worksheetO, "rows", worksheetEntry.getRowCount());
|
||||
JSONUtilities.safePut(worksheetO, "link", worksheetEntry.getSelfLink().getHref());
|
||||
JSONUtilities.safePut(worksheetO, "name", sheet.getProperties().getTitle());
|
||||
JSONUtilities.safePut(worksheetO, "rows", sheet.getProperties().getGridProperties().getRowCount());
|
||||
JSONUtilities.safePut(worksheetO, "link",
|
||||
"https://sheets.googleapis.com/v4/spreadsheets/" + spreadSheetId + "/values/" + sheet.getProperties().getTitle());
|
||||
|
||||
JSONUtilities.append(worksheets, worksheetO);
|
||||
}
|
||||
} else if ("table".equals(type)) {
|
||||
// No metadata for a fusion table.
|
||||
}
|
||||
/* TODO: else */
|
||||
|
||||
HttpUtilities.respond(response, result.toString());
|
||||
} catch (ServiceException e) {
|
||||
e.printStackTrace();
|
||||
HttpUtilities.respond(response, "error", "Internal error: " + e.getLocalizedMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private List<WorksheetEntry> reallyTryToGetWorksheetEntriesForDoc(URL docUrl, String token) throws IOException, ServiceException {
|
||||
try {
|
||||
return getWorksheetEntriesForDoc(docUrl, token);
|
||||
} catch (ServiceException e) {
|
||||
/*
|
||||
* TODO: figure out if we can rewire the URL somehow. This code below
|
||||
* doesn't work but maybe we need to try something similar to it.
|
||||
*
|
||||
String urlString = docUrl.toString();
|
||||
if (urlString.startsWith("https://docs.google.com/spreadsheet/ccc?key=") ||
|
||||
urlString.startsWith("http://docs.google.com/spreadsheet/ccc?key=")) {
|
||||
private List<Sheet> getWorksheetEntriesForDoc(String token, String spreadsheetId) throws IOException {
|
||||
Sheets sheetsService = GoogleAPIExtension.getSheetsService(token);
|
||||
|
||||
String urlString2 = "https://spreadsheets.google.com/spreadsheet/ccc?key=" +
|
||||
urlString.substring(urlString.indexOf("?key=") + 5);
|
||||
boolean includeGridData = true;
|
||||
|
||||
return getWorksheetEntriesForDoc(new URL(urlString2), token);
|
||||
}
|
||||
*/
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
Sheets.Spreadsheets.Get request = sheetsService.spreadsheets().get(spreadsheetId);
|
||||
request.setIncludeGridData(includeGridData);
|
||||
|
||||
private List<WorksheetEntry> getWorksheetEntriesForDoc(URL docUrl, String token) throws IOException, ServiceException {
|
||||
if (token != null) {
|
||||
try {
|
||||
SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token);
|
||||
SpreadsheetEntry spreadsheetEntry = spreadsheetService.getEntry(docUrl, SpreadsheetEntry.class);
|
||||
return spreadsheetEntry.getWorksheets();
|
||||
} catch (ServiceException e) {
|
||||
// Ignore and fall through, pretending that we're not logged in.
|
||||
}
|
||||
}
|
||||
return getWorksheetEntriesForDoc(docUrl);
|
||||
}
|
||||
Spreadsheet response = request.execute();
|
||||
|
||||
private List<WorksheetEntry> getWorksheetEntriesForDoc(URL docUrl) throws IOException, ServiceException {
|
||||
SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(null);
|
||||
String visibility = "public";
|
||||
FeedURLFactory factory = FeedURLFactory.getDefault();
|
||||
String key = GDataExtension.getSpreadsheetID(docUrl);
|
||||
docUrl = factory.getWorksheetFeedUrl(key, visibility, "values");
|
||||
WorksheetFeed feed = spreadsheetService.getFeed(docUrl, WorksheetFeed.class);
|
||||
return feed.getEntries();
|
||||
return response.getSheets();
|
||||
}
|
||||
|
||||
private void doParsePreview(
|
||||
@ -381,6 +313,7 @@ public class GDataImportingController implements ImportingController {
|
||||
pm.setName(JSONUtilities.getString(optionObj, "projectName", "Untitled"));
|
||||
pm.setEncoding(JSONUtilities.getString(optionObj, "encoding", "UTF-8"));
|
||||
|
||||
try {
|
||||
GDataImporter.parse(
|
||||
token,
|
||||
project,
|
||||
@ -390,6 +323,9 @@ public class GDataImportingController implements ImportingController {
|
||||
optionObj,
|
||||
exceptions
|
||||
);
|
||||
} catch (IOException e) {
|
||||
logger.error(ExceptionUtils.getStackTrace(e));
|
||||
}
|
||||
|
||||
if (!job.canceled) {
|
||||
if (exceptions.size() > 0) {
|
||||
|
||||
@ -0,0 +1,205 @@
|
||||
package com.google.refine.extension.gdata;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import com.google.api.client.auth.oauth2.AuthorizationCodeResponseUrl;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeRequestUrl;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeTokenRequest;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse;
|
||||
import com.google.api.client.http.HttpRequest;
|
||||
import com.google.api.client.http.HttpRequestInitializer;
|
||||
import com.google.api.client.http.HttpTransport;
|
||||
import com.google.api.client.http.javanet.NetHttpTransport;
|
||||
import com.google.api.client.json.JsonFactory;
|
||||
import com.google.api.client.json.jackson.JacksonFactory;
|
||||
import com.google.api.services.drive.Drive;
|
||||
import com.google.api.services.drive.DriveScopes;
|
||||
import com.google.api.services.fusiontables.FusiontablesScopes;
|
||||
import com.google.api.services.sheets.v4.Sheets;
|
||||
import com.google.api.services.sheets.v4.SheetsScopes;
|
||||
|
||||
import com.google.refine.util.ParsingUtilities;
|
||||
|
||||
import edu.mit.simile.butterfly.ButterflyModule;
|
||||
|
||||
abstract public class GoogleAPIExtension {
|
||||
protected static final String SERVICE_APP_NAME = "OpenRefine-Google-Service";
|
||||
private static final String CLIENT_ID = "455686949425-d237cmorii0ge8if7it5r1qijce6caf0.apps.googleusercontent.com";
|
||||
private static final String CLIENT_SECRET = "wm5qVtjp3VDfuAx2P2qm6GJb";
|
||||
|
||||
/** Global instance of the HTTP transport. */
|
||||
protected static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
|
||||
|
||||
/** Global instance of the JSON factory. */
|
||||
protected static final JsonFactory JSON_FACTORY = new JacksonFactory();
|
||||
|
||||
private static final String[] SCOPES = {DriveScopes.DRIVE, SheetsScopes.SPREADSHEETS, FusiontablesScopes.FUSIONTABLES};
|
||||
|
||||
static public String getAuthorizationUrl(ButterflyModule module, HttpServletRequest request)
|
||||
throws MalformedURLException {
|
||||
String authorizedUrl = makeRedirectUrl(module, request);
|
||||
|
||||
|
||||
GoogleAuthorizationCodeRequestUrl url = new GoogleAuthorizationCodeRequestUrl(
|
||||
CLIENT_ID,
|
||||
authorizedUrl, // execution continues at authorized on redirect
|
||||
Arrays.asList(SCOPES));
|
||||
|
||||
return url.toString();
|
||||
|
||||
}
|
||||
|
||||
private static String makeRedirectUrl(ButterflyModule module, HttpServletRequest request)
|
||||
throws MalformedURLException {
|
||||
StringBuffer sb = new StringBuffer(module.getMountPoint().getMountPoint());
|
||||
sb.append("authorized?winname=");
|
||||
sb.append(ParsingUtilities.encode(request.getParameter("winname")));
|
||||
sb.append("&cb=");
|
||||
sb.append(ParsingUtilities.encode(request.getParameter("cb")));
|
||||
|
||||
URL thisUrl = new URL(request.getRequestURL().toString());
|
||||
URL authorizedUrl = new URL(thisUrl, sb.toString());
|
||||
|
||||
return authorizedUrl.toExternalForm();
|
||||
}
|
||||
|
||||
static public String getTokenFromCode(ButterflyModule module, HttpServletRequest request)
|
||||
throws IOException {
|
||||
String redirectUrl = makeRedirectUrl(module, request);
|
||||
StringBuffer fullUrlBuf = request.getRequestURL();
|
||||
if (request.getQueryString() != null) {
|
||||
fullUrlBuf.append('?').append(request.getQueryString());
|
||||
}
|
||||
AuthorizationCodeResponseUrl authResponse =
|
||||
new AuthorizationCodeResponseUrl(fullUrlBuf.toString());
|
||||
// check for user-denied error
|
||||
if (authResponse.getError() != null) {
|
||||
// authorization denied...
|
||||
} else {
|
||||
// request access token using authResponse.getCode()...
|
||||
String code = authResponse.getCode();
|
||||
GoogleTokenResponse response = new GoogleAuthorizationCodeTokenRequest(HTTP_TRANSPORT,
|
||||
JSON_FACTORY, CLIENT_ID, CLIENT_SECRET, code, redirectUrl).execute();
|
||||
String tokenAndExpiresInSeconds = response.getAccessToken() + "," + response.getExpiresInSeconds();
|
||||
return tokenAndExpiresInSeconds;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static public Drive getDriveService(String token) {
|
||||
GoogleCredential credential = new GoogleCredential().setAccessToken(token);
|
||||
|
||||
return new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).setHttpRequestInitializer(new HttpRequestInitializer() {
|
||||
@Override
|
||||
public void initialize(HttpRequest httpRequest) throws IOException {
|
||||
credential.initialize(httpRequest);
|
||||
httpRequest.setConnectTimeout(3 * 60000); // 3 minutes connect timeout
|
||||
httpRequest.setReadTimeout(3 * 60000); // 3 minutes read timeout
|
||||
}
|
||||
})
|
||||
.setApplicationName(SERVICE_APP_NAME).build();
|
||||
}
|
||||
|
||||
static boolean isSpreadsheetURL(String url) {
|
||||
try {
|
||||
return url.contains("spreadsheet") && getSpreadsheetID(new URL(url)) != null;
|
||||
} catch (MalformedURLException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static String getSpreadsheetID(URL url) {
|
||||
return getParamValue(url,"key");
|
||||
}
|
||||
|
||||
static private String getParamValue(URL url, String key) {
|
||||
String query = url.getQuery();
|
||||
if (query != null) {
|
||||
String[] parts = query.split("&");
|
||||
for (String part : parts) {
|
||||
if (part.startsWith(key+"=")) {
|
||||
int offset = key.length()+1;
|
||||
String tableId = part.substring(offset);
|
||||
return tableId;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build and return an authorized Sheets API client service.
|
||||
* @return an authorized Sheets API client service
|
||||
* @throws IOException
|
||||
*/
|
||||
public static Sheets getSheetsService(String token) throws IOException {
|
||||
return new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY,
|
||||
new GoogleCredential().setAccessToken(token))
|
||||
.setApplicationName(SERVICE_APP_NAME)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static String extractSpreadSheetId(String url)
|
||||
throws IllegalArgumentException {
|
||||
URL urlAsUrl;
|
||||
|
||||
Matcher matcher = Pattern.compile("(?<=\\/d\\/).*(?=\\/.*)").matcher(url);
|
||||
if (matcher.find()) {
|
||||
return matcher.group(0);
|
||||
}
|
||||
|
||||
try {
|
||||
urlAsUrl = new URL(url);
|
||||
String query = urlAsUrl.getQuery();
|
||||
if ( query != null ) {
|
||||
|
||||
String[] parts = query.split("&");
|
||||
|
||||
int offset = -1;
|
||||
int numParts = 0;
|
||||
String keyOrId = "";
|
||||
|
||||
for (String part : parts) {
|
||||
if (part.startsWith("id=")) {
|
||||
offset = ("id=").length();
|
||||
keyOrId = part.substring(offset);
|
||||
numParts = 4;
|
||||
break;
|
||||
} else if (part.startsWith("key=")) {
|
||||
offset = ("key=").length();
|
||||
keyOrId = part.substring(offset);
|
||||
if (!keyOrId.isEmpty()) {
|
||||
return keyOrId;
|
||||
}
|
||||
numParts = 2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (offset > -1) {
|
||||
String[] dottedParts = keyOrId.split("\\.");
|
||||
if (dottedParts.length == numParts) {
|
||||
return dottedParts[0] + "." + dottedParts[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch ( MalformedURLException e ) {
|
||||
// This is not a URL, maybe it is just an id
|
||||
String[] dottedParts = url.split("\\.");
|
||||
|
||||
if (dottedParts.length == 4 || dottedParts.length == 2) {
|
||||
return dottedParts[0] + "." + dottedParts[1];
|
||||
}
|
||||
}
|
||||
|
||||
throw new IllegalArgumentException("Uknown URL format.");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,124 @@
|
||||
package com.google.refine.extension.gdata;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.api.services.sheets.v4.Sheets;
|
||||
import com.google.api.services.sheets.v4.model.AppendCellsRequest;
|
||||
import com.google.api.services.sheets.v4.model.BatchUpdateSpreadsheetRequest;
|
||||
import com.google.api.services.sheets.v4.model.BatchUpdateSpreadsheetResponse;
|
||||
import com.google.api.services.sheets.v4.model.ExtendedValue;
|
||||
import com.google.api.services.sheets.v4.model.Request;
|
||||
import com.google.api.services.sheets.v4.model.RowData;
|
||||
|
||||
import com.google.refine.exporters.TabularSerializer;
|
||||
|
||||
final class SpreadsheetSerializer implements TabularSerializer {
|
||||
static final Logger logger = LoggerFactory.getLogger("SpreadsheetSerializer");
|
||||
|
||||
private static final int BATCH_SIZE = 1000;
|
||||
|
||||
private Sheets service;
|
||||
private String spreadsheetId;
|
||||
private List<Exception> exceptions;
|
||||
|
||||
// A list of updates to apply to the spreadsheet.
|
||||
private List<Request> requests = new ArrayList<>();
|
||||
|
||||
private Request batchRequest = null;
|
||||
private int row = 0;
|
||||
|
||||
private List<RowData> rows;
|
||||
|
||||
SpreadsheetSerializer(Sheets service, String spreadsheetId, List<Exception> exceptions) {
|
||||
this.service = service;
|
||||
this.spreadsheetId = spreadsheetId;
|
||||
this.exceptions = exceptions;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startFile(JSONObject options) {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void endFile() {
|
||||
if (batchRequest != null) {
|
||||
sendBatch(rows);
|
||||
}
|
||||
|
||||
BatchUpdateSpreadsheetRequest requestBody = new BatchUpdateSpreadsheetRequest();
|
||||
requestBody.setIncludeSpreadsheetInResponse(false);
|
||||
requestBody.setRequests(requests);
|
||||
|
||||
Sheets.Spreadsheets.BatchUpdate request;
|
||||
try {
|
||||
logger.debug("spreadsheetId: " + spreadsheetId);
|
||||
logger.debug("requestBody:" + requestBody.toString());
|
||||
request = service.spreadsheets().batchUpdate(spreadsheetId, requestBody);
|
||||
|
||||
BatchUpdateSpreadsheetResponse response = request.execute();
|
||||
logger.debug("response:" + response.toPrettyString());
|
||||
} catch (IOException e) {
|
||||
exceptions.add(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addRow(List<CellData> cells, boolean isHeader) {
|
||||
if (batchRequest == null) {
|
||||
batchRequest = new Request();
|
||||
rows = new ArrayList<RowData>(BATCH_SIZE);
|
||||
}
|
||||
List<com.google.api.services.sheets.v4.model.CellData> cellDatas = new ArrayList<>();
|
||||
RowData rowData = new RowData();
|
||||
|
||||
for (int c = 0; c < cells.size(); c++) {
|
||||
CellData cellData = cells.get(c);
|
||||
if (cellData != null && cellData.text != null) {
|
||||
cellDatas.add(cellData2sheetCellData(cellData));
|
||||
}
|
||||
}
|
||||
|
||||
rowData.setValues(cellDatas);
|
||||
rows.add(rowData);
|
||||
row++;
|
||||
|
||||
if (row % BATCH_SIZE == 0) {
|
||||
sendBatch(rows);
|
||||
}
|
||||
}
|
||||
|
||||
private com.google.api.services.sheets.v4.model.CellData cellData2sheetCellData(CellData cellData) {
|
||||
com.google.api.services.sheets.v4.model.CellData sheetCellData = new com.google.api.services.sheets.v4.model.CellData();
|
||||
|
||||
ExtendedValue ev = new ExtendedValue();
|
||||
ev.setStringValue(cellData.value.toString());
|
||||
|
||||
sheetCellData.setUserEnteredValue(ev);
|
||||
|
||||
return sheetCellData;
|
||||
}
|
||||
|
||||
private void sendBatch(List<RowData> rows) {
|
||||
AppendCellsRequest acr = new AppendCellsRequest();
|
||||
acr.setFields("*");
|
||||
acr.setSheetId(0);
|
||||
acr.setRows(rows);
|
||||
batchRequest.setAppendCells(acr);
|
||||
|
||||
requests.add(batchRequest);
|
||||
}
|
||||
|
||||
public String getUrl() throws UnsupportedEncodingException {
|
||||
String urlString= "https://docs.google.com/spreadsheets/d/" + spreadsheetId + "/edit#gid=0";
|
||||
return URLEncoder.encode(urlString, "UTF-8");
|
||||
}
|
||||
}
|
||||
@ -7,9 +7,7 @@ import javax.servlet.http.HttpServletResponse;
|
||||
import com.google.refine.util.CookiesUtilities;
|
||||
|
||||
public class TokenCookie {
|
||||
|
||||
private static final String COOKIE_NAME = "oauth2_token";
|
||||
private static int MAX_AGE = 30 * 24 * 60 * 60; // 30 days
|
||||
|
||||
public static String getToken(HttpServletRequest request) {
|
||||
Cookie cookie = CookiesUtilities.getCookie(request, COOKIE_NAME);
|
||||
@ -17,8 +15,9 @@ public class TokenCookie {
|
||||
}
|
||||
|
||||
public static void setToken(HttpServletRequest request,
|
||||
HttpServletResponse response, String token) {
|
||||
CookiesUtilities.setCookie(request, response, COOKIE_NAME, token, MAX_AGE);
|
||||
HttpServletResponse response, String token, String expiresInSeconds) {
|
||||
CookiesUtilities.setCookie(request, response, COOKIE_NAME, token,
|
||||
Integer.parseInt(expiresInSeconds));
|
||||
}
|
||||
|
||||
public static void deleteToken(HttpServletRequest request,
|
||||
|
||||
@ -1,37 +1,10 @@
|
||||
/*
|
||||
* Copyright (c) 2010,2011,2015 Thomas F. Morris <tfmorris@gmail.com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
* - Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
* - Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* Neither the name of Google nor the names of its contributors may be used to
|
||||
* endorse or promote products derived from this software without specific
|
||||
* prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
||||
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
package com.google.refine.extension.gdata;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.Writer;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
@ -41,27 +14,17 @@ import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.json.JSONObject;
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.json.JSONWriter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
|
||||
import com.google.api.client.http.FileContent;
|
||||
import com.google.api.services.drive.Drive;
|
||||
import com.google.api.services.drive.model.File;
|
||||
import com.google.gdata.client.spreadsheet.CellQuery;
|
||||
import com.google.gdata.client.spreadsheet.SpreadsheetService;
|
||||
import com.google.gdata.data.Link;
|
||||
import com.google.gdata.data.PlainTextConstruct;
|
||||
import com.google.gdata.data.batch.BatchOperationType;
|
||||
import com.google.gdata.data.batch.BatchStatus;
|
||||
import com.google.gdata.data.batch.BatchUtils;
|
||||
import com.google.gdata.data.spreadsheet.Cell;
|
||||
import com.google.gdata.data.spreadsheet.CellEntry;
|
||||
import com.google.gdata.data.spreadsheet.CellFeed;
|
||||
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
|
||||
import com.google.gdata.data.spreadsheet.SpreadsheetFeed;
|
||||
import com.google.gdata.data.spreadsheet.WorksheetEntry;
|
||||
import com.google.gdata.util.ServiceException;
|
||||
import com.google.api.services.drive.model.File.ContentHints;
|
||||
import com.google.api.services.drive.model.File.ContentHints.Thumbnail;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.browsing.Engine;
|
||||
@ -69,13 +32,14 @@ import com.google.refine.commands.Command;
|
||||
import com.google.refine.commands.HttpUtilities;
|
||||
import com.google.refine.commands.project.ExportRowsCommand;
|
||||
import com.google.refine.exporters.CustomizableTabularExporterUtilities;
|
||||
import com.google.refine.exporters.TabularSerializer;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.Project;
|
||||
|
||||
public class UploadCommand extends Command {
|
||||
static final Logger logger = LoggerFactory.getLogger("gdata_upload");
|
||||
private static final String SPREADSHEET_FEED = "https://spreadsheets.google.com/feeds/spreadsheets/private/full";
|
||||
|
||||
private static final String METADATA_DESCRIPTION = "OpenRefine project dump";
|
||||
private static final String METADATA_ICONLINK = "https://raw.githubusercontent.com/OpenRefine/OpenRefine/master/main/webapp/modules/core/images/logo-openrefine-550.png";
|
||||
@Override
|
||||
public void doPost(HttpServletRequest request, HttpServletResponse response)
|
||||
throws ServletException, IOException {
|
||||
@ -140,170 +104,94 @@ public class UploadCommand extends Command {
|
||||
return uploadSpreadsheet(project, engine, params, token, name, exceptions);
|
||||
} else if ("gdata/fusion-table".equals(format)) {
|
||||
return uploadFusionTable(project, engine, params, token, name, exceptions);
|
||||
} else if (("raw/openrefine-project").equals(format)) {
|
||||
return uploadOpenRefineProject(project, token, name, exceptions);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static byte[] getImageFromUrl(String urlText) throws IOException {
|
||||
URL url = new URL(urlText);
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
|
||||
try (InputStream inputStream = url.openStream()) {
|
||||
int n = 0;
|
||||
byte [] buffer = new byte[ 1024 ];
|
||||
while (-1 != (n = inputStream.read(buffer))) {
|
||||
output.write(buffer, 0, n);
|
||||
}
|
||||
}
|
||||
|
||||
return output.toByteArray();
|
||||
}
|
||||
|
||||
private static String uploadOpenRefineProject(Project project, String token,
|
||||
String name, List<Exception> exceptions) {
|
||||
FileOutputStream fos = null;
|
||||
|
||||
try {
|
||||
java.io.File filePath = java.io.File.createTempFile(name, ".tgz");
|
||||
filePath.deleteOnExit();
|
||||
|
||||
fos = new FileOutputStream(filePath);
|
||||
FileProjectManager.gzipTarToOutputStream(project, fos);
|
||||
|
||||
File fileMetadata = new File();
|
||||
String asB64 = Base64.encodeBase64URLSafeString(getImageFromUrl(METADATA_ICONLINK));
|
||||
|
||||
Thumbnail tn = new Thumbnail();
|
||||
tn.setMimeType("image/x-icon").setImage(asB64);
|
||||
ContentHints contentHints = new ContentHints();
|
||||
contentHints.setThumbnail(tn);
|
||||
|
||||
fileMetadata.setName(name)
|
||||
.setDescription(METADATA_DESCRIPTION)
|
||||
.setContentHints(contentHints);
|
||||
FileContent projectContent = new FileContent("application/zip", filePath);
|
||||
File file = GoogleAPIExtension.getDriveService(token)
|
||||
.files().create(fileMetadata, projectContent)
|
||||
.setFields("id")
|
||||
.execute();
|
||||
logger.info("File ID: " + file.getId());
|
||||
|
||||
return file.getId();
|
||||
} catch (IOException e) {
|
||||
logger.error(ExceptionUtils.getStackTrace(e));
|
||||
exceptions.add(e);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
static private String uploadSpreadsheet(
|
||||
final Project project, final Engine engine, final Properties params,
|
||||
String token, String name, List<Exception> exceptions) {
|
||||
|
||||
Drive driveService = GDataExtension.getDriveService(token);
|
||||
final SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token);
|
||||
Drive driveService = GoogleAPIExtension.getDriveService(token);
|
||||
|
||||
try {
|
||||
File body = new File();
|
||||
body.setTitle(name);
|
||||
body.setName(name);
|
||||
body.setDescription("Spreadsheet uploaded from OpenRefine project: " + name);
|
||||
body.setMimeType("application/vnd.google-apps.spreadsheet");
|
||||
|
||||
File file = driveService.files().insert(body).execute();
|
||||
String fileID = file.getId();
|
||||
File file = driveService.files().create(body).execute();
|
||||
String spreadsheetId = file.getId();
|
||||
|
||||
// Iterate through all spreadsheets to find one with our ID
|
||||
SpreadsheetEntry spreadsheetEntry2 = null;
|
||||
SpreadsheetFeed feed = spreadsheetService.getFeed(new URL(SPREADSHEET_FEED), SpreadsheetFeed.class);
|
||||
List<com.google.gdata.data.spreadsheet.SpreadsheetEntry> spreadsheets = feed.getEntries();
|
||||
for (com.google.gdata.data.spreadsheet.SpreadsheetEntry spreadsheet : spreadsheets) {
|
||||
if (spreadsheet.getId().endsWith(fileID)) {
|
||||
spreadsheetEntry2 = spreadsheet;
|
||||
}
|
||||
}
|
||||
|
||||
// Bail if we didn't find our spreadsheet (shouldn't happen)
|
||||
if (spreadsheetEntry2 == null) {
|
||||
logger.error("Failed to find match for ID: " + fileID);
|
||||
return null;
|
||||
}
|
||||
|
||||
int[] size = CustomizableTabularExporterUtilities.countColumnsRows(
|
||||
project, engine, params);
|
||||
|
||||
URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl();
|
||||
WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]);
|
||||
worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data"));
|
||||
|
||||
final WorksheetEntry worksheetEntry2 =
|
||||
spreadsheetService.insert(worksheetFeedUrl, worksheetEntry);
|
||||
|
||||
spreadsheetEntry2.getDefaultWorksheet().delete();
|
||||
|
||||
final SpreadsheetEntry spreadsheetEntry3 = spreadsheetEntry2;
|
||||
new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V3);
|
||||
try {
|
||||
uploadToCellFeed(
|
||||
project, engine, params,
|
||||
spreadsheetService,
|
||||
spreadsheetEntry3,
|
||||
worksheetEntry2);
|
||||
} catch (Exception e) {
|
||||
logger.error("Error uploading data to Google Spreadsheets", e);
|
||||
}
|
||||
}
|
||||
}.start();
|
||||
|
||||
return spreadsheetEntry2.getSpreadsheetLink().getHref();
|
||||
} catch (IOException | ServiceException e) {
|
||||
exceptions.add(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static private void uploadToCellFeed(
|
||||
Project project,
|
||||
Engine engine,
|
||||
Properties params,
|
||||
final SpreadsheetService service,
|
||||
final SpreadsheetEntry spreadsheetEntry,
|
||||
final WorksheetEntry worksheetEntry)
|
||||
throws IOException, ServiceException {
|
||||
|
||||
final URL cellFeedUrl = worksheetEntry.getCellFeedUrl();
|
||||
final CellEntry[][] cellEntries =
|
||||
new CellEntry[worksheetEntry.getRowCount()][worksheetEntry.getColCount()];
|
||||
{
|
||||
CellQuery cellQuery = new CellQuery(cellFeedUrl);
|
||||
cellQuery.setReturnEmpty(true);
|
||||
|
||||
CellFeed fetchingCellFeed = service.getFeed(cellQuery, CellFeed.class);
|
||||
for (CellEntry cellEntry : fetchingCellFeed.getEntries()) {
|
||||
Cell cell = cellEntry.getCell();
|
||||
cellEntries[cell.getRow() - 1][cell.getCol() - 1] = cellEntry;
|
||||
}
|
||||
}
|
||||
|
||||
TabularSerializer serializer = new TabularSerializer() {
|
||||
CellFeed cellFeed = service.getFeed(cellFeedUrl, CellFeed.class);
|
||||
CellFeed batchRequest = null;
|
||||
int row = 0;
|
||||
|
||||
@Override
|
||||
public void startFile(JSONObject options) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endFile() {
|
||||
if (batchRequest != null) {
|
||||
sendBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private void sendBatch() {
|
||||
try {
|
||||
Link batchLink = cellFeed.getLink(Link.Rel.FEED_BATCH, Link.Type.ATOM);
|
||||
CellFeed batchResponse = service.batch(new URL(batchLink.getHref()), batchRequest);
|
||||
|
||||
for (CellEntry entry : batchResponse.getEntries()) {
|
||||
String batchId = BatchUtils.getBatchId(entry);
|
||||
if (!BatchUtils.isSuccess(entry)) {
|
||||
BatchStatus status = BatchUtils.getBatchStatus(entry);
|
||||
logger.warn(
|
||||
String.format(
|
||||
"Error: %s failed (%s) %s\n",
|
||||
batchId, status.getReason(), status.getContent()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
batchRequest = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addRow(List<CellData> cells, boolean isHeader) {
|
||||
if (batchRequest == null) {
|
||||
batchRequest = new CellFeed();
|
||||
}
|
||||
for (int c = 0; c < cells.size(); c++) {
|
||||
CellData cellData = cells.get(c);
|
||||
if (cellData != null && cellData.text != null) {
|
||||
String cellId = String.format("R%sC%s", row + 1, c + 1);
|
||||
|
||||
CellEntry cellEntry = cellEntries[row][c];
|
||||
cellEntry.changeInputValueLocal(cellData.text);
|
||||
if (cellData.link != null) {
|
||||
cellEntry.addHtmlLink(cellData.link, null, cellData.text);
|
||||
}
|
||||
cellEntry.setId(cellId);
|
||||
BatchUtils.setBatchId(cellEntry, cellId);
|
||||
BatchUtils.setBatchOperationType(cellEntry, BatchOperationType.UPDATE);
|
||||
|
||||
batchRequest.getEntries().add(cellEntry);
|
||||
}
|
||||
}
|
||||
row++;
|
||||
if (row % 20 == 0) {
|
||||
sendBatch();
|
||||
}
|
||||
}
|
||||
};
|
||||
SpreadsheetSerializer serializer = new SpreadsheetSerializer(
|
||||
GoogleAPIExtension.getSheetsService(token),
|
||||
spreadsheetId,
|
||||
exceptions);
|
||||
|
||||
CustomizableTabularExporterUtilities.exportRows(
|
||||
project, engine, params, serializer);
|
||||
|
||||
return serializer.getUrl();
|
||||
} catch (IOException e) {
|
||||
logger.error(ExceptionUtils.getStackTrace(e));
|
||||
exceptions.add(e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static private String uploadFusionTable(
|
||||
|
||||
@ -35,16 +35,14 @@ package com.google.refine.commands.project;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.tools.tar.TarOutputStream;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.commands.Command;
|
||||
import com.google.refine.io.FileProjectManager;
|
||||
import com.google.refine.model.Project;
|
||||
|
||||
public class ExportProjectCommand extends Command {
|
||||
@ -61,7 +59,7 @@ public class ExportProjectCommand extends Command {
|
||||
|
||||
OutputStream os = response.getOutputStream();
|
||||
try {
|
||||
gzipTarToOutputStream(project, os);
|
||||
FileProjectManager.gzipTarToOutputStream(project, os);
|
||||
} finally {
|
||||
os.close();
|
||||
}
|
||||
@ -69,24 +67,4 @@ public class ExportProjectCommand extends Command {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
protected void gzipTarToOutputStream(Project project, OutputStream os) throws IOException {
|
||||
GZIPOutputStream gos = new GZIPOutputStream(os);
|
||||
try {
|
||||
tarToOutputStream(project, gos);
|
||||
} finally {
|
||||
gos.close();
|
||||
}
|
||||
}
|
||||
|
||||
protected void tarToOutputStream(Project project, OutputStream os) throws IOException {
|
||||
TarOutputStream tos = new TarOutputStream(os);
|
||||
try {
|
||||
ProjectManager.singleton.exportProject(project.id, tos);
|
||||
} finally {
|
||||
tos.close();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -44,6 +44,7 @@ import java.io.OutputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.Properties;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import org.apache.tools.tar.TarEntry;
|
||||
import org.apache.tools.tar.TarInputStream;
|
||||
@ -472,4 +473,15 @@ public class FileProjectManager extends ProjectManager {
|
||||
public HistoryEntryManager getHistoryEntryManager(){
|
||||
return new FileHistoryEntryManager();
|
||||
}
|
||||
|
||||
public static void gzipTarToOutputStream(Project project, OutputStream os) throws IOException {
|
||||
GZIPOutputStream gos = new GZIPOutputStream(os);
|
||||
TarOutputStream tos = new TarOutputStream(gos);
|
||||
try {
|
||||
ProjectManager.singleton.exportProject(project.id, tos);
|
||||
} finally {
|
||||
tos.close();
|
||||
gos.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -273,7 +273,10 @@
|
||||
"char-enc": "Character encoding",
|
||||
"line-sep": "Line separator",
|
||||
"upload-to": "Upload to",
|
||||
"json-text": "The following JSON text encodes the options you have set in the other tabs. You can copy it out and save it for later, and paste it back in and click Apply to re-use the same options."
|
||||
"json-text": "The following JSON text encodes the options you have set in the other tabs. You can copy it out and save it for later, and paste it back in and click Apply to re-use the same options.",
|
||||
"choose-export-destination": "Please choose the destination for project export",
|
||||
"export-to-local": "Export to local",
|
||||
"export-to-google-drive": "Export to Google Drive"
|
||||
},
|
||||
"core-facets": {
|
||||
"remove-facet": "Remove this facet",
|
||||
|
||||
@ -0,0 +1,20 @@
|
||||
<div class="dialog-frame" style="width: 500px;">
|
||||
<div class="dialog-header" bind="dialogHeader"></div>
|
||||
<div class="dialog-body" bind="dialogBody">
|
||||
|
||||
<div id="choose-export-destination" ><div class="grid-layout grid-layout-for-ui layout-loose layout-full"><table>
|
||||
<tr>
|
||||
<td width="60%"><div class="grid-layout grid-layout-for-text layout-tightest"><table>
|
||||
<tr><td width="1%"><input type="radio" name="export-destination" value="local" checked id="$export-to-local" /></td>
|
||||
<td><label for="$export-to-local" bind="toLocalRadio"></label></td></tr>
|
||||
<tr><td width="1%"><input type="radio" name="export-destination" value="google" id="$export-to-google-drive" /></td>
|
||||
<td><label for="$export-to-google-drive" bind="toGoogleDriveRadio"></label></td></tr>
|
||||
</table></div></td>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="dialog-footer" bind="dialogFooter"><div class="grid-layout layout-tightest layout-full"><table><tr>
|
||||
<td><button class="button" bind="exportButton"></button></td>
|
||||
<td><button class="button" bind="cancelButton"></button></td>
|
||||
</tr></table></div></div>
|
||||
</div>
|
||||
@ -169,6 +169,67 @@ ExporterManager.prepareExportRowsForm = function(format, includeEngine, ext) {
|
||||
|
||||
ExporterManager.handlers.exportProject = function() {
|
||||
var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
|
||||
// dialog
|
||||
var dialog = $(DOM.loadHTML("core", "scripts/dialogs/export-project-dialog.html"));
|
||||
var _elmts = DOM.bind(dialog);
|
||||
|
||||
_elmts.dialogHeader.html($.i18n._('core-dialogs')["choose-export-destination"]);
|
||||
_elmts.toLocalRadio.html($.i18n._('core-dialogs')["export-to-local"]);
|
||||
_elmts.toGoogleDriveRadio.html($.i18n._('core-dialogs')["export-to-google-drive"]);
|
||||
_elmts.exportButton.html($.i18n._('core-buttons')["export"]);
|
||||
_elmts.cancelButton.html($.i18n._('core-buttons')["cancel"]);
|
||||
|
||||
_elmts.exportButton.click(function() {
|
||||
if ($("input[name='export-destination']")[0].checked) {
|
||||
exportToLocal(name);
|
||||
} else {
|
||||
exportToGoogleDrive(name);
|
||||
}
|
||||
|
||||
DialogSystem.dismissAll();
|
||||
});
|
||||
|
||||
_elmts.cancelButton.click(function() { DialogSystem.dismissAll(); });
|
||||
|
||||
DialogSystem.showDialog(dialog);
|
||||
|
||||
// save to google drive
|
||||
var doExportToGoogleDrive = function() {
|
||||
var name = window.prompt(prompt, theProject.metadata.name);
|
||||
if (name) {
|
||||
var dismiss = DialogSystem.showBusy($.i18n._('gdata-exporter')["uploading"]);
|
||||
$.post(
|
||||
"command/gdata/upload",
|
||||
{
|
||||
"project" : theProject.id,
|
||||
"name" : name,
|
||||
"format" : "raw/openrefine-project"
|
||||
},
|
||||
function(o) {
|
||||
dismiss();
|
||||
|
||||
if (o.url) {
|
||||
alert($.i18n._('gdata-exporter')["upload-success"] + o.url);
|
||||
} else {
|
||||
alert($.i18n._('gdata-exporter')["upload-error"] + o.message)
|
||||
}
|
||||
onDone();
|
||||
},
|
||||
"json"
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
function exportToGoogleDrive(name) {
|
||||
if (GdataExtension.isAuthorized()) {
|
||||
doExportToGoogleDrive();
|
||||
} else {
|
||||
GdataExtension.showAuthorizationDialog(doExportToGoogleDrive);
|
||||
}
|
||||
}
|
||||
|
||||
// save to local
|
||||
function exportToLocal(name) {
|
||||
var form = document.createElement("form");
|
||||
$(form)
|
||||
.css("display", "none")
|
||||
@ -186,6 +247,7 @@ ExporterManager.handlers.exportProject = function() {
|
||||
form.submit();
|
||||
|
||||
document.body.removeChild(form);
|
||||
}
|
||||
};
|
||||
|
||||
ExporterManager.handlers.projectDataPackage = function() {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user