From fe5871bd51caeed7c79403645fe10d4c630788b6 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 31 Dec 2019 07:17:41 +0100 Subject: [PATCH 1/2] Remove Fusion Tables support in gdata extension. Closes #2113. --- extensions/gdata/module/MOD-INF/controller.js | 1 - .../gdata/module/langs/translation-en.json | 4 +- .../gdata/module/langs/translation-fr.json | 6 +- .../gdata/module/langs/translation-he.json | 4 +- .../gdata/module/langs/translation-it.json | 4 +- .../gdata/module/langs/translation-jp.json | 2 - .../gdata/module/langs/translation-ko.json | 2 - .../gdata/module/langs/translation-sv.json | 2 - .../gdata-fusion-tables-parsing-panel.html | 39 --- .../scripts/index/importing-controller.js | 17 +- extensions/gdata/pom.xml | 5 - .../extension/gdata/FusionTableHandler.java | 166 ---------- .../extension/gdata/FusionTableImporter.java | 283 ------------------ .../gdata/FusionTableSerializer.java | 115 ------- .../refine/extension/gdata/GDataImporter.java | 9 - .../gdata/GDataImportingController.java | 29 -- .../extension/gdata/GoogleAPIExtension.java | 3 +- .../refine/extension/gdata/UploadCommand.java | 15 - 18 files changed, 12 insertions(+), 694 deletions(-) delete mode 100644 extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html delete mode 100644 extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java delete mode 100644 extensions/gdata/src/com/google/refine/extension/gdata/FusionTableImporter.java delete mode 100644 extensions/gdata/src/com/google/refine/extension/gdata/FusionTableSerializer.java diff --git a/extensions/gdata/module/MOD-INF/controller.js b/extensions/gdata/module/MOD-INF/controller.js index a5ef68568..597afa119 100644 --- a/extensions/gdata/module/MOD-INF/controller.js +++ b/extensions/gdata/module/MOD-INF/controller.js @@ -50,7 +50,6 @@ function init() { var RS = Packages.com.google.refine.RefineServlet; RS.registerCommand(module, "deauthorize", Packages.com.google.refine.extension.gdata.DeAuthorizeCommand()); RS.registerCommand(module, "upload", Packages.com.google.refine.extension.gdata.UploadCommand()); - // TODO: Need a new OAUTH2 authorize command for FusionTables // Register importer and exporter var IM = Packages.com.google.refine.importing.ImportingManager; diff --git a/extensions/gdata/module/langs/translation-en.json b/extensions/gdata/module/langs/translation-en.json index c3ef8c452..64a4abf2e 100644 --- a/extensions/gdata/module/langs/translation-en.json +++ b/extensions/gdata/module/langs/translation-en.json @@ -2,7 +2,7 @@ "gdata-import/preparing": "Preparing …", "gdata-import/creating": "Creating project …", "gdata-import/title": "Public Documents", - "gdata-import/import-by-url": "Import a public Google Spreadsheet or Fusion Table by its URL:", + "gdata-import/import-by-url": "Import a public Google Spreadsheet by its URL:", "gdata-import/next->": "Next »", "gdata-import/auth-doc": "Authorized Documents", "gdata-import/please": "Please", @@ -40,8 +40,6 @@ "gdata-exporter/upload-success": "Project was uploaded successfully to Google Drive with Id ", "gdata-exporter/new-spreadsheet": "A new Google spreadsheet", "gdata-exporter/enter-spreadsheet": "Enter a name for the new Google spreadsheet", - "gdata-exporter/new-fusion": "A new Google Fusion table", - "gdata-exporter/enter-fusion": "Enter a name for the new Google Fusion table", "gdata-auth/authorize-label": "OpenRefine - Authorization", "gdata-auth/authorized-label": "Authorization process completed. Close this window and return to OpenRefine." } diff --git a/extensions/gdata/module/langs/translation-fr.json b/extensions/gdata/module/langs/translation-fr.json index b3e4be830..482e96307 100644 --- a/extensions/gdata/module/langs/translation-fr.json +++ b/extensions/gdata/module/langs/translation-fr.json @@ -2,7 +2,7 @@ "gdata-import/preparing": "Préparation...", "gdata-import/creating": "Création du projet...", "gdata-import/title": "Documents publics", - "gdata-import/import-by-url": "Importer un tableau public Google Spreadsheet ou Google Fusion par son URL :", + "gdata-import/import-by-url": "Importer un tableau public Google Spreadsheet par son URL :", "gdata-import/next->": "Suivant »", "gdata-import/auth-doc": "Documents autorisés", "gdata-import/please": "Merci", @@ -39,9 +39,7 @@ "gdata-exporter/upload-error": "Erreur de téléversement : ", "gdata-exporter/new-spreadsheet": "Un nouveau tableau Google Spreadsheet", "gdata-exporter/enter-spreadsheet": "Indiquer un nom pour le nouveau tableau Google Spreadsheet", - "gdata-exporter/new-fusion": "Une nouvelle table Google Fusion", - "gdata-exporter/enter-fusion": "Indiquer un nom pour la nouvelle table Google Fusion", "gdata-exporter/upload-success": "Le projet a été chargé avec succès dans Google Drive avec l'identifiant ", "gdata-auth/authorize-label": "OpenRefine - Autorisation", "gdata-auth/authorized-label": "Procédure d’autorisation terminée. Fermer cette fenêtre et revenir à OpenRefine." -} \ No newline at end of file +} diff --git a/extensions/gdata/module/langs/translation-he.json b/extensions/gdata/module/langs/translation-he.json index a2bc75cde..b078555aa 100644 --- a/extensions/gdata/module/langs/translation-he.json +++ b/extensions/gdata/module/langs/translation-he.json @@ -39,8 +39,6 @@ "gdata-exporter/upload-error": "שגיאת העלאה: ", "gdata-exporter/new-spreadsheet": "גיליון חישוב חדש של גוגל", "gdata-exporter/enter-spreadsheet": "הכנס שם לגיליון חישוב גוגל החדש", - "gdata-exporter/new-fusion": "טבלת מיזוג גוגל חדשה", - "gdata-exporter/enter-fusion": "הכנס שם לטבלת מיזוג גוגל חדשה", "gdata-auth/authorize-label": "OpenRefine - Authorization", "gdata-auth/authorized-label": "תהליך האישור הסתיים. סגור חלון זה וחזור ל- OpenRefine." -} \ No newline at end of file +} diff --git a/extensions/gdata/module/langs/translation-it.json b/extensions/gdata/module/langs/translation-it.json index f32724964..364f292ce 100644 --- a/extensions/gdata/module/langs/translation-it.json +++ b/extensions/gdata/module/langs/translation-it.json @@ -2,7 +2,7 @@ "gdata-import/preparing": "In preparazione ...", "gdata-import/creating": "Creazione il progetto ...", "gdata-import/title": "Documenti Pubblici", - "gdata-import/import-by-url": "Importa un Google Spreadsheet o Fusion Table pubblico inserendo l'URL:", + "gdata-import/import-by-url": "Importa un Google Spreadsheet pubblico inserendo l'URL:", "gdata-import/next->": "Avanti »", "gdata-import/auth-doc": "Documenti Autorizzati", "gdata-import/please": "Per piacere", @@ -39,8 +39,6 @@ "gdata-exporter/upload-error": "Errore durante il caricamento: ", "gdata-exporter/new-spreadsheet": "Un nuovo Google spreadsheet", "gdata-exporter/enter-spreadsheet": "Inserisci un nome per il nuovo Google spreadsheet", - "gdata-exporter/new-fusion": "Una nuova tabella Google Fusion", - "gdata-exporter/enter-fusion": "Inserisci un nome per la nuova tabella Google Fusion", "gdata-auth/authorize-label": "OpenRefine - Autorizzazione", "gdata-auth/authorized-label": "Processo di autorizzazione completato. Chiudi questa finestra e torna ad OpenRefine.", "gdata-exporter/upload-success": "Il progetto è stato caricato con successo su Google Drive con id " diff --git a/extensions/gdata/module/langs/translation-jp.json b/extensions/gdata/module/langs/translation-jp.json index df8f62a28..c773517e7 100644 --- a/extensions/gdata/module/langs/translation-jp.json +++ b/extensions/gdata/module/langs/translation-jp.json @@ -39,8 +39,6 @@ "gdata-exporter/upload-error": "アップロードでエラー : ", "gdata-exporter/new-spreadsheet": "新しいGoogle spreadsheet", "gdata-exporter/enter-spreadsheet": "新しいGoogle spreadsheetの題名を入力してください", - "gdata-exporter/new-fusion": "新しいFusion table", - "gdata-exporter/enter-fusion": "新しいFusion tableの題名を入力してください", "gdata-exporter/upload-success": "プロジェクトがGoogle Driveにアップロードされました ", "gdata-auth/authorize-label": "OpenRefine - 認証", "gdata-auth/authorized-label": "認証が完了しました。このウインドウを閉じてOpenRefineで作業できます." diff --git a/extensions/gdata/module/langs/translation-ko.json b/extensions/gdata/module/langs/translation-ko.json index a6fe75edc..fec6dc8fb 100644 --- a/extensions/gdata/module/langs/translation-ko.json +++ b/extensions/gdata/module/langs/translation-ko.json @@ -40,8 +40,6 @@ "gdata-exporter/upload-success": "다음 Id로 Google Drive에 프로젝트가 성공적으로 업로드 되었습니다 ", "gdata-exporter/new-spreadsheet": "새로운 Google speadsheet", "gdata-exporter/enter-spreadsheet": "새로운 Google Spreadsheet 이름을 입력하세요", - "gdata-exporter/new-fusion": "새로운 Google Fusion 테이블", - "gdata-exporter/enter-fusion": "새로운 Google Fusion 테이블을 위한 이름을 입력하세요", "gdata-auth/authorize-label": "OpenRefine - 검증", "gdata-auth/authorized-label": "검증 프로세스 완료. 이 창을 닫고 OpenRefine으로 돌아가세요." } diff --git a/extensions/gdata/module/langs/translation-sv.json b/extensions/gdata/module/langs/translation-sv.json index e69e37cd7..5a9945428 100644 --- a/extensions/gdata/module/langs/translation-sv.json +++ b/extensions/gdata/module/langs/translation-sv.json @@ -40,8 +40,6 @@ "gdata-exporter/upload-success": "Projektet laddades upp till Google Drive med ID ", "gdata-exporter/new-spreadsheet": "Ett nytt Google-kalkylblad", "gdata-exporter/enter-spreadsheet": "Ange ett namn för det nya Google-kalkylbladet", - "gdata-exporter/new-fusion": "En ny Google Fusion-tabell", - "gdata-exporter/enter-fusion": "Ange ett namn för den nya Google Fusion-tabellen", "gdata-auth/authorize-label": "OpenRefine - godkännande", "gdata-auth/authorized-label": "Godkännandeprocessen avslutad. Stäng det här fönstret och återgå till OpenRefine." } diff --git a/extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html b/extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html deleted file mode 100644 index 2e5b018f5..000000000 --- a/extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html +++ /dev/null @@ -1,39 +0,0 @@ -
- - - - - -
- -
- -
- -
- -
- - - - - - - -
- - - - - - - - - - - - - -
-
-
\ No newline at end of file diff --git a/extensions/gdata/module/scripts/index/importing-controller.js b/extensions/gdata/module/scripts/index/importing-controller.js index d8fbde184..653997683 100644 --- a/extensions/gdata/module/scripts/index/importing-controller.js +++ b/extensions/gdata/module/scripts/index/importing-controller.js @@ -163,19 +163,14 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() { var self = this; this._parsingPanel.unbind().empty().html( - DOM.loadHTML("gdata", - this._doc.type == 'table' ? - 'scripts/index/gdata-fusion-tables-parsing-panel.html' : - 'scripts/index/gdata-parsing-panel.html')); + DOM.loadHTML("gdata", 'scripts/index/gdata-parsing-panel.html')); this._parsingPanelElmts = DOM.bind(this._parsingPanel); - if(this._doc.type != 'table'){ - this._parsingPanelElmts.gdata_worksheet.html($.i18n('gdata-parsing/worksheet')); - this._parsingPanelElmts.gdata_ignore_first.html($.i18n('gdata-parsing/ignore-first')); - this._parsingPanelElmts.gdata_ignore.html($.i18n('gdata-parsing/ignore')); - this._parsingPanelElmts.gdata_parse_next.html($.i18n('gdata-parsing/parse-next')); - this._parsingPanelElmts.gdata_parse.html($.i18n('gdata-parsing/parse')); - } + this._parsingPanelElmts.gdata_worksheet.html($.i18n('gdata-parsing/worksheet')); + this._parsingPanelElmts.gdata_ignore_first.html($.i18n('gdata-parsing/ignore-first')); + this._parsingPanelElmts.gdata_ignore.html($.i18n('gdata-parsing/ignore')); + this._parsingPanelElmts.gdata_parse_next.html($.i18n('gdata-parsing/parse-next')); + this._parsingPanelElmts.gdata_parse.html($.i18n('gdata-parsing/parse')); this._parsingPanelElmts.startOverButton.html($.i18n('gdata-parsing/start-over')); this._parsingPanelElmts.gdata_conf_pars.html($.i18n('gdata-parsing/conf-pars')); this._parsingPanelElmts.gdata_proj_name.html($.i18n('gdata-parsing/proj-name')); diff --git a/extensions/gdata/pom.xml b/extensions/gdata/pom.xml index 934720a9c..8f49a026e 100644 --- a/extensions/gdata/pom.xml +++ b/extensions/gdata/pom.xml @@ -136,11 +136,6 @@ google-oauth-client-jetty 1.23.0 - - com.google.apis - google-api-services-fusiontables - v2-rev21-1.23.0 - com.google.apis google-api-services-drive diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java b/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java deleted file mode 100644 index c7f219f78..000000000 --- a/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2013, Thomas F. Morris and other contributors - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.gdata; - -import java.io.IOException; -import java.net.URL; -import java.util.List; - -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.http.AbstractInputStreamContent; -import com.google.api.services.fusiontables.Fusiontables; -import com.google.api.services.fusiontables.Fusiontables.Query.Sql; -import com.google.api.services.fusiontables.Fusiontables.Query.SqlGet; -import com.google.api.services.fusiontables.Fusiontables.Table.ImportRows; -import com.google.api.services.fusiontables.model.FusiontablesImport; -import com.google.api.services.fusiontables.model.Sqlresponse; - -/** - * @author Tom Morris - * @copyright 2010,2013 Thomas F. Morris - */ - -public class FusionTableHandler { - - static private Sqlresponse executeQuery (Fusiontables service, String query) - throws IOException { - Sql sql = service.query().sql(query); - Sqlresponse response = sql.execute(); - return response; - } - - static String createTable(Fusiontables service, String name, List columnNames) throws IOException { - StringBuffer sb = new StringBuffer(); - sb.append("CREATE TABLE '"); - sb.append(name); - sb.append("' ("); - boolean first = true; - for (String columnName : columnNames) { - if (first) { - first = false; - } else { - sb.append(','); - } - sb.append("'"); - sb.append(columnName); - sb.append("': STRING"); - } - sb.append(")"); - - String createQuery = sb.toString(); - - Sqlresponse response = executeQuery(service, createQuery); -// response.getTableId(); // FIXME: Oh wait, there's no such F*ing method!!! - return getTableId(response); - } - - private static String getTableId(Sqlresponse response) { - List row = response.getRows().get(0); - int i = 0; - for (String colname : response.getColumns()) { - if ("tableid".equals(colname)) { - return (String) row.get(i); - } - } - return null; - } - - - /** - * Insert a set of rows and optionally return the IDs of the new rows. - * - * @param service a Fusiontables object - * @param sql SQL statement to do the inserts - * @param returnIds true to return the IDs of the newly inserted rows - * @return - * @throws IOException - */ - static Long insertRows(Fusiontables service, String tableId, AbstractInputStreamContent mediaContent) throws IOException { - ImportRows importRows = service.table().importRows(tableId, mediaContent); - importRows.setIsStrict(false); - FusiontablesImport response = importRows.execute(); - return response.getNumRowsReceived(); - } - - - static String getFusionTableKey(URL url) { - String tableId = getParamValue(url,"dsrcid"); // old style phased out - if (tableId == null || tableId.isEmpty()) { - tableId = getParamValue(url,"docid"); - } - return tableId; - } - - static public Fusiontables getFusionTablesService(String token) { - Credential credential = new GoogleCredential().setAccessToken(token); - Fusiontables fusiontables = new Fusiontables.Builder( - GoogleAPIExtension.HTTP_TRANSPORT, GoogleAPIExtension.JSON_FACTORY, credential) - .setApplicationName(GoogleAPIExtension.SERVICE_APP_NAME) - .build();; - - return fusiontables; - } - - static boolean isFusionTableURL(URL url) { - // http://www.google.com/fusiontables/DataSource?dsrcid=1219 - String query = url.getQuery(); - if (query == null) { - query = ""; - } - return url.getHost().endsWith(".google.com") - && url.getPath().startsWith("/fusiontables/DataSource") - && (query.contains("dsrcid=")||query.contains("docid=")); - } - - static Sqlresponse runFusionTablesSelect(Fusiontables service, String selectQuery) - throws IOException { - - // FIXME: alt=csv doesn't actually work! It will attempt to parse response as JSON and die - // perhaps use .executeUnparsed() would work? - SqlGet query = service.query().sqlGet(selectQuery);//.setAlt("csv"); - Sqlresponse response = query.execute(); - return response; - } - - static private String getParamValue(URL url, String key) { - String query = url.getQuery(); - if (query != null) { - String[] parts = query.split("&"); - for (String part : parts) { - if (part.startsWith(key+"=")) { - int offset = key.length()+1; - String tableId = part.substring(offset); - return tableId; - } - } - } - return null; - } -} diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableImporter.java b/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableImporter.java deleted file mode 100644 index 998e567fb..000000000 --- a/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableImporter.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Copyright (c) 2010,2013 Thomas F. Morris and other contributors - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of Google nor the names of its contributors may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package com.google.refine.extension.gdata; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.api.services.fusiontables.Fusiontables; -import com.google.api.services.fusiontables.model.Column; -import com.google.api.services.fusiontables.model.Sqlresponse; -import com.google.api.services.fusiontables.model.Table; - -import com.google.refine.ProjectMetadata; -import com.google.refine.importers.TabularImportingParserBase; -import com.google.refine.importers.TabularImportingParserBase.TableDataReader; -import com.google.refine.importing.ImportingJob; -import com.google.refine.model.Project; -import com.google.refine.util.JSONUtilities; - -/** - * OpenRefine parser for Google Spreadsheets. - * - * @author Tom Morris - * @copyright 2010 Thomas F. Morris - * @license New BSD http://www.opensource.org/licenses/bsd-license.php - */ -public class FusionTableImporter { - static public void parse( - String token, - Project project, - ProjectMetadata metadata, - final ImportingJob job, - int limit, - ObjectNode options, - List exceptions) { - - Fusiontables service = FusionTableHandler.getFusionTablesService(token); - parse( - service, - project, - metadata, - job, - limit, - options, - exceptions - ); - - } - - static void setProgress(ImportingJob job, String fileSource, int percent) { - job.setProgress(percent, "Reading " + fileSource); - } - - - static private class FusionTableBatchRowReader implements TableDataReader { - final ImportingJob job; - final String fileSource; - - final Fusiontables service; - final List columns; - final int batchSize; - - final String baseQuery; - - int nextRow = 0; // 0-based - int batchRowStart = 0; // 0-based - boolean end = false; - List> rowsOfCells = null; - boolean usedHeaders = false; - - public FusionTableBatchRowReader(ImportingJob job, String fileSource, - Fusiontables service, String tableId, List columns, - int batchSize) { - this.job = job; - this.fileSource = fileSource; - this.service = service; - this.columns = columns; - this.batchSize = batchSize; - - StringBuffer sb = new StringBuffer(); - sb.append("SELECT "); - - boolean first = true; - for (FTColumnData cd : columns) { - if (first) { - first = false; - } else { - sb.append(","); - } - sb.append("'"); - sb.append(cd.name); - sb.append("'"); - } - sb.append(" FROM "); - sb.append(tableId); - - baseQuery = sb.toString(); - } - - @Override - public List getNextRowOfCells() throws IOException { - if (!usedHeaders) { - List row = new ArrayList(columns.size()); - for (FTColumnData cd : columns) { - row.add(cd.name); - } - usedHeaders = true; - return row; - } - - if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) { - int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size()); - rowsOfCells = getRowsOfCells(newBatchRowStart); - batchRowStart = newBatchRowStart; - - setProgress(job, fileSource, -1 /* batchRowStart * 100 / totalRows */); - } - - if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) { - return rowsOfCells.get(nextRow++ - batchRowStart); - } else { - return null; - } - } - - - private List> getRowsOfCells(int startRow) throws IOException { - List> rowsOfCells = new ArrayList>(batchSize); - - String query = baseQuery + " OFFSET " + startRow + " LIMIT " + batchSize; - - Sqlresponse sqlresponse = FusionTableHandler.runFusionTablesSelect(service, query); - List> rows = sqlresponse.getRows(); - if (rows.size() > 1) { - for (int i = 1; i < rows.size(); i++) { - List row = rows.get(i); - List rowOfCells = new ArrayList(row.size()); - for (int j = 0; j < row.size() && j < columns.size(); j++) { - String text = String.valueOf(row.get(j)); - if (text.isEmpty()) { - rowOfCells.add(null); - } else { - FTColumnData cd = columns.get(j); - if (cd.type == FTColumnType.NUMBER) { - try { - rowOfCells.add(Long.parseLong(text)); - continue; - } catch (NumberFormatException e) { - // ignore - } - try { - double d = Double.parseDouble(text); - if (!Double.isInfinite(d) && !Double.isNaN(d)) { - rowOfCells.add(d); - continue; - } - } catch (NumberFormatException e) { - // ignore - } - } - - rowOfCells.add(text); - } - } - rowsOfCells.add(rowOfCells); - } - } - end = rows.size() < batchSize + 1; - return rowsOfCells; - } - - } - - static public void parse( - Fusiontables service, - Project project, - ProjectMetadata metadata, - final ImportingJob job, - int limit, - ObjectNode options, - List exceptions) { - - String docUrlString = JSONUtilities.getString(options, "docUrl", null); - String id = getFTid(docUrlString); // Use GDataExtension.getFusionTableKey(url) ? - // TODO: Allow arbitrary Fusion Tables URL instead of (in addition to?) constructing our own? - - try { - List columns = new ArrayList(); - Table table = service.table().get(id).execute(); - for (Column col : table.getColumns()) { - FTColumnData cd = new FTColumnData(); - cd.name = col.getName(); - String type = col.getType(); - if (type.equals("STRING")) { - cd.type = FTColumnType.STRING; - } else if (type.equals("NUMBER")) { - cd.type = FTColumnType.NUMBER; - } else if (type.equals("DATETIME")) { - cd.type = FTColumnType.DATETIME; - } else if (type.equals("LOCATION")) { - cd.type = FTColumnType.LOCATION; - } else { - // TODO: unknown type - cd.type = FTColumnType.STRING; - } - columns.add(cd); - } - - setProgress(job, docUrlString, -1); - - // Force these options for the next call because each fusion table - // is strictly structured with a single line of headers. - JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore - JSONUtilities.safePut(options, "headerLines", 1); // number of header lines - - TabularImportingParserBase.readTable( - project, - metadata, - job, - new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100), - docUrlString, - limit, - options, - exceptions - ); - setProgress(job, docUrlString, 100); - } catch (IOException e) { - e.printStackTrace(); - exceptions.add(e); - } - } - - static private String getFTid(String url) { - if (url == null) { - return null; - } - int equal = url.lastIndexOf('='); - if (equal < 0) { - return null; - } - return url.substring(equal + 1); - } - - static enum FTColumnType { - STRING, - NUMBER, - DATETIME, - LOCATION - } - - final static class FTColumnData { - String name; - FTColumnType type; - } -} \ No newline at end of file diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableSerializer.java b/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableSerializer.java deleted file mode 100644 index 86e6cef90..000000000 --- a/extensions/gdata/src/com/google/refine/extension/gdata/FusionTableSerializer.java +++ /dev/null @@ -1,115 +0,0 @@ -package com.google.refine.extension.gdata; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.api.client.http.AbstractInputStreamContent; -import com.google.api.client.http.ByteArrayContent; -import com.google.api.client.http.HttpResponseException; -import com.google.api.services.fusiontables.Fusiontables; -import com.google.refine.exporters.TabularSerializer; - -final class FusionTableSerializer implements TabularSerializer { - private static final int BATCH_SIZE = 20; - Fusiontables service; - String tableName; - List exceptions; - - String tableId; - List columnNames; - StringBuffer sbBatch; - int rows; - - FusionTableSerializer(Fusiontables service, String tableName, List exceptions) { - this.service = service; - this.tableName = tableName; - this.exceptions = exceptions; - } - - @Override - public void startFile(JsonNode options) { - } - - @Override - public void endFile() { - if (sbBatch != null) { - sendBatch(rows % BATCH_SIZE); - } - } - - @Override - public void addRow(List cells, boolean isHeader) { - if (isHeader) { - columnNames = new ArrayList(cells.size()); - for (CellData cellData : cells) { - columnNames.add(cellData.text); - } - try { - tableId = FusionTableHandler.createTable(service, tableName, columnNames); - } catch (Exception e) { - tableId = null; - exceptions.add(e); - } - } else if (tableId != null) { - if (sbBatch == null) { - sbBatch = new StringBuffer(); - } - formatCsv(cells, sbBatch); - rows++; - if (rows % BATCH_SIZE == 0) { - if (!sendBatch(BATCH_SIZE)) { - return; - } - } - } - } - - private boolean sendBatch(int batchSize) { - try { - // TODO: we really want to do GZIP compression here - // FIXME: text/csv doesn't work even though that's what the content is - AbstractInputStreamContent content = ByteArrayContent.fromString("application/octet-stream", sbBatch.toString()); - Long count = FusionTableHandler.insertRows(service, tableId, content); - if (count != null && count.intValue() != batchSize) { - exceptions.add(new IOException("only imported " + count + " of " + batchSize + " rows")); - } - } catch (IOException e) { - exceptions.add(e); - if (e instanceof HttpResponseException) { - int code = ((HttpResponseException)e).getStatusCode(); - if (code >= 400 && code < 500) { - return false; - } - // 500s appear to be retried automatically by li - } - } finally { - sbBatch = null; - } - return true; - } - - private void formatCsv(List cells, StringBuffer sb) { - boolean first = true; - for (int i = 0; i < cells.size() && i < columnNames.size(); i++) { - CellData cellData = cells.get(i); - if (!first) { - sb.append(','); - } else { - first = false; - } - sb.append("\""); - if (cellData != null && cellData.text != null) { - sb.append(cellData.text.replaceAll("\"", "\"\"")); - } - sb.append("\""); - } - sb.append("\n"); - } - - public String getUrl() { - return tableId == null || exceptions.size() > 0 ? null : - "https://www.google.com/fusiontables/DataSource?docid=" + tableId; - } -} diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java index 7238bbd17..17e723ca7 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java @@ -45,15 +45,6 @@ public class GDataImporter { options, exceptions ); - } else if ("table".equals(docType)) { - FusionTableImporter.parse(token, - project, - metadata, - job, - limit, - options, - exceptions - ); } } diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java index 8034d6fc6..8c258504e 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java @@ -21,9 +21,6 @@ import com.google.api.services.drive.Drive; import com.google.api.services.drive.model.File; import com.google.api.services.drive.model.FileList; import com.google.api.services.drive.model.User; -import com.google.api.services.fusiontables.Fusiontables; -import com.google.api.services.fusiontables.model.Table; -import com.google.api.services.fusiontables.model.TableList; import com.google.api.services.sheets.v4.Sheets; import com.google.api.services.sheets.v4.model.Sheet; import com.google.api.services.sheets.v4.model.Spreadsheet; @@ -91,7 +88,6 @@ public class GDataImportingController implements ImportingController { try { listSpreadsheets(GoogleAPIExtension.getDriveService(token), writer); - listFusionTables(FusionTableHandler.getFusionTablesService(token), writer); } catch (Exception e) { logger.error("doListDocuments exception:" + ExceptionUtils.getStackTrace(e)); } finally { @@ -139,31 +135,6 @@ public class GDataImportingController implements ImportingController { } } - private void listFusionTables(Fusiontables service, JsonGenerator writer) - throws IOException { - - Fusiontables.Table.List listTables = service.table().list(); - TableList tablelist = listTables.execute(); - - if (tablelist == null || tablelist.getItems() == null) - return; - - for (Table table : tablelist.getItems()) { - String id = table.getTableId(); - String name = table.getName(); - String link = "https://www.google.com/fusiontables/DataSource?docid=" + id; - - // Add JSON object to our stream - writer.writeStartObject(); - writer.writeStringField("docId", id); - writer.writeStringField("docLink", link); - writer.writeStringField("docSelfLink", link); - writer.writeStringField("title", name); - writer.writeStringField("type", "table"); - writer.writeEndObject(); - } - } - private void doInitializeParserUI( HttpServletRequest request, HttpServletResponse response, Properties parameters) throws ServletException, IOException { diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java b/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java index a7bf3def0..52b4d87f4 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java @@ -22,7 +22,6 @@ import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.drive.Drive; import com.google.api.services.drive.DriveScopes; -import com.google.api.services.fusiontables.FusiontablesScopes; import com.google.api.services.sheets.v4.Sheets; import com.google.api.services.sheets.v4.SheetsScopes; import com.google.refine.ProjectManager; @@ -42,7 +41,7 @@ abstract public class GoogleAPIExtension { /** Global instance of the JSON factory. */ protected static final JsonFactory JSON_FACTORY = new JacksonFactory(); - private static final String[] SCOPES = {DriveScopes.DRIVE, SheetsScopes.SPREADSHEETS, FusiontablesScopes.FUSIONTABLES}; + private static final String[] SCOPES = {DriveScopes.DRIVE, SheetsScopes.SPREADSHEETS}; private static PreferenceStore prefStore = ProjectManager.singleton.getPreferenceStore(); diff --git a/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java b/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java index 122b6f47b..0844ed9f4 100644 --- a/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java +++ b/extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java @@ -108,8 +108,6 @@ public class UploadCommand extends Command { String format = params.getProperty("format"); if ("gdata/google-spreadsheet".equals(format)) { return uploadSpreadsheet(project, engine, params, token, name, exceptions); - } else if ("gdata/fusion-table".equals(format)) { - return uploadFusionTable(project, engine, params, token, name, exceptions); } else if (("raw/openrefine-project").equals(format)) { return uploadOpenRefineProject(project, token, name, exceptions); } @@ -199,17 +197,4 @@ public class UploadCommand extends Command { } return null; } - - static private String uploadFusionTable( - Project project, final Engine engine, final Properties params, - String token, String name, List exceptions) { - - FusionTableSerializer serializer = new FusionTableSerializer( - FusionTableHandler.getFusionTablesService(token), name, exceptions); - - CustomizableTabularExporterUtilities.exportRows( - project, engine, params, serializer); - - return serializer.getUrl(); - } } From 6dc79d965d16e4abcbfbaf080930602a6a608080 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 31 Dec 2019 10:21:07 +0100 Subject: [PATCH 2/2] Use state to keep redirect URI constant --- extensions/gdata/module/MOD-INF/controller.js | 3 +-- extensions/gdata/module/authorized.vt | 6 +++-- .../extension/gdata/GoogleAPIExtension.java | 24 ++++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/extensions/gdata/module/MOD-INF/controller.js b/extensions/gdata/module/MOD-INF/controller.js index 597afa119..6fa5a13b9 100644 --- a/extensions/gdata/module/MOD-INF/controller.js +++ b/extensions/gdata/module/MOD-INF/controller.js @@ -102,8 +102,7 @@ function process(path, request, response) { send(request, response, "authorize.vt", context); } else if (path == "authorized") { var context = {}; - context.winname = request.getParameter("winname"); - context.callback = request.getParameter("cb"); + context.state = request.getParameter("state"); (function() { var tokenAndExpiresInSeconds = Packages.com.google.refine.extension.gdata.GoogleAPIExtension.getTokenFromCode(module,request); diff --git a/extensions/gdata/module/authorized.vt b/extensions/gdata/module/authorized.vt index 69c891bf0..67ed3f571 100644 --- a/extensions/gdata/module/authorized.vt +++ b/extensions/gdata/module/authorized.vt @@ -40,8 +40,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.