s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove Fusion Tables support in gdata extension. Closes #2113.

Browse Source
This commit is contained in:
Antonin Delpeuch 2019-12-31 07:17:41 +01:00
parent 14dd4c0112
commit fe5871bd51
18 changed files with 12 additions and 694 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/gdata/module/MOD-INF/controller.js
View File

@ -50,7 +50,6 @@ function init() {
var RS = Packages.com.google.refine.RefineServlet;
RS.registerCommand(module, "deauthorize", Packages.com.google.refine.extension.gdata.DeAuthorizeCommand());
RS.registerCommand(module, "upload", Packages.com.google.refine.extension.gdata.UploadCommand());
// TODO: Need a new OAUTH2 authorize command for FusionTables
// Register importer and exporter
var IM = Packages.com.google.refine.importing.ImportingManager;

4
extensions/gdata/module/langs/translation-en.json
View File

@ -2,7 +2,7 @@
"gdata-import/preparing": "Preparing …",
"gdata-import/creating": "Creating project …",
"gdata-import/title": "Public Documents",
"gdata-import/import-by-url": "Import a <em>public</em> Google Spreadsheet or Fusion Table by its URL:",
"gdata-import/import-by-url": "Import a <em>public</em> Google Spreadsheet by its URL:",
"gdata-import/next->": "Next &raquo;",
"gdata-import/auth-doc": "Authorized Documents",
"gdata-import/please": "Please",
@ -40,8 +40,6 @@
"gdata-exporter/upload-success": "Project was uploaded successfully to Google Drive with Id ",
"gdata-exporter/new-spreadsheet": "A new Google spreadsheet",
"gdata-exporter/enter-spreadsheet": "Enter a name for the new Google spreadsheet",
"gdata-exporter/new-fusion": "A new Google Fusion table",
"gdata-exporter/enter-fusion": "Enter a name for the new Google Fusion table",
"gdata-auth/authorize-label": "OpenRefine - Authorization",
"gdata-auth/authorized-label": "Authorization process completed. Close this window and return to OpenRefine."
}

4
extensions/gdata/module/langs/translation-fr.json
View File

@ -2,7 +2,7 @@
"gdata-import/preparing": "Préparation...",
"gdata-import/creating": "Création du projet...",
"gdata-import/title": "Documents publics",
"gdata-import/import-by-url": "Importer un tableau <em>public</em> Google Spreadsheet ou Google Fusion par son URL :",
"gdata-import/import-by-url": "Importer un tableau <em>public</em> Google Spreadsheet par son URL :",
"gdata-import/next->": "Suivant &raquo;",
"gdata-import/auth-doc": "Documents autorisés",
"gdata-import/please": "Merci",
@ -39,8 +39,6 @@
"gdata-exporter/upload-error": "Erreur de téléversement : ",
"gdata-exporter/new-spreadsheet": "Un nouveau tableau Google Spreadsheet",
"gdata-exporter/enter-spreadsheet": "Indiquer un nom pour le nouveau tableau Google Spreadsheet",
"gdata-exporter/new-fusion": "Une nouvelle table Google Fusion",
"gdata-exporter/enter-fusion": "Indiquer un nom pour la nouvelle table Google Fusion",
"gdata-exporter/upload-success": "Le projet a été chargé avec succès dans Google Drive avec l'identifiant ",
"gdata-auth/authorize-label": "OpenRefine - Autorisation",
"gdata-auth/authorized-label": "Procédure d’autorisation terminée. Fermer cette fenêtre et revenir à OpenRefine."

2
extensions/gdata/module/langs/translation-he.json
View File

@ -39,8 +39,6 @@
"gdata-exporter/upload-error": "שגיאת העלאה: ",
"gdata-exporter/new-spreadsheet": "גיליון חישוב חדש של גוגל",
"gdata-exporter/enter-spreadsheet": "הכנס שם לגיליון חישוב גוגל החדש",
"gdata-exporter/new-fusion": "טבלת מיזוג גוגל חדשה",
"gdata-exporter/enter-fusion": "הכנס שם לטבלת מיזוג גוגל חדשה",
"gdata-auth/authorize-label": "OpenRefine - Authorization",
"gdata-auth/authorized-label": "תהליך האישור הסתיים. סגור חלון זה וחזור ל- OpenRefine."
}

4
extensions/gdata/module/langs/translation-it.json
View File

@ -2,7 +2,7 @@
"gdata-import/preparing": "In preparazione ...",
"gdata-import/creating": "Creazione il progetto ...",
"gdata-import/title": "Documenti Pubblici",
"gdata-import/import-by-url": "Importa un Google Spreadsheet o Fusion Table <em>pubblico</em> inserendo l'URL:",
"gdata-import/import-by-url": "Importa un Google Spreadsheet <em>pubblico</em> inserendo l'URL:",
"gdata-import/next->": "Avanti &raquo;",
"gdata-import/auth-doc": "Documenti Autorizzati",
"gdata-import/please": "Per piacere",
@ -39,8 +39,6 @@
"gdata-exporter/upload-error": "Errore durante il caricamento: ",
"gdata-exporter/new-spreadsheet": "Un nuovo Google spreadsheet",
"gdata-exporter/enter-spreadsheet": "Inserisci un nome per il nuovo Google spreadsheet",
"gdata-exporter/new-fusion": "Una nuova tabella Google Fusion",
"gdata-exporter/enter-fusion": "Inserisci un nome per la nuova tabella Google Fusion",
"gdata-auth/authorize-label": "OpenRefine - Autorizzazione",
"gdata-auth/authorized-label": "Processo di autorizzazione completato. Chiudi questa finestra e torna ad OpenRefine.",
"gdata-exporter/upload-success": "Il progetto è stato caricato con successo su Google Drive con id "

2
extensions/gdata/module/langs/translation-jp.json
View File

@ -39,8 +39,6 @@
"gdata-exporter/upload-error": "アップロードでエラー : ",
"gdata-exporter/new-spreadsheet": "新しいGoogle spreadsheet",
"gdata-exporter/enter-spreadsheet": "新しいGoogle spreadsheetの題名を入力してください",
"gdata-exporter/new-fusion": "新しいFusion table",
"gdata-exporter/enter-fusion": "新しいFusion tableの題名を入力してください",
"gdata-exporter/upload-success": "プロジェクトがGoogle Driveにアップロードされました ",
"gdata-auth/authorize-label": "OpenRefine - 認証",
"gdata-auth/authorized-label": "認証が完了しました。このウインドウを閉じてOpenRefineで作業できます."

2
extensions/gdata/module/langs/translation-ko.json
View File

@ -40,8 +40,6 @@
"gdata-exporter/upload-success": "다음 Id로 Google Drive에 프로젝트가 성공적으로 업로드 되었습니다 ",
"gdata-exporter/new-spreadsheet": "새로운 Google speadsheet",
"gdata-exporter/enter-spreadsheet": "새로운 Google Spreadsheet 이름을 입력하세요",
"gdata-exporter/new-fusion": "새로운 Google Fusion 테이블",
"gdata-exporter/enter-fusion": "새로운 Google Fusion 테이블을 위한 이름을 입력하세요",
"gdata-auth/authorize-label": "OpenRefine - 검증",
"gdata-auth/authorized-label": "검증 프로세스 완료. 이 창을 닫고 OpenRefine으로 돌아가세요."
}

2
extensions/gdata/module/langs/translation-sv.json
View File

@ -40,8 +40,6 @@
"gdata-exporter/upload-success": "Projektet laddades upp till Google Drive med ID ",
"gdata-exporter/new-spreadsheet": "Ett nytt Google-kalkylblad",
"gdata-exporter/enter-spreadsheet": "Ange ett namn för det nya Google-kalkylbladet",
"gdata-exporter/new-fusion": "En ny Google Fusion-tabell",
"gdata-exporter/enter-fusion": "Ange ett namn för den nya Google Fusion-tabellen",
"gdata-auth/authorize-label": "OpenRefine - godkännande",
"gdata-auth/authorized-label": "Godkännandeprocessen avslutad. Stäng det här fönstret och återgå till OpenRefine."
}

39
extensions/gdata/module/scripts/index/gdata-fusion-tables-parsing-panel.html
View File

@ -1,39 +0,0 @@
<div bind="wizardHeader" class="gdata-importing-wizard-header"><div class="grid-layout layout-tightest layout-full"><table><tr>
<td width="1%"><button bind="startOverButton" class="button">&laquo; Start Over</button></td>
<td width="98%" bind="gdata_conf_pars"></td>
<td style="text-align: right;" bind="gdata_proj_name"></td>
<td width="1%"><input class="inline" type="text" size="30" bind="projectNameInput" /></td>
<td width="1%"><button bind="createProjectButton" class="button button-primary"></button></td>
</tr></table></div></div>
<div bind="dataPanel" class="gdata-importing-parsing-data-panel"></div>
<div bind="progressPanel" class="gdata-importing-progress-data-panel">
<img src="images/large-spinner.gif" /> <span bind="gdata_updating"></span>
</div>
<div bind="controlPanel" class="gdata-importing-parsing-control-panel"><div class="grid-layout layout-normal"><table>
<tr>
<td bind="gdata_options"></td>
<td><button class="button" bind="previewButton"></button></td>
</tr>
<tr>
<td><div class="grid-layout layout-tightest"><table>
<tr><td width="1%"><input type="checkbox" bind="skipCheckbox" id="$skip"/></td>
<td><label for="$skip" bind="gdata_discard_next"></label></td>
<td><input bind="skipInput" type="text" class="lightweight" size="2" value="0" />
<label for="$skip" bind="gdata_discard"></label></td></tr>
<tr><td width="1%"><input type="checkbox" bind="limitCheckbox" id="$limit" /></td>
<td><label for="$limit" bind="gdata_limit_next"></label></td>
<td><input bind="limitInput" type="text" class="lightweight" size="2" value="0" />
<label for="$limit" bind="gdata_limit"></label></td></tr>
<tr><td width="1%"><input type="checkbox" bind="storeBlankRowsCheckbox" id="$store-blank-rows" /></td>
<td colspan="2"><label for="$store-blank-rows" bind="gdata_store_row"></label></td></tr>
<tr><td width="1%"><input type="checkbox" bind="storeBlankCellsAsNullsCheckbox" id="$store-blank-cells" /></td>
<td colspan="2"><label for="$store-blank-cells" bind="gdata_store_cell"></label></td></tr>
</table></div></td>
</tr>
</table></div></div>

7
extensions/gdata/module/scripts/index/importing-controller.js
View File

@ -163,19 +163,14 @@ Refine.GDataImportingController.prototype._showParsingPanel = function() {
var self = this;
this._parsingPanel.unbind().empty().html(
DOM.loadHTML("gdata",
this._doc.type == 'table' ?
'scripts/index/gdata-fusion-tables-parsing-panel.html' :
'scripts/index/gdata-parsing-panel.html'));
DOM.loadHTML("gdata", 'scripts/index/gdata-parsing-panel.html'));
this._parsingPanelElmts = DOM.bind(this._parsingPanel);
if(this._doc.type != 'table'){
this._parsingPanelElmts.gdata_worksheet.html($.i18n('gdata-parsing/worksheet'));
this._parsingPanelElmts.gdata_ignore_first.html($.i18n('gdata-parsing/ignore-first'));
this._parsingPanelElmts.gdata_ignore.html($.i18n('gdata-parsing/ignore'));
this._parsingPanelElmts.gdata_parse_next.html($.i18n('gdata-parsing/parse-next'));
this._parsingPanelElmts.gdata_parse.html($.i18n('gdata-parsing/parse'));
}
this._parsingPanelElmts.startOverButton.html($.i18n('gdata-parsing/start-over'));
this._parsingPanelElmts.gdata_conf_pars.html($.i18n('gdata-parsing/conf-pars'));
this._parsingPanelElmts.gdata_proj_name.html($.i18n('gdata-parsing/proj-name'));

5
extensions/gdata/pom.xml
View File

@ -136,11 +136,6 @@
<artifactId>google-oauth-client-jetty</artifactId>
<version>1.23.0</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-fusiontables</artifactId>
<version>v2-rev21-1.23.0</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-drive</artifactId>

166
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableHandler.java
View File

@ -1,166 +0,0 @@
/*
* Copyright (c) 2013, Thomas F. Morris and other contributors
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.api.services.fusiontables.Fusiontables.Query.Sql;
import com.google.api.services.fusiontables.Fusiontables.Query.SqlGet;
import com.google.api.services.fusiontables.Fusiontables.Table.ImportRows;
import com.google.api.services.fusiontables.model.FusiontablesImport;
import com.google.api.services.fusiontables.model.Sqlresponse;
/**
* @author Tom Morris <tfmorris@gmail.com>
* @copyright 2010,2013 Thomas F. Morris
*/
public class FusionTableHandler {
static private Sqlresponse executeQuery (Fusiontables service, String query)
throws IOException {
Sql sql = service.query().sql(query);
Sqlresponse response = sql.execute();
return response;
}
static String createTable(Fusiontables service, String name, List<String> columnNames) throws IOException {
StringBuffer sb = new StringBuffer();
sb.append("CREATE TABLE '");
sb.append(name);
sb.append("' (");
boolean first = true;
for (String columnName : columnNames) {
if (first) {
first = false;
} else {
sb.append(',');
}
sb.append("'");
sb.append(columnName);
sb.append("': STRING");
}
sb.append(")");
String createQuery = sb.toString();
Sqlresponse response = executeQuery(service, createQuery);
// response.getTableId(); // FIXME: Oh wait, there's no such F*ing method!!!
return getTableId(response);
}
private static String getTableId(Sqlresponse response) {
List<Object> row = response.getRows().get(0);
int i = 0;
for (String colname : response.getColumns()) {
if ("tableid".equals(colname)) {
return (String) row.get(i);
}
}
return null;
}
/**
* Insert a set of rows and optionally return the IDs of the new rows.
*
* @param service a Fusiontables object
* @param sql SQL statement to do the inserts
* @param returnIds true to return the IDs of the newly inserted rows
* @return
* @throws IOException
*/
static Long insertRows(Fusiontables service, String tableId, AbstractInputStreamContent mediaContent) throws IOException {
ImportRows importRows = service.table().importRows(tableId, mediaContent);
importRows.setIsStrict(false);
FusiontablesImport response = importRows.execute();
return response.getNumRowsReceived();
}
static String getFusionTableKey(URL url) {
String tableId = getParamValue(url,"dsrcid"); // old style phased out
if (tableId == null || tableId.isEmpty()) {
tableId = getParamValue(url,"docid");
}
return tableId;
}
static public Fusiontables getFusionTablesService(String token) {
Credential credential = new GoogleCredential().setAccessToken(token);
Fusiontables fusiontables = new Fusiontables.Builder(
GoogleAPIExtension.HTTP_TRANSPORT, GoogleAPIExtension.JSON_FACTORY, credential)
.setApplicationName(GoogleAPIExtension.SERVICE_APP_NAME)
.build();;
return fusiontables;
}
static boolean isFusionTableURL(URL url) {
// http://www.google.com/fusiontables/DataSource?dsrcid=1219
String query = url.getQuery();
if (query == null) {
query = "";
}
return url.getHost().endsWith(".google.com")
&& url.getPath().startsWith("/fusiontables/DataSource")
&& (query.contains("dsrcid=")||query.contains("docid="));
}
static Sqlresponse runFusionTablesSelect(Fusiontables service, String selectQuery)
throws IOException {
// FIXME: alt=csv doesn't actually work! It will attempt to parse response as JSON and die
// perhaps use .executeUnparsed() would work?
SqlGet query = service.query().sqlGet(selectQuery);//.setAlt("csv");
Sqlresponse response = query.execute();
return response;
}
static private String getParamValue(URL url, String key) {
String query = url.getQuery();
if (query != null) {
String[] parts = query.split("&");
for (String part : parts) {
if (part.startsWith(key+"=")) {
int offset = key.length()+1;
String tableId = part.substring(offset);
return tableId;
}
}
}
return null;
}
}

283
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableImporter.java
View File

@ -1,283 +0,0 @@
/*
* Copyright (c) 2010,2013 Thomas F. Morris and other contributors
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of Google nor the names of its contributors may be used to
* endorse or promote products derived from this software without specific
* prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.api.services.fusiontables.model.Column;
import com.google.api.services.fusiontables.model.Sqlresponse;
import com.google.api.services.fusiontables.model.Table;
import com.google.refine.ProjectMetadata;
import com.google.refine.importers.TabularImportingParserBase;
import com.google.refine.importers.TabularImportingParserBase.TableDataReader;
import com.google.refine.importing.ImportingJob;
import com.google.refine.model.Project;
import com.google.refine.util.JSONUtilities;
/**
* OpenRefine parser for Google Spreadsheets.
*
* @author Tom Morris <tfmorris@gmail.com>
* @copyright 2010 Thomas F. Morris
* @license New BSD http://www.opensource.org/licenses/bsd-license.php
*/
public class FusionTableImporter {
static public void parse(
String token,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
ObjectNode options,
List<Exception> exceptions) {
Fusiontables service = FusionTableHandler.getFusionTablesService(token);
parse(
service,
project,
metadata,
job,
limit,
options,
exceptions
);
}
static void setProgress(ImportingJob job, String fileSource, int percent) {
job.setProgress(percent, "Reading " + fileSource);
}
static private class FusionTableBatchRowReader implements TableDataReader {
final ImportingJob job;
final String fileSource;
final Fusiontables service;
final List<FTColumnData> columns;
final int batchSize;
final String baseQuery;
int nextRow = 0; // 0-based
int batchRowStart = 0; // 0-based
boolean end = false;
List<List<Object>> rowsOfCells = null;
boolean usedHeaders = false;
public FusionTableBatchRowReader(ImportingJob job, String fileSource,
Fusiontables service, String tableId, List<FTColumnData> columns,
int batchSize) {
this.job = job;
this.fileSource = fileSource;
this.service = service;
this.columns = columns;
this.batchSize = batchSize;
StringBuffer sb = new StringBuffer();
sb.append("SELECT ");
boolean first = true;
for (FTColumnData cd : columns) {
if (first) {
first = false;
} else {
sb.append(",");
}
sb.append("'");
sb.append(cd.name);
sb.append("'");
}
sb.append(" FROM ");
sb.append(tableId);
baseQuery = sb.toString();
}
@Override
public List<Object> getNextRowOfCells() throws IOException {
if (!usedHeaders) {
List<Object> row = new ArrayList<Object>(columns.size());
for (FTColumnData cd : columns) {
row.add(cd.name);
}
usedHeaders = true;
return row;
}
if (rowsOfCells == null || (nextRow >= batchRowStart + rowsOfCells.size() && !end)) {
int newBatchRowStart = batchRowStart + (rowsOfCells == null ? 0 : rowsOfCells.size());
rowsOfCells = getRowsOfCells(newBatchRowStart);
batchRowStart = newBatchRowStart;
setProgress(job, fileSource, -1 /* batchRowStart * 100 / totalRows */);
}
if (rowsOfCells != null && nextRow - batchRowStart < rowsOfCells.size()) {
return rowsOfCells.get(nextRow++ - batchRowStart);
} else {
return null;
}
}
private List<List<Object>> getRowsOfCells(int startRow) throws IOException {
List<List<Object>> rowsOfCells = new ArrayList<List<Object>>(batchSize);
String query = baseQuery + " OFFSET " + startRow + " LIMIT " + batchSize;
Sqlresponse sqlresponse = FusionTableHandler.runFusionTablesSelect(service, query);
List<List<Object>> rows = sqlresponse.getRows();
if (rows.size() > 1) {
for (int i = 1; i < rows.size(); i++) {
List<Object> row = rows.get(i);
List<Object> rowOfCells = new ArrayList<Object>(row.size());
for (int j = 0; j < row.size() && j < columns.size(); j++) {
String text = String.valueOf(row.get(j));
if (text.isEmpty()) {
rowOfCells.add(null);
} else {
FTColumnData cd = columns.get(j);
if (cd.type == FTColumnType.NUMBER) {
try {
rowOfCells.add(Long.parseLong(text));
continue;
} catch (NumberFormatException e) {
// ignore
}
try {
double d = Double.parseDouble(text);
if (!Double.isInfinite(d) && !Double.isNaN(d)) {
rowOfCells.add(d);
continue;
}
} catch (NumberFormatException e) {
// ignore
}
}
rowOfCells.add(text);
}
}
rowsOfCells.add(rowOfCells);
}
}
end = rows.size() < batchSize + 1;
return rowsOfCells;
}
}
static public void parse(
Fusiontables service,
Project project,
ProjectMetadata metadata,
final ImportingJob job,
int limit,
ObjectNode options,
List<Exception> exceptions) {
String docUrlString = JSONUtilities.getString(options, "docUrl", null);
String id = getFTid(docUrlString); // Use GDataExtension.getFusionTableKey(url) ?
// TODO: Allow arbitrary Fusion Tables URL instead of (in addition to?) constructing our own?
try {
List<FTColumnData> columns = new ArrayList<FusionTableImporter.FTColumnData>();
Table table = service.table().get(id).execute();
for (Column col : table.getColumns()) {
FTColumnData cd = new FTColumnData();
cd.name = col.getName();
String type = col.getType();
if (type.equals("STRING")) {
cd.type = FTColumnType.STRING;
} else if (type.equals("NUMBER")) {
cd.type = FTColumnType.NUMBER;
} else if (type.equals("DATETIME")) {
cd.type = FTColumnType.DATETIME;
} else if (type.equals("LOCATION")) {
cd.type = FTColumnType.LOCATION;
} else {
// TODO: unknown type
cd.type = FTColumnType.STRING;
}
columns.add(cd);
}
setProgress(job, docUrlString, -1);
// Force these options for the next call because each fusion table
// is strictly structured with a single line of headers.
JSONUtilities.safePut(options, "ignoreLines", 0); // number of blank lines at the beginning to ignore
JSONUtilities.safePut(options, "headerLines", 1); // number of header lines
TabularImportingParserBase.readTable(
project,
metadata,
job,
new FusionTableBatchRowReader(job, docUrlString, service, id, columns, 100),
docUrlString,
limit,
options,
exceptions
);
setProgress(job, docUrlString, 100);
} catch (IOException e) {
e.printStackTrace();
exceptions.add(e);
}
}
static private String getFTid(String url) {
if (url == null) {
return null;
}
int equal = url.lastIndexOf('=');
if (equal < 0) {
return null;
}
return url.substring(equal + 1);
}
static enum FTColumnType {
STRING,
NUMBER,
DATETIME,
LOCATION
}
final static class FTColumnData {
String name;
FTColumnType type;
}
}

115
extensions/gdata/src/com/google/refine/extension/gdata/FusionTableSerializer.java
View File

@ -1,115 +0,0 @@
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.HttpResponseException;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.refine.exporters.TabularSerializer;
final class FusionTableSerializer implements TabularSerializer {
private static final int BATCH_SIZE = 20;
Fusiontables service;
String tableName;
List<Exception> exceptions;
String tableId;
List<String> columnNames;
StringBuffer sbBatch;
int rows;
FusionTableSerializer(Fusiontables service, String tableName, List<Exception> exceptions) {
this.service = service;
this.tableName = tableName;
this.exceptions = exceptions;
}
@Override
public void startFile(JsonNode options) {
}
@Override
public void endFile() {
if (sbBatch != null) {
sendBatch(rows % BATCH_SIZE);
}
}
@Override
public void addRow(List<CellData> cells, boolean isHeader) {
if (isHeader) {
columnNames = new ArrayList<String>(cells.size());
for (CellData cellData : cells) {
columnNames.add(cellData.text);
}
try {
tableId = FusionTableHandler.createTable(service, tableName, columnNames);
} catch (Exception e) {
tableId = null;
exceptions.add(e);
}
} else if (tableId != null) {
if (sbBatch == null) {
sbBatch = new StringBuffer();
}
formatCsv(cells, sbBatch);
rows++;
if (rows % BATCH_SIZE == 0) {
if (!sendBatch(BATCH_SIZE)) {
return;
}
}
}
}
private boolean sendBatch(int batchSize) {
try {
// TODO: we really want to do GZIP compression here
// FIXME: text/csv doesn't work even though that's what the content is
AbstractInputStreamContent content = ByteArrayContent.fromString("application/octet-stream", sbBatch.toString());
Long count = FusionTableHandler.insertRows(service, tableId, content);
if (count != null && count.intValue() != batchSize) {
exceptions.add(new IOException("only imported " + count + " of " + batchSize + " rows"));
}
} catch (IOException e) {
exceptions.add(e);
if (e instanceof HttpResponseException) {
int code = ((HttpResponseException)e).getStatusCode();
if (code >= 400 && code < 500) {
return false;
}
// 500s appear to be retried automatically by li
}
} finally {
sbBatch = null;
}
return true;
}
private void formatCsv(List<CellData> cells, StringBuffer sb) {
boolean first = true;
for (int i = 0; i < cells.size() && i < columnNames.size(); i++) {
CellData cellData = cells.get(i);
if (!first) {
sb.append(',');
} else {
first = false;
}
sb.append("\"");
if (cellData != null && cellData.text != null) {
sb.append(cellData.text.replaceAll("\"", "\"\""));
}
sb.append("\"");
}
sb.append("\n");
}
public String getUrl() {
return tableId == null || exceptions.size() > 0 ? null :
"https://www.google.com/fusiontables/DataSource?docid=" + tableId;
}
}

9
extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java
View File

@ -45,15 +45,6 @@ public class GDataImporter {
options,
exceptions
);
} else if ("table".equals(docType)) {
FusionTableImporter.parse(token,
project,
metadata,
job,
limit,
options,
exceptions
);
}
}

29
extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java
View File

@ -21,9 +21,6 @@ import com.google.api.services.drive.Drive;
import com.google.api.services.drive.model.File;
import com.google.api.services.drive.model.FileList;
import com.google.api.services.drive.model.User;
import com.google.api.services.fusiontables.Fusiontables;
import com.google.api.services.fusiontables.model.Table;
import com.google.api.services.fusiontables.model.TableList;
import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.model.Sheet;
import com.google.api.services.sheets.v4.model.Spreadsheet;
@ -91,7 +88,6 @@ public class GDataImportingController implements ImportingController {
try {
listSpreadsheets(GoogleAPIExtension.getDriveService(token), writer);
listFusionTables(FusionTableHandler.getFusionTablesService(token), writer);
} catch (Exception e) {
logger.error("doListDocuments exception:" + ExceptionUtils.getStackTrace(e));
} finally {
@ -139,31 +135,6 @@ public class GDataImportingController implements ImportingController {
}
}
private void listFusionTables(Fusiontables service, JsonGenerator writer)
throws IOException {
Fusiontables.Table.List listTables = service.table().list();
TableList tablelist = listTables.execute();
if (tablelist == null || tablelist.getItems() == null)
return;
for (Table table : tablelist.getItems()) {
String id = table.getTableId();
String name = table.getName();
String link = "https://www.google.com/fusiontables/DataSource?docid=" + id;
// Add JSON object to our stream
writer.writeStartObject();
writer.writeStringField("docId", id);
writer.writeStringField("docLink", link);
writer.writeStringField("docSelfLink", link);
writer.writeStringField("title", name);
writer.writeStringField("type", "table");
writer.writeEndObject();
}
}
private void doInitializeParserUI(
HttpServletRequest request, HttpServletResponse response, Properties parameters)
throws ServletException, IOException {

3
extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java
View File

@ -22,7 +22,6 @@ import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.fusiontables.FusiontablesScopes;
import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.SheetsScopes;
import com.google.refine.ProjectManager;
@ -42,7 +41,7 @@ abstract public class GoogleAPIExtension {
/** Global instance of the JSON factory. */
protected static final JsonFactory JSON_FACTORY = new JacksonFactory();
private static final String[] SCOPES = {DriveScopes.DRIVE, SheetsScopes.SPREADSHEETS, FusiontablesScopes.FUSIONTABLES};
private static final String[] SCOPES = {DriveScopes.DRIVE, SheetsScopes.SPREADSHEETS};
private static PreferenceStore prefStore = ProjectManager.singleton.getPreferenceStore();

15
extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java
View File

@ -108,8 +108,6 @@ public class UploadCommand extends Command {
String format = params.getProperty("format");
if ("gdata/google-spreadsheet".equals(format)) {
return uploadSpreadsheet(project, engine, params, token, name, exceptions);
} else if ("gdata/fusion-table".equals(format)) {
return uploadFusionTable(project, engine, params, token, name, exceptions);
} else if (("raw/openrefine-project").equals(format)) {
return uploadOpenRefineProject(project, token, name, exceptions);
}
@ -199,17 +197,4 @@ public class UploadCommand extends Command {
}
return null;
}
static private String uploadFusionTable(
Project project, final Engine engine, final Properties params,
String token, String name, List<Exception> exceptions) {
FusionTableSerializer serializer = new FusionTableSerializer(
FusionTableHandler.getFusionTablesService(token), name, exceptions);
CustomizableTabularExporterUtilities.exportRows(
project, engine, params, serializer);
return serializer.getUrl();
}
}