s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support uploading directly to a new Google Fusion table.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2244 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2011-09-16 20:59:58 +00:00
parent 5c446d28d0
commit 4c5b2514fd
5 changed files with 266 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
extensions/gdata/module/scripts/project/exporters.js
View File

@ -31,12 +31,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
CustomTabularExporterDialog.uploadTargets.push({
id: 'gdata/google-spreadsheet',
label: 'A new Google spreadsheet',
handler: function(options, exportAllRows, onDone) {
(function() {
var handleUpload = function(options, exportAllRows, onDone, prompt) {
var doUpload = function() {
var name = window.prompt('Enter name of new spreadsheet', theProject.metadata.name);
var name = window.prompt(prompt, theProject.metadata.name);
if (name) {
var dismiss = DialogSystem.showBusy('Uploading...');
$.post(
@ -50,7 +48,7 @@ CustomTabularExporterDialog.uploadTargets.push({
},
function(o) {
dismiss();
if (o.url) {
window.open(o.url, '_blank');
}
@ -60,14 +58,14 @@ CustomTabularExporterDialog.uploadTargets.push({
);
}
};
var messageListener = function(evt) {
window.removeEventListener("message", messageListener, false);
if ($.cookie('authsub_token')) {
doUpload();
}
};
var authenticate = function() {
window.addEventListener("message", messageListener, false);
window.open(
@ -76,7 +74,22 @@ CustomTabularExporterDialog.uploadTargets.push({
"resizable=1,width=600,height=450"
);
};
authenticate();
}
});
};
CustomTabularExporterDialog.uploadTargets.push({
id: 'gdata/google-spreadsheet',
label: 'A new Google spreadsheet',
handler: function(options, exportAllRows, onDone) {
handleUpload(options, exportAllRows, onDone, 'Enter a name for the new Google spreadsheet');
}
});
CustomTabularExporterDialog.uploadTargets.push({
id: 'gdata/fusion-table',
label: 'A new Google Fusion table',
handler: function(options, exportAllRows, onDone) {
handleUpload(options, exportAllRows, onDone, 'Enter a name for the new Google Fusion table');
}
});
})();

34
extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java
View File

@ -29,6 +29,7 @@
package com.google.refine.extension.gdata;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
@ -90,13 +91,36 @@ abstract public class GDataExtension {
static public List<List<String>> runFusionTablesSelect(GoogleService service, String selectQuery)
throws IOException, ServiceException {
URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" +
URLEncoder.encode(selectQuery, "UTF-8"));
GDataRequest request = service.getRequestFactory().getRequest(
RequestType.QUERY, url, ContentType.TEXT_PLAIN);
GDataRequest request = createFusionTablesRequest(service, RequestType.QUERY, selectQuery);
request.execute();
return parseFusionTablesResults(request);
}
static public GDataRequest createFusionTablesRequest(
GoogleService service, RequestType requestType, String query)
throws IOException, ServiceException {
URL url = new URL(FUSION_TABLES_SERVICE_URL + "?sql=" +
URLEncoder.encode(query, "UTF-8"));
return service.getRequestFactory().getRequest(
requestType, url, ContentType.TEXT_PLAIN);
}
static public GDataRequest createFusionTablesPostRequest(
GoogleService service, RequestType requestType, String query)
throws IOException, ServiceException {
URL url = new URL(FUSION_TABLES_SERVICE_URL);
GDataRequest request = service.getRequestFactory().getRequest(
requestType, url, new ContentType("application/x-www-form-urlencoded"));
OutputStreamWriter writer =
new OutputStreamWriter(request.getRequestStream());
writer.append("sql=" + URLEncoder.encode(query, "UTF-8"));
writer.flush();
return request;
}
static public List<List<String>> parseFusionTablesResults(GDataRequest request) throws IOException {
List<List<String>> rows = new ArrayList<List<String>>();
List<String> row = null;

246
extensions/gdata/src/com/google/refine/extension/gdata/UploadCommand.java
View File

@ -33,6 +33,8 @@ import java.io.IOException;
import java.io.Writer;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
@ -45,6 +47,9 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gdata.client.GoogleService;
import com.google.gdata.client.Service.GDataRequest;
import com.google.gdata.client.Service.GDataRequest.RequestType;
import com.google.gdata.client.docs.DocsService;
import com.google.gdata.client.spreadsheet.CellQuery;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
@ -96,13 +101,21 @@ public class UploadCommand extends Command {
JSONWriter writer = new JSONWriter(w);
try {
writer.object();
String url = upload(project, engine, params, token, name);
List<Exception> exceptions = new LinkedList<Exception>();
String url = upload(project, engine, params, token, name, exceptions);
if (url != null) {
writer.key("status"); writer.value("ok");
writer.key("url"); writer.value(url);
} else {
} else if (exceptions.size() == 0) {
writer.key("status"); writer.value("error");
writer.key("message"); writer.value("No such format");
} else {
for (Exception e : exceptions) {
logger.warn(e.getLocalizedMessage(), e);
}
writer.key("status"); writer.value("error");
writer.key("message"); writer.value(exceptions.get(0).getLocalizedMessage());
}
} catch (Exception e) {
e.printStackTrace();
@ -122,56 +135,67 @@ public class UploadCommand extends Command {
static private String upload(
Project project, Engine engine, Properties params,
String token, String name) throws Exception {
String token, String name, List<Exception> exceptions) {
String format = params.getProperty("format");
if ("gdata/google-spreadsheet".equals(format)) {
return uploadSpreadsheet(project, engine, params, token, name);
return uploadSpreadsheet(project, engine, params, token, name, exceptions);
} else if ("gdata/fusion-table".equals(format)) {
return uploadFusionTable(project, engine, params, token, name, exceptions);
}
return null;
}
static private String uploadSpreadsheet(
final Project project, final Engine engine, final Properties params,
String token, String name)
throws MalformedURLException, IOException, ServiceException {
String token, String name, List<Exception> exceptions) {
DocsService docsService = GDataExtension.getDocsService(token);
final SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token);
SpreadsheetEntry spreadsheetEntry = new SpreadsheetEntry();
spreadsheetEntry.setTitle(new PlainTextConstruct(name));
final SpreadsheetEntry spreadsheetEntry2 = docsService.insert(
new URL("https://docs.google.com/feeds/default/private/full/"), spreadsheetEntry);
int[] size = CustomizableTabularExporterUtilities.countColumnsRows(
project, engine, params);
URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl();
WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]);
worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data"));
final WorksheetEntry worksheetEntry2 =
spreadsheetService.insert(worksheetFeedUrl, worksheetEntry);
spreadsheetEntry2.getDefaultWorksheet().delete();
new Thread() {
@Override
public void run() {
spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V1);
try {
uploadToCellFeed(
project, engine, params,
spreadsheetService,
spreadsheetEntry2,
worksheetEntry2);
} catch (Exception e) {
logger.error("Error uploading data to Google Spreadsheets", e);
try {
SpreadsheetEntry spreadsheetEntry = new SpreadsheetEntry();
spreadsheetEntry.setTitle(new PlainTextConstruct(name));
final SpreadsheetEntry spreadsheetEntry2 = docsService.insert(
new URL("https://docs.google.com/feeds/default/private/full/"), spreadsheetEntry);
int[] size = CustomizableTabularExporterUtilities.countColumnsRows(
project, engine, params);
URL worksheetFeedUrl = spreadsheetEntry2.getWorksheetFeedUrl();
WorksheetEntry worksheetEntry = new WorksheetEntry(size[1], size[0]);
worksheetEntry.setTitle(new PlainTextConstruct("Uploaded Data"));
final WorksheetEntry worksheetEntry2 =
spreadsheetService.insert(worksheetFeedUrl, worksheetEntry);
spreadsheetEntry2.getDefaultWorksheet().delete();
new Thread() {
@Override
public void run() {
spreadsheetService.setProtocolVersion(SpreadsheetService.Versions.V1);
try {
uploadToCellFeed(
project, engine, params,
spreadsheetService,
spreadsheetEntry2,
worksheetEntry2);
} catch (Exception e) {
logger.error("Error uploading data to Google Spreadsheets", e);
}
}
}
}.start();
return spreadsheetEntry2.getDocumentLink().getHref();
}.start();
return spreadsheetEntry2.getDocumentLink().getHref();
} catch (MalformedURLException e) {
exceptions.add(e);
} catch (IOException e) {
exceptions.add(e);
} catch (ServiceException e) {
exceptions.add(e);
}
return null;
}
static private void uploadToCellFeed(
@ -267,4 +291,146 @@ public class UploadCommand extends Command {
CustomizableTabularExporterUtilities.exportRows(
project, engine, params, serializer);
}
static private String uploadFusionTable(
Project project, final Engine engine, final Properties params,
String token, String name, List<Exception> exceptions) {
GoogleService service = GDataExtension.getFusionTablesGoogleService(token);
FusionTableSerializer serializer = new FusionTableSerializer(service, name, exceptions);
CustomizableTabularExporterUtilities.exportRows(
project, engine, params, serializer);
return serializer.tableId == null || exceptions.size() > 0 ? null :
"https://www.google.com/fusiontables/DataSource?dsrcid=" + serializer.tableId;
}
final static private class FusionTableSerializer implements TabularSerializer {
GoogleService service;
String tableName;
List<Exception> exceptions;
String tableId;
List<String> columnNames;
StringBuffer sbBatch;
int rows;
FusionTableSerializer(GoogleService service, String tableName, List<Exception> exceptions) {
this.service = service;
this.tableName = tableName;
this.exceptions = exceptions;
}
@Override
public void startFile(JSONObject options) {
}
@Override
public void endFile() {
if (sbBatch != null) {
sendBatch();
}
}
@Override
public void addRow(List<CellData> cells, boolean isHeader) {
if (isHeader) {
columnNames = new ArrayList<String>(cells.size());
StringBuffer sb = new StringBuffer();
sb.append("CREATE TABLE '");
sb.append(tableName);
sb.append("' (");
boolean first = true;
for (CellData cellData : cells) {
columnNames.add(cellData.text);
if (first) {
first = false;
} else {
sb.append(',');
}
sb.append("'");
sb.append(cellData.text);
sb.append("': STRING");
}
sb.append(")");
try {
String createQuery = sb.toString();
GDataRequest createTableRequest = GDataExtension.createFusionTablesPostRequest(
service, RequestType.INSERT, createQuery);
createTableRequest.execute();
List<List<String>> createTableResults =
GDataExtension.parseFusionTablesResults(createTableRequest);
if (createTableResults != null && createTableResults.size() == 2) {
tableId = createTableResults.get(1).get(0);
}
} catch (Exception e) {
exceptions.add(e);
}
} else if (tableId != null) {
if (sbBatch == null) {
sbBatch = new StringBuffer();
}
formulateInsert(cells, sbBatch);
rows++;
if (rows % 20 == 0) {
sendBatch();
}
}
}
void sendBatch() {
try {
GDataRequest createTableRequest = GDataExtension.createFusionTablesPostRequest(
service, RequestType.INSERT, sbBatch.toString());
createTableRequest.execute();
} catch (IOException e) {
exceptions.add(e);
} catch (ServiceException e) {
exceptions.add(e);
} finally {
sbBatch = null;
}
}
void formulateInsert(List<CellData> cells, StringBuffer sb) {
StringBuffer sbColumnNames = new StringBuffer();
StringBuffer sbValues = new StringBuffer();
boolean first = true;
for (int i = 0; i < cells.size() && i < columnNames.size(); i++) {
CellData cellData = cells.get(i);
if (first) {
first = false;
} else {
sbColumnNames.append(',');
sbValues.append(',');
}
sbColumnNames.append("'");
sbColumnNames.append(columnNames.get(i));
sbColumnNames.append("'");
sbValues.append("'");
if (cellData != null && cellData.text != null) {
sbValues.append(cellData.text.replaceAll("'", "\\'"));
}
sbValues.append("'");
}
if (sb.length() > 0) {
sb.append(';');
}
sb.append("INSERT INTO ");
sb.append(tableId);
sb.append("(");
sb.append(sbColumnNames.toString());
sb.append(") values (");
sb.append(sbValues.toString());
sb.append(")");
}
}
}

12
main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.html
View File

@ -139,15 +139,15 @@
</table></div></div>
<div bind="uploadTabBody" id="custom-tabular-exporter-tabs-upload" style="display: none;"><div class="grid-layout grid-layout-for-ui layout-loose layout-full"><table bind="uploadTargetTable">
<div bind="uploadTabBody" id="custom-tabular-exporter-tabs-upload" style="display: none;"><div class="grid-layout grid-layout-for-ui layout-loose layout-full"><table>
<tr>
<th colspan="2">Upload to</th>
<th>Upload to</th>
</tr>
<tr>
<td colspan="2"><div class="grid-layout grid-layout-for-text layout-tightest layout-full"><table><tr>
<td width="100%"> </td>
<td width="1%"><button class="button button-primary" bind="uploadButton">Upload</button></td>
</tr></table></div></td>
<td><div class="grid-layout grid-layout-for-text layout-tightest"><table bind="uploadTargetTable"></table></div></td>
</tr>
<tr>
<td><button class="button button-primary" bind="uploadButton">Upload</button></td>
</tr>
</table></div></div>

2
main/webapp/modules/core/scripts/dialogs/custom-tabular-exporter-dialog.js
View File

@ -131,7 +131,7 @@ CustomTabularExporterDialog.prototype._createDialog = function(options) {
var table = this._elmts.uploadTargetTable[0];
for (var i = 0; i < CustomTabularExporterDialog.uploadTargets.length; i++) {
var target = CustomTabularExporterDialog.uploadTargets[i];
var tr = table.insertRow(table.rows.length - 1);
var tr = table.insertRow(table.rows.length);
var td0 = $(tr.insertCell(0))
.attr('width', '1');