Issue 461 - Add support for public Google Spreadsheets and ability to sign-out.

Kind of messy, but I don't think any of the previously existing functionality is broken.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2375 7d457c2a-affb-35e4-300a-418c747d4874

extensions/gdata/module/MOD-INF/controller.js

@@ -40,9 +40,6 @@ var ClientSideResourceManager = Packages.com.google.refine.ClientSideResourceMan
  * Function invoked to initialize the extension.
  */
 function init() {
-//Packages.java.lang.System.err.println("Initializing gData extension");
-//Packages.java.lang.System.err.println(module.getMountPoint());
-
   var RS = Packages.com.google.refine.RefineServlet;
   RS.registerCommand(module, "deauthorize", Packages.com.google.refine.extension.gdata.DeAuthorizeCommand());
   RS.registerCommand(module, "upload", Packages.com.google.refine.extension.gdata.UploadCommand());

extensions/gdata/module/scripts/index/gdata-source-ui.js

@@ -53,7 +53,46 @@ Refine.GDataSourceUI.prototype.attachUI = function(body) {
       }
     );
   });
+  this._body.find('.gdata-signout.button').click(function() {
+      $.get("/command/gdata/deauthorize" );
+      self._body.find('.gdata-page').hide();
+      self._elmts.signinPage.show();
+  });
   
+  var importURL1 = function(evt) {
+    if ($.trim(self._elmts.urlInput1[0].value).length === 0) {
+	    window.alert("You must specify a web address (URL) to import.");
+    } else {
+	    var doc={}
+	    doc.docSelfLink = self._elmts.urlInput1[0].value;
+	    if (doc.docSelfLink.contains('spreadsheet')) { // TODO: fragile?
+	      doc.type = 'spreadsheet';
+	    } else {
+	      doc.type = 'table';
+	    }
+        self._controller.startImportingDocument(doc);
+	}
+  }
+  // TODO: Consolidate these two URL input forms
+  var importURL2 = function(evt) {
+    if ($.trim(self._elmts.urlInput2[0].value).length === 0) {
+	    window.alert("You must specify a web address (URL) to import.");
+    } else {
+	    var doc={}
+	    doc.docSelfLink = self._elmts.urlInput2[0].value;
+	    if (doc.docSelfLink.contains('spreadsheet')) { // TODO: fragile?
+	      doc.type = 'spreadsheet';
+	    } else {
+	      doc.type = 'table';
+	    }
+        self._controller.startImportingDocument(doc);
+	}
+  }
+  
+  this._elmts.urlNextButton1.click(importURL1);
+  this._elmts.urlNextButton2.click(importURL2);
+  
+   
   this._body.find('.gdata-page').hide();
   this._elmts.signinPage.show();
   

extensions/gdata/module/scripts/index/import-from-gdata-form.html

@@ -1,7 +1,14 @@
 <div>
   <div bind="signinPage" class="gdata-page">
     <p>Please <button class="gdata-signin button button-primary">sign in and authorize</button>
-      access to your Google data.</p>
+      access to your Google data.</p></tr>
+	<form bind="form"><div class="grid-layout layout-normal"><table>
+	  <tr><td colspan="2">Or enter web address (URL) of a <em>public</em> Google Spreadsheet or Fusion Table below and click Next:</td></tr>
+	  <tr bind="urlRow"><td colspan="2"><input bind="urlInput1" name="download" class="default-importing-web-url" /></td></tr>
+	  <tr bind="buttons">
+	    <td><button bind="urlNextButton1" class="button button-primary" type="button">Next &raquo;</button></td>
+	  </tr>
+	</table></div></form>
   </div>
   <div bind="progressPage" class="gdata-page">
     <p><img src="images/large-spinner.gif" /> Retrieving Google Docs documents ...</p>
@@ -9,8 +16,16 @@
   <div bind="listingPage" class="gdata-page grid-layout layout-full layout-normal"><table>
     <tr>
       <td>Google Docs documents</td>
-      <td style="text-align: right;"><button class="gdata-signin button">re-sign in</button> with another account</td>
+      <td style="text-align: center;"><button class="gdata-signin button">re-sign in</button> with another account</td>
+      <td style="text-align: right;"><button class="gdata-signout button">sign out</button></td>
     </tr>
-    <tr><td colspan="2"><div bind="listingContainer" class="grid-layout layout-tight gdata-document-container"></div></td></tr>
+	  <tr><td colspan="2">Click one of the documents below or enter the web address (URL) of a <em>public</em> Google Spreadsheet or Fusion Table and click "Next":</td></tr>
+	<tr bind="urlRow"><td colspan="2">
+	    <form bind="form">
+		  <input bind="urlInput2" name="download" class="default-importing-web-url" /></td>
+		</form>
+		<td bind="buttons"><button bind="urlNextButton2" class="button button-primary" type="button">Next &raquo;</button></td>
+	</tr>
+    <tr><td colspan="3"><div bind="listingContainer" class="grid-layout layout-tight gdata-document-container"></div></td></tr>
   </table></div>
 </div>

extensions/gdata/src/com/google/refine/extension/gdata/GDataExtension.java

@@ -95,19 +95,25 @@ abstract public class GDataExtension {
     
     static public DocsService getDocsService(String token) {
         DocsService service = new DocsService(SERVICE_APP_NAME);
-        service.setAuthSubToken(token);
+        if (token != null) {
+            service.setAuthSubToken(token);
+        }
         return service;
     }
     
     static public SpreadsheetService getSpreadsheetService(String token) {
         SpreadsheetService service = new SpreadsheetService(SERVICE_APP_NAME);
-        service.setAuthSubToken(token);
+        if (token != null) {
+            service.setAuthSubToken(token);
+        }
         return service;
     }
     
     static public GoogleService getFusionTablesGoogleService(String token) {
         GoogleService service = new GoogleService("fusiontables", SERVICE_APP_NAME);
-        service.setAuthSubToken(token);
+        if (token != null) {
+            service.setAuthSubToken(token);
+        }
         return service;
     }
 
@@ -178,4 +184,60 @@ abstract public class GDataExtension {
         }
         return rows;
     }
+
+    static boolean isSpreadsheetURL(String url) {
+        // e.g. http://spreadsheets.google.com/ccc?key=tI36b9Fxk1lFBS83iR_3XQA&hl=en
+        // TODO: The following should work, but the GData implementation is too limited
+//        try {
+//            FeedURLFactory.getSpreadsheetKeyFromUrl(url);
+//            return true;
+//        } catch (IllegalArgumentException e) {
+//            return false;
+//        }
+        try {
+            return url.contains("spreadsheet") && getSpreadsheetID(new URL(url)) != null;
+        } catch (MalformedURLException e) {
+            return false;
+        }
+    }
+    
+    static String getSpreadsheetID(URL url) {
+        return getParamValue(url,"key");
+    }
+    
+    static private String getParamValue(URL url, String key) {
+        String query = url.getQuery();
+        if (query != null) {
+            String[] parts = query.split("&");
+            for (String part : parts) {
+                if (part.startsWith(key+"=")) {
+                    int offset = key.length()+1;
+                    String tableId = part.substring(offset);
+                    return tableId;
+                }
+            }
+        }
+        return null; 
+    }
+    
+    static boolean isFusionTableURL(URL url) {
+        // http://www.google.com/fusiontables/DataSource?dsrcid=1219
+        String query = url.getQuery();
+        if (query == null) {
+            query = "";
+        }
+        return url.getHost().endsWith(".google.com") 
+                && url.getPath().startsWith("/fusiontables/DataSource")
+                && query.contains("dsrcid=");
+    }
+    
+    static String getFusionTableKey(URL url) {
+        String tableId = getParamValue(url,"dsrcid");
+        // TODO: Any special id format considerations to worry about?
+//        if (tableId.startsWith("p") || !tableId.contains(".")) {
+//            return tableId;
+//        }
+        return tableId;
+    }
+
 }

extensions/gdata/src/com/google/refine/extension/gdata/GDataImporter.java

@@ -138,10 +138,16 @@ public class GDataImporter {
         List<Exception> exceptions) {
         
         try {
-            SpreadsheetEntry spreadsheetEntry = service.getEntry(docURL, SpreadsheetEntry.class);
             WorksheetEntry worksheetEntry = service.getEntry(worksheetURL, WorksheetEntry.class);
+            String spreadsheetName = docURL.toExternalForm();
+            try {
+                SpreadsheetEntry spreadsheetEntry = service.getEntry(docURL, SpreadsheetEntry.class);
+                spreadsheetName = spreadsheetEntry.getTitle().getPlainText();
+            } catch (ServiceException e) { // RedirectRequiredException among others
+                // fall back to just using the URL (better for traceability anyway?)
+            }
             
-            String fileSource = spreadsheetEntry.getTitle().getPlainText() + " # " +
+            String fileSource = spreadsheetName + " # " +
                 worksheetEntry.getTitle().getPlainText();
             
             setProgress(job, fileSource, 0);
@@ -283,15 +289,8 @@ public class GDataImporter {
             List<Exception> exceptions) {
         
         String docUrlString = JSONUtilities.getString(options, "docUrl", null);
-        if (docUrlString == null) {
+        String id = getFTid(docUrlString); // Use GDataExtension.getFusionTableKey(url) ?
-            return;
+        // TODO: Allow arbitrary Fusion Tables URL instead of (in addition to?) constructing our own?
-        }
-        int equal = docUrlString.lastIndexOf('=');
-        if (equal < 0) {
-            return;
-        }
-        
-        String id = docUrlString.substring(equal + 1);
         
         try {
             List<FTColumnData> columns = new ArrayList<GDataImporter.FTColumnData>();
@@ -346,6 +345,17 @@ public class GDataImporter {
         }
     }
     
+    static private String getFTid(String url) {
+        if (url == null) {
+            return null;
+        }
+        int equal = url.lastIndexOf('=');
+        if (equal < 0) {
+            return null;
+        }
+        return url.substring(equal + 1);
+    }
+    
     static private enum FTColumnType {
         STRING,
         NUMBER,

extensions/gdata/src/com/google/refine/extension/gdata/GDataImportingController.java

@@ -51,12 +51,14 @@ import org.json.JSONWriter;
 
 import com.google.gdata.client.GoogleService;
 import com.google.gdata.client.docs.DocsService;
+import com.google.gdata.client.spreadsheet.FeedURLFactory;
 import com.google.gdata.client.spreadsheet.SpreadsheetService;
 import com.google.gdata.data.DateTime;
 import com.google.gdata.data.Person;
 import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
 import com.google.gdata.data.spreadsheet.SpreadsheetFeed;
 import com.google.gdata.data.spreadsheet.WorksheetEntry;
+import com.google.gdata.data.spreadsheet.WorksheetFeed;
 import com.google.gdata.util.ServiceException;
 
 import com.google.refine.ProjectManager;
@@ -83,7 +85,7 @@ public class GDataImportingController implements ImportingController {
     @Override
     public void doGet(HttpServletRequest request, HttpServletResponse response)
         throws ServletException, IOException {
-        // TODO Auto-generated method stub
+        HttpUtilities.respond(response, "error", "GET not implemented");
     }
 
     @Override
@@ -193,11 +195,7 @@ public class GDataImportingController implements ImportingController {
         HttpServletRequest request, HttpServletResponse response, Properties parameters)
             throws ServletException, IOException {
     
-        String token = TokenCookie.getToken(request);
+        String token = TokenCookie.getToken(request); // authorization token, if logged in
-        if (token == null) {
-            HttpUtilities.respond(response, "error", "Not authorized");
-            return;
-        }
         
         String type = parameters.getProperty("docType");
         String urlString = parameters.getProperty("docUrl");
@@ -220,8 +218,19 @@ public class GDataImportingController implements ImportingController {
                 JSONUtilities.safePut(options, "worksheets", worksheets);
                 
                 SpreadsheetService spreadsheetService = GDataExtension.getSpreadsheetService(token);
-                SpreadsheetEntry spreadsheetEntry = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
+                List<WorksheetEntry> worksheetEntries;
-                for (WorksheetEntry worksheetEntry : spreadsheetEntry.getWorksheets()) {
+                if (token == null) {
+                    String visibility = "public";
+                    FeedURLFactory factory = FeedURLFactory.getDefault();
+                    String key = GDataExtension.getSpreadsheetID(url);
+                    url = factory.getWorksheetFeedUrl(key, visibility, "values");
+                    WorksheetFeed feed = spreadsheetService.getFeed(url, WorksheetFeed.class);
+                    worksheetEntries = feed.getEntries(); 
+                } else {
+                    SpreadsheetEntry spreadsheetEntry = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
+                    worksheetEntries = spreadsheetEntry.getWorksheets();
+                }
+                for (WorksheetEntry worksheetEntry : worksheetEntries) {
                     JSONObject worksheetO = new JSONObject();
                     JSONUtilities.safePut(worksheetO, "name", worksheetEntry.getTitle().getPlainText());
                     JSONUtilities.safePut(worksheetO, "rows", worksheetEntry.getRowCount());
@@ -246,12 +255,7 @@ public class GDataImportingController implements ImportingController {
             throws ServletException, IOException {
         
         String token = TokenCookie.getToken(request);
-        if (token == null) {
-            HttpUtilities.respond(response, "error", "Not authorized");
-            return;
-        }
         
-
         long jobID = Long.parseLong(parameters.getProperty("jobID"));
         ImportingJob job = ImportingManager.getJob(jobID);
         if (job == null) {
@@ -317,10 +321,6 @@ public class GDataImportingController implements ImportingController {
         throws ServletException, IOException {
     
         final String token = TokenCookie.getToken(request);
-        if (token == null) {
-            HttpUtilities.respond(response, "error", "Not authorized");
-            return;
-        }
         
         long jobID = Long.parseLong(parameters.getProperty("jobID"));
         final ImportingJob job = ImportingManager.getJob(jobID);