implemented more conservative data loading workflow

git-svn-id: http://google-refine.googlecode.com/svn/trunk@554 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/GridworksServlet.java

@@ -29,6 +29,8 @@ import com.metaweb.gridworks.commands.edit.ExportProjectCommand;
 import com.metaweb.gridworks.commands.edit.ExtendDataCommand;
 import com.metaweb.gridworks.commands.edit.ImportProjectCommand;
 import com.metaweb.gridworks.commands.edit.JoinMultiValueCellsCommand;
+import com.metaweb.gridworks.commands.edit.MQLReadCommand;
+import com.metaweb.gridworks.commands.edit.MQLWriteCommand;
 import com.metaweb.gridworks.commands.edit.MassEditCommand;
 import com.metaweb.gridworks.commands.edit.RemoveColumnCommand;
 import com.metaweb.gridworks.commands.edit.RemoveRowsCommand;
@@ -145,6 +147,8 @@ public class GridworksServlet extends HttpServlet {
         _commands.put("deauthorize", new DeAuthorizeCommand());
 
         _commands.put("upload-data", new UploadDataCommand());
+        _commands.put("mqlread", new MQLReadCommand());
+        _commands.put("mqlwrite", new MQLWriteCommand());
     }
 
     @Override

src/main/java/com/metaweb/gridworks/commands/edit/MQLReadCommand.java

@@ -0,0 +1,31 @@
+package com.metaweb.gridworks.commands.edit;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.metaweb.gridworks.commands.Command;
+import com.metaweb.gridworks.oauth.OAuthUtilities;
+import com.metaweb.gridworks.oauth.Provider;
+import com.metaweb.gridworks.util.FreebaseUtils;
+
+public class MQLReadCommand extends Command {
+
+    @Override
+    public void doPost(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        
+        try {
+            Provider provider = OAuthUtilities.getProvider(request);
+            response.setCharacterEncoding("UTF-8");
+            response.setHeader("Content-Type", "application/json");
+            String query = request.getParameter("query");
+            String result = FreebaseUtils.mqlread(provider,query);
+            response.getWriter().write(result);
+        } catch (Exception e) {
+            respondException(response, e);
+        }
+    }
+}

src/main/java/com/metaweb/gridworks/commands/edit/MQLWriteCommand.java

@@ -0,0 +1,41 @@
+package com.metaweb.gridworks.commands.edit;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.metaweb.gridworks.commands.Command;
+import com.metaweb.gridworks.oauth.Credentials;
+import com.metaweb.gridworks.oauth.OAuthUtilities;
+import com.metaweb.gridworks.oauth.Provider;
+import com.metaweb.gridworks.util.FreebaseUtils;
+
+public class MQLWriteCommand extends Command {
+
+    @Override
+    public void doPost(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        
+        try {
+            Provider provider = OAuthUtilities.getProvider(request);
+            
+            Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
+            
+            response.setCharacterEncoding("UTF-8");
+            response.setHeader("Content-Type", "application/json");
+            
+            if (access_credentials != null) {
+                String query = request.getParameter("query");
+                System.out.println(query);
+                String result = FreebaseUtils.mqlwrite(access_credentials, provider, query);
+                response.getWriter().write(result);
+            } else {    
+                respond(response, "401 Unauthorized", "You don't have the right credentials");
+            }
+        } catch (Exception e) {
+            respondException(response, e);
+        }
+    }
+}

src/main/java/com/metaweb/gridworks/commands/edit/UploadDataCommand.java

@@ -32,8 +32,9 @@ public class UploadDataCommand extends Command {
 
             String source_name = request.getParameter("source_name");
             String source_id = request.getParameter("source_id");
+            String graph = request.getParameter("graph");
                         
-            String result = FreebaseUtils.uploadTriples(request, source_name, source_id, triples.toString());
+            String result = FreebaseUtils.uploadTriples(request, graph, source_name, source_id, triples.toString());
 
             response.setCharacterEncoding("UTF-8");
             response.setHeader("Content-Type", "application/json");

src/main/java/com/metaweb/gridworks/util/FreebaseUtils.java

@@ -45,6 +45,14 @@ public class FreebaseUtils {
         return "http://" + host + "/api/service/user_info";
     }
 
+    private static String getMQLWriteURL(String host) {
+        return "http://" + host + "/api/service/mqlwrite";
+    }
+
+    private static String getMQLReadURL(String host) {
+        return "http://" + host + "/api/service/mqlread";
+    }
+    
     public static String getUserInfo(Credentials credentials, Provider provider) 
         throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
         
@@ -66,8 +74,62 @@ public class FreebaseUtils {
         // return the results
         return EntityUtils.toString(httpResponse.getEntity());
     }
+
+    public static String mqlwrite(Credentials credentials, Provider provider, String query) 
+    throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
+        OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
+
+        JSONObject envelope = new JSONObject();
+        envelope.put("query", new JSONObject(query));
+        
+        List<NameValuePair> formparams = new ArrayList<NameValuePair>();
+        formparams.add(new BasicNameValuePair("query", envelope.toString()));
+        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
+
+        HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
+        httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
+        httpRequest.setEntity(entity);
+
+        // this is required by the Metaweb API to avoid XSS
+        httpRequest.setHeader("X-Requested-With", "1");
+
+        // sign the request with the oauth library
+        consumer.sign(httpRequest);
+
+        // execute the request
+        HttpClient httpClient = new DefaultHttpClient();
+        HttpResponse httpResponse = httpClient.execute(httpRequest);
     
-    public static String uploadTriples(HttpServletRequest request, String source_name, String source_id, String triples) 
+        // return the results
+        return EntityUtils.toString(httpResponse.getEntity());
+    }
+
+    public static String mqlread(Provider provider, String query) 
+    throws ClientProtocolException, IOException, JSONException {
+        
+        JSONObject envelope = new JSONObject();
+        envelope.put("query", new JSONObject(query));
+        
+        List<NameValuePair> formparams = new ArrayList<NameValuePair>();
+        formparams.add(new BasicNameValuePair("query", envelope.toString()));
+        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
+
+        HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
+        httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + Gridworks.getVersion());
+        httpRequest.setEntity(entity);
+
+        // this is required by the Metaweb API to avoid XSS
+        httpRequest.setHeader("X-Requested-With", "1");
+
+        // execute the request
+        HttpClient httpClient = new DefaultHttpClient();
+        HttpResponse httpResponse = httpClient.execute(httpRequest);
+    
+        // return the results
+        return EntityUtils.toString(httpResponse.getEntity());
+    }
+        
+    public static String uploadTriples(HttpServletRequest request, String graph, String source_name, String source_id, String triples) 
         throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
         
         Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
@@ -88,7 +150,7 @@ public class FreebaseUtils {
             formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
             formparams.add(new BasicNameValuePair("operator", GRIDWORKS_ID));
             formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
-            formparams.add(new BasicNameValuePair("graphport", provider.getHost().equals(FREEBASE_HOST) ? "otg" : "sandbox"));
+            formparams.add(new BasicNameValuePair("graphport", graph));
             formparams.add(new BasicNameValuePair("payload", triples));
             formparams.add(new BasicNameValuePair("check_params", "false"));
             UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
@@ -121,5 +183,4 @@ public class FreebaseUtils {
             throw new RuntimeException("Invalid credentials");
         }
     }
-
 }

src/main/webapp/scripts/dialogs/freebase-loading-dialog.js

@@ -1,5 +1,6 @@
 function FreebaseLoadingDialog() {
     this._createDialog();
+    this._signedin = false;
 }
 
 FreebaseLoadingDialog.prototype._createDialog = function() {
@@ -33,109 +34,175 @@ FreebaseLoadingDialog.prototype._createDialog = function() {
 
     var selector = $('<span bind="selector">').addClass("freebase-loading-graph-selector").html("Load this data into " +
         '<input type="radio" bind="sandbox" id="freebase-loading-graph-selector-sandbox" name="graph-selector" checked="checked" value="sandbox"/><label class="sandbox" for="freebase-loading-graph-selector-sandbox" title="Load into the sandbox">sandbox</label>' +
-        '<input type="radio" bind="freebase" id="freebase-loading-graph-selector-freebase" name="graph-selector" value="freebase"/><label class="freebase" for="freebase-loading-graph-selector-freebase" title="Load into Freebase">freebase</label>'
+        '<input type="radio" bind="freebase" id="freebase-loading-graph-selector-freebase" name="graph-selector" value="freebase" disabled="disabled"/><label class="freebase" for="freebase-loading-graph-selector-freebase" title="Load into Freebase">freebase</label>'
     ).buttonset().appendTo(right_footer);
 
-    var load_button = $('<button bind="load" id="freebase-loading-load"></button>').text("Load").appendTo(right_footer);
+    var load_button = $('<button bind="load" id="freebase-loading-load" disabled="disabled"></button>').text("Load").appendTo(right_footer);
     
     var provider = "www.freebase.com";
 
-    var check_authorization = function(autoload) {
+    var check_authorization = function(cont) {
         $.get("/command/check-authorization/" + provider, function(data) {
-            if ("status" in data && data.status == "200 OK") {
+            if ("status" in data && data.code == "/api/status/ok") {
                 authorization.html('Signed in as: <a target="_new" href="http://www.freebase.com/view/user/' + data.username + '">' + data.username + '</a> | <a href="javascript:{}" bind="signout">Sign Out</a>').show();
                 DOM.bind(authorization).signout.click(function() {
+                    self._signedin = false;
+                    load_button.attr("disabled","disabled");
                     Sign.signout(check_authorization,provider);
                 });
-                if (autoload) {
-                    self._load();
-                } else {
-                    load_button.unbind().click(function() {
-                        self._load();
-                    });
-                }
-            } else {
-                authorization.html("").hide();
                 load_button.unbind().click(function() {
+                    self._load();
+                });
+                self._signedin = true;
+                $("#freebase-loading-source-name").keyup();
+                if (typeof cont == "function") cont(data);
+            } else {
+                authorization.html('<a href="javascript:{}" bind="signin">Sign into Freebase</a> to enable loading').show();
+                DOM.bind(authorization).signin.click(function() {
                     Sign.signin(function() {
-                        check_authorization(true);
+                        check_authorization(cont);
                     },provider);
                 });                    
             }
         },"json");
     };
     
-    $.post(
+    var check_allowed = function(user_id, cont) {
-        "/command/export-rows", 
+        var mql_query = {
-        { 
+            id : user_id,  
-            project: theProject.id, 
+            "!/type/usergroup/member": [{
-            format : "tripleloader" 
+                "id": "/en/metaweb_staff"
-        },
+            }]
-        function(data) {
+        };        
-            if (data == null || data == "") {
+        
-                body.html(
+        $.post("/command/mqlread/" + provider, 
-                    '<div class="freebase-loading-tripleloader-message">'+
+            { "query" : JSON.stringify(mql_query) }, 
-                        '<h2>This dataset has no triples</h2>' +
+            function(data) {
-                        '<p>Have you aligned it with the Freebase schemas yet?</p>' +
+                if ("status" in data && data.code == "/api/status/ok") {
-                    '</div>'
+                    if (typeof cont == "function") cont((data.result != null));
-                );
+                } else {
-                left_footer.hide();
+                    self._show_error("Error checking if user is a staff member", data);
-                center_footer.hide();
+                }
-                selector.hide();
+            },
-                load_button.text("Close").unbind().click(function() {
+            "json"
-                    self._dismiss();
+        );
-                });
+    };
-            } else {
+    
-                body.html(
+    var make_topic = function(new_topic_id, topic_type, cont) {
-                    '<div class="freebase-loading-tripleloader-data">' + data + '</div>' +
+        var mql_query = {
-                    '<div class="freebase-loading-tripleloader-info">' +
+            "create": "unless_exists",
-                      '<div>Describe the data you\'re about to load &not;</div>' +
+            "name":   new_topic_id,
-                      '<textarea bind="info"></textarea>' +
+            "type":   topic_type,
-                    '</div>'
+            "id":     null,
-                );
+            "guid":   null
-                self._elmts = DOM.bind(frame);
+        };
-                check_authorization(false);
+
+        $.post("/command/mqlwrite/" + provider, 
+            { "query" : JSON.stringify(mql_query) }, 
+            function(data) {
+                if ("status" in data && data.code == "/api/status/ok") {
+                    self._elmts.source_id.val(data.result.id);
+                    if (typeof cont == "function") cont();
+                } else {
+                    self._show_error("Error creating new topic", data);
+                }
+            },
+            "json"
+        );
+    };
+    
+    var show_triples = function(cont) {
+        $.post("/command/export-rows", 
+            { 
+                project: theProject.id, 
+                format : "tripleloader" 
+            },
+            function(data) {
+                if (data == null || data == "") {
+                    body.html(
+                        '<div class="freebase-loading-tripleloader-message">'+
+                            '<h2>This dataset has no triples</h2>' +
+                            '<p>Have you aligned it with the Freebase schemas yet?</p>' +
+                        '</div>'
+                    );
+
+                    self._end();
+                } else {
+                    body.html(
+                        '<div class="freebase-loading-tripleloader-info"><table><tr>' +
+                          '<td><div>Name this data load &not; <sup style="color: red">required</sup></div>' +
+                          '<input type="text" size="40" id="freebase-loading-source-name" bind="source_name"></td>' +
+                          '<td><div>Source ID &not; <sup style="color: #888">optional</sup></div>' +
+                          '<input type="text" size="60" id="freebase-loading-source-id" bind="source_id"></td>' +
+                        '</tr></table></div>' +
+                        '<div class="freebase-loading-tripleloader-data">' + data + '</div>'
+                    );
+                    self._elmts = DOM.bind(frame);
+                    
+                    self._elmts.source_name.keyup(function() {
+                        if (self._signedin && $(this).val() != "") {
+                            load_button.removeAttr("disabled");
+                        } else {
+                            load_button.attr("disabled","disabled");
+                        }
+                    });
+                    
+                    self._elmts.source_id.suggest({
+                        "type": "/dataworld/information_source",
+                        "soft": true,
+                        "suggest_new": "Click here to add a new information source"
+                    }).bind("fb-select-new", function(e, val) {
+                        make_topic(val, "/dataworld/information_source");
+                    });
+
+                    if (typeof cont == "function") cont();
+                }
+
+                self._level = DialogSystem.showDialog(frame);
             }
-            self._level = DialogSystem.showDialog(frame);
+        );
-        }
+    };
-    );
+
+    show_triples(function() {
+        check_authorization(function(data) {
+            check_allowed(data.id, function(is_allowed) {
+                if (is_allowed) {
+                    $("#freebase-loading-graph-selector-freebase").removeAttr("disabled").button("refresh");
+                }
+            });
+        });
+    });
 };
 
 FreebaseLoadingDialog.prototype._load = function() {
     var self = this;
     var freebase = self._elmts.freebase.attr("checked");
 
+    var get_peacock_url = function(url) {
+      return "http://peacock.freebaseapps.com/stats/data.labs/" + url.split("/").slice(-2).join("/");
+    }
+    
     var doLoad = function() {
         $.post("/command/upload-data", 
             {
                 project: theProject.id, 
                 "graph" : (freebase) ? "otg" : "sandbox",
-                "info" : self._elmts.info.val()
+                "source_name" : self._elmts.source_name.val(),
+                "source_id" : self._elmts.source_id.val()
             }, 
             function(data) {
                 var body = $(".dialog-body");
-                if ("status" in data && data.status == "200 OK") {
+                if ("status" in data && "code" in data.status && data.status.code == 200) {
                     body.html(
                         '<div class="freebase-loading-tripleloader-message">' +
-                            '<h2>Data successfully loaded</h2>' + 
+                            '<h2>' + data.result.added + ' triples successfully scheduled for loading</h2>' + 
-                            '<p>' + data.message + '</p>' +
+                            '<p>Follow the loading progress <a href="' + get_peacock_url(data.result.status_url) + '">here</a></p>' +
                         '</div>'
                     );
+                    self._end();
                 } else {
-                    body.html(
+                    self._show_error("Error loading data",error);
-                        '<div class="freebase-loading-tripleloader-message">' +
-                            '<h2>Error loading data</h2>' + 
-                            '<p>' + data.message + '</p>' +
-                            '<pre>' + data.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '</p>' +
-                        '</div>'
-                    );
                 }
-                self._elmts.load.text("Close").unbind().click(function() {
-                    self._dismiss();
-                });
-                self._elmts.cancel.hide();
-                self._elmts.authorization.hide();
-                self._elmts.selector.hide();
             },
             "json"
         );
@@ -144,7 +211,7 @@ FreebaseLoadingDialog.prototype._load = function() {
     if (freebase) {
         var dialog = $(
             '<div id="freebase-confirmation-dialog" title="Are you sure?">' +
-                '<table><tr><td width="30%"><img src="/images/cop.png" width="140px"></td><td width="70%" style="text-align: center; vertical-align: middle; font-size: 120%">Are you sure this data is ready to be uploaded into <a href="http://www.freebase.com/" target="_new">Freebase</a>?</td></tr></table>' +
+                '<table><tr><td width="30%"><img src="/images/cop.png" width="140px"></td><td width="70%" style="text-align: center; vertical-align: middle; font-size: 120%">Are you sure this data is ready to be uploaded into <b>Freebase</b>?</td></tr></table>' +
             '</div>'
         ).dialog({
             resizable: false,
@@ -152,7 +219,7 @@ FreebaseLoadingDialog.prototype._load = function() {
             height: 230,
             modal: true,
             buttons: {
-                'yes': function() {
+                'yes, I know what I\'m doing': function() {
                     $(this).dialog('close');
                     doLoad();
                 },
@@ -170,3 +237,24 @@ FreebaseLoadingDialog.prototype._dismiss = function() {
     DialogSystem.dismissUntil(this._level - 1);
 };
 
+FreebaseLoadingDialog.prototype._show_error = function(msg, error) {
+    var body = $(".dialog-body");
+    body.html(
+        '<div class="freebase-loading-tripleloader-message">' +
+            '<h2>' + msg + '</h2>' + 
+            '<p>' + error.message + '</p>' +
+            (('stack' in error) ? '<pre>' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '</p>' : "") +
+        '</div>'
+    );
+    this._end();
+};
+
+FreebaseLoadingDialog.prototype._end = function() {
+    var self = this;
+    self._elmts.load.text("Close").unbind().click(function() {
+        self._dismiss();
+    });
+    self._elmts.cancel.hide();
+    self._elmts.authorization.hide();
+    self._elmts.selector.hide();
+};

src/main/webapp/styles/dialogs/freebase-loading-dialog.css

@@ -25,7 +25,7 @@
 }
     
 .freebase-loading-tripleloader-info {
-    margin-top: 1em;
+    margin-bottom: 0.5em;
 }
 
 .freebase-loading-tripleloader-info textarea {