From c9760916242925bd1ff98ac2e1c145f800de601d Mon Sep 17 00:00:00 2001 From: Stefano Mazzocchi Date: Tue, 28 Sep 2010 02:19:50 +0000 Subject: [PATCH] new hooks to the Freebase Refinery git-svn-id: http://google-refine.googlecode.com/svn/trunk@1368 7d457c2a-affb-35e4-300a-418c747d4874 --- .../commands/freebase/UploadDataCommand.java | 10 +- .../com/google/refine/util/FreebaseUtils.java | 128 +++++++----------- .../dialogs/freebase-loading-dialog.html | 7 +- .../dialogs/freebase-loading-dialog.js | 62 +++------ .../dialogs/freebase-loading-dialog.css | 24 ++++ 5 files changed, 96 insertions(+), 135 deletions(-) diff --git a/main/src/com/google/refine/commands/freebase/UploadDataCommand.java b/main/src/com/google/refine/commands/freebase/UploadDataCommand.java index 5a87be5cb..db067cc63 100644 --- a/main/src/com/google/refine/commands/freebase/UploadDataCommand.java +++ b/main/src/com/google/refine/commands/freebase/UploadDataCommand.java @@ -41,7 +41,7 @@ public class UploadDataCommand extends Command { String source_name = request.getParameter("source_name"); String source_id = request.getParameter("source_id"); - String graph = request.getParameter("graph"); + String qa = request.getParameter("qa"); String mdo_id = null; preferenceStore.put(s_dataLoadJobNamePref, source_name); @@ -61,7 +61,8 @@ public class UploadDataCommand extends Command { } String uploadResponse = FreebaseUtils.uploadTriples( - request, graph, source_name, source_id, mdo_id, triples.toString()); + request, qa, source_name, source_id, mdo_id, triples.toString() + ); response.setCharacterEncoding("UTF-8"); response.setHeader("Content-Type", "application/json"); @@ -72,16 +73,13 @@ public class UploadDataCommand extends Command { JSONObject result = obj.getJSONObject("result"); if (result.has("job_id") && !result.isNull("job_id")) { Integer jobID = result.getInt("job_id"); - - project.getMetadata().getPreferenceStore().put( - s_dataLoadJobIDPref, jobID); + project.getMetadata().getPreferenceStore().put(s_dataLoadJobIDPref, jobID); } } response.getWriter().write(uploadResponse); } catch (JSONException e) { respond(response,"500 Error", uploadResponse); } - } catch (Exception e) { respondException(response, e); } finally { diff --git a/main/src/com/google/refine/util/FreebaseUtils.java b/main/src/com/google/refine/util/FreebaseUtils.java index bf248a88b..c1bef4c76 100644 --- a/main/src/com/google/refine/util/FreebaseUtils.java +++ b/main/src/com/google/refine/util/FreebaseUtils.java @@ -23,12 +23,11 @@ import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.message.BasicNameValuePair; import org.apache.http.params.CoreProtocolPNames; import org.apache.http.util.EntityUtils; -import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import com.google.refine.RefineServlet; import com.google.refine.ProjectManager; +import com.google.refine.RefineServlet; import com.google.refine.oauth.Credentials; import com.google.refine.oauth.OAuthUtilities; import com.google.refine.oauth.Provider; @@ -39,8 +38,9 @@ public class FreebaseUtils { static final public String FREEBASE_SANDBOX_HOST = "www.sandbox-freebase.com"; static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/gridworks"; + //static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine"; - static final private String GRIDWORKS_ID = "/en/gridworks"; + static final private String AGENT_ID = "/en/google_refine"; private static String getUserInfoURL(String host) { return "http://" + host + "/api/service/user_info"; @@ -54,13 +54,17 @@ public class FreebaseUtils { return "http://" + host + "/api/service/mqlread"; } + private static String getUserAgent() { + return "Google Refine " + RefineServlet.getVersion(); + } + public static String getUserInfo(Credentials credentials, Provider provider) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException { OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + RefineServlet.getVersion()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); // this is required by the Metaweb API to avoid XSS httpRequest.setHeader("X-Requested-With", "1"); @@ -103,7 +107,7 @@ public class FreebaseUtils { UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + RefineServlet.getVersion()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); httpRequest.setEntity(entity); // this is required by the Metaweb API to avoid XSS @@ -129,7 +133,7 @@ public class FreebaseUtils { UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + RefineServlet.getVersion()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); httpRequest.setEntity(entity); // this is required by the Metaweb API to avoid XSS @@ -148,7 +152,7 @@ public class FreebaseUtils { public static String uploadTriples( HttpServletRequest request, - String graph, + String qa, String source_name, String source_id, String mdo_id, @@ -168,82 +172,52 @@ public class FreebaseUtils { JSONObject user_info = new JSONObject(getUserInfo(credentials, provider)); if (user_info.has("username")) { - String user_id = user_info.getString("id"); - boolean allowed = isAllowedToWrite(provider, graph, user_id); - - if (allowed) { - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("user", user_info.getString("id"))); - formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE")); - formparams.add(new BasicNameValuePair("operator", user_info.getString("id"))); - formparams.add(new BasicNameValuePair("software_tool_used", GRIDWORKS_ID)); - formparams.add(new BasicNameValuePair("rabj", "true")); - formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString())); - formparams.add(new BasicNameValuePair("graphport", graph)); - formparams.add(new BasicNameValuePair("payload", triples)); - formparams.add(new BasicNameValuePair("check_params", "false")); - if (mdo_id != null) { - formparams.add(new BasicNameValuePair("mdo_guid", mdo_id)); - } - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - - HttpPost httpRequest = new HttpPost(getFreeQUrl()); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + RefineServlet.getVersion()); - httpRequest.setEntity(entity); - - HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); - surrogateRequest.setEntity(entity); - - OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); - - consumer.sign(surrogateRequest); - - Header[] h = surrogateRequest.getHeaders("Authorization"); - if (h.length > 0) { - httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); - } else { - throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); - } - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } else { - throw new RuntimeException("User '" + user_id + "' is not allowed to write to '" + graph + "' with Gridworks"); + List formparams = new ArrayList(); + formparams.add(new BasicNameValuePair("user", user_info.getString("id"))); + formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE")); + formparams.add(new BasicNameValuePair("operator", user_info.getString("id"))); + formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID)); + formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString())); + formparams.add(new BasicNameValuePair("graphport", "sandbox")); + formparams.add(new BasicNameValuePair("payload", triples)); + formparams.add(new BasicNameValuePair("check_params", "false")); + if (mdo_id != null) { + formparams.add(new BasicNameValuePair("mdo_guid", mdo_id)); } + if (Boolean.parseBoolean(qa)) { + formparams.add(new BasicNameValuePair("rabj", "true")); + } + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); + + HttpPost httpRequest = new HttpPost(getFreeQUrl()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Google Refine " + RefineServlet.getVersion()); + httpRequest.setEntity(entity); + + HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); + surrogateRequest.setEntity(entity); + + OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); + + consumer.sign(surrogateRequest); + + Header[] h = surrogateRequest.getHeaders("Authorization"); + if (h.length > 0) { + httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); + } else { + throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); + } + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); } else { throw new RuntimeException("Invalid credentials"); } } - - private static boolean isAllowedToWrite(Provider provider, String graph, String user_id) throws JSONException, ClientProtocolException, IOException { - if ("sandbox".equals(graph)) return true; - JSONObject user_badges = new JSONObject(getUserBadges(provider, user_id)); - JSONObject result = user_badges.getJSONObject("result"); - - if (result == null) { - throw new RuntimeException("Error evaluating badges for user '" + user_id + "'"); - } - - boolean allowed = false; - - JSONArray badges = result.getJSONArray("!/type/usergroup/member"); - for (int i = 0; i < badges.length(); i++) { - JSONObject o = badges.getJSONObject(i); - String id = o.getString("id"); - if ("/en/metaweb_staff".equals(id)) { - allowed = true; - break; - } - } - - return allowed; - } - static public String getFreeQUrl() { String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); return url != null ? url : FREEQ_URL; diff --git a/main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.html b/main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.html index a1433649f..f0425868f 100644 --- a/main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.html +++ b/main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.html @@ -11,12 +11,7 @@
- - Load this data into - - - - + Ready to perform QA on this data? diff --git a/main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.js b/main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.js index 15885b97e..20878aeb3 100644 --- a/main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.js +++ b/main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.js @@ -8,7 +8,6 @@ FreebaseLoadingDialog.prototype._createDialog = function() { var dialog = $(DOM.loadHTML("core", "scripts/dialogs/freebase-loading-dialog.html")); this._elmts = DOM.bind(dialog); this._elmts.cancelButton.click(function() { self._dismiss(); }); - this._elmts.selector.buttonset(); var provider = "www.freebase.com"; var authorization = this._elmts.authorization; @@ -40,30 +39,7 @@ FreebaseLoadingDialog.prototype._createDialog = function() { } },"json"); }; - - var check_allowed = function(user_id, cont) { - $.get("/command/core/user-badges/" + provider, - { "user_id" : user_id }, - function(data) { - if ("status" in data && data.code == "/api/status/ok") { - var badges = data.result['!/type/usergroup/member']; - var allowed = false; - for (var i = 0; i < badges.length; i++) { - var id = badges[i].id; - if (id == "/en/metaweb_staff") { - allowed = true; - break; - } - } - if (typeof cont == "function") cont(allowed); - } else { - self._show_error("Error checking if user is a staff member", data); - } - }, - "json" - ); - }; - + var make_topic = function(new_topic_id, topic_type, cont) { var mql_query = [{ "create": "unless_exists", @@ -153,22 +129,14 @@ FreebaseLoadingDialog.prototype._createDialog = function() { ); }; - show_triples(function() { - check_authorization(function(data) { - check_allowed(data.id, function(is_allowed) { - if (is_allowed) { - $("#freebase-loading-graph-selector-freebase").removeAttr("disabled").button("refresh"); - } - }); - }); - }); + show_triples(check_authorization); }; FreebaseLoadingDialog.prototype._load = function() { var self = this; - var freebase = self._elmts.freebase.attr("checked"); + var qa = self._elmts.qaCheckbox.is(':checked'); - var get_peacock_url = function(url) { + var get_refinery_url = function(url) { return "http://refinery.freebaseapps.com/load/" + url.split("/").slice(-1)[0]; }; @@ -178,7 +146,7 @@ FreebaseLoadingDialog.prototype._load = function() { $.post("/command/core/upload-data", { project: theProject.id, - "graph" : (freebase) ? "otg" : "sandbox", + "qa" : qa, "engine" : JSON.stringify(ui.browsingEngine.getJSON()), "source_name" : self._elmts.source_name.val(), "source_id" : self._elmts.source_id.val() @@ -190,8 +158,8 @@ FreebaseLoadingDialog.prototype._load = function() { if ("status" in data && typeof data.status == "object" && "code" in data.status && data.status.code == 200) { body.html( '
' + - '

' + data.result.added + ' triples successfully scheduled for loading

' + - '

Follow the loading progress here.

' + + '

' + data.result.added + ' triples successfully scheduled for loading

' + + '

Follow the loading progress in the Freebase Refinery

' + '
' ); self._end(); @@ -203,10 +171,10 @@ FreebaseLoadingDialog.prototype._load = function() { ); }; - if (freebase) { + if (qa) { var dialog = $( '
' + - '
Are you sure this data is ready to be uploaded into Freebase?
' + + '
Are you sure this data is ready to be tested for upload into Freebase?
' + '
' ).dialog({ resizable: false, @@ -214,11 +182,11 @@ FreebaseLoadingDialog.prototype._load = function() { height: 230, modal: true, buttons: { - 'yes, I know what I\'m doing': function() { + 'yes, I\'m sure': function() { $(this).dialog('close'); doLoad(); }, - 'cancel': function() { + 'hmm, not really': function() { $(this).dialog('close'); } } @@ -233,12 +201,14 @@ FreebaseLoadingDialog.prototype._dismiss = function() { }; FreebaseLoadingDialog.prototype._show_error = function(msg, error) { + console.log(error); + var self = this; var body = self._elmts.dialogBody; body.html( '
' + '

' + msg + '

' + - '

' + error.message + '

' + - (('stack' in error) ? '
' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '
' : "") +
+            (('message' in error) ? '
' + error.message + '
' : '
' + JSON.stringify(error, null, 2) + '
') +
+            (('stack' in error) ? '
' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '
' : "") +
         '
' ); this._end(); @@ -250,6 +220,6 @@ FreebaseLoadingDialog.prototype._end = function() { self._dismiss(); }); self._elmts.cancelButton.hide(); + self._elmts.qaCheckboxContainer.hide(); self._elmts.authorization.hide(); - self._elmts.selector.hide(); }; \ No newline at end of file diff --git a/main/webapp/modules/core/styles/dialogs/freebase-loading-dialog.css b/main/webapp/modules/core/styles/dialogs/freebase-loading-dialog.css index 5b7a29990..b5e260823 100644 --- a/main/webapp/modules/core/styles/dialogs/freebase-loading-dialog.css +++ b/main/webapp/modules/core/styles/dialogs/freebase-loading-dialog.css @@ -15,6 +15,30 @@ padding: 0.3em 0.5em 0.5em 0.5em; } +.freebase-loading-tripleloader-message h2 { + text-align: center; + font-size: 150%; + margin-top: 3em; + font-weight: normal; +} + +.freebase-loading-tripleloader-message h2 span { + font-size; 130%; + font-weight: bold; +} + +.freebase-loading-tripleloader-message h4 { + text-align: center; + font-size: 120%; + margin-top: 2em; + font-weight: normal; +} + +.freebase-loading-tripleloader-message a { + font-size: 120%; + font-weight: bold; +} + .freebase-loading-authorization { margin: 0em 2em; padding: 0.6em 0.8em;