s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

new hooks to the Freebase Refinery

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1368 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-09-28 02:19:50 +00:00
parent 823fe989a4
commit c976091624
5 changed files with 96 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
main/src/com/google/refine/commands/freebase/UploadDataCommand.java
View File

@ -41,7 +41,7 @@ public class UploadDataCommand extends Command {
String source_name = request.getParameter("source_name"); String source_name = request.getParameter("source_name");
String source_id = request.getParameter("source_id"); String source_id = request.getParameter("source_id");
String graph = request.getParameter("graph"); String qa = request.getParameter("qa");
String mdo_id = null; String mdo_id = null;
preferenceStore.put(s_dataLoadJobNamePref, source_name); preferenceStore.put(s_dataLoadJobNamePref, source_name);
@ -61,7 +61,8 @@ public class UploadDataCommand extends Command {
} }
String uploadResponse = FreebaseUtils.uploadTriples( String uploadResponse = FreebaseUtils.uploadTriples(
request, graph, source_name, source_id, mdo_id, triples.toString()); request, qa, source_name, source_id, mdo_id, triples.toString()
);
response.setCharacterEncoding("UTF-8"); response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json"); response.setHeader("Content-Type", "application/json");
@ -72,16 +73,13 @@ public class UploadDataCommand extends Command {
JSONObject result = obj.getJSONObject("result"); JSONObject result = obj.getJSONObject("result");
if (result.has("job_id") && !result.isNull("job_id")) { if (result.has("job_id") && !result.isNull("job_id")) {
Integer jobID = result.getInt("job_id"); Integer jobID = result.getInt("job_id");
project.getMetadata().getPreferenceStore().put(s_dataLoadJobIDPref, jobID);
project.getMetadata().getPreferenceStore().put(
s_dataLoadJobIDPref, jobID);
} }
} }
response.getWriter().write(uploadResponse); response.getWriter().write(uploadResponse);
} catch (JSONException e) { } catch (JSONException e) {
respond(response,"500 Error", uploadResponse); respond(response,"500 Error", uploadResponse);
} }
} catch (Exception e) { } catch (Exception e) {
respondException(response, e); respondException(response, e);
} finally { } finally {

128
main/src/com/google/refine/util/FreebaseUtils.java
View File

@ -23,12 +23,11 @@ import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair; import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.CoreProtocolPNames; import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import com.google.refine.RefineServlet;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
import com.google.refine.oauth.Credentials; import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities; import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider; import com.google.refine.oauth.Provider;
@ -39,8 +38,9 @@ public class FreebaseUtils {
static final public String FREEBASE_SANDBOX_HOST = "www.sandbox-freebase.com"; static final public String FREEBASE_SANDBOX_HOST = "www.sandbox-freebase.com";
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/gridworks"; static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/gridworks";
//static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
static final private String GRIDWORKS_ID = "/en/gridworks"; static final private String AGENT_ID = "/en/google_refine";
private static String getUserInfoURL(String host) { private static String getUserInfoURL(String host) {
return "http://" + host + "/api/service/user_info"; return "http://" + host + "/api/service/user_info";
@ -54,13 +54,17 @@ public class FreebaseUtils {
return "http://" + host + "/api/service/mqlread"; return "http://" + host + "/api/service/mqlread";
} }
private static String getUserAgent() {
return "Google Refine " + RefineServlet.getVersion();
}
public static String getUserInfo(Credentials credentials, Provider provider) public static String getUserInfo(Credentials credentials, Provider provider)
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException { throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + RefineServlet.getVersion()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
// this is required by the Metaweb API to avoid XSS // this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1"); httpRequest.setHeader("X-Requested-With", "1");
@ -103,7 +107,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + RefineServlet.getVersion()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity); httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS // this is required by the Metaweb API to avoid XSS
@ -129,7 +133,7 @@ public class FreebaseUtils {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + RefineServlet.getVersion()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity); httpRequest.setEntity(entity);
// this is required by the Metaweb API to avoid XSS // this is required by the Metaweb API to avoid XSS
@ -148,7 +152,7 @@ public class FreebaseUtils {
public static String uploadTriples( public static String uploadTriples(
HttpServletRequest request, HttpServletRequest request,
String graph, String qa,
String source_name, String source_name,
String source_id, String source_id,
String mdo_id, String mdo_id,
@ -168,82 +172,52 @@ public class FreebaseUtils {
JSONObject user_info = new JSONObject(getUserInfo(credentials, provider)); JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
if (user_info.has("username")) { if (user_info.has("username")) {
String user_id = user_info.getString("id"); List<NameValuePair> formparams = new ArrayList<NameValuePair>();
boolean allowed = isAllowedToWrite(provider, graph, user_id); formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
if (allowed) { formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
List<NameValuePair> formparams = new ArrayList<NameValuePair>(); formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
formparams.add(new BasicNameValuePair("user", user_info.getString("id"))); formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE")); formparams.add(new BasicNameValuePair("graphport", "sandbox"));
formparams.add(new BasicNameValuePair("operator", user_info.getString("id"))); formparams.add(new BasicNameValuePair("payload", triples));
formparams.add(new BasicNameValuePair("software_tool_used", GRIDWORKS_ID)); formparams.add(new BasicNameValuePair("check_params", "false"));
formparams.add(new BasicNameValuePair("rabj", "true")); if (mdo_id != null) {
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString())); formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
formparams.add(new BasicNameValuePair("graphport", graph));
formparams.add(new BasicNameValuePair("payload", triples));
formparams.add(new BasicNameValuePair("check_params", "false"));
if (mdo_id != null) {
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
}
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getFreeQUrl());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + RefineServlet.getVersion());
httpRequest.setEntity(entity);
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity);
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
consumer.sign(surrogateRequest);
Header[] h = surrogateRequest.getHeaders("Authorization");
if (h.length > 0) {
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
} else {
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
}
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
} else {
throw new RuntimeException("User '" + user_id + "' is not allowed to write to '" + graph + "' with Gridworks");
} }
if (Boolean.parseBoolean(qa)) {
formparams.add(new BasicNameValuePair("rabj", "true"));
}
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getFreeQUrl());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Google Refine " + RefineServlet.getVersion());
httpRequest.setEntity(entity);
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity);
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
consumer.sign(surrogateRequest);
Header[] h = surrogateRequest.getHeaders("Authorization");
if (h.length > 0) {
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
} else {
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
}
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
} else { } else {
throw new RuntimeException("Invalid credentials"); throw new RuntimeException("Invalid credentials");
} }
} }
private static boolean isAllowedToWrite(Provider provider, String graph, String user_id) throws JSONException, ClientProtocolException, IOException {
if ("sandbox".equals(graph)) return true;
JSONObject user_badges = new JSONObject(getUserBadges(provider, user_id));
JSONObject result = user_badges.getJSONObject("result");
if (result == null) {
throw new RuntimeException("Error evaluating badges for user '" + user_id + "'");
}
boolean allowed = false;
JSONArray badges = result.getJSONArray("!/type/usergroup/member");
for (int i = 0; i < badges.length(); i++) {
JSONObject o = badges.getJSONObject(i);
String id = o.getString("id");
if ("/en/metaweb_staff".equals(id)) {
allowed = true;
break;
}
}
return allowed;
}
static public String getFreeQUrl() { static public String getFreeQUrl() {
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
return url != null ? url : FREEQ_URL; return url != null ? url : FREEQ_URL;

7
main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.html
View File

@ -11,12 +11,7 @@
<div bind="authorization" class="freebase-loading-authorization"></div> <div bind="authorization" class="freebase-loading-authorization"></div>
</td> </td>
<td bind="right" style="text-align: right" width="40%" nowrap="true"> <td bind="right" style="text-align: right" width="40%" nowrap="true">
<span bind="selector" class="freebase-loading-graph-selector"> <span bind="qaCheckboxContainer">Ready to perform QA on this data? <input bind="qaCheckbox" type="checkbox" name="qa" value="qa"></span>
Load this data into
<input type="radio" bind="sandbox" id="freebase-loading-graph-selector-sandbox" name="graph-selector" checked="checked" value="sandbox"/><label class="sandbox" for="freebase-loading-graph-selector-sandbox" title="Load into the sandbox">sandbox</label>
<input type="radio" bind="freebase" id="freebase-loading-graph-selector-freebase" name="graph-selector" value="freebase" disabled="disabled"/><label class="freebase" for="freebase-loading-graph-selector-freebase" title="Load into Freebase">freebase</label>
</span>
<button bind="loadButton" id="freebase-loading-load" disabled="disabled">Load</button> <button bind="loadButton" id="freebase-loading-load" disabled="disabled">Load</button>
</td> </td>
</tr></table> </tr></table>

62
main/webapp/modules/core/scripts/dialogs/freebase-loading-dialog.js
View File

@ -8,7 +8,6 @@ FreebaseLoadingDialog.prototype._createDialog = function() {
var dialog = $(DOM.loadHTML("core", "scripts/dialogs/freebase-loading-dialog.html")); var dialog = $(DOM.loadHTML("core", "scripts/dialogs/freebase-loading-dialog.html"));
this._elmts = DOM.bind(dialog); this._elmts = DOM.bind(dialog);
this._elmts.cancelButton.click(function() { self._dismiss(); }); this._elmts.cancelButton.click(function() { self._dismiss(); });
this._elmts.selector.buttonset();
var provider = "www.freebase.com"; var provider = "www.freebase.com";
var authorization = this._elmts.authorization; var authorization = this._elmts.authorization;
@ -40,30 +39,7 @@ FreebaseLoadingDialog.prototype._createDialog = function() {
} }
},"json"); },"json");
}; };
var check_allowed = function(user_id, cont) {
$.get("/command/core/user-badges/" + provider,
{ "user_id" : user_id },
function(data) {
if ("status" in data && data.code == "/api/status/ok") {
var badges = data.result['!/type/usergroup/member'];
var allowed = false;
for (var i = 0; i < badges.length; i++) {
var id = badges[i].id;
if (id == "/en/metaweb_staff") {
allowed = true;
break;
}
}
if (typeof cont == "function") cont(allowed);
} else {
self._show_error("Error checking if user is a staff member", data);
}
},
"json"
);
};
var make_topic = function(new_topic_id, topic_type, cont) { var make_topic = function(new_topic_id, topic_type, cont) {
var mql_query = [{ var mql_query = [{
"create": "unless_exists", "create": "unless_exists",
@ -153,22 +129,14 @@ FreebaseLoadingDialog.prototype._createDialog = function() {
); );
}; };
show_triples(function() { show_triples(check_authorization);
check_authorization(function(data) {
check_allowed(data.id, function(is_allowed) {
if (is_allowed) {
$("#freebase-loading-graph-selector-freebase").removeAttr("disabled").button("refresh");
}
});
});
});
}; };
FreebaseLoadingDialog.prototype._load = function() { FreebaseLoadingDialog.prototype._load = function() {
var self = this; var self = this;
var freebase = self._elmts.freebase.attr("checked"); var qa = self._elmts.qaCheckbox.is(':checked');
var get_peacock_url = function(url) { var get_refinery_url = function(url) {
return "http://refinery.freebaseapps.com/load/" + url.split("/").slice(-1)[0]; return "http://refinery.freebaseapps.com/load/" + url.split("/").slice(-1)[0];
}; };
@ -178,7 +146,7 @@ FreebaseLoadingDialog.prototype._load = function() {
$.post("/command/core/upload-data", $.post("/command/core/upload-data",
{ {
project: theProject.id, project: theProject.id,
"graph" : (freebase) ? "otg" : "sandbox", "qa" : qa,
"engine" : JSON.stringify(ui.browsingEngine.getJSON()), "engine" : JSON.stringify(ui.browsingEngine.getJSON()),
"source_name" : self._elmts.source_name.val(), "source_name" : self._elmts.source_name.val(),
"source_id" : self._elmts.source_id.val() "source_id" : self._elmts.source_id.val()
@ -190,8 +158,8 @@ FreebaseLoadingDialog.prototype._load = function() {
if ("status" in data && typeof data.status == "object" && "code" in data.status && data.status.code == 200) { if ("status" in data && typeof data.status == "object" && "code" in data.status && data.status.code == 200) {
body.html( body.html(
'<div class="freebase-loading-tripleloader-message">' + '<div class="freebase-loading-tripleloader-message">' +
'<h2>' + data.result.added + ' triples successfully scheduled for loading</h2>' + '<h2><span>' + data.result.added + '</span> triples successfully scheduled for loading</h2>' +
'<h4>Follow the loading progress <a href="' + get_peacock_url(data.result.status_url) + '" target="_new">here</a>.</h4>' + '<h4>Follow the loading progress in the <a href="' + get_refinery_url(data.result.status_url) + '" target="_new">Freebase Refinery</a></h4>' +
'</div>' '</div>'
); );
self._end(); self._end();
@ -203,10 +171,10 @@ FreebaseLoadingDialog.prototype._load = function() {
); );
}; };
if (freebase) { if (qa) {
var dialog = $( var dialog = $(
'<div id="freebase-confirmation-dialog" title="Are you sure?">' + '<div id="freebase-confirmation-dialog" title="Are you sure?">' +
'<table><tr><td width="30%"><img src="/images/cop.png" width="140px"></td><td width="70%" style="text-align: center; vertical-align: middle; font-size: 120%">Are you sure this data is ready to be uploaded into <b>Freebase</b>?</td></tr></table>' + '<table><tr><td width="30%"><img src="/images/cop.png" width="140px"></td><td width="70%" style="text-align: center; vertical-align: middle; font-size: 120%">Are you sure this data is ready to be tested for upload into <b>Freebase</b>?</td></tr></table>' +
'</div>' '</div>'
).dialog({ ).dialog({
resizable: false, resizable: false,
@ -214,11 +182,11 @@ FreebaseLoadingDialog.prototype._load = function() {
height: 230, height: 230,
modal: true, modal: true,
buttons: { buttons: {
'yes, I know what I\'m doing': function() { 'yes, I\'m sure': function() {
$(this).dialog('close'); $(this).dialog('close');
doLoad(); doLoad();
}, },
'cancel': function() { 'hmm, not really': function() {
$(this).dialog('close'); $(this).dialog('close');
} }
} }
@ -233,12 +201,14 @@ FreebaseLoadingDialog.prototype._dismiss = function() {
}; };
FreebaseLoadingDialog.prototype._show_error = function(msg, error) { FreebaseLoadingDialog.prototype._show_error = function(msg, error) {
console.log(error);
var self = this;
var body = self._elmts.dialogBody; var body = self._elmts.dialogBody;
body.html( body.html(
'<div class="freebase-loading-tripleloader-message">' + '<div class="freebase-loading-tripleloader-message">' +
'<h2>' + msg + '</h2>' + '<h2>' + msg + '</h2>' +
'<p>' + error.message + '</p>' + (('message' in error) ? '<p>' + error.message + '</p>' : '<pre>' + JSON.stringify(error, null, 2) + '</pre>') +
(('stack' in error) ? '<pre>' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '</p>' : "") + (('stack' in error) ? '<pre>' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '</pre>' : "") +
'</div>' '</div>'
); );
this._end(); this._end();
@ -250,6 +220,6 @@ FreebaseLoadingDialog.prototype._end = function() {
self._dismiss(); self._dismiss();
}); });
self._elmts.cancelButton.hide(); self._elmts.cancelButton.hide();
self._elmts.qaCheckboxContainer.hide();
self._elmts.authorization.hide(); self._elmts.authorization.hide();
self._elmts.selector.hide();
}; };

24
main/webapp/modules/core/styles/dialogs/freebase-loading-dialog.css
View File

@ -15,6 +15,30 @@
padding: 0.3em 0.5em 0.5em 0.5em; padding: 0.3em 0.5em 0.5em 0.5em;
} }
.freebase-loading-tripleloader-message h2 {
text-align: center;
font-size: 150%;
margin-top: 3em;
font-weight: normal;
}
.freebase-loading-tripleloader-message h2 span {
font-size; 130%;
font-weight: bold;
}
.freebase-loading-tripleloader-message h4 {
text-align: center;
font-size: 120%;
margin-top: 2em;
font-weight: normal;
}
.freebase-loading-tripleloader-message a {
font-size: 120%;
font-weight: bold;
}
.freebase-loading-authorization { .freebase-loading-authorization {
margin: 0em 2em; margin: 0em 2em;
padding: 0.6em 0.8em; padding: 0.6em 0.8em;