s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Centralize Google API key

Browse Source
This commit is contained in:
Tom Morris 2013-03-12 17:34:06 -04:00
parent 054ea643c3
commit 757c3e32ff
2 changed files with 305 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
View File

@ -35,12 +35,13 @@ package com.google.refine.freebase.model.recon;
import org.json.JSONObject; import org.json.JSONObject;
import com.google.refine.freebase.util.FreebaseUtils;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
import com.google.refine.model.Recon.Judgment; import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig; import com.google.refine.model.recon.ReconConfig;
abstract public class StrictReconConfig extends ReconConfig { abstract public class StrictReconConfig extends ReconConfig {
final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M&"; final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=" + FreebaseUtils.API_KEY + "&";
static public ReconConfig reconstruct(JSONObject obj) throws Exception { static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match"); String match = obj.getString("match");

591
extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
View File

@ -1,289 +1,304 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010, Google Inc.
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are modification, are permitted provided that the following conditions are
met: met:
* Redistributions of source code must retain the above copyright * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the in the documentation and/or other materials provided with the
distribution. distribution.
* Neither the name of Google Inc. nor the names of its * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from contributors may be used to endorse or promote products derived from
this software without specific prior written permission. this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package com.google.refine.freebase.util; package com.google.refine.freebase.util;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import oauth.signpost.OAuthConsumer; import oauth.signpost.OAuthConsumer;
import oauth.signpost.exception.OAuthCommunicationException; import oauth.signpost.exception.OAuthCommunicationException;
import oauth.signpost.exception.OAuthExpectationFailedException; import oauth.signpost.exception.OAuthExpectationFailedException;
import oauth.signpost.exception.OAuthMessageSignerException; import oauth.signpost.exception.OAuthMessageSignerException;
import org.apache.http.Header; import org.apache.http.Header;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair; import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException; import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair; import org.apache.http.message.BasicNameValuePair;
import org.apache.http.params.CoreProtocolPNames; import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet; import com.google.refine.RefineServlet;
import com.google.refine.oauth.Credentials; import com.google.refine.oauth.Credentials;
import com.google.refine.oauth.OAuthUtilities; import com.google.refine.oauth.OAuthUtilities;
import com.google.refine.oauth.Provider; import com.google.refine.oauth.Provider;
public class FreebaseUtils { public class FreebaseUtils {
static final Logger logger = LoggerFactory.getLogger("freebase"); static final Logger logger = LoggerFactory.getLogger("freebase");
static final public String FREEBASE_HOST = "freebase.com"; static final public String FREEBASE_HOST = "freebase.com";
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine"; static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
static final private String AGENT_ID = "/en/google_refine"; static final private String AGENT_ID = "/en/google_refine";
static final private int SAMPLE_SIZE = 300; static final private int SAMPLE_SIZE = 300;
static final private int JUDGES = 4; static final private int JUDGES = 4;
private static String getUserInfoURL(String host) { public static final String API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M";
return "http://api." + host + "/api/service/user_info";
} private static String getUserInfoURL(String host) {
// TODO: Needs to be upgraded to new APIs sandbox-freebase.com as host becomes v1sandbox as version
private static String getMQLWriteURL(String host) { return "http://api." + host + "/api/service/user_info";
return "http://api." + host + "/api/service/mqlwrite"; }
}
private static String getMQLWriteURL(String host) {
private static String getMQLReadURL(String host) { // TODO: Needs to be upgraded to new APIs
return "http://api." + host + "/api/service/mqlread"; // String version = "v1";
} // if (host != null && host.contains("sandbox")) {
// version += "sandbox";
private static String getUserAgent() { // }
return RefineServlet.FULLNAME; // return "https://www.googleapis.com/freebase/"+version+"/mqlwrite?key=" + API_KEY + "&";
} return "http://api." + host + "/api/service/mqlwrite";
}
public static String getUserInfo(Credentials credentials, Provider provider)
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException { private static String getMQLReadURL(String host) {
// TODO: Needs to be upgraded to new APIs
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); // String version = "v1";
// if (host != null && host.contains("sandbox")) {
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); // version += "sandbox";
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); // }
// return "https://www.googleapis.com/freebase/"+version+"/mqlread?key=" + API_KEY + "&";
// this is required by the Metaweb API to avoid XSS return "http://api." + host + "/api/service/mqlread";
httpRequest.setHeader("X-Requested-With", "1"); }
// sign the request with the oauth library private static String getUserAgent() {
consumer.sign(httpRequest); return RefineServlet.FULLNAME;
}
// execute the request
HttpClient httpClient = new DefaultHttpClient(); public static String getUserInfo(Credentials credentials, Provider provider)
HttpResponse httpResponse = httpClient.execute(httpRequest); throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
// return the results OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
return EntityUtils.toString(httpResponse.getEntity());
} HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
public static String getUserBadges(Provider provider, String user_id)
throws ClientProtocolException, IOException, JSONException { // this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1");
String query = "{" +
"'id' : '" + user_id + "'," + // sign the request with the oauth library
"'!/type/usergroup/member' : [{" + consumer.sign(httpRequest);
"'id' : null," +
"'key' : [{" + // execute the request
"'namespace' : null" + HttpClient httpClient = new DefaultHttpClient();
"}]" + HttpResponse httpResponse = httpClient.execute(httpRequest);
"}]" +
"}".replace("'", "\""); // return the results
return EntityUtils.toString(httpResponse.getEntity());
return mqlread(provider, query); }
}
public static String getUserBadges(Provider provider, String user_id)
public static String mqlread(Provider provider, String query) throws ClientProtocolException, IOException, JSONException {
throws ClientProtocolException, IOException, JSONException {
String query = "{" +
JSONObject envelope = new JSONObject(); "'id' : '" + user_id + "'," +
envelope.put("query", new JSONObject(query)); "'!/type/usergroup/member' : [{" +
"'id' : null," +
List<NameValuePair> formparams = new ArrayList<NameValuePair>(); "'key' : [{" +
formparams.add(new BasicNameValuePair("query", envelope.toString())); "'namespace' : null" +
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); "}]" +
"}]" +
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); "}".replace("'", "\"");
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity); return mqlread(provider, query);
}
// this is required by the Metaweb API to avoid XSS
httpRequest.setHeader("X-Requested-With", "1"); public static String mqlread(Provider provider, String query)
throws ClientProtocolException, IOException, JSONException {
// execute the request
HttpClient httpClient = new DefaultHttpClient(); JSONObject envelope = new JSONObject();
HttpResponse httpResponse = httpClient.execute(httpRequest); envelope.put("query", new JSONObject(query));
// return the results List<NameValuePair> formparams = new ArrayList<NameValuePair>();
return EntityUtils.toString(httpResponse.getEntity()); formparams.add(new BasicNameValuePair("query", envelope.toString()));
} UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
public static String mqlwrite(Credentials credentials, Provider provider, String query) HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException { httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); httpRequest.setEntity(entity);
JSONObject envelope = new JSONObject(); // this is required by the Metaweb API to avoid XSS
envelope.put("query", new JSONObject(query)); httpRequest.setHeader("X-Requested-With", "1");
List<NameValuePair> formparams = new ArrayList<NameValuePair>(); // execute the request
formparams.add(new BasicNameValuePair("query", envelope.toString())); HttpClient httpClient = new DefaultHttpClient();
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); HttpResponse httpResponse = httpClient.execute(httpRequest);
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); // return the results
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); return EntityUtils.toString(httpResponse.getEntity());
httpRequest.setEntity(entity); }
// this is required by the Metaweb API to avoid XSS public static String mqlwrite(Credentials credentials, Provider provider, String query)
httpRequest.setHeader("X-Requested-With", "1"); throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
// sign the request with the oauth library
consumer.sign(httpRequest); JSONObject envelope = new JSONObject();
envelope.put("query", new JSONObject(query));
// execute the request
HttpClient httpClient = new DefaultHttpClient(); List<NameValuePair> formparams = new ArrayList<NameValuePair>();
HttpResponse httpResponse = httpClient.execute(httpRequest); formparams.add(new BasicNameValuePair("query", envelope.toString()));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
// return the results
return EntityUtils.toString(httpResponse.getEntity()); HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
} httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity);
private static String getTweezersParams(int sample_size, int judges) {
String o = "{" + // this is required by the Metaweb API to avoid XSS
"'sample_size':" + sample_size + "," + httpRequest.setHeader("X-Requested-With", "1");
"'votes':{" +
"'reconciled':" + judges + "," + // sign the request with the oauth library
"'invalid':" + judges + "," + consumer.sign(httpRequest);
"'new':" + judges + "," +
"'skip':" + (judges + 2) + // execute the request
"}" + HttpClient httpClient = new DefaultHttpClient();
"}"; HttpResponse httpResponse = httpClient.execute(httpRequest);
return o.replace('\'', '"');
} // return the results
return EntityUtils.toString(httpResponse.getEntity());
public static String uploadTriples( }
HttpServletRequest request,
String qa, private static String getTweezersParams(int sample_size, int judges) {
String source_name, String o = "{" +
String source_id, "'sample_size':" + sample_size + "," +
String mdo_id, "'votes':{" +
String triples "'reconciled':" + judges + "," +
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException { "'invalid':" + judges + "," +
"'new':" + judges + "," +
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST); "'skip':" + (judges + 2) +
"}" +
Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); "}";
return o.replace('\'', '"');
JSONObject mdo_info = new JSONObject(); }
mdo_info.put("name", source_name);
if (source_id != null) { public static String uploadTriples(
mdo_info.put("info_source",source_id); HttpServletRequest request,
} String qa,
String source_name,
JSONObject user_info = new JSONObject(getUserInfo(credentials, provider)); String source_id,
if (user_info.has("username")) { String mdo_id,
String triples
List<NameValuePair> formparams = new ArrayList<NameValuePair>(); ) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE")); Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID)); Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
formparams.add(new BasicNameValuePair("graphport", "sandbox")); JSONObject mdo_info = new JSONObject();
formparams.add(new BasicNameValuePair("payload", triples)); mdo_info.put("name", source_name);
formparams.add(new BasicNameValuePair("check_params", "false")); if (source_id != null) {
if (mdo_id != null) { mdo_info.put("info_source",source_id);
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id)); }
}
if (Boolean.parseBoolean(qa)) { JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES))); if (user_info.has("username")) {
}
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
String freeqKey = System.getProperty("freeq.key"); formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
if (freeqKey != null) { formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
logger.warn("Found Freeq key, will bypass OAuth signature"); formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
formparams.add(new BasicNameValuePair("apikey", freeqKey)); formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
} formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
formparams.add(new BasicNameValuePair("graphport", "sandbox"));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); formparams.add(new BasicNameValuePair("payload", triples));
formparams.add(new BasicNameValuePair("check_params", "false"));
HttpPost httpRequest = new HttpPost(getFreeQUrl()); if (mdo_id != null) {
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
httpRequest.setEntity(entity); }
if (Boolean.parseBoolean(qa)) {
if (freeqKey == null) { formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
logger.warn("Calculating OAuth signature"); }
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity); String freeqKey = System.getProperty("freeq.key");
if (freeqKey != null) {
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); logger.warn("Found Freeq key, will bypass OAuth signature");
formparams.add(new BasicNameValuePair("apikey", freeqKey));
// TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors. }
// Is there something we can do to generate an oauth signature without consuming so much memory?
consumer.sign(surrogateRequest); UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
Header[] h = surrogateRequest.getHeaders("Authorization"); HttpPost httpRequest = new HttpPost(getFreeQUrl());
if (h.length > 0) { httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); httpRequest.setEntity(entity);
} else {
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); if (freeqKey == null) {
} logger.warn("Calculating OAuth signature");
} HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity);
// execute the request
HttpClient httpClient = new DefaultHttpClient(); OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
HttpResponse httpResponse = httpClient.execute(httpRequest);
// TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors.
// return the results // Is there something we can do to generate an oauth signature without consuming so much memory?
return EntityUtils.toString(httpResponse.getEntity()); consumer.sign(surrogateRequest);
} else {
throw new RuntimeException("Invalid credentials"); Header[] h = surrogateRequest.getHeaders("Authorization");
} if (h.length > 0) {
} httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
} else {
static public String getFreeQUrl() { throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); }
return url != null ? url : FREEQ_URL; }
}
// execute the request
HttpClient httpClient = new DefaultHttpClient();
HttpResponse httpResponse = httpClient.execute(httpRequest);
// return the results
return EntityUtils.toString(httpResponse.getEntity());
} else {
throw new RuntimeException("Invalid credentials");
}
}
static public String getFreeQUrl() {
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
return url != null ? url : FREEQ_URL;
}
} }