s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

allow people with freeq keys to load data bypassing oauth signature (whichi is poorly implemented in SignPost and consumes ridiculous amounts of memory to perform the OAuth signing).

Browse Source 
NOTE: this is only a temporary measure while we figure out a better way to perform authorized loads

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2353 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2011-11-01 21:47:34 +00:00
parent 856ef6a65a
commit 09889e75c5
2 changed files with 39 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
View File

@ -58,6 +58,8 @@ import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet;
@ -67,6 +69,8 @@ import com.google.refine.oauth.Provider;
public class FreebaseUtils {
static final Logger logger = LoggerFactory.getLogger("freebase");
static final public String FREEBASE_HOST = "www.freebase.com";
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
@ -205,7 +209,7 @@ public class FreebaseUtils {
String mdo_id,
String triples
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
@ -234,24 +238,36 @@ public class FreebaseUtils {
if (Boolean.parseBoolean(qa)) {
formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
}
String freeqKey = System.getProperty("freeq.key");
if (freeqKey != null) {
logger.warn("Found Freeq key, will bypass OAuth signature");
formparams.add(new BasicNameValuePair("apikey", freeqKey));
}
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getFreeQUrl());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity);
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity);
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
consumer.sign(surrogateRequest);
Header[] h = surrogateRequest.getHeaders("Authorization");
if (h.length > 0) {
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
} else {
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
if (freeqKey == null) {
logger.warn("Calculating OAuth signature");
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity);
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
// TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors.
// Is there something we can do to generate an oauth signature without consuming so much memory?
consumer.sign(surrogateRequest);
Header[] h = surrogateRequest.getHeaders("Authorization");
if (h.length > 0) {
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
} else {
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
}
}
// execute the request
@ -269,4 +285,5 @@ public class FreebaseUtils {
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
return url != null ? url : FREEQ_URL;
}
}

10
refine
View File

@ -48,6 +48,9 @@ where [options] include:
-v <level> verbosity level [from low to high: error,warn,info,debug,trace]
default: info
-x <name=value> additional configuration parameters to pass to Google Refine
default: [none]
--debug enable JVM debugging (on port 8000)
@ -850,6 +853,7 @@ while [ $# -ne 0 ] ; do
-d) shift; REFINE_DATA_DIR="$1"; shift; continue;;
-m) shift; REFINE_MEMORY="$1"; shift; continue;;
-v) shift; REFINE_VERBOSITY="$1"; shift; continue;;
-x) shift; REFINE_EXTRA_OPTS="$1"; shift; continue;;
--debug) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;;
--jmx) shift; add_option '-Dcom.sun.management.jmxremote'; continue;;
-*) fail "Invalid option: $1";;
@ -875,7 +879,7 @@ add_option "$JAVA_OPTIONS"
if [ -z "$REFINE_MEMORY" ] ; then
REFINE_MEMORY="1024M"
fi
add_option "-Xms256M -Xmx$REFINE_MEMORY -Drefine.memory=$REFINE_MEMORY -XX:MaxPermSize=256m -XX:+CMSClassUnloadingEnabled"
add_option "-Xms256M -Xmx$REFINE_MEMORY -Drefine.memory=$REFINE_MEMORY"
if [ -z "$REFINE_PORT" ] ; then
REFINE_PORT="3333"
@ -918,6 +922,10 @@ if [ -z "$REFINE_VERBOSITY" ] ; then
fi
add_option "-Drefine.verbosity=$REFINE_VERBOSITY"
if [ ! -z "$REFINE_EXTRA_OPTS" ] ; then
add_option "-D$REFINE_EXTRA_OPTS"
fi
# ----- Respond to the action given --------------------------------------------
case "$ACTION" in