allow people with freeq keys to load data bypassing oauth signature (whichi is poorly implemented in SignPost and consumes ridiculous amounts of memory to perform the OAuth signing).

NOTE: this is only a temporary measure while we figure out a better way to perform authorized loads

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2353 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2011-11-01 21:47:34 +00:00
parent 856ef6a65a
commit 09889e75c5
2 changed files with 39 additions and 14 deletions

View File

@ -58,6 +58,8 @@ import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.ProjectManager; import com.google.refine.ProjectManager;
import com.google.refine.RefineServlet; import com.google.refine.RefineServlet;
@ -67,6 +69,8 @@ import com.google.refine.oauth.Provider;
public class FreebaseUtils { public class FreebaseUtils {
static final Logger logger = LoggerFactory.getLogger("freebase");
static final public String FREEBASE_HOST = "www.freebase.com"; static final public String FREEBASE_HOST = "www.freebase.com";
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine"; static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
@ -234,17 +238,28 @@ public class FreebaseUtils {
if (Boolean.parseBoolean(qa)) { if (Boolean.parseBoolean(qa)) {
formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES))); formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
} }
String freeqKey = System.getProperty("freeq.key");
if (freeqKey != null) {
logger.warn("Found Freeq key, will bypass OAuth signature");
formparams.add(new BasicNameValuePair("apikey", freeqKey));
}
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
HttpPost httpRequest = new HttpPost(getFreeQUrl()); HttpPost httpRequest = new HttpPost(getFreeQUrl());
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
httpRequest.setEntity(entity); httpRequest.setEntity(entity);
if (freeqKey == null) {
logger.warn("Calculating OAuth signature");
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
surrogateRequest.setEntity(entity); surrogateRequest.setEntity(entity);
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
// TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors.
// Is there something we can do to generate an oauth signature without consuming so much memory?
consumer.sign(surrogateRequest); consumer.sign(surrogateRequest);
Header[] h = surrogateRequest.getHeaders("Authorization"); Header[] h = surrogateRequest.getHeaders("Authorization");
@ -253,6 +268,7 @@ public class FreebaseUtils {
} else { } else {
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
} }
}
// execute the request // execute the request
HttpClient httpClient = new DefaultHttpClient(); HttpClient httpClient = new DefaultHttpClient();
@ -269,4 +285,5 @@ public class FreebaseUtils {
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
return url != null ? url : FREEQ_URL; return url != null ? url : FREEQ_URL;
} }
} }

10
refine
View File

@ -49,6 +49,9 @@ where [options] include:
-v <level> verbosity level [from low to high: error,warn,info,debug,trace] -v <level> verbosity level [from low to high: error,warn,info,debug,trace]
default: info default: info
-x <name=value> additional configuration parameters to pass to Google Refine
default: [none]
--debug enable JVM debugging (on port 8000) --debug enable JVM debugging (on port 8000)
--jmx enable JMX monitoring (for jconsole and jvisualvm) --jmx enable JMX monitoring (for jconsole and jvisualvm)
@ -850,6 +853,7 @@ while [ $# -ne 0 ] ; do
-d) shift; REFINE_DATA_DIR="$1"; shift; continue;; -d) shift; REFINE_DATA_DIR="$1"; shift; continue;;
-m) shift; REFINE_MEMORY="$1"; shift; continue;; -m) shift; REFINE_MEMORY="$1"; shift; continue;;
-v) shift; REFINE_VERBOSITY="$1"; shift; continue;; -v) shift; REFINE_VERBOSITY="$1"; shift; continue;;
-x) shift; REFINE_EXTRA_OPTS="$1"; shift; continue;;
--debug) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;; --debug) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;;
--jmx) shift; add_option '-Dcom.sun.management.jmxremote'; continue;; --jmx) shift; add_option '-Dcom.sun.management.jmxremote'; continue;;
-*) fail "Invalid option: $1";; -*) fail "Invalid option: $1";;
@ -875,7 +879,7 @@ add_option "$JAVA_OPTIONS"
if [ -z "$REFINE_MEMORY" ] ; then if [ -z "$REFINE_MEMORY" ] ; then
REFINE_MEMORY="1024M" REFINE_MEMORY="1024M"
fi fi
add_option "-Xms256M -Xmx$REFINE_MEMORY -Drefine.memory=$REFINE_MEMORY -XX:MaxPermSize=256m -XX:+CMSClassUnloadingEnabled" add_option "-Xms256M -Xmx$REFINE_MEMORY -Drefine.memory=$REFINE_MEMORY"
if [ -z "$REFINE_PORT" ] ; then if [ -z "$REFINE_PORT" ] ; then
REFINE_PORT="3333" REFINE_PORT="3333"
@ -918,6 +922,10 @@ if [ -z "$REFINE_VERBOSITY" ] ; then
fi fi
add_option "-Drefine.verbosity=$REFINE_VERBOSITY" add_option "-Drefine.verbosity=$REFINE_VERBOSITY"
if [ ! -z "$REFINE_EXTRA_OPTS" ] ; then
add_option "-D$REFINE_EXTRA_OPTS"
fi
# ----- Respond to the action given -------------------------------------------- # ----- Respond to the action given --------------------------------------------
case "$ACTION" in case "$ACTION" in