diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java index ff3ab6b08..ec9d0c0d8 100644 --- a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java +++ b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java @@ -58,6 +58,8 @@ import org.apache.http.params.CoreProtocolPNames; import org.apache.http.util.EntityUtils; import org.json.JSONException; import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.refine.ProjectManager; import com.google.refine.RefineServlet; @@ -67,6 +69,8 @@ import com.google.refine.oauth.Provider; public class FreebaseUtils { + static final Logger logger = LoggerFactory.getLogger("freebase"); + static final public String FREEBASE_HOST = "www.freebase.com"; static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine"; @@ -205,7 +209,7 @@ public class FreebaseUtils { String mdo_id, String triples ) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException { - + Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST); Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); @@ -234,24 +238,36 @@ public class FreebaseUtils { if (Boolean.parseBoolean(qa)) { formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES))); } + + String freeqKey = System.getProperty("freeq.key"); + if (freeqKey != null) { + logger.warn("Found Freeq key, will bypass OAuth signature"); + formparams.add(new BasicNameValuePair("apikey", freeqKey)); + } + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); HttpPost httpRequest = new HttpPost(getFreeQUrl()); httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); httpRequest.setEntity(entity); - - HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); - surrogateRequest.setEntity(entity); - - OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); - consumer.sign(surrogateRequest); - - Header[] h = surrogateRequest.getHeaders("Authorization"); - if (h.length > 0) { - httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); - } else { - throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); + if (freeqKey == null) { + logger.warn("Calculating OAuth signature"); + HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); + surrogateRequest.setEntity(entity); + + OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); + + // TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors. + // Is there something we can do to generate an oauth signature without consuming so much memory? + consumer.sign(surrogateRequest); + + Header[] h = surrogateRequest.getHeaders("Authorization"); + if (h.length > 0) { + httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); + } else { + throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); + } } // execute the request @@ -269,4 +285,5 @@ public class FreebaseUtils { String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); return url != null ? url : FREEQ_URL; } + } \ No newline at end of file diff --git a/refine b/refine index 3ace186e1..b6e05b889 100755 --- a/refine +++ b/refine @@ -48,6 +48,9 @@ where [options] include: -v verbosity level [from low to high: error,warn,info,debug,trace] default: info + + -x additional configuration parameters to pass to Google Refine + default: [none] --debug enable JVM debugging (on port 8000) @@ -850,6 +853,7 @@ while [ $# -ne 0 ] ; do -d) shift; REFINE_DATA_DIR="$1"; shift; continue;; -m) shift; REFINE_MEMORY="$1"; shift; continue;; -v) shift; REFINE_VERBOSITY="$1"; shift; continue;; + -x) shift; REFINE_EXTRA_OPTS="$1"; shift; continue;; --debug) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;; --jmx) shift; add_option '-Dcom.sun.management.jmxremote'; continue;; -*) fail "Invalid option: $1";; @@ -875,7 +879,7 @@ add_option "$JAVA_OPTIONS" if [ -z "$REFINE_MEMORY" ] ; then REFINE_MEMORY="1024M" fi -add_option "-Xms256M -Xmx$REFINE_MEMORY -Drefine.memory=$REFINE_MEMORY -XX:MaxPermSize=256m -XX:+CMSClassUnloadingEnabled" +add_option "-Xms256M -Xmx$REFINE_MEMORY -Drefine.memory=$REFINE_MEMORY" if [ -z "$REFINE_PORT" ] ; then REFINE_PORT="3333" @@ -918,6 +922,10 @@ if [ -z "$REFINE_VERBOSITY" ] ; then fi add_option "-Drefine.verbosity=$REFINE_VERBOSITY" +if [ ! -z "$REFINE_EXTRA_OPTS" ] ; then + add_option "-D$REFINE_EXTRA_OPTS" +fi + # ----- Respond to the action given -------------------------------------------- case "$ACTION" in