allow people with freeq keys to load data bypassing oauth signature (whichi is poorly implemented in SignPost and consumes ridiculous amounts of memory to perform the OAuth signing).

NOTE: this is only a temporary measure while we figure out a better way to perform authorized loads

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2353 7d457c2a-affb-35e4-300a-418c747d4874

extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java

@@ -58,6 +58,8 @@ import org.apache.http.params.CoreProtocolPNames;
 import org.apache.http.util.EntityUtils;
 import org.json.JSONException;
 import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.refine.ProjectManager;
 import com.google.refine.RefineServlet;
@@ -67,6 +69,8 @@ import com.google.refine.oauth.Provider;
 
 public class FreebaseUtils {
 
+    static final Logger logger = LoggerFactory.getLogger("freebase");
+    
     static final public String FREEBASE_HOST = "www.freebase.com";
     
     static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
@@ -234,17 +238,28 @@ public class FreebaseUtils {
             if (Boolean.parseBoolean(qa)) {
                 formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
             }
+
+            String freeqKey = System.getProperty("freeq.key");
+            if (freeqKey != null) {
+                logger.warn("Found Freeq key, will bypass OAuth signature");
+                formparams.add(new BasicNameValuePair("apikey", freeqKey));
+            }
+
             UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
 
             HttpPost httpRequest = new HttpPost(getFreeQUrl());
             httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
             httpRequest.setEntity(entity);
 
+            if (freeqKey == null) {
+                logger.warn("Calculating OAuth signature");
                 HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
                 surrogateRequest.setEntity(entity);
                 
                 OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
     
+                // TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors.
+                // Is there something we can do to generate an oauth signature without consuming so much memory?
                 consumer.sign(surrogateRequest);
                     
                 Header[] h = surrogateRequest.getHeaders("Authorization");
@@ -253,6 +268,7 @@ public class FreebaseUtils {
                 } else {
                     throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
                 }
+            }
             
             // execute the request
             HttpClient httpClient = new DefaultHttpClient();
@@ -269,4 +285,5 @@ public class FreebaseUtils {
         String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
         return url != null ? url : FREEQ_URL;
     }
+    
 }

refine

@@ -49,6 +49,9 @@ where [options] include:
   -v <level> verbosity level [from low to high: error,warn,info,debug,trace]
      default: info
 
+  -x <name=value> additional configuration parameters to pass to Google Refine
+     default: [none]
+  
   --debug enable JVM debugging (on port 8000)
        
   --jmx enable JMX monitoring (for jconsole and jvisualvm)
@@ -850,6 +853,7 @@ while [ $# -ne 0 ] ; do
     -d) shift; REFINE_DATA_DIR="$1"; shift; continue;;
     -m) shift; REFINE_MEMORY="$1"; shift; continue;;
     -v) shift; REFINE_VERBOSITY="$1"; shift; continue;;
+    -x) shift; REFINE_EXTRA_OPTS="$1"; shift; continue;;
     --debug) shift; add_option '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;;
     --jmx) shift; add_option '-Dcom.sun.management.jmxremote'; continue;;
     -*) fail "Invalid option: $1";;
@@ -875,7 +879,7 @@ add_option "$JAVA_OPTIONS"
 if [ -z "$REFINE_MEMORY" ] ; then
     REFINE_MEMORY="1024M"
 fi
-add_option "-Xms256M -Xmx$REFINE_MEMORY -Drefine.memory=$REFINE_MEMORY -XX:MaxPermSize=256m -XX:+CMSClassUnloadingEnabled"
+add_option "-Xms256M -Xmx$REFINE_MEMORY -Drefine.memory=$REFINE_MEMORY"
 
 if [ -z "$REFINE_PORT" ] ; then
     REFINE_PORT="3333"
@@ -918,6 +922,10 @@ if [ -z "$REFINE_VERBOSITY" ] ; then
 fi
 add_option "-Drefine.verbosity=$REFINE_VERBOSITY"
 
+if [ ! -z "$REFINE_EXTRA_OPTS" ] ; then
+    add_option "-D$REFINE_EXTRA_OPTS"
+fi
+
 # ----- Respond to the action given --------------------------------------------
 
 case "$ACTION" in