Refactor to use common mqlread code and eliminate clones

extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java

@@ -44,6 +44,11 @@ import com.google.refine.freebase.util.FreebaseUtils;
 import com.google.refine.oauth.OAuthUtilities;
 import com.google.refine.oauth.Provider;
 
+/**
+ * Perform an MQLread on the server, using the client's credentials.
+ * 
+ * Currently unused.  All client code calls the Freebase API directly.
+ */
 public class MQLReadCommand extends Command {
 
     @Override
@@ -55,6 +60,7 @@ public class MQLReadCommand extends Command {
             response.setCharacterEncoding("UTF-8");
             response.setHeader("Content-Type", "application/json");
             String query = request.getParameter("query");
+            @SuppressWarnings("deprecation")
             String result = FreebaseUtils.mqlread(provider,query);
             response.getWriter().write(result);
         } catch (Exception e) {

extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java

@@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.freebase.model.recon;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.StringWriter;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -49,6 +46,7 @@ import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.JSONWriter;
 
+import com.google.refine.freebase.util.FreebaseUtils;
 import com.google.refine.model.Cell;
 import com.google.refine.model.Project;
 import com.google.refine.model.Recon;
@@ -120,88 +118,42 @@ public class GuidBasedReconConfig extends StrictReconConfig {
         Map<String, Recon> guidToRecon = new HashMap<String, Recon>();
         
         try {
-            String query = null;
+            String query = buildQuery(jobs);
-            {
-                StringWriter stringWriter = new StringWriter();
-                JSONWriter jsonWriter = new JSONWriter(stringWriter);
 
-                jsonWriter.array();
+            String s = FreebaseUtils.mqlread(query);
-                jsonWriter.object();
+            JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
             
-                jsonWriter.key("id"); jsonWriter.value(null);
+            if (o.has("result")) {
-                jsonWriter.key("name"); jsonWriter.value(null);
+                JSONArray results = o.getJSONArray("result");
-                jsonWriter.key("guid"); jsonWriter.value(null);
+                int count = results.length();
-                jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
 
-                jsonWriter.key("guid|=");
+                for (int i = 0; i < count; i++) {
-                jsonWriter.array();
+                    JSONObject result = results.getJSONObject(i);
-                for (ReconJob job : jobs) {
-                    jsonWriter.value(((GuidBasedReconJob) job).guid);
-                }
-                jsonWriter.endArray();
 
-                jsonWriter.endObject();
+                    String guid = result.getString("guid");
-                jsonWriter.endArray();
 
-                query = stringWriter.toString();
+                    JSONArray types = result.getJSONArray("type");
-            }
+                    String[] typeIDs = new String[types.length()];
-            
+                    for (int j = 0; j < typeIDs.length; j++) {
-            StringBuffer sb = new StringBuffer(1024);
+                        typeIDs[j] = types.getString(j);
-            sb.append(s_mqlreadService);
-            sb.append("query=");
-            sb.append(ParsingUtilities.encode(query));
-            
-            URL url = new URL(sb.toString());
-            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-            connection.setConnectTimeout(5000);
-            connection.connect();
-            
-            if (connection.getResponseCode() >= 400) {
-                String responseMessage = connection.getResponseMessage();
-                String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
-                LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() 
-                        + " : " + responseMessage + " : " + errorString);
-            } else {
-                InputStream is = connection.getInputStream();
-
-                try {
-                    String s = ParsingUtilities.inputStreamToString(is);
-                    JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
-                    if (o.has("result")) {
-                        JSONArray results = o.getJSONArray("result");
-                        int count = results.length();
-
-                        for (int i = 0; i < count; i++) {
-                            JSONObject result = results.getJSONObject(i);
-
-                            String guid = result.getString("guid");
-
-                            JSONArray types = result.getJSONArray("type");
-                            String[] typeIDs = new String[types.length()];
-                            for (int j = 0; j < typeIDs.length; j++) {
-                                typeIDs[j] = types.getString(j);
-                            }
-
-                            ReconCandidate candidate = new ReconCandidate(
-                                    result.getString("id"),
-                                    result.getString("name"),
-                                    typeIDs,
-                                    100
-                                    );
-
-                            Recon recon = Recon.makeFreebaseRecon(historyEntryID);
-                            recon.addCandidate(candidate);
-                            recon.service = "mql";
-                            recon.judgment = Judgment.Matched;
-                            recon.judgmentAction = "auto";
-                            recon.match = candidate;
-                            recon.matchRank = 0;
-
-                            guidToRecon.put(guid, recon);
-                        }
                     }
-                } finally {
+
-                    is.close();
+                    ReconCandidate candidate = new ReconCandidate(
+                            result.getString("id"),
+                            result.getString("name"),
+                            typeIDs,
+                            100
+                            );
+
+                    Recon recon = Recon.makeFreebaseRecon(historyEntryID);
+                    recon.addCandidate(candidate);
+                    recon.service = "mql";
+                    recon.judgment = Judgment.Matched;
+                    recon.judgmentAction = "auto";
+                    recon.match = candidate;
+                    recon.matchRank = 0;
+
+                    guidToRecon.put(guid, recon);
                 }
             }
         } catch (IOException e) {
@@ -221,4 +173,33 @@ public class GuidBasedReconConfig extends StrictReconConfig {
         
         return recons;
     }
+
+    private String buildQuery(List<ReconJob> jobs)
+            throws JSONException {
+        String query = null;
+
+        StringWriter stringWriter = new StringWriter();
+        JSONWriter jsonWriter = new JSONWriter(stringWriter);
+
+        jsonWriter.array();
+        jsonWriter.object();
+
+        jsonWriter.key("id"); jsonWriter.value(null);
+        jsonWriter.key("name"); jsonWriter.value(null);
+        jsonWriter.key("guid"); jsonWriter.value(null);
+        jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
+
+        jsonWriter.key("guid|=");
+        jsonWriter.array();
+        for (ReconJob job : jobs) {
+            jsonWriter.value(((GuidBasedReconJob) job).guid);
+        }
+        jsonWriter.endArray();
+
+        jsonWriter.endObject();
+        jsonWriter.endArray();
+
+        query = stringWriter.toString();
+        return query;
+    }
 }

extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java

@@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.freebase.model.recon;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.StringWriter;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -49,6 +46,7 @@ import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.JSONWriter;
 
+import com.google.refine.freebase.util.FreebaseUtils;
 import com.google.refine.model.Cell;
 import com.google.refine.model.Project;
 import com.google.refine.model.Recon;
@@ -124,87 +122,41 @@ public class IdBasedReconConfig extends StrictReconConfig {
         Map<String, Recon> idToRecon = new HashMap<String, Recon>();
         
         try {
-            String query = null;
+            String query = buildQuery(jobs);
-            {
+            String s = FreebaseUtils.mqlread(query);
-                StringWriter stringWriter = new StringWriter();
-                JSONWriter jsonWriter = new JSONWriter(stringWriter);
 
-                    jsonWriter.array();
+            JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
-                    jsonWriter.object();
+            if (o.has("result")) {
+                JSONArray results = o.getJSONArray("result");
+                int count = results.length();
 
-                        jsonWriter.key("id"); jsonWriter.value(null);
+                for (int i = 0; i < count; i++) {
-                        jsonWriter.key("name"); jsonWriter.value(null);
+                    JSONObject result = results.getJSONObject(i);
-                        jsonWriter.key("guid"); jsonWriter.value(null);
-                        jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
 
-                        jsonWriter.key("id|=");
+                    String id = result.getString("id");
-                            jsonWriter.array();
-                            for (ReconJob job : jobs) {
-                                jsonWriter.value(((IdBasedReconJob) job).id);
-                            }
-                            jsonWriter.endArray();
 
-                    jsonWriter.endObject();
+                    JSONArray types = result.getJSONArray("type");
-                    jsonWriter.endArray();
+                    String[] typeIDs = new String[types.length()];
-                
+                    for (int j = 0; j < typeIDs.length; j++) {
-                query = stringWriter.toString();
+                        typeIDs[j] = types.getString(j);
-            }
-            
-            StringBuffer sb = new StringBuffer(1024);
-            sb.append(s_mqlreadService);
-            sb.append("query=");
-            sb.append(ParsingUtilities.encode(query));
-            
-            URL url = new URL(sb.toString());
-            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-            connection.setConnectTimeout(5000);
-            connection.connect();
-            
-            if (connection.getResponseCode() >= 400) {
-                String responseMessage = connection.getResponseMessage();
-                String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
-                LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() 
-                        + " : " + responseMessage + " : " + errorString);
-            } else {
-                InputStream is = connection.getInputStream();
-                try {
-                    String s = ParsingUtilities.inputStreamToString(is);
-                    JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
-                    if (o.has("result")) {
-                        JSONArray results = o.getJSONArray("result");
-                        int count = results.length();
-
-                        for (int i = 0; i < count; i++) {
-                            JSONObject result = results.getJSONObject(i);
-
-                            String id = result.getString("id");
-
-                            JSONArray types = result.getJSONArray("type");
-                            String[] typeIDs = new String[types.length()];
-                            for (int j = 0; j < typeIDs.length; j++) {
-                                typeIDs[j] = types.getString(j);
-                            }
-
-                            ReconCandidate candidate = new ReconCandidate(
-                                    id,
-                                    result.getString("name"),
-                                    typeIDs,
-                                    100
-                                    );
-
-                            Recon recon = Recon.makeFreebaseRecon(historyEntryID);
-                            recon.addCandidate(candidate);
-                            recon.service = "mql";
-                            recon.judgment = Judgment.Matched;
-                            recon.judgmentAction = "auto";
-                            recon.match = candidate;
-                            recon.matchRank = 0;
-
-                            idToRecon.put(id, recon);
-                        }
                     }
-                } finally {
+
-                    is.close();
+                    ReconCandidate candidate = new ReconCandidate(
+                            id,
+                            result.getString("name"),
+                            typeIDs,
+                            100
+                            );
+
+                    Recon recon = Recon.makeFreebaseRecon(historyEntryID);
+                    recon.addCandidate(candidate);
+                    recon.service = "mql";
+                    recon.judgment = Judgment.Matched;
+                    recon.judgmentAction = "auto";
+                    recon.match = candidate;
+                    recon.matchRank = 0;
+
+                    idToRecon.put(id, recon);
                 }
             }
         } catch (IOException e) {
@@ -225,4 +177,34 @@ public class IdBasedReconConfig extends StrictReconConfig {
         return recons;
     }
 
+    private String buildQuery(List<ReconJob> jobs)
+            throws JSONException {
+        String query = null;
+        {
+            StringWriter stringWriter = new StringWriter();
+            JSONWriter jsonWriter = new JSONWriter(stringWriter);
+            
+                jsonWriter.array();
+                jsonWriter.object();
+                
+                    jsonWriter.key("id"); jsonWriter.value(null);
+                    jsonWriter.key("name"); jsonWriter.value(null);
+                    jsonWriter.key("guid"); jsonWriter.value(null);
+                    jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
+                    
+                    jsonWriter.key("id|=");
+                        jsonWriter.array();
+                        for (ReconJob job : jobs) {
+                            jsonWriter.value(((IdBasedReconJob) job).id);
+                        }
+                        jsonWriter.endArray();
+                    
+                jsonWriter.endObject();
+                jsonWriter.endArray();
+            
+            query = stringWriter.toString();
+        }
+        return query;
+    }
+
 }

extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java

@@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.freebase.model.recon;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.StringWriter;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -50,6 +47,7 @@ import org.json.JSONObject;
 import org.json.JSONWriter;
 
 import com.google.refine.freebase.FreebaseTopic;
+import com.google.refine.freebase.util.FreebaseUtils;
 import com.google.refine.model.Cell;
 import com.google.refine.model.Project;
 import com.google.refine.model.Recon;
@@ -125,99 +123,41 @@ public class KeyBasedReconConfig extends StrictReconConfig {
         Map<String, Recon> keyToRecon = new HashMap<String, Recon>();
         
         try {
-            String query = null;
+            String query = buildQuery(jobs);
-            {
+            String s = FreebaseUtils.mqlread(query);
-                StringWriter stringWriter = new StringWriter();
-                JSONWriter jsonWriter = new JSONWriter(stringWriter);
             
-                    jsonWriter.array();
+            JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
-                    jsonWriter.object();
+            if (o.has("result")) {
+                JSONArray results = o.getJSONArray("result");
+                int count = results.length();
 
-                        jsonWriter.key("id"); jsonWriter.value(null);
+                for (int i = 0; i < count; i++) {
-                        jsonWriter.key("name"); jsonWriter.value(null);
+                    JSONObject result = results.getJSONObject(i);
-                        jsonWriter.key("guid"); jsonWriter.value(null);
-                        jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
 
-                        jsonWriter.key("key");
+                    String key = result.getJSONArray("key").getJSONObject(0).getString("value");
-                            jsonWriter.array();
-                            jsonWriter.object();
 
-                            jsonWriter.key("namespace");
+                    JSONArray types = result.getJSONArray("type");
-                                jsonWriter.object();
+                    String[] typeIDs = new String[types.length()];
-                                jsonWriter.key("id"); jsonWriter.value(namespace.id);
+                    for (int j = 0; j < typeIDs.length; j++) {
-                                jsonWriter.endObject();
+                        typeIDs[j] = types.getString(j);
-                                
-                            jsonWriter.key("value"); jsonWriter.value(null);
-                            jsonWriter.key("value|=");
-                                jsonWriter.array();
-                                for (ReconJob job : jobs) {
-                                    jsonWriter.value(((KeyBasedReconJob) job).key);
-                                }
-                                jsonWriter.endArray();
-                                
-                            jsonWriter.endObject();
-                            jsonWriter.endArray();
-                        
-                    jsonWriter.endObject();
-                    jsonWriter.endArray();
-                
-                query = stringWriter.toString();
-            }
-            
-            StringBuffer sb = new StringBuffer(1024);
-            sb.append(s_mqlreadService);
-            sb.append("query=");
-            sb.append(ParsingUtilities.encode(query));
-            
-            URL url = new URL(sb.toString());
-            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-            connection.setConnectTimeout(5000);
-            connection.connect();
-            if (connection.getResponseCode() >= 400) {
-                String responseMessage = connection.getResponseMessage();
-                String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
-                LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() 
-                        + " : " + responseMessage + " : " + errorString);
-            } else {
-                InputStream is = connection.getInputStream();
-                try {
-                    String s = ParsingUtilities.inputStreamToString(is);
-                    JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
-                    if (o.has("result")) {
-                        JSONArray results = o.getJSONArray("result");
-                        int count = results.length();
-
-                        for (int i = 0; i < count; i++) {
-                            JSONObject result = results.getJSONObject(i);
-
-                            String key = result.getJSONArray("key").getJSONObject(0).getString("value");
-
-                            JSONArray types = result.getJSONArray("type");
-                            String[] typeIDs = new String[types.length()];
-                            for (int j = 0; j < typeIDs.length; j++) {
-                                typeIDs[j] = types.getString(j);
-                            }
-
-                            ReconCandidate candidate = new ReconCandidate(
-                                    result.getString("id"),
-                                    result.getString("name"),
-                                    typeIDs,
-                                    100
-                                    );
-
-                            Recon recon = Recon.makeFreebaseRecon(historyEntryID);
-                            recon.addCandidate(candidate);
-                            recon.service = "mql";
-                            recon.judgment = Judgment.Matched;
-                            recon.judgmentAction = "auto";
-                            recon.match = candidate;
-                            recon.matchRank = 0;
-
-                            keyToRecon.put(key, recon);
-                        }
                     }
-                } finally {
+
-                    is.close();
+                    ReconCandidate candidate = new ReconCandidate(
+                            result.getString("id"),
+                            result.getString("name"),
+                            typeIDs,
+                            100
+                            );
+
+                    Recon recon = Recon.makeFreebaseRecon(historyEntryID);
+                    recon.addCandidate(candidate);
+                    recon.service = "mql";
+                    recon.judgment = Judgment.Matched;
+                    recon.judgmentAction = "auto";
+                    recon.match = candidate;
+                    recon.matchRank = 0;
+
+                    keyToRecon.put(key, recon);
                 }
             }
         } catch (IOException e) {
@@ -226,7 +166,6 @@ public class KeyBasedReconConfig extends StrictReconConfig {
             LOGGER.error("JSONException during recon : ",e);
         }
         
-
         for (ReconJob job : jobs) {
             String key = ((KeyBasedReconJob) job).key;
             Recon recon = keyToRecon.get(key);
@@ -239,4 +178,47 @@ public class KeyBasedReconConfig extends StrictReconConfig {
         return recons;
     }
 
+    private String buildQuery(List<ReconJob> jobs)
+            throws JSONException {
+        String query = null;
+        {
+            StringWriter stringWriter = new StringWriter();
+            JSONWriter jsonWriter = new JSONWriter(stringWriter);
+            
+                jsonWriter.array();
+                jsonWriter.object();
+                
+                    jsonWriter.key("id"); jsonWriter.value(null);
+                    jsonWriter.key("name"); jsonWriter.value(null);
+                    jsonWriter.key("guid"); jsonWriter.value(null);
+                    jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
+                    
+                    jsonWriter.key("key");
+                        jsonWriter.array();
+                        jsonWriter.object();
+                        
+                        jsonWriter.key("namespace");
+                            jsonWriter.object();
+                            jsonWriter.key("id"); jsonWriter.value(namespace.id);
+                            jsonWriter.endObject();
+                            
+                        jsonWriter.key("value"); jsonWriter.value(null);
+                        jsonWriter.key("value|=");
+                            jsonWriter.array();
+                            for (ReconJob job : jobs) {
+                                jsonWriter.value(((KeyBasedReconJob) job).key);
+                            }
+                            jsonWriter.endArray();
+                            
+                        jsonWriter.endObject();
+                        jsonWriter.endArray();
+                    
+                jsonWriter.endObject();
+                jsonWriter.endArray();
+            
+            query = stringWriter.toString();
+        }
+        return query;
+    }
+
 }

extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java

@@ -35,13 +35,11 @@ package com.google.refine.freebase.model.recon;
 
 import org.json.JSONObject;
 
-import com.google.refine.freebase.util.FreebaseUtils;
 import com.google.refine.model.Recon;
 import com.google.refine.model.Recon.Judgment;
 import com.google.refine.model.recon.ReconConfig;
 
 abstract public class StrictReconConfig extends ReconConfig {
-    final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=" + FreebaseUtils.API_KEY + "&";
 
     static public ReconConfig reconstruct(JSONObject obj) throws Exception {
         String match = obj.getString("match");

extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java

@@ -31,49 +31,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 */
 
-/**
- * 
- */
 package com.google.refine.freebase.util;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
 import java.io.Serializable;
 import java.io.StringWriter;
-import java.io.UnsupportedEncodingException;
 import java.io.Writer;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import javax.mail.MessagingException;
-import javax.mail.internet.MimeBodyPart;
-import javax.mail.internet.MimeMultipart;
-
-import org.apache.http.NameValuePair;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.message.BasicNameValuePair;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.JSONWriter;
 
-import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
-import com.google.api.client.googleapis.json.GoogleJsonError;
-import com.google.api.client.http.HttpHeaders;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.NetHttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson.JacksonFactory;
-import com.google.api.services.freebase.Freebase;
-import com.google.api.services.freebase.FreebaseRequestInitializer;
-
 import com.google.refine.freebase.FreebaseType;
 import com.google.refine.model.ReconCandidate;
 import com.google.refine.util.JSONUtilities;
@@ -116,9 +89,11 @@ public class FreebaseDataExtensionJob {
     ) throws Exception {
         StringWriter writer = new StringWriter();
         formulateQuery(ids, extension, writer);
-        
         String query = writer.toString();
-        JSONObject o = doMqlRead(query);
+        
+        String result = FreebaseUtils.mqlread(query);
+        
+        JSONObject o = ParsingUtilities.evaluateJsonStringToObject(result);
         Map<String, FreebaseDataExtensionJob.DataExtension> map = new HashMap<String, FreebaseDataExtensionJob.DataExtension>();
         if (o.has("result")) {
             JSONArray a = o.getJSONArray("result");
@@ -325,163 +300,6 @@ public class FreebaseDataExtensionJob {
     }
 
 
-    /**
-     * This RPC call works for the Reconcile API, but MQLread is not supported by JSONRPC
-     */
-    static private JSONObject rpcCall(String query) throws JSONException, UnsupportedEncodingException, IOException {
-        URL url = new URL("https://www.googleapis.com/rpc");
-
-        JSONObject params = new JSONObject();
-        params.put("query",query);
-        params.put("key", FreebaseUtils.API_KEY);
-       
-        JSONObject req1 = new JSONObject();
-        req1.put("jsonrpc","2.0");    
-        req1.put("id","q0");
-        req1.put("method","freebase.mqlread");
-        req1.put("apiVersion", "v1");
-        req1.put("params",params);
-
-        JSONArray body = new JSONArray();
-        body.put(req1);
-        
-        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-        connection.setRequestProperty("Content-Type", "application/json"); //
-        connection.setConnectTimeout(5000);
-        connection.setDoOutput(true);
-        
-        OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8");
-        try {
-            writer.write(body.toString());
-        } finally {
-            writer.flush();
-            writer.close();
-        }
-        
-        connection.connect();
-        JSONArray result = null;
-        if (connection.getResponseCode() >= 400) {
-           String responseMessage = connection.getResponseMessage();
-           String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
-        } else {
-            InputStream is = connection.getInputStream();
-            try {
-                String s = ParsingUtilities.inputStreamToString(is);
-                result = ParsingUtilities.evaluateJsonStringToArray(s);
-            } finally {
-                is.close();
-            }
-        }
-        return result.getJSONObject(0);
-    }
-
-    private static MimeBodyPart queryToMimeBodyPart(String query_name, 
-            String query, String service_url, String api_key) 
-                    throws MessagingException, IOException {
-        MimeBodyPart mbp = new MimeBodyPart();
-        mbp.setHeader("Content-Transfer-Encoding", "binary");
-        mbp.setHeader("Content-ID", "<" + query_name + ">");
-        mbp.setHeader("Content-type", "application/http"); 
-        
-        List<NameValuePair> params = new ArrayList<NameValuePair>();
-        params.add(new BasicNameValuePair("query",query));
-        params.add(new BasicNameValuePair("key", api_key));
-        UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8");
-        
-        String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n";
-        mbp.setText(body, "UTF-8");
-        mbp.getLineCount();
-        mbp.getAllHeaderLines();
-        return mbp;
-    }
-
-    
-    /**
-     * The beginnings of a homebrew attempt to generate multi-part MIME messages
-     * so that we can parse response bodies ourselves and avoid the limitations
-     * of the Google Client library.
-     */
-    static private JSONObject batchCallOld(String query) throws JSONException, MessagingException, IOException {
-        URL url = new URL("https://www.googleapis.com/batch");
-        String service_url = "https://www.googleapis.com/freebase/v1/mqlread";
-        MimeMultipart mp = new MimeMultipart("mixed");
-
-        MimeBodyPart mbp = queryToMimeBodyPart("q0", query, service_url, FreebaseUtils.API_KEY);
-        mp.addBodyPart(mbp);
-        mp.getPreamble();
-        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
-        connection.setRequestProperty("Content-Type", mp.getContentType()); 
-        connection.setConnectTimeout(5000);
-        connection.setDoOutput(true);
-        
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        mp.writeTo(baos);
-        String foo = baos.toString("UTF-8");
-        
-        mp.writeTo(connection.getOutputStream());
-        
-        connection.connect();
-        JSONArray result = null;
-        if (connection.getResponseCode() >= 400) {
-           String responseMessage = connection.getResponseMessage();
-           String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
-        } else {
-            InputStream is = connection.getInputStream();
-            try {
-                String s = ParsingUtilities.inputStreamToString(is);
-                result = ParsingUtilities.evaluateJsonStringToArray(s);
-            } finally {
-                is.close();
-            }
-        }
-        return result.getJSONObject(0);        
-    }
-
-    
-    private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
-
-    /** Global instance of the JSON factory. */
-    private static final JsonFactory JSON_FACTORY = new JacksonFactory();
-    
-
-    private static final FreebaseRequestInitializer REQUEST_INITIALIZER = 
-            new FreebaseRequestInitializer(FreebaseUtils.API_KEY);
-  
-    static private JSONObject batchCall1(String query) throws IOException, JSONException {
-
-        JsonBatchCallback<Freebase> callback = new JsonBatchCallback<Freebase>() {
-            public void onSuccess(Freebase res, HttpHeaders responseHeaders) {
-                System.out.println(res);
-            }
-            public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
-              System.out.println("Error Message: " + e.getMessage());
-            }
-          };
-
-          Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
-            .setApplicationName("OpenRefine")
-            .setFreebaseRequestInitializer(REQUEST_INITIALIZER)
-            .build();
-          
-          // TODO: Batch doesn't work with MqlRead since it extends FreebaseRequest<Void>
-//          BatchRequest batch = client.batch();
-//          client.mqlread(query).queue(batch, callback); 
-//          batch.execute();
-
-          String s = ParsingUtilities.inputStreamToString(client.mqlread(query).executeAsInputStream());
-          JSONObject response = ParsingUtilities.evaluateJsonStringToObject(s);
-
-           return response;
-    }
-    
-    static protected JSONObject doMqlRead(String query) 
-            throws IOException, JSONException, MessagingException {
-//        JSONObject result = rpcCall(query);
-//        JSONObject resutl =  batchCallOld(query);
-        JSONObject result = batchCall1(query);
-
-        return result;
-    }
   
     static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException {
         JSONWriter jsonWriter = new JSONWriter(writer);

extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java

@@ -1,6 +1,6 @@
 /*
 
-Copyright 2010, Google Inc.
+Copyright 2010,2013 Google Inc. and other contributors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -34,6 +34,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.freebase.util;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -56,19 +62,41 @@ import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.params.CoreProtocolPNames;
 import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.api.client.googleapis.batch.BatchRequest;
+import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
+import com.google.api.client.googleapis.json.GoogleJsonError;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson.JacksonFactory;
+import com.google.api.services.freebase.Freebase;
+import com.google.api.services.freebase.FreebaseRequestInitializer;
+
 import com.google.refine.ProjectManager;
 import com.google.refine.RefineServlet;
 import com.google.refine.oauth.Credentials;
 import com.google.refine.oauth.OAuthUtilities;
 import com.google.refine.oauth.Provider;
+import com.google.refine.util.ParsingUtilities;
 
 public class FreebaseUtils {
 
+    private static final String FREEBASE_API_VERSION = "v1";
+//    private static final String FREEBASE_SANDBOX_API_VERSION = "v1sandbox";
+
+    private static final String GOOGLE_RPC_URL = "https://www.googleapis.com/rpc";
+
+    private static final String FREEBASE_SERVICE_URL = "https://www.googleapis.com/freebase/" + FREEBASE_API_VERSION;
+
+    private static final String GOOGLE_BATCH_URL = "https://www.googleapis.com/batch";
+
     static final Logger logger = LoggerFactory.getLogger("freebase");
     
     static final public String FREEBASE_HOST = "freebase.com";
@@ -88,22 +116,10 @@ public class FreebaseUtils {
     }
 
     private static String getMQLWriteURL(String host) {
-        // TODO: Needs to be upgraded to new APIs
-//      String version = "v1";
-//      if (host != null && host.contains("sandbox")) {
-//          version += "sandbox";
-//      }
-//      return "https://www.googleapis.com/freebase/"+version+"/mqlwrite?key=" + API_KEY + "&";
         return "http://api." + host + "/api/service/mqlwrite";
     }
 
     private static String getMQLReadURL(String host) {
-        // TODO: Needs to be upgraded to new APIs
-//      String version = "v1";
-//      if (host != null && host.contains("sandbox")) {
-//          version += "sandbox";
-//      }
-//      return "https://www.googleapis.com/freebase/"+version+"/mqlread?key=" + API_KEY + "&";
         return "http://api." + host + "/api/service/mqlread";
     }
     
@@ -149,6 +165,12 @@ public class FreebaseUtils {
         return mqlread(provider, query);
     }
 
+    /**
+     * Perform an MQLREAD operation using the credentials of the given OAuth provider
+     * 
+     * @deprecated This will go away when we switch to Google authentication.
+     */
+    @Deprecated
     public static String mqlread(Provider provider, String query) 
     throws ClientProtocolException, IOException, JSONException {
         
@@ -174,6 +196,24 @@ public class FreebaseUtils {
         return EntityUtils.toString(httpResponse.getEntity());
     }
 
+    
+    /**
+     * Perform a single unauthenticated MQLread.
+     * 
+     * (wrapper method for a bunch of alternative implementations)
+     */
+    static public String mqlread(String query) 
+            throws IOException, JSONException {
+        // A bunch of implementations which don't work for MQLread, but do for other methods
+        // String result = rpcCall(query);
+        // String result = googleCall(query);
+        // String result = batchCall1(query);
+
+        String result =  mqlreadBatchMime(query);
+        return result;
+    }
+    
+    
     public static String mqlwrite(Credentials credentials, Provider provider, String query) 
     throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
         OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
@@ -301,4 +341,190 @@ public class FreebaseUtils {
         return url != null ? url : FREEQ_URL;
     }
 
+
+    static final String BOUNDARY = "---theOpenRefineBoundary--=";
+    
+    /**
+     * A hand rolled MIME multipart/mixed implementation for Google's Batch API
+     */
+    static private String mqlreadBatchMime(String query) throws JSONException, IOException {
+        URL url = new URL(GOOGLE_BATCH_URL);
+        String service_url = FREEBASE_SERVICE_URL+"/mqlread";
+        
+        // We could use the javax.mail package, but it's actually more trouble than it's worth
+        String body = "--" + BOUNDARY + "\n" 
+            + queryToMimeBodyPart("0", query, service_url, API_KEY)
+            + "\n--" + BOUNDARY + "\n" ;
+        
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        connection.setRequestProperty("Content-Type","multipart/mixed; boundary="+ BOUNDARY); 
+        connection.setConnectTimeout(5000);
+        connection.setDoOutput(true);
+    
+        Writer writer = new OutputStreamWriter(connection.getOutputStream());
+        try {
+            writer.write(body);
+        } finally {
+            writer.flush();
+            writer.close();
+        }
+    
+        connection.connect();
+        String result = null;
+        if (connection.getResponseCode() >= 400) {
+            String responseMessage = connection.getResponseMessage();
+            String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
+            LoggerFactory.getLogger("freebase").error(
+                    "Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
+                            + errorStream);
+        } else {
+            InputStream is = connection.getInputStream();
+            try {
+                String s = ParsingUtilities.inputStreamToString(is);
+                String boundary = s.substring(0,s.indexOf("\n"));
+                boundary = boundary.split("\r")[0]; 
+                String[] part = s.split(boundary); // part 0 is empty because of leading boundary
+                String[] sections = part[1].split("\r\n\r\n");
+                // Mime headers, followed by HTTP headers, followd by actual response
+                result = sections[2];
+            } finally {
+                is.close();
+            }
+        }
+        return result;        
+    }
+
+    static String queryToMimeBodyPart(String query_name, 
+            String query, String service_url, String api_key) 
+                    throws IOException {
+        // We could use the javax.mail package, but it's actually more trouble than it's worth
+        StringBuilder sb = new StringBuilder();
+        sb.append("Content-Type: application/http\n"); 
+        sb.append("Content-Transfer-Encoding: binary\n");
+        sb.append("Content-ID: " + query_name + "\n");
+        sb.append("\n");
+        
+        List<NameValuePair> params = new ArrayList<NameValuePair>();
+        params.add(new BasicNameValuePair("query",query));
+        params.add(new BasicNameValuePair("key", api_key));
+        UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8");
+        
+        String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n";
+        sb.append(body);
+        sb.append("\n");
+        
+        return sb.toString();
+    }
+
+    //////////////////////// Unused methods for future use /////////////////////
+
+    /**
+     * This RPC call works for the Reconcile API, but MQLread is not supported over JSONRPC
+     */
+    @SuppressWarnings("unused")
+    static private JSONObject mqlreadRpc(String query) throws JSONException, UnsupportedEncodingException, IOException {
+        URL url = new URL(GOOGLE_RPC_URL);
+    
+        JSONObject params = new JSONObject();
+        params.put("query",query);
+        params.put("key", API_KEY);
+    
+        JSONObject req1 = new JSONObject();
+        req1.put("jsonrpc","2.0");    
+        req1.put("id","q0");
+        req1.put("method","freebase.mqlread");
+        req1.put("apiVersion", FREEBASE_API_VERSION);
+        req1.put("params",params);
+    
+        JSONArray body = new JSONArray();
+        body.put(req1);
+        
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        connection.setRequestProperty("Content-Type", "application/json"); //
+        connection.setConnectTimeout(5000);
+        connection.setDoOutput(true);
+        
+        OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8");
+        try {
+            writer.write(body.toString());
+        } finally {
+            writer.flush();
+            writer.close();
+        }
+        
+        connection.connect();
+        JSONArray result = null;
+        if (connection.getResponseCode() >= 400) {
+           String responseMessage = connection.getResponseMessage();
+           String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
+           LoggerFactory.getLogger("freebase").error(
+                   "Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
+                           + errorStream);
+        } else {
+            InputStream is = connection.getInputStream();
+            try {
+                String s = ParsingUtilities.inputStreamToString(is);
+                result = ParsingUtilities.evaluateJsonStringToArray(s);
+            } finally {
+                is.close();
+            }
+        }
+        return result.getJSONObject(0);
+    }
+    
+    
+    private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
+    private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+    private static final FreebaseRequestInitializer REQUEST_INITIALIZER = 
+            new FreebaseRequestInitializer(FreebaseUtils.API_KEY);
+    
+    /**
+     * Submit a single MQL read query via the standard Google client library
+     */
+    @SuppressWarnings("unused")
+    static private String mqlreadFreebaseClient(String query)
+            throws IOException, JSONException {
+        
+        Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
+            .setApplicationName("OpenRefine")
+            .setFreebaseRequestInitializer(REQUEST_INITIALIZER)
+            .build();
+
+        InputStream is = client.mqlread(query).executeAsInputStream();
+        String result = ParsingUtilities.inputStreamToString(is);
+        return result;
+    }
+    
+    
+    /**
+     * Submit a single MQL query via the Batch endpoint
+     * (not supported by Google's Java client)
+     */
+    @SuppressWarnings("unused")
+    static private JSONObject mqlreadBatchFreebaseClient(String query) throws IOException, JSONException {
+        JSONObject response = null;
+        
+        // FIXME: We really want JsonBatchCallback<Freebase> here, but it's not supported right now
+        JsonBatchCallback<Void> callback = new JsonBatchCallback<Void>() {
+            public void onSuccess(Void res, HttpHeaders responseHeaders) {
+                System.out.println(res);
+            }
+            public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
+              System.out.println("Error Message: " + e.getMessage());
+            }
+          };
+    
+          Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
+            .setApplicationName("OpenRefine")
+            .setFreebaseRequestInitializer(REQUEST_INITIALIZER)
+            .build();
+          
+          // FIXME: Batch doesn't work with MqlRead since it extends FreebaseRequest<Void>
+          BatchRequest batch = client.batch();
+          client.mqlread(query).queue(batch, callback); 
+          batch.execute();
+    
+           return response;
+    }
+
 }