From 4074d3267c224f42556eb513a9499cf0bf79d769 Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Wed, 13 Mar 2013 16:45:37 -0400 Subject: [PATCH] Refactor to use common mqlread code and eliminate clones --- .../freebase/commands/MQLReadCommand.java | 6 + .../model/recon/GuidBasedReconConfig.java | 141 ++- .../model/recon/IdBasedReconConfig.java | 140 ++- .../model/recon/KeyBasedReconConfig.java | 166 ++-- .../model/recon/StrictReconConfig.java | 2 - .../util/FreebaseDataExtensionJob.java | 190 +--- .../refine/freebase/util/FreebaseUtils.java | 832 +++++++++++------- 7 files changed, 735 insertions(+), 742 deletions(-) diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java index 0c2d08edd..f6763ca11 100644 --- a/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java +++ b/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java @@ -44,6 +44,11 @@ import com.google.refine.freebase.util.FreebaseUtils; import com.google.refine.oauth.OAuthUtilities; import com.google.refine.oauth.Provider; +/** + * Perform an MQLread on the server, using the client's credentials. + * + * Currently unused. All client code calls the Freebase API directly. + */ public class MQLReadCommand extends Command { @Override @@ -55,6 +60,7 @@ public class MQLReadCommand extends Command { response.setCharacterEncoding("UTF-8"); response.setHeader("Content-Type", "application/json"); String query = request.getParameter("query"); + @SuppressWarnings("deprecation") String result = FreebaseUtils.mqlread(provider,query); response.getWriter().write(result); } catch (Exception e) { diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java index ada53809d..28fceee2b 100644 --- a/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java @@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.freebase.model.recon; import java.io.IOException; -import java.io.InputStream; import java.io.StringWriter; -import java.net.HttpURLConnection; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -49,6 +46,7 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.google.refine.freebase.util.FreebaseUtils; import com.google.refine.model.Cell; import com.google.refine.model.Project; import com.google.refine.model.Recon; @@ -120,88 +118,42 @@ public class GuidBasedReconConfig extends StrictReconConfig { Map guidToRecon = new HashMap(); try { - String query = null; - { - StringWriter stringWriter = new StringWriter(); - JSONWriter jsonWriter = new JSONWriter(stringWriter); - - jsonWriter.array(); - jsonWriter.object(); + String query = buildQuery(jobs); - jsonWriter.key("id"); jsonWriter.value(null); - jsonWriter.key("name"); jsonWriter.value(null); - jsonWriter.key("guid"); jsonWriter.value(null); - jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); - - jsonWriter.key("guid|="); - jsonWriter.array(); - for (ReconJob job : jobs) { - jsonWriter.value(((GuidBasedReconJob) job).guid); - } - jsonWriter.endArray(); - - jsonWriter.endObject(); - jsonWriter.endArray(); - - query = stringWriter.toString(); - } + String s = FreebaseUtils.mqlread(query); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - StringBuffer sb = new StringBuffer(1024); - sb.append(s_mqlreadService); - sb.append("query="); - sb.append(ParsingUtilities.encode(query)); - - URL url = new URL(sb.toString()); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - - if (connection.getResponseCode() >= 400) { - String responseMessage = connection.getResponseMessage(); - String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream()); - LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() - + " : " + responseMessage + " : " + errorString); - } else { - InputStream is = connection.getInputStream(); + if (o.has("result")) { + JSONArray results = o.getJSONArray("result"); + int count = results.length(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - if (o.has("result")) { - JSONArray results = o.getJSONArray("result"); - int count = results.length(); + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); + String guid = result.getString("guid"); - String guid = result.getString("guid"); - - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); - } - - ReconCandidate candidate = new ReconCandidate( - result.getString("id"), - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = Recon.makeFreebaseRecon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - guidToRecon.put(guid, recon); - } + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); } - } finally { - is.close(); + + ReconCandidate candidate = new ReconCandidate( + result.getString("id"), + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + guidToRecon.put(guid, recon); } } } catch (IOException e) { @@ -209,7 +161,7 @@ public class GuidBasedReconConfig extends StrictReconConfig { } catch (JSONException e) { LOGGER.error("JSONException during recon : ",e); } - + for (ReconJob job : jobs) { String guid = ((GuidBasedReconJob) job).guid; Recon recon = guidToRecon.get(guid); @@ -221,4 +173,33 @@ public class GuidBasedReconConfig extends StrictReconConfig { return recons; } + + private String buildQuery(List jobs) + throws JSONException { + String query = null; + + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("guid|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((GuidBasedReconJob) job).guid); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + + query = stringWriter.toString(); + return query; + } } diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java index e50c10d2e..1fdd02ad6 100644 --- a/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java @@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.freebase.model.recon; import java.io.IOException; -import java.io.InputStream; import java.io.StringWriter; -import java.net.HttpURLConnection; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -49,6 +46,7 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import com.google.refine.freebase.util.FreebaseUtils; import com.google.refine.model.Cell; import com.google.refine.model.Project; import com.google.refine.model.Recon; @@ -124,87 +122,41 @@ public class IdBasedReconConfig extends StrictReconConfig { Map idToRecon = new HashMap(); try { - String query = null; - { - StringWriter stringWriter = new StringWriter(); - JSONWriter jsonWriter = new JSONWriter(stringWriter); - - jsonWriter.array(); - jsonWriter.object(); - - jsonWriter.key("id"); jsonWriter.value(null); - jsonWriter.key("name"); jsonWriter.value(null); - jsonWriter.key("guid"); jsonWriter.value(null); - jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); - - jsonWriter.key("id|="); - jsonWriter.array(); - for (ReconJob job : jobs) { - jsonWriter.value(((IdBasedReconJob) job).id); - } - jsonWriter.endArray(); - - jsonWriter.endObject(); - jsonWriter.endArray(); - - query = stringWriter.toString(); - } - - StringBuffer sb = new StringBuffer(1024); - sb.append(s_mqlreadService); - sb.append("query="); - sb.append(ParsingUtilities.encode(query)); - - URL url = new URL(sb.toString()); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - - if (connection.getResponseCode() >= 400) { - String responseMessage = connection.getResponseMessage(); - String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream()); - LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() - + " : " + responseMessage + " : " + errorString); - } else { - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - if (o.has("result")) { - JSONArray results = o.getJSONArray("result"); - int count = results.length(); + String query = buildQuery(jobs); + String s = FreebaseUtils.mqlread(query); - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + if (o.has("result")) { + JSONArray results = o.getJSONArray("result"); + int count = results.length(); - String id = result.getString("id"); + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); - } + String id = result.getString("id"); - ReconCandidate candidate = new ReconCandidate( - id, - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = Recon.makeFreebaseRecon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - idToRecon.put(id, recon); - } + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); } - } finally { - is.close(); + + ReconCandidate candidate = new ReconCandidate( + id, + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + idToRecon.put(id, recon); } } } catch (IOException e) { @@ -225,4 +177,34 @@ public class IdBasedReconConfig extends StrictReconConfig { return recons; } + private String buildQuery(List jobs) + throws JSONException { + String query = null; + { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("id|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((IdBasedReconJob) job).id); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + + query = stringWriter.toString(); + } + return query; + } + } diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java index e7b264202..7f682b111 100644 --- a/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java @@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.freebase.model.recon; import java.io.IOException; -import java.io.InputStream; import java.io.StringWriter; -import java.net.HttpURLConnection; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -50,6 +47,7 @@ import org.json.JSONObject; import org.json.JSONWriter; import com.google.refine.freebase.FreebaseTopic; +import com.google.refine.freebase.util.FreebaseUtils; import com.google.refine.model.Cell; import com.google.refine.model.Project; import com.google.refine.model.Recon; @@ -125,99 +123,41 @@ public class KeyBasedReconConfig extends StrictReconConfig { Map keyToRecon = new HashMap(); try { - String query = null; - { - StringWriter stringWriter = new StringWriter(); - JSONWriter jsonWriter = new JSONWriter(stringWriter); - - jsonWriter.array(); - jsonWriter.object(); - - jsonWriter.key("id"); jsonWriter.value(null); - jsonWriter.key("name"); jsonWriter.value(null); - jsonWriter.key("guid"); jsonWriter.value(null); - jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); - - jsonWriter.key("key"); - jsonWriter.array(); - jsonWriter.object(); - - jsonWriter.key("namespace"); - jsonWriter.object(); - jsonWriter.key("id"); jsonWriter.value(namespace.id); - jsonWriter.endObject(); - - jsonWriter.key("value"); jsonWriter.value(null); - jsonWriter.key("value|="); - jsonWriter.array(); - for (ReconJob job : jobs) { - jsonWriter.value(((KeyBasedReconJob) job).key); - } - jsonWriter.endArray(); - - jsonWriter.endObject(); - jsonWriter.endArray(); - - jsonWriter.endObject(); - jsonWriter.endArray(); - - query = stringWriter.toString(); - } + String query = buildQuery(jobs); + String s = FreebaseUtils.mqlread(query); - StringBuffer sb = new StringBuffer(1024); - sb.append(s_mqlreadService); - sb.append("query="); - sb.append(ParsingUtilities.encode(query)); - - URL url = new URL(sb.toString()); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setConnectTimeout(5000); - connection.connect(); - if (connection.getResponseCode() >= 400) { - String responseMessage = connection.getResponseMessage(); - String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream()); - LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() - + " : " + responseMessage + " : " + errorString); - } else { - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - if (o.has("result")) { - JSONArray results = o.getJSONArray("result"); - int count = results.length(); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + if (o.has("result")) { + JSONArray results = o.getJSONArray("result"); + int count = results.length(); - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); - String key = result.getJSONArray("key").getJSONObject(0).getString("value"); + String key = result.getJSONArray("key").getJSONObject(0).getString("value"); - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); - } - - ReconCandidate candidate = new ReconCandidate( - result.getString("id"), - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = Recon.makeFreebaseRecon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - keyToRecon.put(key, recon); - } + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); } - } finally { - is.close(); + + ReconCandidate candidate = new ReconCandidate( + result.getString("id"), + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + keyToRecon.put(key, recon); } } } catch (IOException e) { @@ -226,7 +166,6 @@ public class KeyBasedReconConfig extends StrictReconConfig { LOGGER.error("JSONException during recon : ",e); } - for (ReconJob job : jobs) { String key = ((KeyBasedReconJob) job).key; Recon recon = keyToRecon.get(key); @@ -239,4 +178,47 @@ public class KeyBasedReconConfig extends StrictReconConfig { return recons; } + private String buildQuery(List jobs) + throws JSONException { + String query = null; + { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("key"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("namespace"); + jsonWriter.object(); + jsonWriter.key("id"); jsonWriter.value(namespace.id); + jsonWriter.endObject(); + + jsonWriter.key("value"); jsonWriter.value(null); + jsonWriter.key("value|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((KeyBasedReconJob) job).key); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + + query = stringWriter.toString(); + } + return query; + } + } diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java index 7b81b4aa8..338983f1b 100644 --- a/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java @@ -35,13 +35,11 @@ package com.google.refine.freebase.model.recon; import org.json.JSONObject; -import com.google.refine.freebase.util.FreebaseUtils; import com.google.refine.model.Recon; import com.google.refine.model.Recon.Judgment; import com.google.refine.model.recon.ReconConfig; abstract public class StrictReconConfig extends ReconConfig { - final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=" + FreebaseUtils.API_KEY + "&"; static public ReconConfig reconstruct(JSONObject obj) throws Exception { String match = obj.getString("match"); diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java index 94e9fa098..5fa31a35f 100644 --- a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java +++ b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java @@ -31,49 +31,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/** - * - */ package com.google.refine.freebase.util; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; import java.io.Serializable; import java.io.StringWriter; -import java.io.UnsupportedEncodingException; import java.io.Writer; -import java.net.HttpURLConnection; -import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import javax.mail.MessagingException; -import javax.mail.internet.MimeBodyPart; -import javax.mail.internet.MimeMultipart; - -import org.apache.http.NameValuePair; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.message.BasicNameValuePair; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; -import com.google.api.client.googleapis.batch.json.JsonBatchCallback; -import com.google.api.client.googleapis.json.GoogleJsonError; -import com.google.api.client.http.HttpHeaders; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson.JacksonFactory; -import com.google.api.services.freebase.Freebase; -import com.google.api.services.freebase.FreebaseRequestInitializer; - import com.google.refine.freebase.FreebaseType; import com.google.refine.model.ReconCandidate; import com.google.refine.util.JSONUtilities; @@ -116,9 +89,11 @@ public class FreebaseDataExtensionJob { ) throws Exception { StringWriter writer = new StringWriter(); formulateQuery(ids, extension, writer); - String query = writer.toString(); - JSONObject o = doMqlRead(query); + + String result = FreebaseUtils.mqlread(query); + + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(result); Map map = new HashMap(); if (o.has("result")) { JSONArray a = o.getJSONArray("result"); @@ -325,164 +300,7 @@ public class FreebaseDataExtensionJob { } - /** - * This RPC call works for the Reconcile API, but MQLread is not supported by JSONRPC - */ - static private JSONObject rpcCall(String query) throws JSONException, UnsupportedEncodingException, IOException { - URL url = new URL("https://www.googleapis.com/rpc"); - - JSONObject params = new JSONObject(); - params.put("query",query); - params.put("key", FreebaseUtils.API_KEY); - - JSONObject req1 = new JSONObject(); - req1.put("jsonrpc","2.0"); - req1.put("id","q0"); - req1.put("method","freebase.mqlread"); - req1.put("apiVersion", "v1"); - req1.put("params",params); - - JSONArray body = new JSONArray(); - body.put(req1); - - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setRequestProperty("Content-Type", "application/json"); // - connection.setConnectTimeout(5000); - connection.setDoOutput(true); - - OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8"); - try { - writer.write(body.toString()); - } finally { - writer.flush(); - writer.close(); - } - - connection.connect(); - JSONArray result = null; - if (connection.getResponseCode() >= 400) { - String responseMessage = connection.getResponseMessage(); - String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream()); - } else { - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - result = ParsingUtilities.evaluateJsonStringToArray(s); - } finally { - is.close(); - } - } - return result.getJSONObject(0); - } - - private static MimeBodyPart queryToMimeBodyPart(String query_name, - String query, String service_url, String api_key) - throws MessagingException, IOException { - MimeBodyPart mbp = new MimeBodyPart(); - mbp.setHeader("Content-Transfer-Encoding", "binary"); - mbp.setHeader("Content-ID", "<" + query_name + ">"); - mbp.setHeader("Content-type", "application/http"); - - List params = new ArrayList(); - params.add(new BasicNameValuePair("query",query)); - params.add(new BasicNameValuePair("key", api_key)); - UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8"); - - String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n"; - mbp.setText(body, "UTF-8"); - mbp.getLineCount(); - mbp.getAllHeaderLines(); - return mbp; - } - - - /** - * The beginnings of a homebrew attempt to generate multi-part MIME messages - * so that we can parse response bodies ourselves and avoid the limitations - * of the Google Client library. - */ - static private JSONObject batchCallOld(String query) throws JSONException, MessagingException, IOException { - URL url = new URL("https://www.googleapis.com/batch"); - String service_url = "https://www.googleapis.com/freebase/v1/mqlread"; - MimeMultipart mp = new MimeMultipart("mixed"); - - MimeBodyPart mbp = queryToMimeBodyPart("q0", query, service_url, FreebaseUtils.API_KEY); - mp.addBodyPart(mbp); - mp.getPreamble(); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setRequestProperty("Content-Type", mp.getContentType()); - connection.setConnectTimeout(5000); - connection.setDoOutput(true); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - mp.writeTo(baos); - String foo = baos.toString("UTF-8"); - - mp.writeTo(connection.getOutputStream()); - - connection.connect(); - JSONArray result = null; - if (connection.getResponseCode() >= 400) { - String responseMessage = connection.getResponseMessage(); - String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream()); - } else { - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - result = ParsingUtilities.evaluateJsonStringToArray(s); - } finally { - is.close(); - } - } - return result.getJSONObject(0); - } - - - private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); - - /** Global instance of the JSON factory. */ - private static final JsonFactory JSON_FACTORY = new JacksonFactory(); - - - private static final FreebaseRequestInitializer REQUEST_INITIALIZER = - new FreebaseRequestInitializer(FreebaseUtils.API_KEY); - static private JSONObject batchCall1(String query) throws IOException, JSONException { - - JsonBatchCallback callback = new JsonBatchCallback() { - public void onSuccess(Freebase res, HttpHeaders responseHeaders) { - System.out.println(res); - } - public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { - System.out.println("Error Message: " + e.getMessage()); - } - }; - - Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) - .setApplicationName("OpenRefine") - .setFreebaseRequestInitializer(REQUEST_INITIALIZER) - .build(); - - // TODO: Batch doesn't work with MqlRead since it extends FreebaseRequest -// BatchRequest batch = client.batch(); -// client.mqlread(query).queue(batch, callback); -// batch.execute(); - - String s = ParsingUtilities.inputStreamToString(client.mqlread(query).executeAsInputStream()); - JSONObject response = ParsingUtilities.evaluateJsonStringToObject(s); - - return response; - } - - static protected JSONObject doMqlRead(String query) - throws IOException, JSONException, MessagingException { -// JSONObject result = rpcCall(query); -// JSONObject resutl = batchCallOld(query); - JSONObject result = batchCall1(query); - - return result; - } - static protected void formulateQuery(Set ids, JSONObject node, Writer writer) throws JSONException { JSONWriter jsonWriter = new JSONWriter(writer); diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java index 39bb56dfa..96a203fe9 100644 --- a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java +++ b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java @@ -1,304 +1,530 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -package com.google.refine.freebase.util; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import javax.servlet.http.HttpServletRequest; - -import oauth.signpost.OAuthConsumer; -import oauth.signpost.exception.OAuthCommunicationException; -import oauth.signpost.exception.OAuthExpectationFailedException; -import oauth.signpost.exception.OAuthMessageSignerException; - -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.HttpClient; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.params.CoreProtocolPNames; -import org.apache.http.util.EntityUtils; -import org.json.JSONException; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.ProjectManager; -import com.google.refine.RefineServlet; -import com.google.refine.oauth.Credentials; -import com.google.refine.oauth.OAuthUtilities; -import com.google.refine.oauth.Provider; - -public class FreebaseUtils { - - static final Logger logger = LoggerFactory.getLogger("freebase"); - - static final public String FREEBASE_HOST = "freebase.com"; - - static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine"; - - static final private String AGENT_ID = "/en/google_refine"; - - static final private int SAMPLE_SIZE = 300; - static final private int JUDGES = 4; - - public static final String API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M"; - - private static String getUserInfoURL(String host) { - // TODO: Needs to be upgraded to new APIs sandbox-freebase.com as host becomes v1sandbox as version - return "http://api." + host + "/api/service/user_info"; - } - - private static String getMQLWriteURL(String host) { - // TODO: Needs to be upgraded to new APIs -// String version = "v1"; -// if (host != null && host.contains("sandbox")) { -// version += "sandbox"; -// } -// return "https://www.googleapis.com/freebase/"+version+"/mqlwrite?key=" + API_KEY + "&"; - return "http://api." + host + "/api/service/mqlwrite"; - } - - private static String getMQLReadURL(String host) { - // TODO: Needs to be upgraded to new APIs -// String version = "v1"; -// if (host != null && host.contains("sandbox")) { -// version += "sandbox"; -// } -// return "https://www.googleapis.com/freebase/"+version+"/mqlread?key=" + API_KEY + "&"; - return "http://api." + host + "/api/service/mqlread"; - } - - private static String getUserAgent() { - return RefineServlet.FULLNAME; - } - - public static String getUserInfo(Credentials credentials, Provider provider) - throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException { - - OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); - - HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); - - // this is required by the Metaweb API to avoid XSS - httpRequest.setHeader("X-Requested-With", "1"); - - // sign the request with the oauth library - consumer.sign(httpRequest); - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } - - public static String getUserBadges(Provider provider, String user_id) - throws ClientProtocolException, IOException, JSONException { - - String query = "{" + - "'id' : '" + user_id + "'," + - "'!/type/usergroup/member' : [{" + - "'id' : null," + - "'key' : [{" + - "'namespace' : null" + - "}]" + - "}]" + - "}".replace("'", "\""); - - return mqlread(provider, query); - } - - public static String mqlread(Provider provider, String query) - throws ClientProtocolException, IOException, JSONException { - - JSONObject envelope = new JSONObject(); - envelope.put("query", new JSONObject(query)); - - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("query", envelope.toString())); - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - - HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); - httpRequest.setEntity(entity); - - // this is required by the Metaweb API to avoid XSS - httpRequest.setHeader("X-Requested-With", "1"); - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } - - public static String mqlwrite(Credentials credentials, Provider provider, String query) - throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException { - OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); - - JSONObject envelope = new JSONObject(); - envelope.put("query", new JSONObject(query)); - - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("query", envelope.toString())); - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - - HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); - httpRequest.setEntity(entity); - - // this is required by the Metaweb API to avoid XSS - httpRequest.setHeader("X-Requested-With", "1"); - - // sign the request with the oauth library - consumer.sign(httpRequest); - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } - - private static String getTweezersParams(int sample_size, int judges) { - String o = "{" + - "'sample_size':" + sample_size + "," + - "'votes':{" + - "'reconciled':" + judges + "," + - "'invalid':" + judges + "," + - "'new':" + judges + "," + - "'skip':" + (judges + 2) + - "}" + - "}"; - return o.replace('\'', '"'); - } - - public static String uploadTriples( - HttpServletRequest request, - String qa, - String source_name, - String source_id, - String mdo_id, - String triples - ) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException { - - Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST); - - Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); - - JSONObject mdo_info = new JSONObject(); - mdo_info.put("name", source_name); - if (source_id != null) { - mdo_info.put("info_source",source_id); - } - - JSONObject user_info = new JSONObject(getUserInfo(credentials, provider)); - if (user_info.has("username")) { - - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("user", user_info.getString("id"))); - formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE")); - formparams.add(new BasicNameValuePair("operator", user_info.getString("id"))); - formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID)); - formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString())); - formparams.add(new BasicNameValuePair("graphport", "sandbox")); - formparams.add(new BasicNameValuePair("payload", triples)); - formparams.add(new BasicNameValuePair("check_params", "false")); - if (mdo_id != null) { - formparams.add(new BasicNameValuePair("mdo_guid", mdo_id)); - } - if (Boolean.parseBoolean(qa)) { - formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES))); - } - - String freeqKey = System.getProperty("freeq.key"); - if (freeqKey != null) { - logger.warn("Found Freeq key, will bypass OAuth signature"); - formparams.add(new BasicNameValuePair("apikey", freeqKey)); - } - - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - - HttpPost httpRequest = new HttpPost(getFreeQUrl()); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); - httpRequest.setEntity(entity); - - if (freeqKey == null) { - logger.warn("Calculating OAuth signature"); - HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); - surrogateRequest.setEntity(entity); - - OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); - - // TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors. - // Is there something we can do to generate an oauth signature without consuming so much memory? - consumer.sign(surrogateRequest); - - Header[] h = surrogateRequest.getHeaders("Authorization"); - if (h.length > 0) { - httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); - } else { - throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); - } - } - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } else { - throw new RuntimeException("Invalid credentials"); - } - } - - static public String getFreeQUrl() { - String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); - return url != null ? url : FREEQ_URL; - } - +/* + +Copyright 2010,2013 Google Inc. and other contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.freebase.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.io.Writer; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; + +import oauth.signpost.OAuthConsumer; +import oauth.signpost.exception.OAuthCommunicationException; +import oauth.signpost.exception.OAuthExpectationFailedException; +import oauth.signpost.exception.OAuthMessageSignerException; + +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.params.CoreProtocolPNames; +import org.apache.http.util.EntityUtils; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.api.client.googleapis.batch.BatchRequest; +import com.google.api.client.googleapis.batch.json.JsonBatchCallback; +import com.google.api.client.googleapis.json.GoogleJsonError; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.services.freebase.Freebase; +import com.google.api.services.freebase.FreebaseRequestInitializer; + +import com.google.refine.ProjectManager; +import com.google.refine.RefineServlet; +import com.google.refine.oauth.Credentials; +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; +import com.google.refine.util.ParsingUtilities; + +public class FreebaseUtils { + + private static final String FREEBASE_API_VERSION = "v1"; +// private static final String FREEBASE_SANDBOX_API_VERSION = "v1sandbox"; + + private static final String GOOGLE_RPC_URL = "https://www.googleapis.com/rpc"; + + private static final String FREEBASE_SERVICE_URL = "https://www.googleapis.com/freebase/" + FREEBASE_API_VERSION; + + private static final String GOOGLE_BATCH_URL = "https://www.googleapis.com/batch"; + + static final Logger logger = LoggerFactory.getLogger("freebase"); + + static final public String FREEBASE_HOST = "freebase.com"; + + static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine"; + + static final private String AGENT_ID = "/en/google_refine"; + + static final private int SAMPLE_SIZE = 300; + static final private int JUDGES = 4; + + public static final String API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M"; + + private static String getUserInfoURL(String host) { + // TODO: Needs to be upgraded to new APIs sandbox-freebase.com as host becomes v1sandbox as version + return "http://api." + host + "/api/service/user_info"; + } + + private static String getMQLWriteURL(String host) { + return "http://api." + host + "/api/service/mqlwrite"; + } + + private static String getMQLReadURL(String host) { + return "http://api." + host + "/api/service/mqlread"; + } + + private static String getUserAgent() { + return RefineServlet.FULLNAME; + } + + public static String getUserInfo(Credentials credentials, Provider provider) + throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException { + + OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); + + HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); + + // this is required by the Metaweb API to avoid XSS + httpRequest.setHeader("X-Requested-With", "1"); + + // sign the request with the oauth library + consumer.sign(httpRequest); + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); + } + + public static String getUserBadges(Provider provider, String user_id) + throws ClientProtocolException, IOException, JSONException { + + String query = "{" + + "'id' : '" + user_id + "'," + + "'!/type/usergroup/member' : [{" + + "'id' : null," + + "'key' : [{" + + "'namespace' : null" + + "}]" + + "}]" + + "}".replace("'", "\""); + + return mqlread(provider, query); + } + + /** + * Perform an MQLREAD operation using the credentials of the given OAuth provider + * + * @deprecated This will go away when we switch to Google authentication. + */ + @Deprecated + public static String mqlread(Provider provider, String query) + throws ClientProtocolException, IOException, JSONException { + + JSONObject envelope = new JSONObject(); + envelope.put("query", new JSONObject(query)); + + List formparams = new ArrayList(); + formparams.add(new BasicNameValuePair("query", envelope.toString())); + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); + + HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); + httpRequest.setEntity(entity); + + // this is required by the Metaweb API to avoid XSS + httpRequest.setHeader("X-Requested-With", "1"); + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); + } + + + /** + * Perform a single unauthenticated MQLread. + * + * (wrapper method for a bunch of alternative implementations) + */ + static public String mqlread(String query) + throws IOException, JSONException { + // A bunch of implementations which don't work for MQLread, but do for other methods + // String result = rpcCall(query); + // String result = googleCall(query); + // String result = batchCall1(query); + + String result = mqlreadBatchMime(query); + return result; + } + + + public static String mqlwrite(Credentials credentials, Provider provider, String query) + throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException { + OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); + + JSONObject envelope = new JSONObject(); + envelope.put("query", new JSONObject(query)); + + List formparams = new ArrayList(); + formparams.add(new BasicNameValuePair("query", envelope.toString())); + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); + + HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); + httpRequest.setEntity(entity); + + // this is required by the Metaweb API to avoid XSS + httpRequest.setHeader("X-Requested-With", "1"); + + // sign the request with the oauth library + consumer.sign(httpRequest); + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); + } + + private static String getTweezersParams(int sample_size, int judges) { + String o = "{" + + "'sample_size':" + sample_size + "," + + "'votes':{" + + "'reconciled':" + judges + "," + + "'invalid':" + judges + "," + + "'new':" + judges + "," + + "'skip':" + (judges + 2) + + "}" + + "}"; + return o.replace('\'', '"'); + } + + public static String uploadTriples( + HttpServletRequest request, + String qa, + String source_name, + String source_id, + String mdo_id, + String triples + ) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException { + + Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST); + + Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); + + JSONObject mdo_info = new JSONObject(); + mdo_info.put("name", source_name); + if (source_id != null) { + mdo_info.put("info_source",source_id); + } + + JSONObject user_info = new JSONObject(getUserInfo(credentials, provider)); + if (user_info.has("username")) { + + List formparams = new ArrayList(); + formparams.add(new BasicNameValuePair("user", user_info.getString("id"))); + formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE")); + formparams.add(new BasicNameValuePair("operator", user_info.getString("id"))); + formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID)); + formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString())); + formparams.add(new BasicNameValuePair("graphport", "sandbox")); + formparams.add(new BasicNameValuePair("payload", triples)); + formparams.add(new BasicNameValuePair("check_params", "false")); + if (mdo_id != null) { + formparams.add(new BasicNameValuePair("mdo_guid", mdo_id)); + } + if (Boolean.parseBoolean(qa)) { + formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES))); + } + + String freeqKey = System.getProperty("freeq.key"); + if (freeqKey != null) { + logger.warn("Found Freeq key, will bypass OAuth signature"); + formparams.add(new BasicNameValuePair("apikey", freeqKey)); + } + + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); + + HttpPost httpRequest = new HttpPost(getFreeQUrl()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); + httpRequest.setEntity(entity); + + if (freeqKey == null) { + logger.warn("Calculating OAuth signature"); + HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); + surrogateRequest.setEntity(entity); + + OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); + + // TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors. + // Is there something we can do to generate an oauth signature without consuming so much memory? + consumer.sign(surrogateRequest); + + Header[] h = surrogateRequest.getHeaders("Authorization"); + if (h.length > 0) { + httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); + } else { + throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); + } + } + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); + } else { + throw new RuntimeException("Invalid credentials"); + } + } + + static public String getFreeQUrl() { + String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); + return url != null ? url : FREEQ_URL; + } + + + static final String BOUNDARY = "---theOpenRefineBoundary--="; + + /** + * A hand rolled MIME multipart/mixed implementation for Google's Batch API + */ + static private String mqlreadBatchMime(String query) throws JSONException, IOException { + URL url = new URL(GOOGLE_BATCH_URL); + String service_url = FREEBASE_SERVICE_URL+"/mqlread"; + + // We could use the javax.mail package, but it's actually more trouble than it's worth + String body = "--" + BOUNDARY + "\n" + + queryToMimeBodyPart("0", query, service_url, API_KEY) + + "\n--" + BOUNDARY + "\n" ; + + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-Type","multipart/mixed; boundary="+ BOUNDARY); + connection.setConnectTimeout(5000); + connection.setDoOutput(true); + + Writer writer = new OutputStreamWriter(connection.getOutputStream()); + try { + writer.write(body); + } finally { + writer.flush(); + writer.close(); + } + + connection.connect(); + String result = null; + if (connection.getResponseCode() >= 400) { + String responseMessage = connection.getResponseMessage(); + String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream()); + LoggerFactory.getLogger("freebase").error( + "Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : " + + errorStream); + } else { + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + String boundary = s.substring(0,s.indexOf("\n")); + boundary = boundary.split("\r")[0]; + String[] part = s.split(boundary); // part 0 is empty because of leading boundary + String[] sections = part[1].split("\r\n\r\n"); + // Mime headers, followed by HTTP headers, followd by actual response + result = sections[2]; + } finally { + is.close(); + } + } + return result; + } + + static String queryToMimeBodyPart(String query_name, + String query, String service_url, String api_key) + throws IOException { + // We could use the javax.mail package, but it's actually more trouble than it's worth + StringBuilder sb = new StringBuilder(); + sb.append("Content-Type: application/http\n"); + sb.append("Content-Transfer-Encoding: binary\n"); + sb.append("Content-ID: " + query_name + "\n"); + sb.append("\n"); + + List params = new ArrayList(); + params.add(new BasicNameValuePair("query",query)); + params.add(new BasicNameValuePair("key", api_key)); + UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8"); + + String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n"; + sb.append(body); + sb.append("\n"); + + return sb.toString(); + } + + //////////////////////// Unused methods for future use ///////////////////// + + /** + * This RPC call works for the Reconcile API, but MQLread is not supported over JSONRPC + */ + @SuppressWarnings("unused") + static private JSONObject mqlreadRpc(String query) throws JSONException, UnsupportedEncodingException, IOException { + URL url = new URL(GOOGLE_RPC_URL); + + JSONObject params = new JSONObject(); + params.put("query",query); + params.put("key", API_KEY); + + JSONObject req1 = new JSONObject(); + req1.put("jsonrpc","2.0"); + req1.put("id","q0"); + req1.put("method","freebase.mqlread"); + req1.put("apiVersion", FREEBASE_API_VERSION); + req1.put("params",params); + + JSONArray body = new JSONArray(); + body.put(req1); + + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-Type", "application/json"); // + connection.setConnectTimeout(5000); + connection.setDoOutput(true); + + OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8"); + try { + writer.write(body.toString()); + } finally { + writer.flush(); + writer.close(); + } + + connection.connect(); + JSONArray result = null; + if (connection.getResponseCode() >= 400) { + String responseMessage = connection.getResponseMessage(); + String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream()); + LoggerFactory.getLogger("freebase").error( + "Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : " + + errorStream); + } else { + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + result = ParsingUtilities.evaluateJsonStringToArray(s); + } finally { + is.close(); + } + } + return result.getJSONObject(0); + } + + + private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); + private static final JsonFactory JSON_FACTORY = new JacksonFactory(); + private static final FreebaseRequestInitializer REQUEST_INITIALIZER = + new FreebaseRequestInitializer(FreebaseUtils.API_KEY); + + /** + * Submit a single MQL read query via the standard Google client library + */ + @SuppressWarnings("unused") + static private String mqlreadFreebaseClient(String query) + throws IOException, JSONException { + + Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) + .setApplicationName("OpenRefine") + .setFreebaseRequestInitializer(REQUEST_INITIALIZER) + .build(); + + InputStream is = client.mqlread(query).executeAsInputStream(); + String result = ParsingUtilities.inputStreamToString(is); + return result; + } + + + /** + * Submit a single MQL query via the Batch endpoint + * (not supported by Google's Java client) + */ + @SuppressWarnings("unused") + static private JSONObject mqlreadBatchFreebaseClient(String query) throws IOException, JSONException { + JSONObject response = null; + + // FIXME: We really want JsonBatchCallback here, but it's not supported right now + JsonBatchCallback callback = new JsonBatchCallback() { + public void onSuccess(Void res, HttpHeaders responseHeaders) { + System.out.println(res); + } + public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { + System.out.println("Error Message: " + e.getMessage()); + } + }; + + Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) + .setApplicationName("OpenRefine") + .setFreebaseRequestInitializer(REQUEST_INITIALIZER) + .build(); + + // FIXME: Batch doesn't work with MqlRead since it extends FreebaseRequest + BatchRequest batch = client.batch(); + client.mqlread(query).queue(batch, callback); + batch.execute(); + + return response; + } + } \ No newline at end of file