From 7b9f6836e1403785e2306bf1b703ddf0066fe9e0 Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Tue, 12 Mar 2013 16:50:23 -0400 Subject: [PATCH] Update key & id recon to new Freebase APIs - part of #696 --- .../model/recon/GuidBasedReconConfig.java | 92 +++--- .../model/recon/IdBasedReconConfig.java | 91 +++--- .../model/recon/KeyBasedReconConfig.java | 91 +++--- .../model/recon/StrictReconConfig.java | 2 +- .../refine/model/recon/ReconConfig.java | 264 +++++++++--------- 5 files changed, 281 insertions(+), 259 deletions(-) diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java index ec467c262..ada53809d 100644 --- a/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java @@ -1,6 +1,6 @@ /* -Copyright 2010, Google Inc. +Copyright 2010,2013 Google Inc. and other contributors All rights reserved. Redistribution and use in source and binary forms, with or without @@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.freebase.model.recon; +import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; +import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLConnection; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -124,8 +125,6 @@ public class GuidBasedReconConfig extends StrictReconConfig { StringWriter stringWriter = new StringWriter(); JSONWriter jsonWriter = new JSONWriter(stringWriter); - jsonWriter.object(); - jsonWriter.key("query"); jsonWriter.array(); jsonWriter.object(); @@ -143,63 +142,72 @@ public class GuidBasedReconConfig extends StrictReconConfig { jsonWriter.endObject(); jsonWriter.endArray(); - jsonWriter.endObject(); query = stringWriter.toString(); } StringBuffer sb = new StringBuffer(1024); sb.append(s_mqlreadService); - sb.append("?query="); + sb.append("query="); sb.append(ParsingUtilities.encode(query)); URL url = new URL(sb.toString()); - URLConnection connection = url.openConnection(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setConnectTimeout(5000); connection.connect(); - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - if (o.has("result")) { - JSONArray results = o.getJSONArray("result"); - int count = results.length(); + if (connection.getResponseCode() >= 400) { + String responseMessage = connection.getResponseMessage(); + String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream()); + LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() + + " : " + responseMessage + " : " + errorString); + } else { + InputStream is = connection.getInputStream(); - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + if (o.has("result")) { + JSONArray results = o.getJSONArray("result"); + int count = results.length(); - String guid = result.getString("guid"); + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); + String guid = result.getString("guid"); + + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + result.getString("id"), + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + guidToRecon.put(guid, recon); } - - ReconCandidate candidate = new ReconCandidate( - result.getString("id"), - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = Recon.makeFreebaseRecon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - guidToRecon.put(guid, recon); } + } finally { + is.close(); } - } finally { - is.close(); } - } catch (Exception e) { - e.printStackTrace(); + } catch (IOException e) { + LOGGER.error("IOException during recon : ",e); + } catch (JSONException e) { + LOGGER.error("JSONException during recon : ",e); } for (ReconJob job : jobs) { diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java index 7da13c97f..e50c10d2e 100644 --- a/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java @@ -1,6 +1,6 @@ /* -Copyright 2010, Google Inc. +Copyright 2010,2013 Google Inc. and other contributors All rights reserved. Redistribution and use in source and binary forms, with or without @@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.freebase.model.recon; +import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; +import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLConnection; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -128,8 +129,6 @@ public class IdBasedReconConfig extends StrictReconConfig { StringWriter stringWriter = new StringWriter(); JSONWriter jsonWriter = new JSONWriter(stringWriter); - jsonWriter.object(); - jsonWriter.key("query"); jsonWriter.array(); jsonWriter.object(); @@ -147,63 +146,71 @@ public class IdBasedReconConfig extends StrictReconConfig { jsonWriter.endObject(); jsonWriter.endArray(); - jsonWriter.endObject(); query = stringWriter.toString(); } StringBuffer sb = new StringBuffer(1024); sb.append(s_mqlreadService); - sb.append("?query="); + sb.append("query="); sb.append(ParsingUtilities.encode(query)); URL url = new URL(sb.toString()); - URLConnection connection = url.openConnection(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setConnectTimeout(5000); connection.connect(); - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - if (o.has("result")) { - JSONArray results = o.getJSONArray("result"); - int count = results.length(); + if (connection.getResponseCode() >= 400) { + String responseMessage = connection.getResponseMessage(); + String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream()); + LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() + + " : " + responseMessage + " : " + errorString); + } else { + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + if (o.has("result")) { + JSONArray results = o.getJSONArray("result"); + int count = results.length(); - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); - String id = result.getString("id"); + String id = result.getString("id"); - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + id, + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + idToRecon.put(id, recon); } - - ReconCandidate candidate = new ReconCandidate( - id, - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = Recon.makeFreebaseRecon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - idToRecon.put(id, recon); } + } finally { + is.close(); } - } finally { - is.close(); } - } catch (Exception e) { - e.printStackTrace(); + } catch (IOException e) { + LOGGER.error("IOException during recon : ",e); + } catch (JSONException e) { + LOGGER.error("JSONException during recon : ",e); } for (ReconJob job : jobs) { diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java index 682338454..e7b264202 100644 --- a/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java @@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.freebase.model.recon; +import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; +import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLConnection; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -129,8 +130,6 @@ public class KeyBasedReconConfig extends StrictReconConfig { StringWriter stringWriter = new StringWriter(); JSONWriter jsonWriter = new JSONWriter(stringWriter); - jsonWriter.object(); - jsonWriter.key("query"); jsonWriter.array(); jsonWriter.object(); @@ -161,64 +160,72 @@ public class KeyBasedReconConfig extends StrictReconConfig { jsonWriter.endObject(); jsonWriter.endArray(); - jsonWriter.endObject(); query = stringWriter.toString(); } StringBuffer sb = new StringBuffer(1024); sb.append(s_mqlreadService); - sb.append("?query="); + sb.append("query="); sb.append(ParsingUtilities.encode(query)); URL url = new URL(sb.toString()); - URLConnection connection = url.openConnection(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setConnectTimeout(5000); connection.connect(); - - InputStream is = connection.getInputStream(); - try { - String s = ParsingUtilities.inputStreamToString(is); - JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); - if (o.has("result")) { - JSONArray results = o.getJSONArray("result"); - int count = results.length(); + if (connection.getResponseCode() >= 400) { + String responseMessage = connection.getResponseMessage(); + String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream()); + LOGGER.error("HTTP response error during recon: " + connection.getResponseCode() + + " : " + responseMessage + " : " + errorString); + } else { + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + if (o.has("result")) { + JSONArray results = o.getJSONArray("result"); + int count = results.length(); - for (int i = 0; i < count; i++) { - JSONObject result = results.getJSONObject(i); + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); - String key = result.getJSONArray("key").getJSONObject(0).getString("value"); + String key = result.getJSONArray("key").getJSONObject(0).getString("value"); - JSONArray types = result.getJSONArray("type"); - String[] typeIDs = new String[types.length()]; - for (int j = 0; j < typeIDs.length; j++) { - typeIDs[j] = types.getString(j); + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + result.getString("id"), + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + keyToRecon.put(key, recon); } - - ReconCandidate candidate = new ReconCandidate( - result.getString("id"), - result.getString("name"), - typeIDs, - 100 - ); - - Recon recon = Recon.makeFreebaseRecon(historyEntryID); - recon.addCandidate(candidate); - recon.service = "mql"; - recon.judgment = Judgment.Matched; - recon.judgmentAction = "auto"; - recon.match = candidate; - recon.matchRank = 0; - - keyToRecon.put(key, recon); } + } finally { + is.close(); } - } finally { - is.close(); } - } catch (Exception e) { - e.printStackTrace(); + } catch (IOException e) { + LOGGER.error("IOException during recon : ",e); + } catch (JSONException e) { + LOGGER.error("JSONException during recon : ",e); } + for (ReconJob job : jobs) { String key = ((KeyBasedReconJob) job).key; diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java index 3fe200a7e..38c0620e7 100644 --- a/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java @@ -40,7 +40,7 @@ import com.google.refine.model.Recon.Judgment; import com.google.refine.model.recon.ReconConfig; abstract public class StrictReconConfig extends ReconConfig { - final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread"; + final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M&"; static public ReconConfig reconstruct(JSONObject obj) throws Exception { String match = obj.getString("match"); diff --git a/main/src/com/google/refine/model/recon/ReconConfig.java b/main/src/com/google/refine/model/recon/ReconConfig.java index 9ce95a66a..718b360e0 100644 --- a/main/src/com/google/refine/model/recon/ReconConfig.java +++ b/main/src/com/google/refine/model/recon/ReconConfig.java @@ -1,132 +1,132 @@ -/* - -Copyright 2010, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -package com.google.refine.model.recon; - -import java.io.Writer; -import java.lang.reflect.Method; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.Jsonizable; -import com.google.refine.model.Cell; -import com.google.refine.model.Project; -import com.google.refine.model.Recon; -import com.google.refine.model.Row; - -import edu.mit.simile.butterfly.ButterflyModule; - -abstract public class ReconConfig implements Jsonizable { - final static protected Logger LOGGER = LoggerFactory.getLogger("recon-config"); - - static final public Map>> s_opNameToClass = - new HashMap>>(); - - static final public Map, String> s_opClassToName = - new HashMap, String>(); - - static public void registerReconConfig(ButterflyModule module, String name, Class klass) { - String key = module.getName() + "/" + name; - - s_opClassToName.put(klass, key); - - List> classes = s_opNameToClass.get(key); - if (classes == null) { - classes = new LinkedList>(); - s_opNameToClass.put(key, classes); - } - classes.add(klass); - } - - static public ReconConfig reconstruct(JSONObject obj) throws Exception { - try { - String mode = obj.getString("mode"); - - // Backward compatibility - if ("extend".equals(mode) || "strict".equals(mode)) { - mode = "freebase/" + mode; - } else if ("heuristic".equals(mode)) { - mode = "core/standard-service"; // legacy - } else if (!mode.contains("/")) { - mode = "core/" + mode; - } - - List> classes = s_opNameToClass.get(mode); - if (classes != null && classes.size() > 0) { - Class klass = classes.get(classes.size() - 1); - - Method reconstruct = klass.getMethod("reconstruct", JSONObject.class); - if (reconstruct != null) { - return (ReconConfig) reconstruct.invoke(null, obj); - } - } - } catch (Exception e) { - LOGGER.error("Reconstruct failed",e); - } - return null; - } - - abstract public int getBatchSize(); - - abstract public String getBriefDescription(Project project, String columnName); - - abstract public ReconJob createJob( - Project project, - int rowIndex, - Row row, - String columnName, - Cell cell - ); - - abstract public List batchRecon(List jobs, long historyEntryID); - - abstract public Recon createNewRecon(long historyEntryID); - - public void save(Writer writer) { - JSONWriter jsonWriter = new JSONWriter(writer); - try { - write(jsonWriter, new Properties()); - } catch (JSONException e) { - LOGGER.error("Save failed",e); - } - } -} +/* + +Copyright 2010,2013 Google Inc. and other contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package com.google.refine.model.recon; + +import java.io.Writer; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.Jsonizable; +import com.google.refine.model.Cell; +import com.google.refine.model.Project; +import com.google.refine.model.Recon; +import com.google.refine.model.Row; + +import edu.mit.simile.butterfly.ButterflyModule; + +abstract public class ReconConfig implements Jsonizable { + final static protected Logger LOGGER = LoggerFactory.getLogger("recon-config"); + + static final public Map>> s_opNameToClass = + new HashMap>>(); + + static final public Map, String> s_opClassToName = + new HashMap, String>(); + + static public void registerReconConfig(ButterflyModule module, String name, Class klass) { + String key = module.getName() + "/" + name; + + s_opClassToName.put(klass, key); + + List> classes = s_opNameToClass.get(key); + if (classes == null) { + classes = new LinkedList>(); + s_opNameToClass.put(key, classes); + } + classes.add(klass); + } + + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + try { + String mode = obj.getString("mode"); + + // Backward compatibility + if ("extend".equals(mode) || "strict".equals(mode)) { + mode = "freebase/" + mode; + } else if ("heuristic".equals(mode)) { + mode = "core/standard-service"; // legacy + } else if (!mode.contains("/")) { + mode = "core/" + mode; + } + + List> classes = s_opNameToClass.get(mode); + if (classes != null && classes.size() > 0) { + Class klass = classes.get(classes.size() - 1); + + Method reconstruct = klass.getMethod("reconstruct", JSONObject.class); + if (reconstruct != null) { + return (ReconConfig) reconstruct.invoke(null, obj); + } + } + } catch (Exception e) { + LOGGER.error("Reconstruct failed",e); + } + return null; + } + + abstract public int getBatchSize(); + + abstract public String getBriefDescription(Project project, String columnName); + + abstract public ReconJob createJob( + Project project, + int rowIndex, + Row row, + String columnName, + Cell cell + ); + + abstract public List batchRecon(List jobs, long historyEntryID); + + abstract public Recon createNewRecon(long historyEntryID); + + public void save(Writer writer) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + write(jsonWriter, new Properties()); + } catch (JSONException e) { + LOGGER.error("Save failed",e); + } + } +}