Update key & id recon to new Freebase APIs - part of #696

This commit is contained in:
Tom Morris 2013-03-12 16:50:23 -04:00
parent 7578d3375f
commit 7b9f6836e1
5 changed files with 281 additions and 259 deletions

View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -124,8 +125,6 @@ public class GuidBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -143,63 +142,72 @@ public class GuidBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
InputStream is = connection.getInputStream(); if (connection.getResponseCode() >= 400) {
try { String responseMessage = connection.getResponseMessage();
String s = ParsingUtilities.inputStreamToString(is); String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
if (o.has("result")) { + " : " + responseMessage + " : " + errorString);
JSONArray results = o.getJSONArray("result"); } else {
int count = results.length(); InputStream is = connection.getInputStream();
for (int i = 0; i < count; i++) { try {
JSONObject result = results.getJSONObject(i); String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
String guid = result.getString("guid"); for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
JSONArray types = result.getJSONArray("type"); String guid = result.getString("guid");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) { JSONArray types = result.getJSONArray("type");
typeIDs[j] = types.getString(j); String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
} }
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
} }
} finally {
is.close();
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {

View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -128,8 +129,6 @@ public class IdBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -147,63 +146,71 @@ public class IdBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
InputStream is = connection.getInputStream(); if (connection.getResponseCode() >= 400) {
try { String responseMessage = connection.getResponseMessage();
String s = ParsingUtilities.inputStreamToString(is); String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
if (o.has("result")) { + " : " + responseMessage + " : " + errorString);
JSONArray results = o.getJSONArray("result"); } else {
int count = results.length(); InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i); JSONObject result = results.getJSONObject(i);
String id = result.getString("id"); String id = result.getString("id");
JSONArray types = result.getJSONArray("type"); JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()]; String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) { for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j); typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
} }
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
} }
} finally {
is.close();
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {

View File

@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -129,8 +130,6 @@ public class KeyBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -161,64 +160,72 @@ public class KeyBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
if (connection.getResponseCode() >= 400) {
InputStream is = connection.getInputStream(); String responseMessage = connection.getResponseMessage();
try { String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
String s = ParsingUtilities.inputStreamToString(is); LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + " : " + responseMessage + " : " + errorString);
if (o.has("result")) { } else {
JSONArray results = o.getJSONArray("result"); InputStream is = connection.getInputStream();
int count = results.length(); try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i); JSONObject result = results.getJSONObject(i);
String key = result.getJSONArray("key").getJSONObject(0).getString("value"); String key = result.getJSONArray("key").getJSONObject(0).getString("value");
JSONArray types = result.getJSONArray("type"); JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()]; String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) { for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j); typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
} }
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
} }
} finally {
is.close();
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {
String key = ((KeyBasedReconJob) job).key; String key = ((KeyBasedReconJob) job).key;

View File

@ -40,7 +40,7 @@ import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig; import com.google.refine.model.recon.ReconConfig;
abstract public class StrictReconConfig extends ReconConfig { abstract public class StrictReconConfig extends ReconConfig {
final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread"; final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M&";
static public ReconConfig reconstruct(JSONObject obj) throws Exception { static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match"); String match = obj.getString("match");

View File

@ -1,132 +1,132 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are modification, are permitted provided that the following conditions are
met: met:
* Redistributions of source code must retain the above copyright * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the in the documentation and/or other materials provided with the
distribution. distribution.
* Neither the name of Google Inc. nor the names of its * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from contributors may be used to endorse or promote products derived from
this software without specific prior written permission. this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/ */
package com.google.refine.model.recon; package com.google.refine.model.recon;
import java.io.Writer; import java.io.Writer;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.refine.Jsonizable; import com.google.refine.Jsonizable;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Recon; import com.google.refine.model.Recon;
import com.google.refine.model.Row; import com.google.refine.model.Row;
import edu.mit.simile.butterfly.ButterflyModule; import edu.mit.simile.butterfly.ButterflyModule;
abstract public class ReconConfig implements Jsonizable { abstract public class ReconConfig implements Jsonizable {
final static protected Logger LOGGER = LoggerFactory.getLogger("recon-config"); final static protected Logger LOGGER = LoggerFactory.getLogger("recon-config");
static final public Map<String, List<Class<? extends ReconConfig>>> s_opNameToClass = static final public Map<String, List<Class<? extends ReconConfig>>> s_opNameToClass =
new HashMap<String, List<Class<? extends ReconConfig>>>(); new HashMap<String, List<Class<? extends ReconConfig>>>();
static final public Map<Class<? extends ReconConfig>, String> s_opClassToName = static final public Map<Class<? extends ReconConfig>, String> s_opClassToName =
new HashMap<Class<? extends ReconConfig>, String>(); new HashMap<Class<? extends ReconConfig>, String>();
static public void registerReconConfig(ButterflyModule module, String name, Class<? extends ReconConfig> klass) { static public void registerReconConfig(ButterflyModule module, String name, Class<? extends ReconConfig> klass) {
String key = module.getName() + "/" + name; String key = module.getName() + "/" + name;
s_opClassToName.put(klass, key); s_opClassToName.put(klass, key);
List<Class<? extends ReconConfig>> classes = s_opNameToClass.get(key); List<Class<? extends ReconConfig>> classes = s_opNameToClass.get(key);
if (classes == null) { if (classes == null) {
classes = new LinkedList<Class<? extends ReconConfig>>(); classes = new LinkedList<Class<? extends ReconConfig>>();
s_opNameToClass.put(key, classes); s_opNameToClass.put(key, classes);
} }
classes.add(klass); classes.add(klass);
} }
static public ReconConfig reconstruct(JSONObject obj) throws Exception { static public ReconConfig reconstruct(JSONObject obj) throws Exception {
try { try {
String mode = obj.getString("mode"); String mode = obj.getString("mode");
// Backward compatibility // Backward compatibility
if ("extend".equals(mode) || "strict".equals(mode)) { if ("extend".equals(mode) || "strict".equals(mode)) {
mode = "freebase/" + mode; mode = "freebase/" + mode;
} else if ("heuristic".equals(mode)) { } else if ("heuristic".equals(mode)) {
mode = "core/standard-service"; // legacy mode = "core/standard-service"; // legacy
} else if (!mode.contains("/")) { } else if (!mode.contains("/")) {
mode = "core/" + mode; mode = "core/" + mode;
} }
List<Class<? extends ReconConfig>> classes = s_opNameToClass.get(mode); List<Class<? extends ReconConfig>> classes = s_opNameToClass.get(mode);
if (classes != null && classes.size() > 0) { if (classes != null && classes.size() > 0) {
Class<? extends ReconConfig> klass = classes.get(classes.size() - 1); Class<? extends ReconConfig> klass = classes.get(classes.size() - 1);
Method reconstruct = klass.getMethod("reconstruct", JSONObject.class); Method reconstruct = klass.getMethod("reconstruct", JSONObject.class);
if (reconstruct != null) { if (reconstruct != null) {
return (ReconConfig) reconstruct.invoke(null, obj); return (ReconConfig) reconstruct.invoke(null, obj);
} }
} }
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("Reconstruct failed",e); LOGGER.error("Reconstruct failed",e);
} }
return null; return null;
} }
abstract public int getBatchSize(); abstract public int getBatchSize();
abstract public String getBriefDescription(Project project, String columnName); abstract public String getBriefDescription(Project project, String columnName);
abstract public ReconJob createJob( abstract public ReconJob createJob(
Project project, Project project,
int rowIndex, int rowIndex,
Row row, Row row,
String columnName, String columnName,
Cell cell Cell cell
); );
abstract public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID); abstract public List<Recon> batchRecon(List<ReconJob> jobs, long historyEntryID);
abstract public Recon createNewRecon(long historyEntryID); abstract public Recon createNewRecon(long historyEntryID);
public void save(Writer writer) { public void save(Writer writer) {
JSONWriter jsonWriter = new JSONWriter(writer); JSONWriter jsonWriter = new JSONWriter(writer);
try { try {
write(jsonWriter, new Properties()); write(jsonWriter, new Properties());
} catch (JSONException e) { } catch (JSONException e) {
LOGGER.error("Save failed",e); LOGGER.error("Save failed",e);
} }
} }
} }