s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update key & id recon to new Freebase APIs - part of #696

Browse Source
This commit is contained in:
Tom Morris 2013-03-12 16:50:23 -04:00
parent 7578d3375f
commit 7b9f6836e1
5 changed files with 281 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -124,8 +125,6 @@ public class GuidBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -143,63 +142,72 @@ public class GuidBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
InputStream is = connection.getInputStream(); if (connection.getResponseCode() >= 400) {
try { String responseMessage = connection.getResponseMessage();
String s = ParsingUtilities.inputStreamToString(is); String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
if (o.has("result")) { + " : " + responseMessage + " : " + errorString);
JSONArray results = o.getJSONArray("result"); } else {
int count = results.length(); InputStream is = connection.getInputStream();
for (int i = 0; i < count; i++) { try {
JSONObject result = results.getJSONObject(i); String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
String guid = result.getString("guid"); for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
JSONArray types = result.getJSONArray("type"); String guid = result.getString("guid");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) { JSONArray types = result.getJSONArray("type");
typeIDs[j] = types.getString(j); String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
} }
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
} }
} finally {
is.close();
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {

91
extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -128,8 +129,6 @@ public class IdBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -147,63 +146,71 @@ public class IdBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
InputStream is = connection.getInputStream(); if (connection.getResponseCode() >= 400) {
try { String responseMessage = connection.getResponseMessage();
String s = ParsingUtilities.inputStreamToString(is); String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
if (o.has("result")) { + " : " + responseMessage + " : " + errorString);
JSONArray results = o.getJSONArray("result"); } else {
int count = results.length(); InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i); JSONObject result = results.getJSONObject(i);
String id = result.getString("id"); String id = result.getString("id");
JSONArray types = result.getJSONArray("type"); JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()]; String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) { for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j); typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
} }
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
} }
} finally {
is.close();
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {

89
extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
View File

@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -129,8 +130,6 @@ public class KeyBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -161,65 +160,73 @@ public class KeyBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
+ " : " + responseMessage + " : " + errorString);
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
InputStream is = connection.getInputStream(); for (int i = 0; i < count; i++) {
try { JSONObject result = results.getJSONObject(i);
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) { String key = result.getJSONArray("key").getJSONObject(0).getString("value");
JSONObject result = results.getJSONObject(i);
String key = result.getJSONArray("key").getJSONObject(0).getString("value"); JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
JSONArray types = result.getJSONArray("type"); ReconCandidate candidate = new ReconCandidate(
String[] typeIDs = new String[types.length()]; result.getString("id"),
for (int j = 0; j < typeIDs.length; j++) { result.getString("name"),
typeIDs[j] = types.getString(j); typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
} }
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
} }
} finally {
is.close();
} }
} finally {
is.close();
} }
} catch (Exception e) { } catch (IOException e) {
e.printStackTrace(); LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {
String key = ((KeyBasedReconJob) job).key; String key = ((KeyBasedReconJob) job).key;
Recon recon = keyToRecon.get(key); Recon recon = keyToRecon.get(key);

2
extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
View File

@ -40,7 +40,7 @@ import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig; import com.google.refine.model.recon.ReconConfig;
abstract public class StrictReconConfig extends ReconConfig { abstract public class StrictReconConfig extends ReconConfig {
final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread"; final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M&";
static public ReconConfig reconstruct(JSONObject obj) throws Exception { static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match"); String match = obj.getString("match");

2
main/src/com/google/refine/model/recon/ReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without