s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update key & id recon to new Freebase APIs - part of #696

Browse Source
This commit is contained in:
Tom Morris 2013-03-12 16:50:23 -04:00
parent 7578d3375f
commit 7b9f6836e1
5 changed files with 281 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
View File

@ -1,6 +1,6 @@
/*
Copyright 2010, Google Inc.
Copyright 2010,2013 Google Inc. and other contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -124,8 +125,6 @@ public class GuidBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
@ -143,63 +142,72 @@ public class GuidBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append("query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
+ " : " + responseMessage + " : " + errorString);
} else {
InputStream is = connection.getInputStream();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
String guid = result.getString("guid");
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
String guid = result.getString("guid");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
guidToRecon.put(guid, recon);
}
} finally {
is.close();
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
} catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
}
for (ReconJob job : jobs) {

91
extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
View File

@ -1,6 +1,6 @@
/*
Copyright 2010, Google Inc.
Copyright 2010,2013 Google Inc. and other contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -128,8 +129,6 @@ public class IdBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
@ -147,63 +146,71 @@ public class IdBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append("query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
+ " : " + responseMessage + " : " + errorString);
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String id = result.getString("id");
String id = result.getString("id");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
}
ReconCandidate candidate = new ReconCandidate(
id,
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
idToRecon.put(id, recon);
}
} finally {
is.close();
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
} catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
}
for (ReconJob job : jobs) {

89
extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
View File

@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -129,8 +130,6 @@ public class KeyBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array();
jsonWriter.object();
@ -161,65 +160,73 @@ public class KeyBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject();
jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString();
}
StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService);
sb.append("?query=");
sb.append("query=");
sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
+ " : " + responseMessage + " : " + errorString);
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
if (o.has("result")) {
JSONArray results = o.getJSONArray("result");
int count = results.length();
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
for (int i = 0; i < count; i++) {
JSONObject result = results.getJSONObject(i);
String key = result.getJSONArray("key").getJSONObject(0).getString("value");
String key = result.getJSONArray("key").getJSONObject(0).getString("value");
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
}
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getString(j);
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("name"),
typeIDs,
100
);
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
recon.addCandidate(candidate);
recon.service = "mql";
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.match = candidate;
recon.matchRank = 0;
keyToRecon.put(key, recon);
}
} finally {
is.close();
}
} finally {
is.close();
}
} catch (Exception e) {
e.printStackTrace();
} catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
}
for (ReconJob job : jobs) {
String key = ((KeyBasedReconJob) job).key;
Recon recon = keyToRecon.get(key);

2
extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
View File

@ -40,7 +40,7 @@ import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig;
abstract public class StrictReconConfig extends ReconConfig {
final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread";
final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M&";
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match");

2
main/src/com/google/refine/model/recon/ReconConfig.java
View File

@ -1,6 +1,6 @@
/*
Copyright 2010, Google Inc.
Copyright 2010,2013 Google Inc. and other contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without