s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update key & id recon to new Freebase APIs - part of #696

Browse Source
This commit is contained in:
Tom Morris 2013-03-12 16:50:23 -04:00
parent 7578d3375f
commit 7b9f6836e1
5 changed files with 281 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -124,8 +125,6 @@ public class GuidBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -143,22 +142,28 @@ public class GuidBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
+ " : " + responseMessage + " : " + errorString);
} else {
InputStream is = connection.getInputStream(); InputStream is = connection.getInputStream();
try { try {
String s = ParsingUtilities.inputStreamToString(is); String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
@ -198,8 +203,11 @@ public class GuidBasedReconConfig extends StrictReconConfig {
} finally { } finally {
is.close(); is.close();
} }
} catch (Exception e) { }
e.printStackTrace(); } catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {

25
extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -128,8 +129,6 @@ public class IdBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -147,21 +146,26 @@ public class IdBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
+ " : " + responseMessage + " : " + errorString);
} else {
InputStream is = connection.getInputStream(); InputStream is = connection.getInputStream();
try { try {
String s = ParsingUtilities.inputStreamToString(is); String s = ParsingUtilities.inputStreamToString(is);
@ -202,8 +206,11 @@ public class IdBasedReconConfig extends StrictReconConfig {
} finally { } finally {
is.close(); is.close();
} }
} catch (Exception e) { }
e.printStackTrace(); } catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
} }
for (ReconJob job : jobs) { for (ReconJob job : jobs) {

25
extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
View File

@ -33,10 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.freebase.model.recon; package com.google.refine.freebase.model.recon;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -129,8 +130,6 @@ public class KeyBasedReconConfig extends StrictReconConfig {
StringWriter stringWriter = new StringWriter(); StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter); JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
jsonWriter.key("query");
jsonWriter.array(); jsonWriter.array();
jsonWriter.object(); jsonWriter.object();
@ -161,21 +160,25 @@ public class KeyBasedReconConfig extends StrictReconConfig {
jsonWriter.endObject(); jsonWriter.endObject();
jsonWriter.endArray(); jsonWriter.endArray();
jsonWriter.endObject();
query = stringWriter.toString(); query = stringWriter.toString();
} }
StringBuffer sb = new StringBuffer(1024); StringBuffer sb = new StringBuffer(1024);
sb.append(s_mqlreadService); sb.append(s_mqlreadService);
sb.append("?query="); sb.append("query=");
sb.append(ParsingUtilities.encode(query)); sb.append(ParsingUtilities.encode(query));
URL url = new URL(sb.toString()); URL url = new URL(sb.toString());
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setConnectTimeout(5000); connection.setConnectTimeout(5000);
connection.connect(); connection.connect();
if (connection.getResponseCode() >= 400) {
String responseMessage = connection.getResponseMessage();
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
+ " : " + responseMessage + " : " + errorString);
} else {
InputStream is = connection.getInputStream(); InputStream is = connection.getInputStream();
try { try {
String s = ParsingUtilities.inputStreamToString(is); String s = ParsingUtilities.inputStreamToString(is);
@ -216,9 +219,13 @@ public class KeyBasedReconConfig extends StrictReconConfig {
} finally { } finally {
is.close(); is.close();
} }
} catch (Exception e) {
e.printStackTrace();
} }
} catch (IOException e) {
LOGGER.error("IOException during recon : ",e);
} catch (JSONException e) {
LOGGER.error("JSONException during recon : ",e);
}
for (ReconJob job : jobs) { for (ReconJob job : jobs) {
String key = ((KeyBasedReconJob) job).key; String key = ((KeyBasedReconJob) job).key;

2
extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
View File

@ -40,7 +40,7 @@ import com.google.refine.model.Recon.Judgment;
import com.google.refine.model.recon.ReconConfig; import com.google.refine.model.recon.ReconConfig;
abstract public class StrictReconConfig extends ReconConfig { abstract public class StrictReconConfig extends ReconConfig {
final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread"; final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M&";
static public ReconConfig reconstruct(JSONObject obj) throws Exception { static public ReconConfig reconstruct(JSONObject obj) throws Exception {
String match = obj.getString("match"); String match = obj.getString("match");

2
main/src/com/google/refine/model/recon/ReconConfig.java
View File

@ -1,6 +1,6 @@
/* /*
Copyright 2010, Google Inc. Copyright 2010,2013 Google Inc. and other contributors
All rights reserved. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without