Refactor to use common mqlread code and eliminate clones
This commit is contained in:
Tom Morris
parent
01de9e65c0
commit
4074d3267c
@ -44,6 +44,11 @@ import com.google.refine.freebase.util.FreebaseUtils;
|
|||||||
import com.google.refine.oauth.OAuthUtilities;
|
import com.google.refine.oauth.OAuthUtilities;
|
||||||
import com.google.refine.oauth.Provider;
|
import com.google.refine.oauth.Provider;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Perform an MQLread on the server, using the client's credentials.
|
||||||
|
*
|
||||||
|
* Currently unused. All client code calls the Freebase API directly.
|
||||||
|
*/
|
||||||
public class MQLReadCommand extends Command {
|
public class MQLReadCommand extends Command {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -55,6 +60,7 @@ public class MQLReadCommand extends Command {
|
|||||||
response.setCharacterEncoding("UTF-8");
|
response.setCharacterEncoding("UTF-8");
|
||||||
response.setHeader("Content-Type", "application/json");
|
response.setHeader("Content-Type", "application/json");
|
||||||
String query = request.getParameter("query");
|
String query = request.getParameter("query");
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
String result = FreebaseUtils.mqlread(provider,query);
|
String result = FreebaseUtils.mqlread(provider,query);
|
||||||
response.getWriter().write(result);
|
response.getWriter().write(result);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|||||||
@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||||||
package com.google.refine.freebase.model.recon;
|
package com.google.refine.freebase.model.recon;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
import java.net.HttpURLConnection;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -49,6 +46,7 @@ import org.json.JSONException;
|
|||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.json.JSONWriter;
|
import org.json.JSONWriter;
|
||||||
|
|
||||||
|
import com.google.refine.freebase.util.FreebaseUtils;
|
||||||
import com.google.refine.model.Cell;
|
import com.google.refine.model.Cell;
|
||||||
import com.google.refine.model.Project;
|
import com.google.refine.model.Project;
|
||||||
import com.google.refine.model.Recon;
|
import com.google.refine.model.Recon;
|
||||||
@ -120,88 +118,42 @@ public class GuidBasedReconConfig extends StrictReconConfig {
|
|||||||
Map<String, Recon> guidToRecon = new HashMap<String, Recon>();
|
Map<String, Recon> guidToRecon = new HashMap<String, Recon>();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
String query = null;
|
String query = buildQuery(jobs);
|
||||||
{
|
|
||||||
StringWriter stringWriter = new StringWriter();
|
|
||||||
JSONWriter jsonWriter = new JSONWriter(stringWriter);
|
|
||||||
|
|
||||||
jsonWriter.array();
|
|
||||||
jsonWriter.object();
|
|
||||||
|
|
||||||
jsonWriter.key("id"); jsonWriter.value(null);
|
String s = FreebaseUtils.mqlread(query);
|
||||||
jsonWriter.key("name"); jsonWriter.value(null);
|
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
|
||||||
jsonWriter.key("guid"); jsonWriter.value(null);
|
|
||||||
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
|
|
||||||
|
|
||||||
jsonWriter.key("guid|=");
|
|
||||||
jsonWriter.array();
|
|
||||||
for (ReconJob job : jobs) {
|
|
||||||
jsonWriter.value(((GuidBasedReconJob) job).guid);
|
|
||||||
}
|
|
||||||
jsonWriter.endArray();
|
|
||||||
|
|
||||||
jsonWriter.endObject();
|
|
||||||
jsonWriter.endArray();
|
|
||||||
|
|
||||||
query = stringWriter.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
StringBuffer sb = new StringBuffer(1024);
|
if (o.has("result")) {
|
||||||
sb.append(s_mqlreadService);
|
JSONArray results = o.getJSONArray("result");
|
||||||
sb.append("query=");
|
int count = results.length();
|
||||||
sb.append(ParsingUtilities.encode(query));
|
|
||||||
|
|
||||||
URL url = new URL(sb.toString());
|
|
||||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
|
||||||
connection.setConnectTimeout(5000);
|
|
||||||
connection.connect();
|
|
||||||
|
|
||||||
if (connection.getResponseCode() >= 400) {
|
|
||||||
String responseMessage = connection.getResponseMessage();
|
|
||||||
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
|
|
||||||
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
|
|
||||||
+ " : " + responseMessage + " : " + errorString);
|
|
||||||
} else {
|
|
||||||
InputStream is = connection.getInputStream();
|
|
||||||
|
|
||||||
try {
|
for (int i = 0; i < count; i++) {
|
||||||
String s = ParsingUtilities.inputStreamToString(is);
|
JSONObject result = results.getJSONObject(i);
|
||||||
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
|
|
||||||
if (o.has("result")) {
|
|
||||||
JSONArray results = o.getJSONArray("result");
|
|
||||||
int count = results.length();
|
|
||||||
|
|
||||||
for (int i = 0; i < count; i++) {
|
String guid = result.getString("guid");
|
||||||
JSONObject result = results.getJSONObject(i);
|
|
||||||
|
|
||||||
String guid = result.getString("guid");
|
JSONArray types = result.getJSONArray("type");
|
||||||
|
String[] typeIDs = new String[types.length()];
|
||||||
JSONArray types = result.getJSONArray("type");
|
for (int j = 0; j < typeIDs.length; j++) {
|
||||||
String[] typeIDs = new String[types.length()];
|
typeIDs[j] = types.getString(j);
|
||||||
for (int j = 0; j < typeIDs.length; j++) {
|
|
||||||
typeIDs[j] = types.getString(j);
|
|
||||||
}
|
|
||||||
|
|
||||||
ReconCandidate candidate = new ReconCandidate(
|
|
||||||
result.getString("id"),
|
|
||||||
result.getString("name"),
|
|
||||||
typeIDs,
|
|
||||||
100
|
|
||||||
);
|
|
||||||
|
|
||||||
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
|
|
||||||
recon.addCandidate(candidate);
|
|
||||||
recon.service = "mql";
|
|
||||||
recon.judgment = Judgment.Matched;
|
|
||||||
recon.judgmentAction = "auto";
|
|
||||||
recon.match = candidate;
|
|
||||||
recon.matchRank = 0;
|
|
||||||
|
|
||||||
guidToRecon.put(guid, recon);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
is.close();
|
ReconCandidate candidate = new ReconCandidate(
|
||||||
|
result.getString("id"),
|
||||||
|
result.getString("name"),
|
||||||
|
typeIDs,
|
||||||
|
100
|
||||||
|
);
|
||||||
|
|
||||||
|
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
|
||||||
|
recon.addCandidate(candidate);
|
||||||
|
recon.service = "mql";
|
||||||
|
recon.judgment = Judgment.Matched;
|
||||||
|
recon.judgmentAction = "auto";
|
||||||
|
recon.match = candidate;
|
||||||
|
recon.matchRank = 0;
|
||||||
|
|
||||||
|
guidToRecon.put(guid, recon);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
@ -209,7 +161,7 @@ public class GuidBasedReconConfig extends StrictReconConfig {
|
|||||||
} catch (JSONException e) {
|
} catch (JSONException e) {
|
||||||
LOGGER.error("JSONException during recon : ",e);
|
LOGGER.error("JSONException during recon : ",e);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (ReconJob job : jobs) {
|
for (ReconJob job : jobs) {
|
||||||
String guid = ((GuidBasedReconJob) job).guid;
|
String guid = ((GuidBasedReconJob) job).guid;
|
||||||
Recon recon = guidToRecon.get(guid);
|
Recon recon = guidToRecon.get(guid);
|
||||||
@ -221,4 +173,33 @@ public class GuidBasedReconConfig extends StrictReconConfig {
|
|||||||
|
|
||||||
return recons;
|
return recons;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String buildQuery(List<ReconJob> jobs)
|
||||||
|
throws JSONException {
|
||||||
|
String query = null;
|
||||||
|
|
||||||
|
StringWriter stringWriter = new StringWriter();
|
||||||
|
JSONWriter jsonWriter = new JSONWriter(stringWriter);
|
||||||
|
|
||||||
|
jsonWriter.array();
|
||||||
|
jsonWriter.object();
|
||||||
|
|
||||||
|
jsonWriter.key("id"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("name"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("guid"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
|
||||||
|
|
||||||
|
jsonWriter.key("guid|=");
|
||||||
|
jsonWriter.array();
|
||||||
|
for (ReconJob job : jobs) {
|
||||||
|
jsonWriter.value(((GuidBasedReconJob) job).guid);
|
||||||
|
}
|
||||||
|
jsonWriter.endArray();
|
||||||
|
|
||||||
|
jsonWriter.endObject();
|
||||||
|
jsonWriter.endArray();
|
||||||
|
|
||||||
|
query = stringWriter.toString();
|
||||||
|
return query;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||||||
package com.google.refine.freebase.model.recon;
|
package com.google.refine.freebase.model.recon;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
import java.net.HttpURLConnection;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -49,6 +46,7 @@ import org.json.JSONException;
|
|||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.json.JSONWriter;
|
import org.json.JSONWriter;
|
||||||
|
|
||||||
|
import com.google.refine.freebase.util.FreebaseUtils;
|
||||||
import com.google.refine.model.Cell;
|
import com.google.refine.model.Cell;
|
||||||
import com.google.refine.model.Project;
|
import com.google.refine.model.Project;
|
||||||
import com.google.refine.model.Recon;
|
import com.google.refine.model.Recon;
|
||||||
@ -124,87 +122,41 @@ public class IdBasedReconConfig extends StrictReconConfig {
|
|||||||
Map<String, Recon> idToRecon = new HashMap<String, Recon>();
|
Map<String, Recon> idToRecon = new HashMap<String, Recon>();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
String query = null;
|
String query = buildQuery(jobs);
|
||||||
{
|
String s = FreebaseUtils.mqlread(query);
|
||||||
StringWriter stringWriter = new StringWriter();
|
|
||||||
JSONWriter jsonWriter = new JSONWriter(stringWriter);
|
|
||||||
|
|
||||||
jsonWriter.array();
|
|
||||||
jsonWriter.object();
|
|
||||||
|
|
||||||
jsonWriter.key("id"); jsonWriter.value(null);
|
|
||||||
jsonWriter.key("name"); jsonWriter.value(null);
|
|
||||||
jsonWriter.key("guid"); jsonWriter.value(null);
|
|
||||||
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
|
|
||||||
|
|
||||||
jsonWriter.key("id|=");
|
|
||||||
jsonWriter.array();
|
|
||||||
for (ReconJob job : jobs) {
|
|
||||||
jsonWriter.value(((IdBasedReconJob) job).id);
|
|
||||||
}
|
|
||||||
jsonWriter.endArray();
|
|
||||||
|
|
||||||
jsonWriter.endObject();
|
|
||||||
jsonWriter.endArray();
|
|
||||||
|
|
||||||
query = stringWriter.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
StringBuffer sb = new StringBuffer(1024);
|
|
||||||
sb.append(s_mqlreadService);
|
|
||||||
sb.append("query=");
|
|
||||||
sb.append(ParsingUtilities.encode(query));
|
|
||||||
|
|
||||||
URL url = new URL(sb.toString());
|
|
||||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
|
||||||
connection.setConnectTimeout(5000);
|
|
||||||
connection.connect();
|
|
||||||
|
|
||||||
if (connection.getResponseCode() >= 400) {
|
|
||||||
String responseMessage = connection.getResponseMessage();
|
|
||||||
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
|
|
||||||
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
|
|
||||||
+ " : " + responseMessage + " : " + errorString);
|
|
||||||
} else {
|
|
||||||
InputStream is = connection.getInputStream();
|
|
||||||
try {
|
|
||||||
String s = ParsingUtilities.inputStreamToString(is);
|
|
||||||
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
|
|
||||||
if (o.has("result")) {
|
|
||||||
JSONArray results = o.getJSONArray("result");
|
|
||||||
int count = results.length();
|
|
||||||
|
|
||||||
for (int i = 0; i < count; i++) {
|
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
|
||||||
JSONObject result = results.getJSONObject(i);
|
if (o.has("result")) {
|
||||||
|
JSONArray results = o.getJSONArray("result");
|
||||||
|
int count = results.length();
|
||||||
|
|
||||||
String id = result.getString("id");
|
for (int i = 0; i < count; i++) {
|
||||||
|
JSONObject result = results.getJSONObject(i);
|
||||||
|
|
||||||
JSONArray types = result.getJSONArray("type");
|
String id = result.getString("id");
|
||||||
String[] typeIDs = new String[types.length()];
|
|
||||||
for (int j = 0; j < typeIDs.length; j++) {
|
|
||||||
typeIDs[j] = types.getString(j);
|
|
||||||
}
|
|
||||||
|
|
||||||
ReconCandidate candidate = new ReconCandidate(
|
JSONArray types = result.getJSONArray("type");
|
||||||
id,
|
String[] typeIDs = new String[types.length()];
|
||||||
result.getString("name"),
|
for (int j = 0; j < typeIDs.length; j++) {
|
||||||
typeIDs,
|
typeIDs[j] = types.getString(j);
|
||||||
100
|
|
||||||
);
|
|
||||||
|
|
||||||
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
|
|
||||||
recon.addCandidate(candidate);
|
|
||||||
recon.service = "mql";
|
|
||||||
recon.judgment = Judgment.Matched;
|
|
||||||
recon.judgmentAction = "auto";
|
|
||||||
recon.match = candidate;
|
|
||||||
recon.matchRank = 0;
|
|
||||||
|
|
||||||
idToRecon.put(id, recon);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
is.close();
|
ReconCandidate candidate = new ReconCandidate(
|
||||||
|
id,
|
||||||
|
result.getString("name"),
|
||||||
|
typeIDs,
|
||||||
|
100
|
||||||
|
);
|
||||||
|
|
||||||
|
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
|
||||||
|
recon.addCandidate(candidate);
|
||||||
|
recon.service = "mql";
|
||||||
|
recon.judgment = Judgment.Matched;
|
||||||
|
recon.judgmentAction = "auto";
|
||||||
|
recon.match = candidate;
|
||||||
|
recon.matchRank = 0;
|
||||||
|
|
||||||
|
idToRecon.put(id, recon);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
@ -225,4 +177,34 @@ public class IdBasedReconConfig extends StrictReconConfig {
|
|||||||
return recons;
|
return recons;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String buildQuery(List<ReconJob> jobs)
|
||||||
|
throws JSONException {
|
||||||
|
String query = null;
|
||||||
|
{
|
||||||
|
StringWriter stringWriter = new StringWriter();
|
||||||
|
JSONWriter jsonWriter = new JSONWriter(stringWriter);
|
||||||
|
|
||||||
|
jsonWriter.array();
|
||||||
|
jsonWriter.object();
|
||||||
|
|
||||||
|
jsonWriter.key("id"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("name"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("guid"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
|
||||||
|
|
||||||
|
jsonWriter.key("id|=");
|
||||||
|
jsonWriter.array();
|
||||||
|
for (ReconJob job : jobs) {
|
||||||
|
jsonWriter.value(((IdBasedReconJob) job).id);
|
||||||
|
}
|
||||||
|
jsonWriter.endArray();
|
||||||
|
|
||||||
|
jsonWriter.endObject();
|
||||||
|
jsonWriter.endArray();
|
||||||
|
|
||||||
|
query = stringWriter.toString();
|
||||||
|
}
|
||||||
|
return query;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -34,10 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||||||
package com.google.refine.freebase.model.recon;
|
package com.google.refine.freebase.model.recon;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
import java.net.HttpURLConnection;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -50,6 +47,7 @@ import org.json.JSONObject;
|
|||||||
import org.json.JSONWriter;
|
import org.json.JSONWriter;
|
||||||
|
|
||||||
import com.google.refine.freebase.FreebaseTopic;
|
import com.google.refine.freebase.FreebaseTopic;
|
||||||
|
import com.google.refine.freebase.util.FreebaseUtils;
|
||||||
import com.google.refine.model.Cell;
|
import com.google.refine.model.Cell;
|
||||||
import com.google.refine.model.Project;
|
import com.google.refine.model.Project;
|
||||||
import com.google.refine.model.Recon;
|
import com.google.refine.model.Recon;
|
||||||
@ -125,99 +123,41 @@ public class KeyBasedReconConfig extends StrictReconConfig {
|
|||||||
Map<String, Recon> keyToRecon = new HashMap<String, Recon>();
|
Map<String, Recon> keyToRecon = new HashMap<String, Recon>();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
String query = null;
|
String query = buildQuery(jobs);
|
||||||
{
|
String s = FreebaseUtils.mqlread(query);
|
||||||
StringWriter stringWriter = new StringWriter();
|
|
||||||
JSONWriter jsonWriter = new JSONWriter(stringWriter);
|
|
||||||
|
|
||||||
jsonWriter.array();
|
|
||||||
jsonWriter.object();
|
|
||||||
|
|
||||||
jsonWriter.key("id"); jsonWriter.value(null);
|
|
||||||
jsonWriter.key("name"); jsonWriter.value(null);
|
|
||||||
jsonWriter.key("guid"); jsonWriter.value(null);
|
|
||||||
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
|
|
||||||
|
|
||||||
jsonWriter.key("key");
|
|
||||||
jsonWriter.array();
|
|
||||||
jsonWriter.object();
|
|
||||||
|
|
||||||
jsonWriter.key("namespace");
|
|
||||||
jsonWriter.object();
|
|
||||||
jsonWriter.key("id"); jsonWriter.value(namespace.id);
|
|
||||||
jsonWriter.endObject();
|
|
||||||
|
|
||||||
jsonWriter.key("value"); jsonWriter.value(null);
|
|
||||||
jsonWriter.key("value|=");
|
|
||||||
jsonWriter.array();
|
|
||||||
for (ReconJob job : jobs) {
|
|
||||||
jsonWriter.value(((KeyBasedReconJob) job).key);
|
|
||||||
}
|
|
||||||
jsonWriter.endArray();
|
|
||||||
|
|
||||||
jsonWriter.endObject();
|
|
||||||
jsonWriter.endArray();
|
|
||||||
|
|
||||||
jsonWriter.endObject();
|
|
||||||
jsonWriter.endArray();
|
|
||||||
|
|
||||||
query = stringWriter.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
StringBuffer sb = new StringBuffer(1024);
|
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
|
||||||
sb.append(s_mqlreadService);
|
if (o.has("result")) {
|
||||||
sb.append("query=");
|
JSONArray results = o.getJSONArray("result");
|
||||||
sb.append(ParsingUtilities.encode(query));
|
int count = results.length();
|
||||||
|
|
||||||
URL url = new URL(sb.toString());
|
|
||||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
|
||||||
connection.setConnectTimeout(5000);
|
|
||||||
connection.connect();
|
|
||||||
if (connection.getResponseCode() >= 400) {
|
|
||||||
String responseMessage = connection.getResponseMessage();
|
|
||||||
String errorString = ParsingUtilities.inputStreamToString(connection.getErrorStream());
|
|
||||||
LOGGER.error("HTTP response error during recon: " + connection.getResponseCode()
|
|
||||||
+ " : " + responseMessage + " : " + errorString);
|
|
||||||
} else {
|
|
||||||
InputStream is = connection.getInputStream();
|
|
||||||
try {
|
|
||||||
String s = ParsingUtilities.inputStreamToString(is);
|
|
||||||
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
|
|
||||||
if (o.has("result")) {
|
|
||||||
JSONArray results = o.getJSONArray("result");
|
|
||||||
int count = results.length();
|
|
||||||
|
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
JSONObject result = results.getJSONObject(i);
|
JSONObject result = results.getJSONObject(i);
|
||||||
|
|
||||||
String key = result.getJSONArray("key").getJSONObject(0).getString("value");
|
String key = result.getJSONArray("key").getJSONObject(0).getString("value");
|
||||||
|
|
||||||
JSONArray types = result.getJSONArray("type");
|
JSONArray types = result.getJSONArray("type");
|
||||||
String[] typeIDs = new String[types.length()];
|
String[] typeIDs = new String[types.length()];
|
||||||
for (int j = 0; j < typeIDs.length; j++) {
|
for (int j = 0; j < typeIDs.length; j++) {
|
||||||
typeIDs[j] = types.getString(j);
|
typeIDs[j] = types.getString(j);
|
||||||
}
|
|
||||||
|
|
||||||
ReconCandidate candidate = new ReconCandidate(
|
|
||||||
result.getString("id"),
|
|
||||||
result.getString("name"),
|
|
||||||
typeIDs,
|
|
||||||
100
|
|
||||||
);
|
|
||||||
|
|
||||||
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
|
|
||||||
recon.addCandidate(candidate);
|
|
||||||
recon.service = "mql";
|
|
||||||
recon.judgment = Judgment.Matched;
|
|
||||||
recon.judgmentAction = "auto";
|
|
||||||
recon.match = candidate;
|
|
||||||
recon.matchRank = 0;
|
|
||||||
|
|
||||||
keyToRecon.put(key, recon);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
is.close();
|
ReconCandidate candidate = new ReconCandidate(
|
||||||
|
result.getString("id"),
|
||||||
|
result.getString("name"),
|
||||||
|
typeIDs,
|
||||||
|
100
|
||||||
|
);
|
||||||
|
|
||||||
|
Recon recon = Recon.makeFreebaseRecon(historyEntryID);
|
||||||
|
recon.addCandidate(candidate);
|
||||||
|
recon.service = "mql";
|
||||||
|
recon.judgment = Judgment.Matched;
|
||||||
|
recon.judgmentAction = "auto";
|
||||||
|
recon.match = candidate;
|
||||||
|
recon.matchRank = 0;
|
||||||
|
|
||||||
|
keyToRecon.put(key, recon);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
@ -226,7 +166,6 @@ public class KeyBasedReconConfig extends StrictReconConfig {
|
|||||||
LOGGER.error("JSONException during recon : ",e);
|
LOGGER.error("JSONException during recon : ",e);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for (ReconJob job : jobs) {
|
for (ReconJob job : jobs) {
|
||||||
String key = ((KeyBasedReconJob) job).key;
|
String key = ((KeyBasedReconJob) job).key;
|
||||||
Recon recon = keyToRecon.get(key);
|
Recon recon = keyToRecon.get(key);
|
||||||
@ -239,4 +178,47 @@ public class KeyBasedReconConfig extends StrictReconConfig {
|
|||||||
return recons;
|
return recons;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String buildQuery(List<ReconJob> jobs)
|
||||||
|
throws JSONException {
|
||||||
|
String query = null;
|
||||||
|
{
|
||||||
|
StringWriter stringWriter = new StringWriter();
|
||||||
|
JSONWriter jsonWriter = new JSONWriter(stringWriter);
|
||||||
|
|
||||||
|
jsonWriter.array();
|
||||||
|
jsonWriter.object();
|
||||||
|
|
||||||
|
jsonWriter.key("id"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("name"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("guid"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
|
||||||
|
|
||||||
|
jsonWriter.key("key");
|
||||||
|
jsonWriter.array();
|
||||||
|
jsonWriter.object();
|
||||||
|
|
||||||
|
jsonWriter.key("namespace");
|
||||||
|
jsonWriter.object();
|
||||||
|
jsonWriter.key("id"); jsonWriter.value(namespace.id);
|
||||||
|
jsonWriter.endObject();
|
||||||
|
|
||||||
|
jsonWriter.key("value"); jsonWriter.value(null);
|
||||||
|
jsonWriter.key("value|=");
|
||||||
|
jsonWriter.array();
|
||||||
|
for (ReconJob job : jobs) {
|
||||||
|
jsonWriter.value(((KeyBasedReconJob) job).key);
|
||||||
|
}
|
||||||
|
jsonWriter.endArray();
|
||||||
|
|
||||||
|
jsonWriter.endObject();
|
||||||
|
jsonWriter.endArray();
|
||||||
|
|
||||||
|
jsonWriter.endObject();
|
||||||
|
jsonWriter.endArray();
|
||||||
|
|
||||||
|
query = stringWriter.toString();
|
||||||
|
}
|
||||||
|
return query;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -35,13 +35,11 @@ package com.google.refine.freebase.model.recon;
|
|||||||
|
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
|
|
||||||
import com.google.refine.freebase.util.FreebaseUtils;
|
|
||||||
import com.google.refine.model.Recon;
|
import com.google.refine.model.Recon;
|
||||||
import com.google.refine.model.Recon.Judgment;
|
import com.google.refine.model.Recon.Judgment;
|
||||||
import com.google.refine.model.recon.ReconConfig;
|
import com.google.refine.model.recon.ReconConfig;
|
||||||
|
|
||||||
abstract public class StrictReconConfig extends ReconConfig {
|
abstract public class StrictReconConfig extends ReconConfig {
|
||||||
final static protected String s_mqlreadService = "https://www.googleapis.com/freebase/v1/mqlread?key=" + FreebaseUtils.API_KEY + "&";
|
|
||||||
|
|
||||||
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
|
static public ReconConfig reconstruct(JSONObject obj) throws Exception {
|
||||||
String match = obj.getString("match");
|
String match = obj.getString("match");
|
||||||
|
|||||||
@ -31,49 +31,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
package com.google.refine.freebase.util;
|
package com.google.refine.freebase.util;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.OutputStreamWriter;
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
import java.io.UnsupportedEncodingException;
|
|
||||||
import java.io.Writer;
|
import java.io.Writer;
|
||||||
import java.net.HttpURLConnection;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import javax.mail.MessagingException;
|
|
||||||
import javax.mail.internet.MimeBodyPart;
|
|
||||||
import javax.mail.internet.MimeMultipart;
|
|
||||||
|
|
||||||
import org.apache.http.NameValuePair;
|
|
||||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
|
||||||
import org.apache.http.message.BasicNameValuePair;
|
|
||||||
import org.json.JSONArray;
|
import org.json.JSONArray;
|
||||||
import org.json.JSONException;
|
import org.json.JSONException;
|
||||||
import org.json.JSONObject;
|
import org.json.JSONObject;
|
||||||
import org.json.JSONWriter;
|
import org.json.JSONWriter;
|
||||||
|
|
||||||
import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
|
|
||||||
import com.google.api.client.googleapis.json.GoogleJsonError;
|
|
||||||
import com.google.api.client.http.HttpHeaders;
|
|
||||||
import com.google.api.client.http.HttpTransport;
|
|
||||||
import com.google.api.client.http.javanet.NetHttpTransport;
|
|
||||||
import com.google.api.client.json.JsonFactory;
|
|
||||||
import com.google.api.client.json.jackson.JacksonFactory;
|
|
||||||
import com.google.api.services.freebase.Freebase;
|
|
||||||
import com.google.api.services.freebase.FreebaseRequestInitializer;
|
|
||||||
|
|
||||||
import com.google.refine.freebase.FreebaseType;
|
import com.google.refine.freebase.FreebaseType;
|
||||||
import com.google.refine.model.ReconCandidate;
|
import com.google.refine.model.ReconCandidate;
|
||||||
import com.google.refine.util.JSONUtilities;
|
import com.google.refine.util.JSONUtilities;
|
||||||
@ -116,9 +89,11 @@ public class FreebaseDataExtensionJob {
|
|||||||
) throws Exception {
|
) throws Exception {
|
||||||
StringWriter writer = new StringWriter();
|
StringWriter writer = new StringWriter();
|
||||||
formulateQuery(ids, extension, writer);
|
formulateQuery(ids, extension, writer);
|
||||||
|
|
||||||
String query = writer.toString();
|
String query = writer.toString();
|
||||||
JSONObject o = doMqlRead(query);
|
|
||||||
|
String result = FreebaseUtils.mqlread(query);
|
||||||
|
|
||||||
|
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(result);
|
||||||
Map<String, FreebaseDataExtensionJob.DataExtension> map = new HashMap<String, FreebaseDataExtensionJob.DataExtension>();
|
Map<String, FreebaseDataExtensionJob.DataExtension> map = new HashMap<String, FreebaseDataExtensionJob.DataExtension>();
|
||||||
if (o.has("result")) {
|
if (o.has("result")) {
|
||||||
JSONArray a = o.getJSONArray("result");
|
JSONArray a = o.getJSONArray("result");
|
||||||
@ -325,164 +300,7 @@ public class FreebaseDataExtensionJob {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This RPC call works for the Reconcile API, but MQLread is not supported by JSONRPC
|
|
||||||
*/
|
|
||||||
static private JSONObject rpcCall(String query) throws JSONException, UnsupportedEncodingException, IOException {
|
|
||||||
URL url = new URL("https://www.googleapis.com/rpc");
|
|
||||||
|
|
||||||
JSONObject params = new JSONObject();
|
|
||||||
params.put("query",query);
|
|
||||||
params.put("key", FreebaseUtils.API_KEY);
|
|
||||||
|
|
||||||
JSONObject req1 = new JSONObject();
|
|
||||||
req1.put("jsonrpc","2.0");
|
|
||||||
req1.put("id","q0");
|
|
||||||
req1.put("method","freebase.mqlread");
|
|
||||||
req1.put("apiVersion", "v1");
|
|
||||||
req1.put("params",params);
|
|
||||||
|
|
||||||
JSONArray body = new JSONArray();
|
|
||||||
body.put(req1);
|
|
||||||
|
|
||||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
|
||||||
connection.setRequestProperty("Content-Type", "application/json"); //
|
|
||||||
connection.setConnectTimeout(5000);
|
|
||||||
connection.setDoOutput(true);
|
|
||||||
|
|
||||||
OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8");
|
|
||||||
try {
|
|
||||||
writer.write(body.toString());
|
|
||||||
} finally {
|
|
||||||
writer.flush();
|
|
||||||
writer.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
connection.connect();
|
|
||||||
JSONArray result = null;
|
|
||||||
if (connection.getResponseCode() >= 400) {
|
|
||||||
String responseMessage = connection.getResponseMessage();
|
|
||||||
String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
|
|
||||||
} else {
|
|
||||||
InputStream is = connection.getInputStream();
|
|
||||||
try {
|
|
||||||
String s = ParsingUtilities.inputStreamToString(is);
|
|
||||||
result = ParsingUtilities.evaluateJsonStringToArray(s);
|
|
||||||
} finally {
|
|
||||||
is.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result.getJSONObject(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static MimeBodyPart queryToMimeBodyPart(String query_name,
|
|
||||||
String query, String service_url, String api_key)
|
|
||||||
throws MessagingException, IOException {
|
|
||||||
MimeBodyPart mbp = new MimeBodyPart();
|
|
||||||
mbp.setHeader("Content-Transfer-Encoding", "binary");
|
|
||||||
mbp.setHeader("Content-ID", "<" + query_name + ">");
|
|
||||||
mbp.setHeader("Content-type", "application/http");
|
|
||||||
|
|
||||||
List<NameValuePair> params = new ArrayList<NameValuePair>();
|
|
||||||
params.add(new BasicNameValuePair("query",query));
|
|
||||||
params.add(new BasicNameValuePair("key", api_key));
|
|
||||||
UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8");
|
|
||||||
|
|
||||||
String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n";
|
|
||||||
mbp.setText(body, "UTF-8");
|
|
||||||
mbp.getLineCount();
|
|
||||||
mbp.getAllHeaderLines();
|
|
||||||
return mbp;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The beginnings of a homebrew attempt to generate multi-part MIME messages
|
|
||||||
* so that we can parse response bodies ourselves and avoid the limitations
|
|
||||||
* of the Google Client library.
|
|
||||||
*/
|
|
||||||
static private JSONObject batchCallOld(String query) throws JSONException, MessagingException, IOException {
|
|
||||||
URL url = new URL("https://www.googleapis.com/batch");
|
|
||||||
String service_url = "https://www.googleapis.com/freebase/v1/mqlread";
|
|
||||||
MimeMultipart mp = new MimeMultipart("mixed");
|
|
||||||
|
|
||||||
MimeBodyPart mbp = queryToMimeBodyPart("q0", query, service_url, FreebaseUtils.API_KEY);
|
|
||||||
mp.addBodyPart(mbp);
|
|
||||||
mp.getPreamble();
|
|
||||||
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
|
||||||
connection.setRequestProperty("Content-Type", mp.getContentType());
|
|
||||||
connection.setConnectTimeout(5000);
|
|
||||||
connection.setDoOutput(true);
|
|
||||||
|
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
||||||
mp.writeTo(baos);
|
|
||||||
String foo = baos.toString("UTF-8");
|
|
||||||
|
|
||||||
mp.writeTo(connection.getOutputStream());
|
|
||||||
|
|
||||||
connection.connect();
|
|
||||||
JSONArray result = null;
|
|
||||||
if (connection.getResponseCode() >= 400) {
|
|
||||||
String responseMessage = connection.getResponseMessage();
|
|
||||||
String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
|
|
||||||
} else {
|
|
||||||
InputStream is = connection.getInputStream();
|
|
||||||
try {
|
|
||||||
String s = ParsingUtilities.inputStreamToString(is);
|
|
||||||
result = ParsingUtilities.evaluateJsonStringToArray(s);
|
|
||||||
} finally {
|
|
||||||
is.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result.getJSONObject(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
|
|
||||||
|
|
||||||
/** Global instance of the JSON factory. */
|
|
||||||
private static final JsonFactory JSON_FACTORY = new JacksonFactory();
|
|
||||||
|
|
||||||
|
|
||||||
private static final FreebaseRequestInitializer REQUEST_INITIALIZER =
|
|
||||||
new FreebaseRequestInitializer(FreebaseUtils.API_KEY);
|
|
||||||
|
|
||||||
static private JSONObject batchCall1(String query) throws IOException, JSONException {
|
|
||||||
|
|
||||||
JsonBatchCallback<Freebase> callback = new JsonBatchCallback<Freebase>() {
|
|
||||||
public void onSuccess(Freebase res, HttpHeaders responseHeaders) {
|
|
||||||
System.out.println(res);
|
|
||||||
}
|
|
||||||
public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
|
|
||||||
System.out.println("Error Message: " + e.getMessage());
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
|
|
||||||
.setApplicationName("OpenRefine")
|
|
||||||
.setFreebaseRequestInitializer(REQUEST_INITIALIZER)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
// TODO: Batch doesn't work with MqlRead since it extends FreebaseRequest<Void>
|
|
||||||
// BatchRequest batch = client.batch();
|
|
||||||
// client.mqlread(query).queue(batch, callback);
|
|
||||||
// batch.execute();
|
|
||||||
|
|
||||||
String s = ParsingUtilities.inputStreamToString(client.mqlread(query).executeAsInputStream());
|
|
||||||
JSONObject response = ParsingUtilities.evaluateJsonStringToObject(s);
|
|
||||||
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
static protected JSONObject doMqlRead(String query)
|
|
||||||
throws IOException, JSONException, MessagingException {
|
|
||||||
// JSONObject result = rpcCall(query);
|
|
||||||
// JSONObject resutl = batchCallOld(query);
|
|
||||||
JSONObject result = batchCall1(query);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException {
|
static protected void formulateQuery(Set<String> ids, JSONObject node, Writer writer) throws JSONException {
|
||||||
JSONWriter jsonWriter = new JSONWriter(writer);
|
JSONWriter jsonWriter = new JSONWriter(writer);
|
||||||
|
|
||||||
|
|||||||
@ -1,304 +1,530 @@
|
|||||||
/*
|
/*
|
||||||
|
|
||||||
Copyright 2010, Google Inc.
|
Copyright 2010,2013 Google Inc. and other contributors
|
||||||
All rights reserved.
|
All rights reserved.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
Redistribution and use in source and binary forms, with or without
|
||||||
modification, are permitted provided that the following conditions are
|
modification, are permitted provided that the following conditions are
|
||||||
met:
|
met:
|
||||||
|
|
||||||
* Redistributions of source code must retain the above copyright
|
* Redistributions of source code must retain the above copyright
|
||||||
notice, this list of conditions and the following disclaimer.
|
notice, this list of conditions and the following disclaimer.
|
||||||
* Redistributions in binary form must reproduce the above
|
* Redistributions in binary form must reproduce the above
|
||||||
copyright notice, this list of conditions and the following disclaimer
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
in the documentation and/or other materials provided with the
|
in the documentation and/or other materials provided with the
|
||||||
distribution.
|
distribution.
|
||||||
* Neither the name of Google Inc. nor the names of its
|
* Neither the name of Google Inc. nor the names of its
|
||||||
contributors may be used to endorse or promote products derived from
|
contributors may be used to endorse or promote products derived from
|
||||||
this software without specific prior written permission.
|
this software without specific prior written permission.
|
||||||
|
|
||||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package com.google.refine.freebase.util;
|
package com.google.refine.freebase.util;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.io.InputStream;
|
||||||
import java.util.List;
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import java.io.Writer;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
import oauth.signpost.OAuthConsumer;
|
import java.net.URL;
|
||||||
import oauth.signpost.exception.OAuthCommunicationException;
|
import java.util.ArrayList;
|
||||||
import oauth.signpost.exception.OAuthExpectationFailedException;
|
import java.util.List;
|
||||||
import oauth.signpost.exception.OAuthMessageSignerException;
|
|
||||||
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import org.apache.http.Header;
|
|
||||||
import org.apache.http.HttpResponse;
|
import oauth.signpost.OAuthConsumer;
|
||||||
import org.apache.http.NameValuePair;
|
import oauth.signpost.exception.OAuthCommunicationException;
|
||||||
import org.apache.http.client.ClientProtocolException;
|
import oauth.signpost.exception.OAuthExpectationFailedException;
|
||||||
import org.apache.http.client.HttpClient;
|
import oauth.signpost.exception.OAuthMessageSignerException;
|
||||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
|
||||||
import org.apache.http.client.methods.HttpGet;
|
import org.apache.http.Header;
|
||||||
import org.apache.http.client.methods.HttpPost;
|
import org.apache.http.HttpResponse;
|
||||||
import org.apache.http.impl.client.DefaultHttpClient;
|
import org.apache.http.NameValuePair;
|
||||||
import org.apache.http.message.BasicNameValuePair;
|
import org.apache.http.client.ClientProtocolException;
|
||||||
import org.apache.http.params.CoreProtocolPNames;
|
import org.apache.http.client.HttpClient;
|
||||||
import org.apache.http.util.EntityUtils;
|
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||||
import org.json.JSONException;
|
import org.apache.http.client.methods.HttpGet;
|
||||||
import org.json.JSONObject;
|
import org.apache.http.client.methods.HttpPost;
|
||||||
import org.slf4j.Logger;
|
import org.apache.http.impl.client.DefaultHttpClient;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.apache.http.message.BasicNameValuePair;
|
||||||
|
import org.apache.http.params.CoreProtocolPNames;
|
||||||
import com.google.refine.ProjectManager;
|
import org.apache.http.util.EntityUtils;
|
||||||
import com.google.refine.RefineServlet;
|
import org.json.JSONArray;
|
||||||
import com.google.refine.oauth.Credentials;
|
import org.json.JSONException;
|
||||||
import com.google.refine.oauth.OAuthUtilities;
|
import org.json.JSONObject;
|
||||||
import com.google.refine.oauth.Provider;
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
public class FreebaseUtils {
|
|
||||||
|
import com.google.api.client.googleapis.batch.BatchRequest;
|
||||||
static final Logger logger = LoggerFactory.getLogger("freebase");
|
import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
|
||||||
|
import com.google.api.client.googleapis.json.GoogleJsonError;
|
||||||
static final public String FREEBASE_HOST = "freebase.com";
|
import com.google.api.client.http.HttpHeaders;
|
||||||
|
import com.google.api.client.http.HttpTransport;
|
||||||
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
|
import com.google.api.client.http.javanet.NetHttpTransport;
|
||||||
|
import com.google.api.client.json.JsonFactory;
|
||||||
static final private String AGENT_ID = "/en/google_refine";
|
import com.google.api.client.json.jackson.JacksonFactory;
|
||||||
|
import com.google.api.services.freebase.Freebase;
|
||||||
static final private int SAMPLE_SIZE = 300;
|
import com.google.api.services.freebase.FreebaseRequestInitializer;
|
||||||
static final private int JUDGES = 4;
|
|
||||||
|
import com.google.refine.ProjectManager;
|
||||||
public static final String API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M";
|
import com.google.refine.RefineServlet;
|
||||||
|
import com.google.refine.oauth.Credentials;
|
||||||
private static String getUserInfoURL(String host) {
|
import com.google.refine.oauth.OAuthUtilities;
|
||||||
// TODO: Needs to be upgraded to new APIs sandbox-freebase.com as host becomes v1sandbox as version
|
import com.google.refine.oauth.Provider;
|
||||||
return "http://api." + host + "/api/service/user_info";
|
import com.google.refine.util.ParsingUtilities;
|
||||||
}
|
|
||||||
|
public class FreebaseUtils {
|
||||||
private static String getMQLWriteURL(String host) {
|
|
||||||
// TODO: Needs to be upgraded to new APIs
|
private static final String FREEBASE_API_VERSION = "v1";
|
||||||
// String version = "v1";
|
// private static final String FREEBASE_SANDBOX_API_VERSION = "v1sandbox";
|
||||||
// if (host != null && host.contains("sandbox")) {
|
|
||||||
// version += "sandbox";
|
private static final String GOOGLE_RPC_URL = "https://www.googleapis.com/rpc";
|
||||||
// }
|
|
||||||
// return "https://www.googleapis.com/freebase/"+version+"/mqlwrite?key=" + API_KEY + "&";
|
private static final String FREEBASE_SERVICE_URL = "https://www.googleapis.com/freebase/" + FREEBASE_API_VERSION;
|
||||||
return "http://api." + host + "/api/service/mqlwrite";
|
|
||||||
}
|
private static final String GOOGLE_BATCH_URL = "https://www.googleapis.com/batch";
|
||||||
|
|
||||||
private static String getMQLReadURL(String host) {
|
static final Logger logger = LoggerFactory.getLogger("freebase");
|
||||||
// TODO: Needs to be upgraded to new APIs
|
|
||||||
// String version = "v1";
|
static final public String FREEBASE_HOST = "freebase.com";
|
||||||
// if (host != null && host.contains("sandbox")) {
|
|
||||||
// version += "sandbox";
|
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
|
||||||
// }
|
|
||||||
// return "https://www.googleapis.com/freebase/"+version+"/mqlread?key=" + API_KEY + "&";
|
static final private String AGENT_ID = "/en/google_refine";
|
||||||
return "http://api." + host + "/api/service/mqlread";
|
|
||||||
}
|
static final private int SAMPLE_SIZE = 300;
|
||||||
|
static final private int JUDGES = 4;
|
||||||
private static String getUserAgent() {
|
|
||||||
return RefineServlet.FULLNAME;
|
public static final String API_KEY = "AIzaSyBAZ_EjMPKlOzyyZXv6JKXPPwJFISVji3M";
|
||||||
}
|
|
||||||
|
private static String getUserInfoURL(String host) {
|
||||||
public static String getUserInfo(Credentials credentials, Provider provider)
|
// TODO: Needs to be upgraded to new APIs sandbox-freebase.com as host becomes v1sandbox as version
|
||||||
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
|
return "http://api." + host + "/api/service/user_info";
|
||||||
|
}
|
||||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
|
||||||
|
private static String getMQLWriteURL(String host) {
|
||||||
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
|
return "http://api." + host + "/api/service/mqlwrite";
|
||||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
}
|
||||||
|
|
||||||
// this is required by the Metaweb API to avoid XSS
|
private static String getMQLReadURL(String host) {
|
||||||
httpRequest.setHeader("X-Requested-With", "1");
|
return "http://api." + host + "/api/service/mqlread";
|
||||||
|
}
|
||||||
// sign the request with the oauth library
|
|
||||||
consumer.sign(httpRequest);
|
private static String getUserAgent() {
|
||||||
|
return RefineServlet.FULLNAME;
|
||||||
// execute the request
|
}
|
||||||
HttpClient httpClient = new DefaultHttpClient();
|
|
||||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
public static String getUserInfo(Credentials credentials, Provider provider)
|
||||||
|
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
|
||||||
// return the results
|
|
||||||
return EntityUtils.toString(httpResponse.getEntity());
|
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
||||||
}
|
|
||||||
|
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
|
||||||
public static String getUserBadges(Provider provider, String user_id)
|
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
||||||
throws ClientProtocolException, IOException, JSONException {
|
|
||||||
|
// this is required by the Metaweb API to avoid XSS
|
||||||
String query = "{" +
|
httpRequest.setHeader("X-Requested-With", "1");
|
||||||
"'id' : '" + user_id + "'," +
|
|
||||||
"'!/type/usergroup/member' : [{" +
|
// sign the request with the oauth library
|
||||||
"'id' : null," +
|
consumer.sign(httpRequest);
|
||||||
"'key' : [{" +
|
|
||||||
"'namespace' : null" +
|
// execute the request
|
||||||
"}]" +
|
HttpClient httpClient = new DefaultHttpClient();
|
||||||
"}]" +
|
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
||||||
"}".replace("'", "\"");
|
|
||||||
|
// return the results
|
||||||
return mqlread(provider, query);
|
return EntityUtils.toString(httpResponse.getEntity());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String mqlread(Provider provider, String query)
|
public static String getUserBadges(Provider provider, String user_id)
|
||||||
throws ClientProtocolException, IOException, JSONException {
|
throws ClientProtocolException, IOException, JSONException {
|
||||||
|
|
||||||
JSONObject envelope = new JSONObject();
|
String query = "{" +
|
||||||
envelope.put("query", new JSONObject(query));
|
"'id' : '" + user_id + "'," +
|
||||||
|
"'!/type/usergroup/member' : [{" +
|
||||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
"'id' : null," +
|
||||||
formparams.add(new BasicNameValuePair("query", envelope.toString()));
|
"'key' : [{" +
|
||||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
"'namespace' : null" +
|
||||||
|
"}]" +
|
||||||
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
|
"}]" +
|
||||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
"}".replace("'", "\"");
|
||||||
httpRequest.setEntity(entity);
|
|
||||||
|
return mqlread(provider, query);
|
||||||
// this is required by the Metaweb API to avoid XSS
|
}
|
||||||
httpRequest.setHeader("X-Requested-With", "1");
|
|
||||||
|
/**
|
||||||
// execute the request
|
* Perform an MQLREAD operation using the credentials of the given OAuth provider
|
||||||
HttpClient httpClient = new DefaultHttpClient();
|
*
|
||||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
* @deprecated This will go away when we switch to Google authentication.
|
||||||
|
*/
|
||||||
// return the results
|
@Deprecated
|
||||||
return EntityUtils.toString(httpResponse.getEntity());
|
public static String mqlread(Provider provider, String query)
|
||||||
}
|
throws ClientProtocolException, IOException, JSONException {
|
||||||
|
|
||||||
public static String mqlwrite(Credentials credentials, Provider provider, String query)
|
JSONObject envelope = new JSONObject();
|
||||||
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
|
envelope.put("query", new JSONObject(query));
|
||||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
|
||||||
|
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
||||||
JSONObject envelope = new JSONObject();
|
formparams.add(new BasicNameValuePair("query", envelope.toString()));
|
||||||
envelope.put("query", new JSONObject(query));
|
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
||||||
|
|
||||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
|
||||||
formparams.add(new BasicNameValuePair("query", envelope.toString()));
|
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
||||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
httpRequest.setEntity(entity);
|
||||||
|
|
||||||
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
|
// this is required by the Metaweb API to avoid XSS
|
||||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
httpRequest.setHeader("X-Requested-With", "1");
|
||||||
httpRequest.setEntity(entity);
|
|
||||||
|
// execute the request
|
||||||
// this is required by the Metaweb API to avoid XSS
|
HttpClient httpClient = new DefaultHttpClient();
|
||||||
httpRequest.setHeader("X-Requested-With", "1");
|
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
||||||
|
|
||||||
// sign the request with the oauth library
|
// return the results
|
||||||
consumer.sign(httpRequest);
|
return EntityUtils.toString(httpResponse.getEntity());
|
||||||
|
}
|
||||||
// execute the request
|
|
||||||
HttpClient httpClient = new DefaultHttpClient();
|
|
||||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
/**
|
||||||
|
* Perform a single unauthenticated MQLread.
|
||||||
// return the results
|
*
|
||||||
return EntityUtils.toString(httpResponse.getEntity());
|
* (wrapper method for a bunch of alternative implementations)
|
||||||
}
|
*/
|
||||||
|
static public String mqlread(String query)
|
||||||
private static String getTweezersParams(int sample_size, int judges) {
|
throws IOException, JSONException {
|
||||||
String o = "{" +
|
// A bunch of implementations which don't work for MQLread, but do for other methods
|
||||||
"'sample_size':" + sample_size + "," +
|
// String result = rpcCall(query);
|
||||||
"'votes':{" +
|
// String result = googleCall(query);
|
||||||
"'reconciled':" + judges + "," +
|
// String result = batchCall1(query);
|
||||||
"'invalid':" + judges + "," +
|
|
||||||
"'new':" + judges + "," +
|
String result = mqlreadBatchMime(query);
|
||||||
"'skip':" + (judges + 2) +
|
return result;
|
||||||
"}" +
|
}
|
||||||
"}";
|
|
||||||
return o.replace('\'', '"');
|
|
||||||
}
|
public static String mqlwrite(Credentials credentials, Provider provider, String query)
|
||||||
|
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
|
||||||
public static String uploadTriples(
|
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
||||||
HttpServletRequest request,
|
|
||||||
String qa,
|
JSONObject envelope = new JSONObject();
|
||||||
String source_name,
|
envelope.put("query", new JSONObject(query));
|
||||||
String source_id,
|
|
||||||
String mdo_id,
|
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
||||||
String triples
|
formparams.add(new BasicNameValuePair("query", envelope.toString()));
|
||||||
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
|
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
||||||
|
|
||||||
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
|
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
|
||||||
|
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
||||||
Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
httpRequest.setEntity(entity);
|
||||||
|
|
||||||
JSONObject mdo_info = new JSONObject();
|
// this is required by the Metaweb API to avoid XSS
|
||||||
mdo_info.put("name", source_name);
|
httpRequest.setHeader("X-Requested-With", "1");
|
||||||
if (source_id != null) {
|
|
||||||
mdo_info.put("info_source",source_id);
|
// sign the request with the oauth library
|
||||||
}
|
consumer.sign(httpRequest);
|
||||||
|
|
||||||
JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
|
// execute the request
|
||||||
if (user_info.has("username")) {
|
HttpClient httpClient = new DefaultHttpClient();
|
||||||
|
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
||||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
|
||||||
formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
|
// return the results
|
||||||
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
|
return EntityUtils.toString(httpResponse.getEntity());
|
||||||
formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
|
}
|
||||||
formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
|
|
||||||
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
|
private static String getTweezersParams(int sample_size, int judges) {
|
||||||
formparams.add(new BasicNameValuePair("graphport", "sandbox"));
|
String o = "{" +
|
||||||
formparams.add(new BasicNameValuePair("payload", triples));
|
"'sample_size':" + sample_size + "," +
|
||||||
formparams.add(new BasicNameValuePair("check_params", "false"));
|
"'votes':{" +
|
||||||
if (mdo_id != null) {
|
"'reconciled':" + judges + "," +
|
||||||
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
|
"'invalid':" + judges + "," +
|
||||||
}
|
"'new':" + judges + "," +
|
||||||
if (Boolean.parseBoolean(qa)) {
|
"'skip':" + (judges + 2) +
|
||||||
formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
|
"}" +
|
||||||
}
|
"}";
|
||||||
|
return o.replace('\'', '"');
|
||||||
String freeqKey = System.getProperty("freeq.key");
|
}
|
||||||
if (freeqKey != null) {
|
|
||||||
logger.warn("Found Freeq key, will bypass OAuth signature");
|
public static String uploadTriples(
|
||||||
formparams.add(new BasicNameValuePair("apikey", freeqKey));
|
HttpServletRequest request,
|
||||||
}
|
String qa,
|
||||||
|
String source_name,
|
||||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
String source_id,
|
||||||
|
String mdo_id,
|
||||||
HttpPost httpRequest = new HttpPost(getFreeQUrl());
|
String triples
|
||||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
|
||||||
httpRequest.setEntity(entity);
|
|
||||||
|
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
|
||||||
if (freeqKey == null) {
|
|
||||||
logger.warn("Calculating OAuth signature");
|
Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
||||||
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
|
|
||||||
surrogateRequest.setEntity(entity);
|
JSONObject mdo_info = new JSONObject();
|
||||||
|
mdo_info.put("name", source_name);
|
||||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
if (source_id != null) {
|
||||||
|
mdo_info.put("info_source",source_id);
|
||||||
// TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors.
|
}
|
||||||
// Is there something we can do to generate an oauth signature without consuming so much memory?
|
|
||||||
consumer.sign(surrogateRequest);
|
JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
|
||||||
|
if (user_info.has("username")) {
|
||||||
Header[] h = surrogateRequest.getHeaders("Authorization");
|
|
||||||
if (h.length > 0) {
|
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
||||||
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
|
formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
|
||||||
} else {
|
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
|
||||||
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
|
formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
|
||||||
}
|
formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
|
||||||
}
|
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
|
||||||
|
formparams.add(new BasicNameValuePair("graphport", "sandbox"));
|
||||||
// execute the request
|
formparams.add(new BasicNameValuePair("payload", triples));
|
||||||
HttpClient httpClient = new DefaultHttpClient();
|
formparams.add(new BasicNameValuePair("check_params", "false"));
|
||||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
if (mdo_id != null) {
|
||||||
|
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
|
||||||
// return the results
|
}
|
||||||
return EntityUtils.toString(httpResponse.getEntity());
|
if (Boolean.parseBoolean(qa)) {
|
||||||
} else {
|
formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
|
||||||
throw new RuntimeException("Invalid credentials");
|
}
|
||||||
}
|
|
||||||
}
|
String freeqKey = System.getProperty("freeq.key");
|
||||||
|
if (freeqKey != null) {
|
||||||
static public String getFreeQUrl() {
|
logger.warn("Found Freeq key, will bypass OAuth signature");
|
||||||
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
|
formparams.add(new BasicNameValuePair("apikey", freeqKey));
|
||||||
return url != null ? url : FREEQ_URL;
|
}
|
||||||
}
|
|
||||||
|
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
||||||
|
|
||||||
|
HttpPost httpRequest = new HttpPost(getFreeQUrl());
|
||||||
|
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
||||||
|
httpRequest.setEntity(entity);
|
||||||
|
|
||||||
|
if (freeqKey == null) {
|
||||||
|
logger.warn("Calculating OAuth signature");
|
||||||
|
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
|
||||||
|
surrogateRequest.setEntity(entity);
|
||||||
|
|
||||||
|
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
||||||
|
|
||||||
|
// TODO(SM) This method uses a lot of memory and often results in OutOfMemoryErrors.
|
||||||
|
// Is there something we can do to generate an oauth signature without consuming so much memory?
|
||||||
|
consumer.sign(surrogateRequest);
|
||||||
|
|
||||||
|
Header[] h = surrogateRequest.getHeaders("Authorization");
|
||||||
|
if (h.length > 0) {
|
||||||
|
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
|
||||||
|
} else {
|
||||||
|
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// execute the request
|
||||||
|
HttpClient httpClient = new DefaultHttpClient();
|
||||||
|
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
||||||
|
|
||||||
|
// return the results
|
||||||
|
return EntityUtils.toString(httpResponse.getEntity());
|
||||||
|
} else {
|
||||||
|
throw new RuntimeException("Invalid credentials");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static public String getFreeQUrl() {
|
||||||
|
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
|
||||||
|
return url != null ? url : FREEQ_URL;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static final String BOUNDARY = "---theOpenRefineBoundary--=";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A hand rolled MIME multipart/mixed implementation for Google's Batch API
|
||||||
|
*/
|
||||||
|
static private String mqlreadBatchMime(String query) throws JSONException, IOException {
|
||||||
|
URL url = new URL(GOOGLE_BATCH_URL);
|
||||||
|
String service_url = FREEBASE_SERVICE_URL+"/mqlread";
|
||||||
|
|
||||||
|
// We could use the javax.mail package, but it's actually more trouble than it's worth
|
||||||
|
String body = "--" + BOUNDARY + "\n"
|
||||||
|
+ queryToMimeBodyPart("0", query, service_url, API_KEY)
|
||||||
|
+ "\n--" + BOUNDARY + "\n" ;
|
||||||
|
|
||||||
|
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
||||||
|
connection.setRequestProperty("Content-Type","multipart/mixed; boundary="+ BOUNDARY);
|
||||||
|
connection.setConnectTimeout(5000);
|
||||||
|
connection.setDoOutput(true);
|
||||||
|
|
||||||
|
Writer writer = new OutputStreamWriter(connection.getOutputStream());
|
||||||
|
try {
|
||||||
|
writer.write(body);
|
||||||
|
} finally {
|
||||||
|
writer.flush();
|
||||||
|
writer.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
connection.connect();
|
||||||
|
String result = null;
|
||||||
|
if (connection.getResponseCode() >= 400) {
|
||||||
|
String responseMessage = connection.getResponseMessage();
|
||||||
|
String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
|
||||||
|
LoggerFactory.getLogger("freebase").error(
|
||||||
|
"Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
|
||||||
|
+ errorStream);
|
||||||
|
} else {
|
||||||
|
InputStream is = connection.getInputStream();
|
||||||
|
try {
|
||||||
|
String s = ParsingUtilities.inputStreamToString(is);
|
||||||
|
String boundary = s.substring(0,s.indexOf("\n"));
|
||||||
|
boundary = boundary.split("\r")[0];
|
||||||
|
String[] part = s.split(boundary); // part 0 is empty because of leading boundary
|
||||||
|
String[] sections = part[1].split("\r\n\r\n");
|
||||||
|
// Mime headers, followed by HTTP headers, followd by actual response
|
||||||
|
result = sections[2];
|
||||||
|
} finally {
|
||||||
|
is.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
static String queryToMimeBodyPart(String query_name,
|
||||||
|
String query, String service_url, String api_key)
|
||||||
|
throws IOException {
|
||||||
|
// We could use the javax.mail package, but it's actually more trouble than it's worth
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append("Content-Type: application/http\n");
|
||||||
|
sb.append("Content-Transfer-Encoding: binary\n");
|
||||||
|
sb.append("Content-ID: " + query_name + "\n");
|
||||||
|
sb.append("\n");
|
||||||
|
|
||||||
|
List<NameValuePair> params = new ArrayList<NameValuePair>();
|
||||||
|
params.add(new BasicNameValuePair("query",query));
|
||||||
|
params.add(new BasicNameValuePair("key", api_key));
|
||||||
|
UrlEncodedFormEntity param_string = new UrlEncodedFormEntity(params, "UTF-8");
|
||||||
|
|
||||||
|
String body = "GET " + service_url + "?" + ParsingUtilities.inputStreamToString(param_string.getContent()) + "\n";
|
||||||
|
sb.append(body);
|
||||||
|
sb.append("\n");
|
||||||
|
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
//////////////////////// Unused methods for future use /////////////////////
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This RPC call works for the Reconcile API, but MQLread is not supported over JSONRPC
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
static private JSONObject mqlreadRpc(String query) throws JSONException, UnsupportedEncodingException, IOException {
|
||||||
|
URL url = new URL(GOOGLE_RPC_URL);
|
||||||
|
|
||||||
|
JSONObject params = new JSONObject();
|
||||||
|
params.put("query",query);
|
||||||
|
params.put("key", API_KEY);
|
||||||
|
|
||||||
|
JSONObject req1 = new JSONObject();
|
||||||
|
req1.put("jsonrpc","2.0");
|
||||||
|
req1.put("id","q0");
|
||||||
|
req1.put("method","freebase.mqlread");
|
||||||
|
req1.put("apiVersion", FREEBASE_API_VERSION);
|
||||||
|
req1.put("params",params);
|
||||||
|
|
||||||
|
JSONArray body = new JSONArray();
|
||||||
|
body.put(req1);
|
||||||
|
|
||||||
|
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
||||||
|
connection.setRequestProperty("Content-Type", "application/json"); //
|
||||||
|
connection.setConnectTimeout(5000);
|
||||||
|
connection.setDoOutput(true);
|
||||||
|
|
||||||
|
OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(),"utf-8");
|
||||||
|
try {
|
||||||
|
writer.write(body.toString());
|
||||||
|
} finally {
|
||||||
|
writer.flush();
|
||||||
|
writer.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
connection.connect();
|
||||||
|
JSONArray result = null;
|
||||||
|
if (connection.getResponseCode() >= 400) {
|
||||||
|
String responseMessage = connection.getResponseMessage();
|
||||||
|
String errorStream = ParsingUtilities.inputStreamToString(connection.getErrorStream());
|
||||||
|
LoggerFactory.getLogger("freebase").error(
|
||||||
|
"Error in mqlreadMime: " + connection.getResponseCode() + ":" + responseMessage + " : "
|
||||||
|
+ errorStream);
|
||||||
|
} else {
|
||||||
|
InputStream is = connection.getInputStream();
|
||||||
|
try {
|
||||||
|
String s = ParsingUtilities.inputStreamToString(is);
|
||||||
|
result = ParsingUtilities.evaluateJsonStringToArray(s);
|
||||||
|
} finally {
|
||||||
|
is.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result.getJSONObject(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
|
||||||
|
private static final JsonFactory JSON_FACTORY = new JacksonFactory();
|
||||||
|
private static final FreebaseRequestInitializer REQUEST_INITIALIZER =
|
||||||
|
new FreebaseRequestInitializer(FreebaseUtils.API_KEY);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Submit a single MQL read query via the standard Google client library
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
static private String mqlreadFreebaseClient(String query)
|
||||||
|
throws IOException, JSONException {
|
||||||
|
|
||||||
|
Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
|
||||||
|
.setApplicationName("OpenRefine")
|
||||||
|
.setFreebaseRequestInitializer(REQUEST_INITIALIZER)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
InputStream is = client.mqlread(query).executeAsInputStream();
|
||||||
|
String result = ParsingUtilities.inputStreamToString(is);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Submit a single MQL query via the Batch endpoint
|
||||||
|
* (not supported by Google's Java client)
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
static private JSONObject mqlreadBatchFreebaseClient(String query) throws IOException, JSONException {
|
||||||
|
JSONObject response = null;
|
||||||
|
|
||||||
|
// FIXME: We really want JsonBatchCallback<Freebase> here, but it's not supported right now
|
||||||
|
JsonBatchCallback<Void> callback = new JsonBatchCallback<Void>() {
|
||||||
|
public void onSuccess(Void res, HttpHeaders responseHeaders) {
|
||||||
|
System.out.println(res);
|
||||||
|
}
|
||||||
|
public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
|
||||||
|
System.out.println("Error Message: " + e.getMessage());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
Freebase client = new Freebase.Builder(HTTP_TRANSPORT, JSON_FACTORY, null)
|
||||||
|
.setApplicationName("OpenRefine")
|
||||||
|
.setFreebaseRequestInitializer(REQUEST_INITIALIZER)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
// FIXME: Batch doesn't work with MqlRead since it extends FreebaseRequest<Void>
|
||||||
|
BatchRequest batch = client.batch();
|
||||||
|
client.mqlread(query).queue(batch, callback);
|
||||||
|
batch.execute();
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
Loading…
Reference in New Issue
Block a user