Remove remaining Freebase related functionalities.
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1453 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh
parent
6ddd945a80
commit
ad0d227ab3
@ -1,135 +0,0 @@
|
||||
package com.google.refine.commands.auth;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import oauth.signpost.OAuthConsumer;
|
||||
import oauth.signpost.OAuthProvider;
|
||||
|
||||
import com.google.refine.commands.Command;
|
||||
import com.google.refine.oauth.Credentials;
|
||||
import com.google.refine.oauth.OAuthUtilities;
|
||||
import com.google.refine.oauth.Provider;
|
||||
|
||||
public class AuthorizeCommand extends Command {
|
||||
|
||||
private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier";
|
||||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
// get the provider from the request
|
||||
Provider provider = OAuthUtilities.getProvider(request);
|
||||
|
||||
try {
|
||||
|
||||
// see if the request comes with access credentials
|
||||
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
||||
|
||||
// prepare the continuation URL that the OAuth provider will redirect the user to
|
||||
// (we need to make sure this URL points back to this code or the dance will never complete)
|
||||
String callbackURL = getBaseURL(request,provider);
|
||||
|
||||
if (access_credentials == null) {
|
||||
// access credentials are not available so we need to check
|
||||
// to see at what stage of the OAuth dance we are
|
||||
|
||||
// get the request token credentials
|
||||
Credentials request_credentials = Credentials.getCredentials(request, provider, Credentials.Type.REQUEST);
|
||||
|
||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(request_credentials, provider);
|
||||
OAuthProvider pp = OAuthUtilities.getOAuthProvider(provider);
|
||||
|
||||
if (request_credentials == null) {
|
||||
// no credentials were found, so let's start the dance
|
||||
|
||||
// get the request token
|
||||
|
||||
String url = pp.retrieveRequestToken(consumer, callbackURL);
|
||||
|
||||
request_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
|
||||
|
||||
// and set them to that we can retrieve them later in the second part of the dance
|
||||
Credentials.setCredentials(request, response, request_credentials, Credentials.Type.REQUEST, 3600);
|
||||
|
||||
// now redirect the user to the Authorize URL where she can authenticate against the
|
||||
// service provider and authorize us.
|
||||
// The provider will bounce the user back here for us to continue the dance.
|
||||
|
||||
response.sendRedirect(url);
|
||||
} else {
|
||||
// we are at the second stage of the dance, so we need need to obtain the access credentials now
|
||||
|
||||
// if we got here, it means that the user performed a valid authentication against the
|
||||
// service provider and authorized us, so now we can request more permanent credentials
|
||||
// to the service provider and save those as well for later use.
|
||||
|
||||
// this is set only for OAuth 1.0a
|
||||
String verificationCode = request.getParameter(OAUTH_VERIFIER_PARAM);
|
||||
|
||||
pp.retrieveAccessToken(consumer, verificationCode);
|
||||
|
||||
access_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
|
||||
|
||||
// no matter the result, we need to remove the request token
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
||||
|
||||
Credentials.setCredentials(request, response, access_credentials, Credentials.Type.ACCESS, 30 * 24 * 3600);
|
||||
|
||||
finish(response);
|
||||
}
|
||||
} else {
|
||||
finish(response);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void finish(HttpServletResponse response) throws IOException {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "text/html");
|
||||
|
||||
PrintWriter writer = response.getWriter();
|
||||
writer.write(
|
||||
"<html>" +
|
||||
"<body></body>" +
|
||||
"<script type='text/javascript'>" +
|
||||
"if (top.opener && top.opener.onauthorization) {" +
|
||||
" top.opener.onauthorization(window);" +
|
||||
"}" +
|
||||
"self.close();" +
|
||||
"</script>" +
|
||||
"</html>"
|
||||
);
|
||||
writer.flush();
|
||||
}
|
||||
|
||||
private String getBaseURL(HttpServletRequest request, Provider provider) {
|
||||
String host = request.getHeader("host");
|
||||
if (host == null) {
|
||||
String referrer = request.getHeader("referer");
|
||||
if (referrer != null) {
|
||||
URI url;
|
||||
try {
|
||||
url = new URI(referrer);
|
||||
int port = url.getPort();
|
||||
host = url.getHost() + ((port > -1) ? ":" + url.getPort() : "");
|
||||
} catch (URISyntaxException e) {
|
||||
throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL");
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL Google Refine is listening to.");
|
||||
}
|
||||
}
|
||||
return "http://" + host + "/command/core/authorize/" + provider.getHost();
|
||||
}
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
package com.google.refine.commands.auth;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.refine.commands.Command;
|
||||
import com.google.refine.oauth.Credentials;
|
||||
import com.google.refine.oauth.OAuthUtilities;
|
||||
import com.google.refine.oauth.Provider;
|
||||
import com.google.refine.util.FreebaseUtils;
|
||||
|
||||
public class CheckAuthorizationCommand extends Command {
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("check-authorization_command");
|
||||
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Provider provider = OAuthUtilities.getProvider(request);
|
||||
|
||||
// this cookie should not be there, but this is good hygiene practice
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
||||
|
||||
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
||||
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
if (access_credentials != null) {
|
||||
String user_info = FreebaseUtils.getUserInfo(access_credentials, provider);
|
||||
response.getWriter().write(user_info);
|
||||
} else {
|
||||
respond(response, "401 Unauthorized", "You don't have the right credentials");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.info("error",e);
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
package com.google.refine.commands.auth;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import com.google.refine.commands.Command;
|
||||
import com.google.refine.oauth.Credentials;
|
||||
import com.google.refine.oauth.OAuthUtilities;
|
||||
import com.google.refine.oauth.Provider;
|
||||
|
||||
public class DeAuthorizeCommand extends Command {
|
||||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
|
||||
Provider provider = OAuthUtilities.getProvider(request);
|
||||
|
||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
|
||||
|
||||
respond(response, "200 OK", "");
|
||||
} catch (Exception e) {
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,37 +0,0 @@
|
||||
package com.google.refine.commands.auth;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.refine.commands.Command;
|
||||
import com.google.refine.oauth.OAuthUtilities;
|
||||
import com.google.refine.oauth.Provider;
|
||||
import com.google.refine.util.FreebaseUtils;
|
||||
|
||||
public class GetUserBadgesCommand extends Command {
|
||||
|
||||
final static Logger logger = LoggerFactory.getLogger("get-version_command");
|
||||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
try {
|
||||
Provider provider = OAuthUtilities.getProvider(request);
|
||||
String user_id = request.getParameter("user_id");
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
String user_badges = FreebaseUtils.getUserBadges(provider, user_id);
|
||||
response.getWriter().write(user_badges);
|
||||
} catch (Exception e) {
|
||||
logger.info("error",e);
|
||||
respondException(response, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
public class AnonymousNode implements Node, NodeWithLinks {
|
||||
final public FreebaseType type;
|
||||
final public List<Link> links = new LinkedList<Link>();
|
||||
|
||||
public AnonymousNode(FreebaseType type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("nodeType"); writer.value("anonymous");
|
||||
writer.key("type"); type.write(writer, options);
|
||||
if (links != null) {
|
||||
writer.key("links"); writer.array();
|
||||
for (Link link : links) {
|
||||
link.write(writer, options);
|
||||
}
|
||||
writer.endArray();
|
||||
}
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
public void addLink(Link link) {
|
||||
links.add(link);
|
||||
}
|
||||
|
||||
public Link getLink(int index) {
|
||||
return links.get(index);
|
||||
}
|
||||
|
||||
public int getLinkCount() {
|
||||
return links.size();
|
||||
}
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
|
||||
|
||||
public class BooleanColumnCondition implements Condition {
|
||||
final public String columnName;
|
||||
|
||||
public BooleanColumnCondition(String columnName) {
|
||||
this.columnName = columnName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean test(Project project, int rowIndex, Row row) {
|
||||
Column column = project.columnModel.getColumnByName(columnName);
|
||||
if (column != null) {
|
||||
Object o = row.getCellValue(column.getCellIndex());
|
||||
if (o != null) {
|
||||
if (o instanceof Boolean) {
|
||||
return ((Boolean) o).booleanValue();
|
||||
} else {
|
||||
return Boolean.parseBoolean(o.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(JSONWriter writer, Properties options) throws JSONException {
|
||||
writer.object();
|
||||
writer.key("columnName"); writer.value(columnName);
|
||||
writer.endObject();
|
||||
}
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
public class CellKeyNode extends CellNode {
|
||||
final public FreebaseTopic namespace;
|
||||
|
||||
public CellKeyNode(
|
||||
FreebaseTopic namespace
|
||||
) {
|
||||
this.namespace = namespace;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("nodeType"); writer.value("cell-as-key");
|
||||
|
||||
writer.key("columnNames");
|
||||
writer.array();
|
||||
for (String name : columnNames) {
|
||||
writer.value(name);
|
||||
}
|
||||
writer.endArray();
|
||||
|
||||
writer.key("namespace"); namespace.write(writer, options);
|
||||
writer.endObject();
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
abstract public class CellNode implements Node {
|
||||
final public List<String> columnNames = new LinkedList<String>();
|
||||
}
|
||||
@ -1,56 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
public class CellTopicNode extends CellNode implements NodeWithLinks {
|
||||
final public FreebaseType type;
|
||||
final public List<Link> links = new LinkedList<Link>();
|
||||
|
||||
public CellTopicNode(
|
||||
FreebaseType type
|
||||
) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("nodeType"); writer.value("cell-as-topic");
|
||||
writer.key("columnNames");
|
||||
writer.array();
|
||||
for (String name : columnNames) {
|
||||
writer.value(name);
|
||||
}
|
||||
writer.endArray();
|
||||
if (type != null) {
|
||||
writer.key("type"); type.write(writer, options);
|
||||
}
|
||||
if (links != null) {
|
||||
writer.key("links"); writer.array();
|
||||
for (Link link : links) {
|
||||
link.write(writer, options);
|
||||
}
|
||||
writer.endArray();
|
||||
}
|
||||
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
public void addLink(Link link) {
|
||||
links.add(link);
|
||||
}
|
||||
|
||||
public Link getLink(int index) {
|
||||
return links.get(index);
|
||||
}
|
||||
|
||||
public int getLinkCount() {
|
||||
return links.size();
|
||||
}
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
public class CellValueNode extends CellNode {
|
||||
final public String valueType;
|
||||
final public String lang;
|
||||
|
||||
public CellValueNode(
|
||||
String valueType,
|
||||
String lang
|
||||
) {
|
||||
this.valueType = valueType;
|
||||
this.lang = lang;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("nodeType"); writer.value("cell-as-value");
|
||||
writer.key("columnNames");
|
||||
writer.array();
|
||||
for (String name : columnNames) {
|
||||
writer.value(name);
|
||||
}
|
||||
writer.endArray();
|
||||
writer.key("valueType"); writer.value(valueType);
|
||||
writer.key("lang"); writer.value(lang);
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import com.google.refine.Jsonizable;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
|
||||
public interface Condition extends Jsonizable {
|
||||
public boolean test(Project project, int rowIndex, Row row);
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
public class FreebaseProperty extends FreebaseTopic {
|
||||
//final protected FreebaseType _expectedType;
|
||||
|
||||
public FreebaseProperty(String id, String name) {
|
||||
super(id, name);
|
||||
}
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.refine.Jsonizable;
|
||||
|
||||
public class FreebaseTopic implements Jsonizable {
|
||||
final public String id;
|
||||
final public String name;
|
||||
|
||||
public FreebaseTopic(String id, String name) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("id"); writer.value(id);
|
||||
writer.key("name"); writer.value(name);
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
public class FreebaseTopicNode implements Node, NodeWithLinks {
|
||||
final public FreebaseTopic topic;
|
||||
final public List<Link> links = new LinkedList<Link>();
|
||||
|
||||
public FreebaseTopicNode(FreebaseTopic topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("nodeType"); writer.value("topic");
|
||||
writer.key("topic"); topic.write(writer, options);
|
||||
if (links != null) {
|
||||
writer.key("links"); writer.array();
|
||||
for (Link link : links) {
|
||||
link.write(writer, options);
|
||||
}
|
||||
writer.endArray();
|
||||
}
|
||||
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
public void addLink(Link link) {
|
||||
links.add(link);
|
||||
}
|
||||
|
||||
public Link getLink(int index) {
|
||||
return links.get(index);
|
||||
}
|
||||
|
||||
public int getLinkCount() {
|
||||
return links.size();
|
||||
}
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.refine.Jsonizable;
|
||||
|
||||
public class FreebaseType extends FreebaseTopic implements Jsonizable {
|
||||
public FreebaseType(String id, String name) {
|
||||
super(id, name);
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("id"); writer.value(id);
|
||||
writer.key("name"); writer.value(name);
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
static public FreebaseType load(JSONObject obj) throws Exception {
|
||||
if (obj == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
FreebaseType type = new FreebaseType(
|
||||
obj.getString("id"),
|
||||
obj.getString("name")
|
||||
);
|
||||
return type;
|
||||
}
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.refine.Jsonizable;
|
||||
|
||||
public class Link implements Jsonizable {
|
||||
final public FreebaseProperty property;
|
||||
final public Node target;
|
||||
final public Condition condition;
|
||||
final public boolean load;
|
||||
|
||||
public Link(FreebaseProperty property, Node target, Condition condition, boolean load) {
|
||||
this.property = property;
|
||||
this.target = target;
|
||||
this.condition = condition;
|
||||
this.load = load;
|
||||
}
|
||||
|
||||
public FreebaseProperty getProperty() {
|
||||
return property;
|
||||
}
|
||||
|
||||
public Node getTarget() {
|
||||
return target;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("property"); property.write(writer, options);
|
||||
if (target != null) {
|
||||
writer.key("target");
|
||||
target.write(writer, options);
|
||||
}
|
||||
if (condition != null) {
|
||||
writer.key("condition");
|
||||
condition.write(writer, options);
|
||||
}
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import com.google.refine.Jsonizable;
|
||||
|
||||
public interface Node extends Jsonizable {
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
public interface NodeWithLinks {
|
||||
public void addLink(Link link);
|
||||
|
||||
public int getLinkCount();
|
||||
|
||||
public Link getLink(int index);
|
||||
}
|
||||
@ -1,168 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.refine.model.OverlayModel;
|
||||
import com.google.refine.model.Project;
|
||||
|
||||
public class Protograph implements OverlayModel {
|
||||
final protected List<Node> _rootNodes = new LinkedList<Node>();
|
||||
|
||||
public int getRootNodeCount() {
|
||||
return _rootNodes.size();
|
||||
}
|
||||
|
||||
public Node getRootNode(int index) {
|
||||
return _rootNodes.get(index);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onBeforeSave(Project project) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onAfterSave(Project project) {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void dispose(Project project) {
|
||||
}
|
||||
|
||||
static public Protograph reconstruct(JSONObject o) throws JSONException {
|
||||
Protograph g = new Protograph();
|
||||
|
||||
JSONArray rootNodes = o.getJSONArray("rootNodes");
|
||||
int count = rootNodes.length();
|
||||
|
||||
for (int i = 0; i < count; i++) {
|
||||
JSONObject o2 = rootNodes.getJSONObject(i);
|
||||
Node node = reconstructNode(o2);
|
||||
if (node != null) {
|
||||
g._rootNodes.add(node);
|
||||
}
|
||||
}
|
||||
|
||||
return g;
|
||||
}
|
||||
|
||||
static protected Node reconstructNode(JSONObject o) throws JSONException {
|
||||
Node node = null;
|
||||
|
||||
String nodeType = o.getString("nodeType");
|
||||
if (nodeType.startsWith("cell-as-")) {
|
||||
if ("cell-as-topic".equals(nodeType)) {
|
||||
if (o.has("type")) {
|
||||
node = new CellTopicNode(
|
||||
reconstructType(o.getJSONObject("type"))
|
||||
);
|
||||
}
|
||||
} else if ("cell-as-value".equals(nodeType)) {
|
||||
node = new CellValueNode(
|
||||
o.getString("valueType"),
|
||||
o.getString("lang")
|
||||
);
|
||||
} else if ("cell-as-key".equals(nodeType)) {
|
||||
node = new CellKeyNode(
|
||||
reconstructTopic(o.getJSONObject("namespace"))
|
||||
);
|
||||
}
|
||||
|
||||
if (o.has("columnName") && !o.isNull("columnName")) {
|
||||
((CellNode) node).columnNames.add(o.getString("columnName"));
|
||||
}
|
||||
if (o.has("columnNames") && !o.isNull("columnNames")) {
|
||||
JSONArray columnNames = o.getJSONArray("columnNames");
|
||||
int count = columnNames.length();
|
||||
|
||||
for (int c = 0; c < count; c++) {
|
||||
((CellNode) node).columnNames.add(columnNames.getString(c));
|
||||
}
|
||||
}
|
||||
} else if ("topic".equals(nodeType)) {
|
||||
node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic")));
|
||||
} else if ("value".equals(nodeType)) {
|
||||
node = new ValueNode(
|
||||
o.get("value"),
|
||||
o.getString("valueType"),
|
||||
o.getString("lang")
|
||||
);
|
||||
} else if ("anonymous".equals(nodeType)) {
|
||||
node = new AnonymousNode(reconstructType(o.getJSONObject("type")));
|
||||
}
|
||||
|
||||
if (node != null && node instanceof NodeWithLinks && o.has("links")) {
|
||||
NodeWithLinks node2 = (NodeWithLinks) node;
|
||||
|
||||
JSONArray links = o.getJSONArray("links");
|
||||
int linkCount = links.length();
|
||||
|
||||
for (int j = 0; j < linkCount; j++) {
|
||||
JSONObject oLink = links.getJSONObject(j);
|
||||
Condition condition = null;
|
||||
|
||||
if (oLink.has("condition") && !oLink.isNull("condition")) {
|
||||
JSONObject oCondition = oLink.getJSONObject("condition");
|
||||
if (oCondition.has("columnName") && !oCondition.isNull("columnName")) {
|
||||
condition = new BooleanColumnCondition(oCondition.getString("columnName"));
|
||||
}
|
||||
}
|
||||
|
||||
node2.addLink(new Link(
|
||||
reconstructProperty(oLink.getJSONObject("property")),
|
||||
oLink.has("target") && !oLink.isNull("target") ?
|
||||
reconstructNode(oLink.getJSONObject("target")) : null,
|
||||
condition,
|
||||
oLink.has("load") && !oLink.isNull("load") ?
|
||||
oLink.getBoolean("load") : true
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException {
|
||||
return new FreebaseProperty(
|
||||
o.getString("id"),
|
||||
o.getString("name")
|
||||
);
|
||||
}
|
||||
|
||||
static protected FreebaseType reconstructType(JSONObject o) throws JSONException {
|
||||
return new FreebaseType(
|
||||
o.getString("id"),
|
||||
o.getString("name")
|
||||
);
|
||||
}
|
||||
|
||||
static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException {
|
||||
return new FreebaseTopic(
|
||||
o.getString("id"),
|
||||
o.getString("name")
|
||||
);
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options) throws JSONException {
|
||||
writer.object();
|
||||
writer.key("rootNodes"); writer.array();
|
||||
|
||||
for (Node node : _rootNodes) {
|
||||
node.write(writer, options);
|
||||
}
|
||||
|
||||
writer.endArray();
|
||||
writer.endObject();
|
||||
}
|
||||
|
||||
static public Protograph load(Project project, JSONObject obj) throws Exception {
|
||||
return reconstruct(obj);
|
||||
}
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
package com.google.refine.protograph;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
public class ValueNode implements Node {
|
||||
final public Object value;
|
||||
final public String valueType;
|
||||
final public String lang;
|
||||
|
||||
public ValueNode(Object value, String valueType, String lang) {
|
||||
this.value = value;
|
||||
this.valueType = valueType;
|
||||
this.lang = lang;
|
||||
}
|
||||
|
||||
public void write(JSONWriter writer, Properties options)
|
||||
throws JSONException {
|
||||
|
||||
writer.object();
|
||||
writer.key("nodeType"); writer.value("value");
|
||||
writer.key("value"); writer.value(value);
|
||||
writer.key("valueType"); writer.value(valueType);
|
||||
writer.key("lang"); writer.value(lang);
|
||||
writer.endObject();
|
||||
}
|
||||
}
|
||||
@ -1,346 +0,0 @@
|
||||
package com.google.refine.protograph.transpose;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.json.JSONWriter;
|
||||
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Recon;
|
||||
import com.google.refine.protograph.AnonymousNode;
|
||||
import com.google.refine.protograph.CellKeyNode;
|
||||
import com.google.refine.protograph.CellNode;
|
||||
import com.google.refine.protograph.CellTopicNode;
|
||||
import com.google.refine.protograph.CellValueNode;
|
||||
import com.google.refine.protograph.FreebaseProperty;
|
||||
import com.google.refine.protograph.FreebaseTopicNode;
|
||||
import com.google.refine.protograph.Link;
|
||||
import com.google.refine.protograph.ValueNode;
|
||||
import com.google.refine.util.JSONUtilities;
|
||||
|
||||
public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory {
|
||||
protected Writer writer;
|
||||
protected List<JSONObject> rootObjects = new LinkedList<JSONObject>();
|
||||
|
||||
private static final String TYPE = "type";
|
||||
private static final String ID = "id";
|
||||
private static final String NAME = "name";
|
||||
private static final String CREATE = "create";
|
||||
private static final String VALUE = "value";
|
||||
private static final String CONNECT = "connect";
|
||||
private static final String LANG = "lang";
|
||||
|
||||
public MqlwriteLikeTransposedNodeFactory(Writer writer) {
|
||||
this.writer = writer;
|
||||
}
|
||||
|
||||
protected JSONArray getJSON() {
|
||||
return new JSONArray(rootObjects);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
try {
|
||||
JSONWriter jsonWriter = new JSONWriter(writer);
|
||||
|
||||
jsonWriter.array();
|
||||
for (JSONObject obj : rootObjects) {
|
||||
jsonWriter.value(obj);
|
||||
}
|
||||
jsonWriter.endArray();
|
||||
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
writer.flush();
|
||||
}
|
||||
|
||||
abstract protected class JsonTransposedNode implements TransposedNode {
|
||||
abstract public Object getJSON();
|
||||
}
|
||||
|
||||
abstract protected class JsonObjectTransposedNode extends JsonTransposedNode {
|
||||
abstract public JSONObject getJSONObject();
|
||||
|
||||
protected JSONObject obj;
|
||||
|
||||
public Object getJSON() {
|
||||
return getJSONObject();
|
||||
}
|
||||
}
|
||||
|
||||
protected class AnonymousTransposedNode extends JsonObjectTransposedNode {
|
||||
JsonObjectTransposedNode parent;
|
||||
FreebaseProperty property;
|
||||
AnonymousNode node;
|
||||
|
||||
protected AnonymousTransposedNode(
|
||||
JsonObjectTransposedNode parent,
|
||||
FreebaseProperty property,
|
||||
AnonymousNode node
|
||||
) {
|
||||
this.parent = parent;
|
||||
this.property = property;
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
public JSONObject getJSONObject() {
|
||||
if (obj == null) {
|
||||
obj = new JSONObject();
|
||||
try {
|
||||
obj.put(TYPE, this.node.type.id);
|
||||
obj.put(ID, (String) null);
|
||||
obj.put(CREATE, "unconditional");
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
linkTransposedNodeJSON(obj, parent, property);
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
protected class CellTopicTransposedNode extends JsonObjectTransposedNode {
|
||||
protected CellTopicNode node;
|
||||
protected Cell cell;
|
||||
|
||||
public CellTopicTransposedNode(CellTopicNode node, Cell cell) {
|
||||
this.node = node;
|
||||
this.cell = cell;
|
||||
}
|
||||
|
||||
@Override
|
||||
public JSONObject getJSONObject() {
|
||||
if (obj == null) {
|
||||
obj = new JSONObject();
|
||||
try {
|
||||
if (cell.recon != null &&
|
||||
cell.recon.judgment == Recon.Judgment.Matched &&
|
||||
cell.recon.match != null) {
|
||||
obj.put(ID, cell.recon.match.id);
|
||||
} else {
|
||||
obj.put(ID, (String) null);
|
||||
obj.put(NAME, cell.value.toString());
|
||||
obj.put(TYPE, node.type.id);
|
||||
obj.put(CREATE, "unless_exists");
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
protected class CellValueTransposedNode extends JsonTransposedNode {
|
||||
protected JSONObject obj;
|
||||
protected CellValueNode node;
|
||||
protected Cell cell;
|
||||
|
||||
public CellValueTransposedNode(CellValueNode node, Cell cell) {
|
||||
this.node = node;
|
||||
this.cell = cell;
|
||||
}
|
||||
|
||||
public Object getJSON() {
|
||||
if (obj == null) {
|
||||
obj = new JSONObject();
|
||||
try {
|
||||
JSONUtilities.putField(obj, VALUE, cell.value);
|
||||
|
||||
obj.put(TYPE, node.valueType);
|
||||
if ("/type/text".equals(node.valueType)) {
|
||||
obj.put(LANG, node.lang);
|
||||
}
|
||||
|
||||
obj.put(CONNECT, "insert");
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
protected class CellKeyTransposedNode extends JsonTransposedNode {
|
||||
protected JSONObject obj;
|
||||
protected CellKeyNode node;
|
||||
protected Cell cell;
|
||||
|
||||
public CellKeyTransposedNode(CellKeyNode node, Cell cell) {
|
||||
this.node = node;
|
||||
this.cell = cell;
|
||||
}
|
||||
|
||||
public Object getJSON() {
|
||||
if (obj == null) {
|
||||
obj = new JSONObject();
|
||||
try {
|
||||
obj.put(VALUE, cell.value.toString());
|
||||
|
||||
JSONObject nsObj = new JSONObject();
|
||||
nsObj.put(ID, node.namespace.id);
|
||||
|
||||
obj.put("namespace", nsObj);
|
||||
obj.put(CONNECT, "insert");
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
protected class TopicTransposedNode extends JsonObjectTransposedNode {
|
||||
protected FreebaseTopicNode node;
|
||||
|
||||
public TopicTransposedNode(FreebaseTopicNode node) {
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
@Override
|
||||
public JSONObject getJSONObject() {
|
||||
if (obj == null) {
|
||||
obj = new JSONObject();
|
||||
try {
|
||||
obj.put(ID, node.topic.id);
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
|
||||
protected class ValueTransposedNode extends JsonTransposedNode {
|
||||
protected JSONObject obj;
|
||||
protected ValueNode node;
|
||||
|
||||
public ValueTransposedNode(ValueNode node) {
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
public Object getJSON() {
|
||||
if (obj == null) {
|
||||
obj = new JSONObject();
|
||||
try {
|
||||
obj.put(VALUE, node.value);
|
||||
obj.put(TYPE, node.valueType);
|
||||
if ("/type/text".equals(node.valueType)) {
|
||||
obj.put(LANG, node.lang);
|
||||
}
|
||||
|
||||
obj.put(CONNECT, "insert");
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
public TransposedNode transposeAnonymousNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
AnonymousNode node, int rowIndex) {
|
||||
|
||||
return new AnonymousTransposedNode(
|
||||
parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null,
|
||||
link != null ? link.property : null,
|
||||
node
|
||||
);
|
||||
}
|
||||
|
||||
public TransposedNode transposeCellNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
CellNode node,
|
||||
int rowIndex,
|
||||
int cellIndex,
|
||||
Cell cell) {
|
||||
|
||||
JsonTransposedNode tnode = null;
|
||||
if (node instanceof CellTopicNode) {
|
||||
tnode = new CellTopicTransposedNode((CellTopicNode) node, cell);
|
||||
} else if (node instanceof CellValueNode) {
|
||||
tnode = new CellValueTransposedNode((CellValueNode) node, cell);
|
||||
} else if (node instanceof CellKeyNode) {
|
||||
tnode = new CellKeyTransposedNode((CellKeyNode) node, cell);
|
||||
}
|
||||
|
||||
if (tnode != null) {
|
||||
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
|
||||
}
|
||||
return tnode;
|
||||
}
|
||||
|
||||
public TransposedNode transposeTopicNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
FreebaseTopicNode node, int rowIndex) {
|
||||
|
||||
JsonTransposedNode tnode = new TopicTransposedNode(node);
|
||||
|
||||
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
|
||||
|
||||
return tnode;
|
||||
}
|
||||
|
||||
public TransposedNode transposeValueNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
ValueNode node, int rowIndex) {
|
||||
|
||||
JsonTransposedNode tnode = new ValueTransposedNode(node);
|
||||
|
||||
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
|
||||
|
||||
return tnode;
|
||||
}
|
||||
|
||||
protected void processTransposedNode(
|
||||
JsonTransposedNode tnode,
|
||||
TransposedNode parentNode,
|
||||
FreebaseProperty property
|
||||
) {
|
||||
|
||||
if (!(tnode instanceof AnonymousTransposedNode)) {
|
||||
linkTransposedNodeJSON(tnode.getJSON(), parentNode, property);
|
||||
}
|
||||
}
|
||||
|
||||
protected void linkTransposedNodeJSON(
|
||||
Object obj,
|
||||
TransposedNode parentNode,
|
||||
FreebaseProperty property
|
||||
) {
|
||||
|
||||
if (parentNode == null) {
|
||||
if (obj instanceof JSONObject) {
|
||||
rootObjects.add((JSONObject) obj);
|
||||
}
|
||||
} else if (parentNode instanceof JsonTransposedNode) {
|
||||
JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject();
|
||||
|
||||
try {
|
||||
JSONArray a = null;
|
||||
if (parentObj.has(property.id)) {
|
||||
a = parentObj.getJSONArray(property.id);
|
||||
} else {
|
||||
a = new JSONArray();
|
||||
parentObj.put(property.id, a);
|
||||
}
|
||||
|
||||
a.put(a.length(), obj);
|
||||
} catch (JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,4 +0,0 @@
|
||||
package com.google.refine.protograph.transpose;
|
||||
|
||||
public interface TransposedNode {
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
package com.google.refine.protograph.transpose;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.protograph.AnonymousNode;
|
||||
import com.google.refine.protograph.CellNode;
|
||||
import com.google.refine.protograph.FreebaseTopicNode;
|
||||
import com.google.refine.protograph.Link;
|
||||
import com.google.refine.protograph.ValueNode;
|
||||
|
||||
public interface TransposedNodeFactory {
|
||||
public TransposedNode transposeAnonymousNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
AnonymousNode node, int rowIndex
|
||||
);
|
||||
|
||||
public TransposedNode transposeCellNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
CellNode node,
|
||||
int rowIndex,
|
||||
int cellIndex,
|
||||
Cell cell
|
||||
);
|
||||
|
||||
public TransposedNode transposeValueNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
ValueNode node,
|
||||
int rowIndex
|
||||
);
|
||||
|
||||
public TransposedNode transposeTopicNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
FreebaseTopicNode node,
|
||||
int rowIndex
|
||||
);
|
||||
|
||||
public void flush() throws IOException;
|
||||
}
|
||||
@ -1,222 +0,0 @@
|
||||
package com.google.refine.protograph.transpose;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.refine.browsing.FilteredRows;
|
||||
import com.google.refine.browsing.RowVisitor;
|
||||
import com.google.refine.expr.ExpressionUtils;
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Row;
|
||||
import com.google.refine.model.Recon.Judgment;
|
||||
import com.google.refine.protograph.AnonymousNode;
|
||||
import com.google.refine.protograph.CellNode;
|
||||
import com.google.refine.protograph.CellTopicNode;
|
||||
import com.google.refine.protograph.FreebaseTopicNode;
|
||||
import com.google.refine.protograph.Link;
|
||||
import com.google.refine.protograph.Node;
|
||||
import com.google.refine.protograph.NodeWithLinks;
|
||||
import com.google.refine.protograph.Protograph;
|
||||
import com.google.refine.protograph.ValueNode;
|
||||
|
||||
public class Transposer {
|
||||
static public void transpose(
|
||||
Project project,
|
||||
FilteredRows filteredRows,
|
||||
Protograph protograph,
|
||||
Node rootNode,
|
||||
TransposedNodeFactory nodeFactory
|
||||
) {
|
||||
transpose(project, filteredRows, protograph, rootNode, nodeFactory, 20);
|
||||
}
|
||||
|
||||
static public void transpose(
|
||||
Project project,
|
||||
FilteredRows filteredRows,
|
||||
Protograph protograph,
|
||||
Node rootNode,
|
||||
TransposedNodeFactory nodeFactory,
|
||||
int limit
|
||||
) {
|
||||
Context rootContext = new Context(rootNode, null, null, limit);
|
||||
|
||||
filteredRows.accept(project, new RowVisitor() {
|
||||
Context rootContext;
|
||||
Protograph protograph;
|
||||
Node rootNode;
|
||||
TransposedNodeFactory nodeFactory;
|
||||
|
||||
@Override
|
||||
public boolean visit(Project project, int rowIndex, Row row) {
|
||||
if (rootContext.limit <= 0 || rootContext.count < rootContext.limit) {
|
||||
descend(project, protograph, nodeFactory, rowIndex, row, rootNode, rootContext);
|
||||
}
|
||||
|
||||
if (rootContext.limit > 0 && rootContext.count > rootContext.limit) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(Project project) {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void end(Project project) {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
}
|
||||
|
||||
public RowVisitor init(
|
||||
Context rootContext,
|
||||
Protograph protograph,
|
||||
Node rootNode,
|
||||
TransposedNodeFactory nodeFactory
|
||||
) {
|
||||
this.rootContext = rootContext;
|
||||
this.protograph = protograph;
|
||||
this.rootNode = rootNode;
|
||||
this.nodeFactory = nodeFactory;
|
||||
|
||||
return this;
|
||||
}
|
||||
}.init(rootContext, protograph, rootNode, nodeFactory));
|
||||
}
|
||||
|
||||
static protected void descend(
|
||||
Project project,
|
||||
Protograph protograph,
|
||||
TransposedNodeFactory nodeFactory,
|
||||
int rowIndex,
|
||||
Row row,
|
||||
Node node,
|
||||
Context context
|
||||
) {
|
||||
List<TransposedNode> tnodes = new LinkedList<TransposedNode>();
|
||||
|
||||
TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode;
|
||||
Link link = context.parent == null ? null : context.link;
|
||||
|
||||
if (node instanceof CellNode) {
|
||||
CellNode node2 = (CellNode) node;
|
||||
for (String columnName : node2.columnNames) {
|
||||
Column column = project.columnModel.getColumnByName(columnName);
|
||||
if (column != null) {
|
||||
int cellIndex = column.getCellIndex();
|
||||
|
||||
Cell cell = row.getCell(cellIndex);
|
||||
if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
|
||||
if (node2 instanceof CellTopicNode &&
|
||||
(cell.recon == null || cell.recon.judgment == Judgment.None)) {
|
||||
return;
|
||||
}
|
||||
|
||||
context.count++;
|
||||
if (context.limit > 0 && context.count > context.limit) {
|
||||
return;
|
||||
}
|
||||
|
||||
tnodes.add(nodeFactory.transposeCellNode(
|
||||
parentNode,
|
||||
link,
|
||||
node2,
|
||||
rowIndex,
|
||||
cellIndex,
|
||||
cell
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (node instanceof AnonymousNode) {
|
||||
tnodes.add(nodeFactory.transposeAnonymousNode(
|
||||
parentNode,
|
||||
link,
|
||||
(AnonymousNode) node,
|
||||
rowIndex
|
||||
));
|
||||
} else if (node instanceof FreebaseTopicNode) {
|
||||
tnodes.add(nodeFactory.transposeTopicNode(
|
||||
parentNode,
|
||||
link,
|
||||
(FreebaseTopicNode) node,
|
||||
rowIndex
|
||||
));
|
||||
} else if (node instanceof ValueNode) {
|
||||
tnodes.add(nodeFactory.transposeValueNode(
|
||||
parentNode,
|
||||
link,
|
||||
(ValueNode) node,
|
||||
rowIndex
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if (node instanceof NodeWithLinks) {
|
||||
NodeWithLinks node2 = (NodeWithLinks) node;
|
||||
int linkCount = node2.getLinkCount();
|
||||
|
||||
for (int i = 0; i < linkCount; i++) {
|
||||
Link link2 = node2.getLink(i);
|
||||
if (link2.condition == null || link2.condition.test(project, rowIndex, row)) {
|
||||
for (TransposedNode tnode : tnodes) {
|
||||
context.transposedNode = tnode;
|
||||
context.nullifySubContextNodes();
|
||||
|
||||
descend(
|
||||
project,
|
||||
protograph,
|
||||
nodeFactory,
|
||||
rowIndex,
|
||||
row,
|
||||
link2.getTarget(),
|
||||
context.subContexts.get(i)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static class Context {
|
||||
TransposedNode transposedNode;
|
||||
List<Context> subContexts;
|
||||
Context parent;
|
||||
Link link;
|
||||
int count;
|
||||
int limit;
|
||||
|
||||
Context(Node node, Context parent, Link link, int limit) {
|
||||
this.parent = parent;
|
||||
this.link = link;
|
||||
this.limit = limit;
|
||||
|
||||
if (node instanceof NodeWithLinks) {
|
||||
NodeWithLinks node2 = (NodeWithLinks) node;
|
||||
|
||||
int subContextCount = node2.getLinkCount();
|
||||
|
||||
subContexts = new LinkedList<Context>();
|
||||
for (int i = 0; i < subContextCount; i++) {
|
||||
Link link2 = node2.getLink(i);
|
||||
subContexts.add(
|
||||
new Context(link2.getTarget(), this, link2, -1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void nullifySubContextNodes() {
|
||||
if (subContexts != null) {
|
||||
for (Context context : subContexts) {
|
||||
context.transposedNode = null;
|
||||
context.nullifySubContextNodes();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,725 +0,0 @@
|
||||
package com.google.refine.protograph.transpose;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.refine.model.Cell;
|
||||
import com.google.refine.model.Column;
|
||||
import com.google.refine.model.Project;
|
||||
import com.google.refine.model.Recon;
|
||||
import com.google.refine.model.Recon.Judgment;
|
||||
import com.google.refine.model.recon.ReconConfig;
|
||||
import com.google.refine.model.recon.StandardReconConfig;
|
||||
import com.google.refine.protograph.AnonymousNode;
|
||||
import com.google.refine.protograph.CellKeyNode;
|
||||
import com.google.refine.protograph.CellNode;
|
||||
import com.google.refine.protograph.CellTopicNode;
|
||||
import com.google.refine.protograph.CellValueNode;
|
||||
import com.google.refine.protograph.FreebaseProperty;
|
||||
import com.google.refine.protograph.FreebaseTopic;
|
||||
import com.google.refine.protograph.FreebaseTopicNode;
|
||||
import com.google.refine.protograph.Link;
|
||||
import com.google.refine.protograph.ValueNode;
|
||||
|
||||
public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory {
|
||||
protected Project project;
|
||||
|
||||
protected boolean start = true;
|
||||
protected Writer writer;
|
||||
protected WritingTransposedNode lastRootNode;
|
||||
protected Map<String, Long> varPool = new HashMap<String, Long>();
|
||||
protected Map<Long, String> newTopicVars = new HashMap<Long, String>();
|
||||
protected Set<Long> serializedRecons = new HashSet<Long>();
|
||||
|
||||
protected long contextID = 0;
|
||||
protected int contextRowIndex;
|
||||
protected int contextRefCount = 0;
|
||||
protected JSONObject contextTreeRoot;
|
||||
|
||||
public TripleLoaderTransposedNodeFactory(Project project, Writer writer) {
|
||||
this.project = project;
|
||||
this.writer = writer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
if (lastRootNode != null) {
|
||||
lastRootNode.write(null, null, project, -1, -1, null);
|
||||
lastRootNode = null;
|
||||
|
||||
writeContextTreeNode();
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeLine(String line) {
|
||||
try {
|
||||
if (start) {
|
||||
start = false;
|
||||
} else {
|
||||
writer.write('\n');
|
||||
}
|
||||
writer.write(line);
|
||||
} catch (IOException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeRecon(
|
||||
StringBuffer sb,
|
||||
Project project,
|
||||
int rowIndex,
|
||||
int cellIndex,
|
||||
Cell cell
|
||||
) {
|
||||
Recon recon = cell.recon;
|
||||
|
||||
sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\"");
|
||||
contextRefCount++;
|
||||
|
||||
if (!serializedRecons.contains(recon.id)) {
|
||||
serializedRecons.add(recon.id);
|
||||
|
||||
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
|
||||
|
||||
// qa:sample_group
|
||||
{
|
||||
StringBuffer sb2 = new StringBuffer();
|
||||
|
||||
sb2.append("{ \"s\" : \"rec");
|
||||
sb2.append(Long.toString(recon.id));
|
||||
sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : ");
|
||||
sb2.append(JSONObject.quote(column.getName()));
|
||||
sb2.append(", \"ignore\" : true }");
|
||||
|
||||
writeLine(sb2.toString());
|
||||
}
|
||||
|
||||
// qa:recon_data
|
||||
{
|
||||
StringBuffer sb2 = new StringBuffer();
|
||||
|
||||
String s = cell.value instanceof String ? (String) cell.value : cell.value.toString();
|
||||
|
||||
sb2.append("{ \"s\" : \"rec");
|
||||
sb2.append(Long.toString(recon.id));
|
||||
sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { ");
|
||||
|
||||
sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry));
|
||||
sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s));
|
||||
sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName()));
|
||||
sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service));
|
||||
sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction));
|
||||
sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize));
|
||||
|
||||
if (recon.judgment == Judgment.Matched) {
|
||||
sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank));
|
||||
sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id));
|
||||
}
|
||||
|
||||
ReconConfig reconConfig = column.getReconConfig();
|
||||
if (reconConfig != null && reconConfig instanceof StandardReconConfig) {
|
||||
StandardReconConfig standardReconConfig = (StandardReconConfig) reconConfig;
|
||||
sb2.append(", \"type\" : "); sb2.append(JSONObject.quote(standardReconConfig.typeID));
|
||||
}
|
||||
|
||||
sb2.append(" } }");
|
||||
|
||||
writeLine(sb2.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeLine(
|
||||
String subject, String predicate, Object object,
|
||||
Project project,
|
||||
int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
|
||||
int objectRowIndex, int objectCellIndex, Cell objectCell,
|
||||
boolean ignore
|
||||
) {
|
||||
if (subject != null && object != null) {
|
||||
String s = object instanceof String ?
|
||||
JSONObject.quote((String) object) : object.toString();
|
||||
|
||||
StringBuffer sb = new StringBuffer();
|
||||
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
|
||||
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
|
||||
sb.append(", \"o\" : "); sb.append(s);
|
||||
if (subjectCell != null || objectCell != null) {
|
||||
sb.append(", \"meta\" : { ");
|
||||
|
||||
sb.append("\"recon\" : { ");
|
||||
if (subjectCell != null) {
|
||||
sb.append("\"s\" : ");
|
||||
writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
|
||||
}
|
||||
if (objectCell != null) {
|
||||
if (subjectCell != null) {
|
||||
sb.append(", ");
|
||||
}
|
||||
sb.append("\"o\" : ");
|
||||
writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell);
|
||||
}
|
||||
sb.append(" }");
|
||||
|
||||
sb.append(" }");
|
||||
}
|
||||
if (ignore) {
|
||||
sb.append(", \"ignore\" : true");
|
||||
}
|
||||
sb.append(" }");
|
||||
|
||||
writeLine(sb.toString());
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeLine(
|
||||
String subject, String predicate, Object object, String lang,
|
||||
Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
|
||||
boolean ignore
|
||||
) {
|
||||
if (subject != null && object != null) {
|
||||
String s = object instanceof String ?
|
||||
JSONObject.quote((String) object) : object.toString();
|
||||
|
||||
StringBuffer sb = new StringBuffer();
|
||||
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
|
||||
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
|
||||
sb.append(", \"o\" : "); sb.append(s);
|
||||
sb.append(", \"lang\" : "); sb.append(lang);
|
||||
|
||||
if (subjectCell != null) {
|
||||
sb.append(", \"meta\" : { ");
|
||||
sb.append("\"recon\" : { ");
|
||||
sb.append("\"s\" : ");
|
||||
writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
|
||||
sb.append(" }");
|
||||
sb.append(" }");
|
||||
}
|
||||
if (ignore) {
|
||||
sb.append(", \"ignore\" : true");
|
||||
}
|
||||
sb.append(" }");
|
||||
|
||||
writeLine(sb.toString());
|
||||
}
|
||||
}
|
||||
|
||||
abstract protected class WritingTransposedNode implements TransposedNode {
|
||||
JSONObject jsonContextNode;
|
||||
boolean load;
|
||||
|
||||
public Object write(
|
||||
String subject, String predicate, Project project,
|
||||
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
||||
|
||||
return internalWrite(
|
||||
subject, predicate, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell);
|
||||
}
|
||||
|
||||
abstract public Object internalWrite(
|
||||
String subject, String predicate, Project project,
|
||||
int subjectRowIndex, int subjectCellIndex, Cell subjectCell);
|
||||
}
|
||||
|
||||
abstract protected class TransposedNodeWithChildren extends WritingTransposedNode {
|
||||
public List<Link> links = new LinkedList<Link>();
|
||||
public List<Integer> rowIndices = new LinkedList<Integer>();
|
||||
public List<WritingTransposedNode> children = new LinkedList<WritingTransposedNode>();
|
||||
|
||||
protected void writeChildren(
|
||||
String subject, Project project,
|
||||
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
||||
|
||||
for (int i = 0; i < children.size(); i++) {
|
||||
WritingTransposedNode child = children.get(i);
|
||||
Link link = links.get(i);
|
||||
String predicate = link.property.id;
|
||||
|
||||
child.write(subject, predicate, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected class AnonymousTransposedNode extends TransposedNodeWithChildren {
|
||||
|
||||
//protected AnonymousTransposedNode(AnonymousNode node) { }
|
||||
|
||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
||||
if (children.size() == 0 || subject == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
StringBuffer sb = new StringBuffer();
|
||||
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
|
||||
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
|
||||
sb.append(", \"o\" : { ");
|
||||
|
||||
StringBuffer sbRecon = new StringBuffer();
|
||||
|
||||
boolean first = true;
|
||||
boolean firstRecon = true;
|
||||
|
||||
if (subjectCell.recon != null) {
|
||||
sbRecon.append("\"s\" : ");
|
||||
writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell);
|
||||
|
||||
firstRecon = false;
|
||||
}
|
||||
|
||||
for (int i = 0; i < children.size(); i++) {
|
||||
WritingTransposedNode child = children.get(i);
|
||||
Link link = links.get(i);
|
||||
|
||||
FreebaseProperty property = link.property;
|
||||
|
||||
Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null);
|
||||
if (c != null) {
|
||||
if (first) {
|
||||
first = false;
|
||||
} else {
|
||||
sb.append(", ");
|
||||
}
|
||||
sb.append("\"" + property.id + "\": ");
|
||||
sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
|
||||
}
|
||||
|
||||
if (child instanceof CellTopicTransposedNode) {
|
||||
CellTopicTransposedNode child2 = (CellTopicTransposedNode) child;
|
||||
Recon recon = child2.cell.recon;
|
||||
|
||||
if (recon != null &&
|
||||
(recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) {
|
||||
|
||||
if (firstRecon) {
|
||||
firstRecon = false;
|
||||
} else {
|
||||
sbRecon.append(", ");
|
||||
}
|
||||
|
||||
sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
|
||||
|
||||
writeRecon(sbRecon, project,
|
||||
rowIndices.get(i), child2.cellIndex, child2.cell);
|
||||
}
|
||||
}
|
||||
}
|
||||
sb.append(" }, \"meta\" : { \"recon\" : { ");
|
||||
sb.append(sbRecon.toString());
|
||||
sb.append(" } } }");
|
||||
|
||||
writeLine(sb.toString());
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected class CellTopicTransposedNode extends TransposedNodeWithChildren {
|
||||
protected CellTopicNode node;
|
||||
protected int rowIndex;
|
||||
protected int cellIndex;
|
||||
protected Cell cell;
|
||||
|
||||
public CellTopicTransposedNode(CellTopicNode node, int rowIndex, int cellIndex, Cell cell) {
|
||||
this.node = node;
|
||||
this.rowIndex = rowIndex;
|
||||
this.cellIndex = cellIndex;
|
||||
this.cell = cell;
|
||||
}
|
||||
|
||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
||||
String id = null;
|
||||
if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) {
|
||||
int objectRowIndex = rowIndex;
|
||||
int objectCellIndex = cellIndex;
|
||||
Cell objectCell = cell;
|
||||
|
||||
if (cell.recon.judgment == Recon.Judgment.Matched) {
|
||||
id = cell.recon.match.id;
|
||||
|
||||
} else if (cell.recon.judgment == Judgment.New) {
|
||||
if (newTopicVars.containsKey(cell.recon.id)) {
|
||||
id = newTopicVars.get(cell.recon.id);
|
||||
} else {
|
||||
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
|
||||
String columnName = column.getName();
|
||||
|
||||
long var = 0;
|
||||
if (varPool.containsKey(columnName)) {
|
||||
var = varPool.get(columnName);
|
||||
}
|
||||
varPool.put(columnName, var + 1);
|
||||
|
||||
id = "$" + columnName.replaceAll("\\W+", "_") + "_" + var;
|
||||
|
||||
String typeID = node.type.id;
|
||||
|
||||
ReconConfig reconConfig = column.getReconConfig();
|
||||
if (reconConfig instanceof StandardReconConfig) {
|
||||
typeID = ((StandardReconConfig) reconConfig).typeID;
|
||||
}
|
||||
|
||||
writeLine(id, "type", typeID, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load);
|
||||
writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load);
|
||||
|
||||
if (cell.recon != null) {
|
||||
newTopicVars.put(cell.recon.id, id);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (subject != null) {
|
||||
writeLine(subject, predicate, id, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
||||
objectRowIndex, objectCellIndex, objectCell, !load);
|
||||
}
|
||||
|
||||
writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell);
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
protected class CellValueTransposedNode extends WritingTransposedNode {
|
||||
protected JSONObject obj;
|
||||
protected CellValueNode node;
|
||||
protected int rowIndex;
|
||||
protected int cellIndex;
|
||||
protected Cell cell;
|
||||
|
||||
public CellValueTransposedNode(CellValueNode node, int rowIndex, int cellIndex, Cell cell) {
|
||||
this.node = node;
|
||||
this.rowIndex = rowIndex;
|
||||
this.cellIndex = cellIndex;
|
||||
this.cell = cell;
|
||||
}
|
||||
|
||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
||||
if (subject != null) {
|
||||
if ("/type/text".equals(node.lang)) {
|
||||
writeLine(subject, predicate, cell.value, node.lang, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell, !load);
|
||||
} else {
|
||||
writeLine(subject, predicate, cell.value, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
||||
-1, -1, null, !load);
|
||||
}
|
||||
}
|
||||
|
||||
return cell.value;
|
||||
}
|
||||
}
|
||||
|
||||
protected class CellKeyTransposedNode extends WritingTransposedNode {
|
||||
protected CellKeyNode node;
|
||||
protected int rowIndex;
|
||||
protected int cellIndex;
|
||||
protected Cell cell;
|
||||
|
||||
public CellKeyTransposedNode(CellKeyNode node, int rowIndex, int cellIndex, Cell cell) {
|
||||
this.node = node;
|
||||
this.rowIndex = rowIndex;
|
||||
this.cellIndex = cellIndex;
|
||||
this.cell = cell;
|
||||
}
|
||||
|
||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
||||
writeLine(subject, "key", node.namespace.id + "/" + cell.value, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
||||
-1, -1, null, !load);
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected class TopicTransposedNode extends TransposedNodeWithChildren {
|
||||
protected FreebaseTopicNode node;
|
||||
|
||||
public TopicTransposedNode(FreebaseTopicNode node) {
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
||||
writeLine(subject, predicate, node.topic.id, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
||||
-1, -1, null, !load);
|
||||
|
||||
writeChildren(node.topic.id, project, -1, -1, null);
|
||||
|
||||
return node.topic.id;
|
||||
}
|
||||
}
|
||||
|
||||
protected class ValueTransposedNode extends WritingTransposedNode {
|
||||
protected ValueNode node;
|
||||
|
||||
public ValueTransposedNode(ValueNode node) {
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
||||
if ("/type/text".equals(node.lang)) {
|
||||
writeLine(subject, predicate, node.value, node.lang, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell, !load);
|
||||
} else {
|
||||
writeLine(subject, predicate, node.value, project,
|
||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
||||
-1, -1, null, !load);
|
||||
}
|
||||
|
||||
return node.value;
|
||||
}
|
||||
}
|
||||
|
||||
public TransposedNode transposeAnonymousNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
AnonymousNode node, int rowIndex) {
|
||||
|
||||
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
|
||||
WritingTransposedNode tnode = new AnonymousTransposedNode();
|
||||
|
||||
tnode.load =
|
||||
(parentNode2 == null || parentNode2.load) &&
|
||||
(link == null || link.load);
|
||||
|
||||
processTransposedNode(tnode, parentNode, link, rowIndex);
|
||||
|
||||
tnode.jsonContextNode = addJsonContext(
|
||||
parentNode2 != null ? parentNode2.jsonContextNode : null,
|
||||
link != null ? link.property.id : null,
|
||||
null
|
||||
);
|
||||
|
||||
return tnode;
|
||||
}
|
||||
|
||||
public TransposedNode transposeCellNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
CellNode node,
|
||||
int rowIndex,
|
||||
int cellIndex,
|
||||
Cell cell) {
|
||||
|
||||
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
|
||||
|
||||
WritingTransposedNode tnode = null;
|
||||
if (node instanceof CellTopicNode) {
|
||||
if (cell.recon != null &&
|
||||
(cell.recon.judgment == Judgment.Matched ||
|
||||
cell.recon.judgment == Judgment.New)) {
|
||||
|
||||
tnode = new CellTopicTransposedNode(
|
||||
(CellTopicNode) node, rowIndex, cellIndex, cell);
|
||||
}
|
||||
} else if (node instanceof CellValueNode) {
|
||||
tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell);
|
||||
} else if (node instanceof CellKeyNode) {
|
||||
tnode = new CellKeyTransposedNode((CellKeyNode) node, rowIndex, cellIndex, cell);
|
||||
}
|
||||
|
||||
if (tnode != null) {
|
||||
tnode.load =
|
||||
(parentNode2 == null || parentNode2.load) &&
|
||||
(link == null || link.load);
|
||||
|
||||
processTransposedNode(tnode, parentNode, link, rowIndex);
|
||||
|
||||
tnode.jsonContextNode = addJsonContext(
|
||||
parentNode2 != null ? parentNode2.jsonContextNode : null,
|
||||
link != null ? link.property.id : null,
|
||||
cell,
|
||||
rowIndex
|
||||
);
|
||||
}
|
||||
return tnode;
|
||||
}
|
||||
|
||||
public TransposedNode transposeTopicNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
FreebaseTopicNode node,
|
||||
int rowIndex) {
|
||||
|
||||
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
|
||||
WritingTransposedNode tnode = new TopicTransposedNode(node);
|
||||
|
||||
tnode.load =
|
||||
(parentNode2 == null || parentNode2.load) &&
|
||||
(link == null || link.load);
|
||||
|
||||
processTransposedNode(tnode, parentNode, link, rowIndex);
|
||||
|
||||
tnode.jsonContextNode = addJsonContext(
|
||||
parentNode2 != null ? parentNode2.jsonContextNode : null,
|
||||
link != null ? link.property.id : null,
|
||||
node.topic
|
||||
);
|
||||
|
||||
return tnode;
|
||||
}
|
||||
|
||||
public TransposedNode transposeValueNode(
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
ValueNode node,
|
||||
int rowIndex) {
|
||||
|
||||
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
|
||||
WritingTransposedNode tnode = new ValueTransposedNode(node);
|
||||
|
||||
tnode.load =
|
||||
(parentNode2 == null || parentNode2.load) &&
|
||||
(link == null || link.load);
|
||||
|
||||
processTransposedNode(tnode, parentNode, link, rowIndex);
|
||||
|
||||
tnode.jsonContextNode = addJsonContext(
|
||||
parentNode2 != null ? parentNode2.jsonContextNode : null,
|
||||
link != null ? link.property.id : null,
|
||||
node.value
|
||||
);
|
||||
|
||||
return tnode;
|
||||
}
|
||||
|
||||
protected void processTransposedNode(
|
||||
WritingTransposedNode tnode,
|
||||
TransposedNode parentNode,
|
||||
Link link,
|
||||
int rowIndex
|
||||
) {
|
||||
if (parentNode != null) {
|
||||
if (parentNode instanceof TransposedNodeWithChildren) {
|
||||
TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode;
|
||||
parentNode2.rowIndices.add(rowIndex);
|
||||
parentNode2.children.add(tnode);
|
||||
parentNode2.links.add(link);
|
||||
}
|
||||
} else {
|
||||
addRootNode(tnode, rowIndex);
|
||||
}
|
||||
}
|
||||
|
||||
protected JSONObject addJsonContext(JSONObject parent, String key, Object value) {
|
||||
JSONObject o = new JSONObject();
|
||||
|
||||
try {
|
||||
if (value instanceof FreebaseTopic) {
|
||||
FreebaseTopic topic = (FreebaseTopic) value;
|
||||
o.put("id", topic.id);
|
||||
o.put("name", topic.name);
|
||||
} else {
|
||||
o.put("v", value);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
connectJsonContext(parent, o, key);
|
||||
return o;
|
||||
}
|
||||
|
||||
protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) {
|
||||
JSONObject o = new JSONObject();
|
||||
|
||||
connectJsonContext(parent, o, key);
|
||||
|
||||
try {
|
||||
if (cell != null) {
|
||||
o.put("v", cell.value);
|
||||
if (cell.recon != null) {
|
||||
o.put("recon", "rec" + cell.recon.id);
|
||||
|
||||
if (cell.recon.judgment == Judgment.Matched) {
|
||||
o.put("id", cell.recon.match.id);
|
||||
o.put("name", cell.recon.match.name);
|
||||
}
|
||||
|
||||
// qa:display_context
|
||||
{
|
||||
StringBuffer sb2 = new StringBuffer();
|
||||
|
||||
sb2.append("{ \"ignore\" : true, \"s\" : \"rec");
|
||||
sb2.append(Long.toString(cell.recon.id));
|
||||
sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx");
|
||||
sb2.append(Long.toString(contextID));
|
||||
sb2.append("\", \"meta\" : { \"row\" : ");
|
||||
sb2.append(Integer.toString(rowIndex));
|
||||
sb2.append(" } }");
|
||||
|
||||
writeLine(sb2.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
return o;
|
||||
}
|
||||
|
||||
protected void connectJsonContext(JSONObject parent, JSONObject o, String key) {
|
||||
try {
|
||||
if (parent == null) {
|
||||
contextTreeRoot = o;
|
||||
} else {
|
||||
JSONArray a = null;
|
||||
if (parent.has(key)) {
|
||||
a = parent.getJSONArray(key);
|
||||
} else {
|
||||
a = new JSONArray();
|
||||
parent.put(key, a);
|
||||
}
|
||||
|
||||
a.put(o);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
protected void addRootNode(WritingTransposedNode tnode, int rowIndex) {
|
||||
if (lastRootNode != null) {
|
||||
lastRootNode.write(null, null, project, -1, -1, null);
|
||||
writeContextTreeNode();
|
||||
}
|
||||
lastRootNode = tnode;
|
||||
|
||||
contextTreeRoot = null;
|
||||
contextRowIndex = rowIndex;
|
||||
contextRefCount = 0;
|
||||
contextID++;
|
||||
}
|
||||
|
||||
protected void writeContextTreeNode() {
|
||||
if (contextTreeRoot != null && contextRefCount > 0) {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
|
||||
sb.append("{ \"ignore\" : true, \"s\" : \"ctx");
|
||||
sb.append(Long.toString(contextID));
|
||||
sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : ");
|
||||
sb.append(Integer.toString(contextRowIndex));
|
||||
sb.append(", \"data\" : ");
|
||||
sb.append(contextTreeRoot.toString());
|
||||
sb.append(" } }");
|
||||
|
||||
writeLine(sb.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,239 +0,0 @@
|
||||
package com.google.refine.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import oauth.signpost.OAuthConsumer;
|
||||
import oauth.signpost.exception.OAuthCommunicationException;
|
||||
import oauth.signpost.exception.OAuthExpectationFailedException;
|
||||
import oauth.signpost.exception.OAuthMessageSignerException;
|
||||
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.ClientProtocolException;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.params.CoreProtocolPNames;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import com.google.refine.ProjectManager;
|
||||
import com.google.refine.RefineServlet;
|
||||
import com.google.refine.oauth.Credentials;
|
||||
import com.google.refine.oauth.OAuthUtilities;
|
||||
import com.google.refine.oauth.Provider;
|
||||
|
||||
public class FreebaseUtils {
|
||||
|
||||
static final public String FREEBASE_HOST = "www.freebase.com";
|
||||
|
||||
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
|
||||
|
||||
static final private String AGENT_ID = "/en/google_refine";
|
||||
|
||||
static final private int SAMPLE_SIZE = 300;
|
||||
static final private int JUDGES = 4;
|
||||
|
||||
private static String getUserInfoURL(String host) {
|
||||
return "http://" + host + "/api/service/user_info";
|
||||
}
|
||||
|
||||
private static String getMQLWriteURL(String host) {
|
||||
return "http://" + host + "/api/service/mqlwrite";
|
||||
}
|
||||
|
||||
private static String getMQLReadURL(String host) {
|
||||
return "http://" + host + "/api/service/mqlread";
|
||||
}
|
||||
|
||||
private static String getUserAgent() {
|
||||
return RefineServlet.FULLNAME;
|
||||
}
|
||||
|
||||
public static String getUserInfo(Credentials credentials, Provider provider)
|
||||
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
|
||||
|
||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
||||
|
||||
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
|
||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
||||
|
||||
// this is required by the Metaweb API to avoid XSS
|
||||
httpRequest.setHeader("X-Requested-With", "1");
|
||||
|
||||
// sign the request with the oauth library
|
||||
consumer.sign(httpRequest);
|
||||
|
||||
// execute the request
|
||||
HttpClient httpClient = new DefaultHttpClient();
|
||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
||||
|
||||
// return the results
|
||||
return EntityUtils.toString(httpResponse.getEntity());
|
||||
}
|
||||
|
||||
public static String getUserBadges(Provider provider, String user_id)
|
||||
throws ClientProtocolException, IOException, JSONException {
|
||||
|
||||
String query = "{" +
|
||||
"'id' : '" + user_id + "'," +
|
||||
"'!/type/usergroup/member' : [{" +
|
||||
"'id' : null," +
|
||||
"'key' : [{" +
|
||||
"'namespace' : null" +
|
||||
"}]" +
|
||||
"}]" +
|
||||
"}".replace("'", "\"");
|
||||
|
||||
return mqlread(provider, query);
|
||||
}
|
||||
|
||||
public static String mqlread(Provider provider, String query)
|
||||
throws ClientProtocolException, IOException, JSONException {
|
||||
|
||||
JSONObject envelope = new JSONObject();
|
||||
envelope.put("query", new JSONObject(query));
|
||||
|
||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
||||
formparams.add(new BasicNameValuePair("query", envelope.toString()));
|
||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
||||
|
||||
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
|
||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
||||
httpRequest.setEntity(entity);
|
||||
|
||||
// this is required by the Metaweb API to avoid XSS
|
||||
httpRequest.setHeader("X-Requested-With", "1");
|
||||
|
||||
// execute the request
|
||||
HttpClient httpClient = new DefaultHttpClient();
|
||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
||||
|
||||
// return the results
|
||||
return EntityUtils.toString(httpResponse.getEntity());
|
||||
}
|
||||
|
||||
public static String mqlwrite(Credentials credentials, Provider provider, String query)
|
||||
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
|
||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
||||
|
||||
JSONObject envelope = new JSONObject();
|
||||
envelope.put("query", new JSONObject(query));
|
||||
|
||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
||||
formparams.add(new BasicNameValuePair("query", envelope.toString()));
|
||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
||||
|
||||
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
|
||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
||||
httpRequest.setEntity(entity);
|
||||
|
||||
// this is required by the Metaweb API to avoid XSS
|
||||
httpRequest.setHeader("X-Requested-With", "1");
|
||||
|
||||
// sign the request with the oauth library
|
||||
consumer.sign(httpRequest);
|
||||
|
||||
// execute the request
|
||||
HttpClient httpClient = new DefaultHttpClient();
|
||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
||||
|
||||
// return the results
|
||||
return EntityUtils.toString(httpResponse.getEntity());
|
||||
}
|
||||
|
||||
private static String getTweezersParams(int sample_size, int judges) {
|
||||
String o = "{" +
|
||||
"'sample_size':" + sample_size + "," +
|
||||
"'votes':{" +
|
||||
"'reconciled':" + judges + "," +
|
||||
"'invalid':" + judges + "," +
|
||||
"'new':" + judges + "," +
|
||||
"'skip':" + (judges + 2) +
|
||||
"}" +
|
||||
"}";
|
||||
return o.replace('\'', '"');
|
||||
}
|
||||
|
||||
public static String uploadTriples(
|
||||
HttpServletRequest request,
|
||||
String qa,
|
||||
String source_name,
|
||||
String source_id,
|
||||
String mdo_id,
|
||||
String triples
|
||||
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
|
||||
|
||||
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
|
||||
|
||||
Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
||||
|
||||
JSONObject mdo_info = new JSONObject();
|
||||
mdo_info.put("name", source_name);
|
||||
if (source_id != null) {
|
||||
mdo_info.put("info_source",source_id);
|
||||
}
|
||||
|
||||
JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
|
||||
if (user_info.has("username")) {
|
||||
|
||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
||||
formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
|
||||
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
|
||||
formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
|
||||
formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
|
||||
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
|
||||
formparams.add(new BasicNameValuePair("graphport", "sandbox"));
|
||||
formparams.add(new BasicNameValuePair("payload", triples));
|
||||
formparams.add(new BasicNameValuePair("check_params", "false"));
|
||||
if (mdo_id != null) {
|
||||
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
|
||||
}
|
||||
if (Boolean.parseBoolean(qa)) {
|
||||
formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
|
||||
}
|
||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
||||
|
||||
HttpPost httpRequest = new HttpPost(getFreeQUrl());
|
||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
||||
httpRequest.setEntity(entity);
|
||||
|
||||
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
|
||||
surrogateRequest.setEntity(entity);
|
||||
|
||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
||||
|
||||
consumer.sign(surrogateRequest);
|
||||
|
||||
Header[] h = surrogateRequest.getHeaders("Authorization");
|
||||
if (h.length > 0) {
|
||||
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
|
||||
} else {
|
||||
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
|
||||
}
|
||||
|
||||
// execute the request
|
||||
HttpClient httpClient = new DefaultHttpClient();
|
||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
||||
|
||||
// return the results
|
||||
return EntityUtils.toString(httpResponse.getEntity());
|
||||
} else {
|
||||
throw new RuntimeException("Invalid credentials");
|
||||
}
|
||||
}
|
||||
|
||||
static public String getFreeQUrl() {
|
||||
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
|
||||
return url != null ? url : FREEQ_URL;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user