Remove remaining Freebase related functionalities.
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1453 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh
parent
6ddd945a80
commit
ad0d227ab3
@ -1,135 +0,0 @@
|
|||||||
package com.google.refine.commands.auth;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.PrintWriter;
|
|
||||||
import java.net.URI;
|
|
||||||
import java.net.URISyntaxException;
|
|
||||||
|
|
||||||
import javax.servlet.ServletException;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
|
|
||||||
import oauth.signpost.OAuthConsumer;
|
|
||||||
import oauth.signpost.OAuthProvider;
|
|
||||||
|
|
||||||
import com.google.refine.commands.Command;
|
|
||||||
import com.google.refine.oauth.Credentials;
|
|
||||||
import com.google.refine.oauth.OAuthUtilities;
|
|
||||||
import com.google.refine.oauth.Provider;
|
|
||||||
|
|
||||||
public class AuthorizeCommand extends Command {
|
|
||||||
|
|
||||||
private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier";
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
|
||||||
|
|
||||||
// get the provider from the request
|
|
||||||
Provider provider = OAuthUtilities.getProvider(request);
|
|
||||||
|
|
||||||
try {
|
|
||||||
|
|
||||||
// see if the request comes with access credentials
|
|
||||||
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
|
||||||
|
|
||||||
// prepare the continuation URL that the OAuth provider will redirect the user to
|
|
||||||
// (we need to make sure this URL points back to this code or the dance will never complete)
|
|
||||||
String callbackURL = getBaseURL(request,provider);
|
|
||||||
|
|
||||||
if (access_credentials == null) {
|
|
||||||
// access credentials are not available so we need to check
|
|
||||||
// to see at what stage of the OAuth dance we are
|
|
||||||
|
|
||||||
// get the request token credentials
|
|
||||||
Credentials request_credentials = Credentials.getCredentials(request, provider, Credentials.Type.REQUEST);
|
|
||||||
|
|
||||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(request_credentials, provider);
|
|
||||||
OAuthProvider pp = OAuthUtilities.getOAuthProvider(provider);
|
|
||||||
|
|
||||||
if (request_credentials == null) {
|
|
||||||
// no credentials were found, so let's start the dance
|
|
||||||
|
|
||||||
// get the request token
|
|
||||||
|
|
||||||
String url = pp.retrieveRequestToken(consumer, callbackURL);
|
|
||||||
|
|
||||||
request_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
|
|
||||||
|
|
||||||
// and set them to that we can retrieve them later in the second part of the dance
|
|
||||||
Credentials.setCredentials(request, response, request_credentials, Credentials.Type.REQUEST, 3600);
|
|
||||||
|
|
||||||
// now redirect the user to the Authorize URL where she can authenticate against the
|
|
||||||
// service provider and authorize us.
|
|
||||||
// The provider will bounce the user back here for us to continue the dance.
|
|
||||||
|
|
||||||
response.sendRedirect(url);
|
|
||||||
} else {
|
|
||||||
// we are at the second stage of the dance, so we need need to obtain the access credentials now
|
|
||||||
|
|
||||||
// if we got here, it means that the user performed a valid authentication against the
|
|
||||||
// service provider and authorized us, so now we can request more permanent credentials
|
|
||||||
// to the service provider and save those as well for later use.
|
|
||||||
|
|
||||||
// this is set only for OAuth 1.0a
|
|
||||||
String verificationCode = request.getParameter(OAUTH_VERIFIER_PARAM);
|
|
||||||
|
|
||||||
pp.retrieveAccessToken(consumer, verificationCode);
|
|
||||||
|
|
||||||
access_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
|
|
||||||
|
|
||||||
// no matter the result, we need to remove the request token
|
|
||||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
|
||||||
|
|
||||||
Credentials.setCredentials(request, response, access_credentials, Credentials.Type.ACCESS, 30 * 24 * 3600);
|
|
||||||
|
|
||||||
finish(response);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
finish(response);
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
|
||||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
|
|
||||||
respondException(response, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void finish(HttpServletResponse response) throws IOException {
|
|
||||||
response.setCharacterEncoding("UTF-8");
|
|
||||||
response.setHeader("Content-Type", "text/html");
|
|
||||||
|
|
||||||
PrintWriter writer = response.getWriter();
|
|
||||||
writer.write(
|
|
||||||
"<html>" +
|
|
||||||
"<body></body>" +
|
|
||||||
"<script type='text/javascript'>" +
|
|
||||||
"if (top.opener && top.opener.onauthorization) {" +
|
|
||||||
" top.opener.onauthorization(window);" +
|
|
||||||
"}" +
|
|
||||||
"self.close();" +
|
|
||||||
"</script>" +
|
|
||||||
"</html>"
|
|
||||||
);
|
|
||||||
writer.flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getBaseURL(HttpServletRequest request, Provider provider) {
|
|
||||||
String host = request.getHeader("host");
|
|
||||||
if (host == null) {
|
|
||||||
String referrer = request.getHeader("referer");
|
|
||||||
if (referrer != null) {
|
|
||||||
URI url;
|
|
||||||
try {
|
|
||||||
url = new URI(referrer);
|
|
||||||
int port = url.getPort();
|
|
||||||
host = url.getHost() + ((port > -1) ? ":" + url.getPort() : "");
|
|
||||||
} catch (URISyntaxException e) {
|
|
||||||
throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL Google Refine is listening to.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "http://" + host + "/command/core/authorize/" + provider.getHost();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
package com.google.refine.commands.auth;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import javax.servlet.ServletException;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import com.google.refine.commands.Command;
|
|
||||||
import com.google.refine.oauth.Credentials;
|
|
||||||
import com.google.refine.oauth.OAuthUtilities;
|
|
||||||
import com.google.refine.oauth.Provider;
|
|
||||||
import com.google.refine.util.FreebaseUtils;
|
|
||||||
|
|
||||||
public class CheckAuthorizationCommand extends Command {
|
|
||||||
|
|
||||||
final static Logger logger = LoggerFactory.getLogger("check-authorization_command");
|
|
||||||
|
|
||||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
|
||||||
|
|
||||||
try {
|
|
||||||
Provider provider = OAuthUtilities.getProvider(request);
|
|
||||||
|
|
||||||
// this cookie should not be there, but this is good hygiene practice
|
|
||||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
|
|
||||||
|
|
||||||
Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
|
||||||
|
|
||||||
response.setCharacterEncoding("UTF-8");
|
|
||||||
response.setHeader("Content-Type", "application/json");
|
|
||||||
|
|
||||||
if (access_credentials != null) {
|
|
||||||
String user_info = FreebaseUtils.getUserInfo(access_credentials, provider);
|
|
||||||
response.getWriter().write(user_info);
|
|
||||||
} else {
|
|
||||||
respond(response, "401 Unauthorized", "You don't have the right credentials");
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.info("error",e);
|
|
||||||
respondException(response, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,32 +0,0 @@
|
|||||||
package com.google.refine.commands.auth;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import javax.servlet.ServletException;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
|
|
||||||
import com.google.refine.commands.Command;
|
|
||||||
import com.google.refine.oauth.Credentials;
|
|
||||||
import com.google.refine.oauth.OAuthUtilities;
|
|
||||||
import com.google.refine.oauth.Provider;
|
|
||||||
|
|
||||||
public class DeAuthorizeCommand extends Command {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
|
||||||
|
|
||||||
try {
|
|
||||||
response.setCharacterEncoding("UTF-8");
|
|
||||||
response.setHeader("Content-Type", "application/json");
|
|
||||||
|
|
||||||
Provider provider = OAuthUtilities.getProvider(request);
|
|
||||||
|
|
||||||
Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
|
|
||||||
|
|
||||||
respond(response, "200 OK", "");
|
|
||||||
} catch (Exception e) {
|
|
||||||
respondException(response, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,37 +0,0 @@
|
|||||||
package com.google.refine.commands.auth;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import javax.servlet.ServletException;
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
import javax.servlet.http.HttpServletResponse;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import com.google.refine.commands.Command;
|
|
||||||
import com.google.refine.oauth.OAuthUtilities;
|
|
||||||
import com.google.refine.oauth.Provider;
|
|
||||||
import com.google.refine.util.FreebaseUtils;
|
|
||||||
|
|
||||||
public class GetUserBadgesCommand extends Command {
|
|
||||||
|
|
||||||
final static Logger logger = LoggerFactory.getLogger("get-version_command");
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
|
||||||
|
|
||||||
try {
|
|
||||||
Provider provider = OAuthUtilities.getProvider(request);
|
|
||||||
String user_id = request.getParameter("user_id");
|
|
||||||
response.setCharacterEncoding("UTF-8");
|
|
||||||
response.setHeader("Content-Type", "application/json");
|
|
||||||
String user_badges = FreebaseUtils.getUserBadges(provider, user_id);
|
|
||||||
response.getWriter().write(user_badges);
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.info("error",e);
|
|
||||||
respondException(response, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,45 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
public class AnonymousNode implements Node, NodeWithLinks {
|
|
||||||
final public FreebaseType type;
|
|
||||||
final public List<Link> links = new LinkedList<Link>();
|
|
||||||
|
|
||||||
public AnonymousNode(FreebaseType type) {
|
|
||||||
this.type = type;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("nodeType"); writer.value("anonymous");
|
|
||||||
writer.key("type"); type.write(writer, options);
|
|
||||||
if (links != null) {
|
|
||||||
writer.key("links"); writer.array();
|
|
||||||
for (Link link : links) {
|
|
||||||
link.write(writer, options);
|
|
||||||
}
|
|
||||||
writer.endArray();
|
|
||||||
}
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addLink(Link link) {
|
|
||||||
links.add(link);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Link getLink(int index) {
|
|
||||||
return links.get(index);
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getLinkCount() {
|
|
||||||
return links.size();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
import com.google.refine.model.Column;
|
|
||||||
import com.google.refine.model.Project;
|
|
||||||
import com.google.refine.model.Row;
|
|
||||||
|
|
||||||
|
|
||||||
public class BooleanColumnCondition implements Condition {
|
|
||||||
final public String columnName;
|
|
||||||
|
|
||||||
public BooleanColumnCondition(String columnName) {
|
|
||||||
this.columnName = columnName;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean test(Project project, int rowIndex, Row row) {
|
|
||||||
Column column = project.columnModel.getColumnByName(columnName);
|
|
||||||
if (column != null) {
|
|
||||||
Object o = row.getCellValue(column.getCellIndex());
|
|
||||||
if (o != null) {
|
|
||||||
if (o instanceof Boolean) {
|
|
||||||
return ((Boolean) o).booleanValue();
|
|
||||||
} else {
|
|
||||||
return Boolean.parseBoolean(o.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void write(JSONWriter writer, Properties options) throws JSONException {
|
|
||||||
writer.object();
|
|
||||||
writer.key("columnName"); writer.value(columnName);
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,33 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
public class CellKeyNode extends CellNode {
|
|
||||||
final public FreebaseTopic namespace;
|
|
||||||
|
|
||||||
public CellKeyNode(
|
|
||||||
FreebaseTopic namespace
|
|
||||||
) {
|
|
||||||
this.namespace = namespace;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("nodeType"); writer.value("cell-as-key");
|
|
||||||
|
|
||||||
writer.key("columnNames");
|
|
||||||
writer.array();
|
|
||||||
for (String name : columnNames) {
|
|
||||||
writer.value(name);
|
|
||||||
}
|
|
||||||
writer.endArray();
|
|
||||||
|
|
||||||
writer.key("namespace"); namespace.write(writer, options);
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,8 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
abstract public class CellNode implements Node {
|
|
||||||
final public List<String> columnNames = new LinkedList<String>();
|
|
||||||
}
|
|
||||||
@ -1,56 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
public class CellTopicNode extends CellNode implements NodeWithLinks {
|
|
||||||
final public FreebaseType type;
|
|
||||||
final public List<Link> links = new LinkedList<Link>();
|
|
||||||
|
|
||||||
public CellTopicNode(
|
|
||||||
FreebaseType type
|
|
||||||
) {
|
|
||||||
this.type = type;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("nodeType"); writer.value("cell-as-topic");
|
|
||||||
writer.key("columnNames");
|
|
||||||
writer.array();
|
|
||||||
for (String name : columnNames) {
|
|
||||||
writer.value(name);
|
|
||||||
}
|
|
||||||
writer.endArray();
|
|
||||||
if (type != null) {
|
|
||||||
writer.key("type"); type.write(writer, options);
|
|
||||||
}
|
|
||||||
if (links != null) {
|
|
||||||
writer.key("links"); writer.array();
|
|
||||||
for (Link link : links) {
|
|
||||||
link.write(writer, options);
|
|
||||||
}
|
|
||||||
writer.endArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addLink(Link link) {
|
|
||||||
links.add(link);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Link getLink(int index) {
|
|
||||||
return links.get(index);
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getLinkCount() {
|
|
||||||
return links.size();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,36 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
public class CellValueNode extends CellNode {
|
|
||||||
final public String valueType;
|
|
||||||
final public String lang;
|
|
||||||
|
|
||||||
public CellValueNode(
|
|
||||||
String valueType,
|
|
||||||
String lang
|
|
||||||
) {
|
|
||||||
this.valueType = valueType;
|
|
||||||
this.lang = lang;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("nodeType"); writer.value("cell-as-value");
|
|
||||||
writer.key("columnNames");
|
|
||||||
writer.array();
|
|
||||||
for (String name : columnNames) {
|
|
||||||
writer.value(name);
|
|
||||||
}
|
|
||||||
writer.endArray();
|
|
||||||
writer.key("valueType"); writer.value(valueType);
|
|
||||||
writer.key("lang"); writer.value(lang);
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import com.google.refine.Jsonizable;
|
|
||||||
import com.google.refine.model.Project;
|
|
||||||
import com.google.refine.model.Row;
|
|
||||||
|
|
||||||
public interface Condition extends Jsonizable {
|
|
||||||
public boolean test(Project project, int rowIndex, Row row);
|
|
||||||
}
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
public class FreebaseProperty extends FreebaseTopic {
|
|
||||||
//final protected FreebaseType _expectedType;
|
|
||||||
|
|
||||||
public FreebaseProperty(String id, String name) {
|
|
||||||
super(id, name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,28 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
import com.google.refine.Jsonizable;
|
|
||||||
|
|
||||||
public class FreebaseTopic implements Jsonizable {
|
|
||||||
final public String id;
|
|
||||||
final public String name;
|
|
||||||
|
|
||||||
public FreebaseTopic(String id, String name) {
|
|
||||||
this.id = id;
|
|
||||||
this.name = name;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("id"); writer.value(id);
|
|
||||||
writer.key("name"); writer.value(name);
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,46 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
public class FreebaseTopicNode implements Node, NodeWithLinks {
|
|
||||||
final public FreebaseTopic topic;
|
|
||||||
final public List<Link> links = new LinkedList<Link>();
|
|
||||||
|
|
||||||
public FreebaseTopicNode(FreebaseTopic topic) {
|
|
||||||
this.topic = topic;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("nodeType"); writer.value("topic");
|
|
||||||
writer.key("topic"); topic.write(writer, options);
|
|
||||||
if (links != null) {
|
|
||||||
writer.key("links"); writer.array();
|
|
||||||
for (Link link : links) {
|
|
||||||
link.write(writer, options);
|
|
||||||
}
|
|
||||||
writer.endArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addLink(Link link) {
|
|
||||||
links.add(link);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Link getLink(int index) {
|
|
||||||
return links.get(index);
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getLinkCount() {
|
|
||||||
return links.size();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,36 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONObject;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
import com.google.refine.Jsonizable;
|
|
||||||
|
|
||||||
public class FreebaseType extends FreebaseTopic implements Jsonizable {
|
|
||||||
public FreebaseType(String id, String name) {
|
|
||||||
super(id, name);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("id"); writer.value(id);
|
|
||||||
writer.key("name"); writer.value(name);
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
static public FreebaseType load(JSONObject obj) throws Exception {
|
|
||||||
if (obj == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
FreebaseType type = new FreebaseType(
|
|
||||||
obj.getString("id"),
|
|
||||||
obj.getString("name")
|
|
||||||
);
|
|
||||||
return type;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
import com.google.refine.Jsonizable;
|
|
||||||
|
|
||||||
public class Link implements Jsonizable {
|
|
||||||
final public FreebaseProperty property;
|
|
||||||
final public Node target;
|
|
||||||
final public Condition condition;
|
|
||||||
final public boolean load;
|
|
||||||
|
|
||||||
public Link(FreebaseProperty property, Node target, Condition condition, boolean load) {
|
|
||||||
this.property = property;
|
|
||||||
this.target = target;
|
|
||||||
this.condition = condition;
|
|
||||||
this.load = load;
|
|
||||||
}
|
|
||||||
|
|
||||||
public FreebaseProperty getProperty() {
|
|
||||||
return property;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Node getTarget() {
|
|
||||||
return target;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("property"); property.write(writer, options);
|
|
||||||
if (target != null) {
|
|
||||||
writer.key("target");
|
|
||||||
target.write(writer, options);
|
|
||||||
}
|
|
||||||
if (condition != null) {
|
|
||||||
writer.key("condition");
|
|
||||||
condition.write(writer, options);
|
|
||||||
}
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import com.google.refine.Jsonizable;
|
|
||||||
|
|
||||||
public interface Node extends Jsonizable {
|
|
||||||
}
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
public interface NodeWithLinks {
|
|
||||||
public void addLink(Link link);
|
|
||||||
|
|
||||||
public int getLinkCount();
|
|
||||||
|
|
||||||
public Link getLink(int index);
|
|
||||||
}
|
|
||||||
@ -1,168 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONArray;
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONObject;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
import com.google.refine.model.OverlayModel;
|
|
||||||
import com.google.refine.model.Project;
|
|
||||||
|
|
||||||
public class Protograph implements OverlayModel {
|
|
||||||
final protected List<Node> _rootNodes = new LinkedList<Node>();
|
|
||||||
|
|
||||||
public int getRootNodeCount() {
|
|
||||||
return _rootNodes.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
public Node getRootNode(int index) {
|
|
||||||
return _rootNodes.get(index);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onBeforeSave(Project project) {
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onAfterSave(Project project) {
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void dispose(Project project) {
|
|
||||||
}
|
|
||||||
|
|
||||||
static public Protograph reconstruct(JSONObject o) throws JSONException {
|
|
||||||
Protograph g = new Protograph();
|
|
||||||
|
|
||||||
JSONArray rootNodes = o.getJSONArray("rootNodes");
|
|
||||||
int count = rootNodes.length();
|
|
||||||
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
JSONObject o2 = rootNodes.getJSONObject(i);
|
|
||||||
Node node = reconstructNode(o2);
|
|
||||||
if (node != null) {
|
|
||||||
g._rootNodes.add(node);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return g;
|
|
||||||
}
|
|
||||||
|
|
||||||
static protected Node reconstructNode(JSONObject o) throws JSONException {
|
|
||||||
Node node = null;
|
|
||||||
|
|
||||||
String nodeType = o.getString("nodeType");
|
|
||||||
if (nodeType.startsWith("cell-as-")) {
|
|
||||||
if ("cell-as-topic".equals(nodeType)) {
|
|
||||||
if (o.has("type")) {
|
|
||||||
node = new CellTopicNode(
|
|
||||||
reconstructType(o.getJSONObject("type"))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else if ("cell-as-value".equals(nodeType)) {
|
|
||||||
node = new CellValueNode(
|
|
||||||
o.getString("valueType"),
|
|
||||||
o.getString("lang")
|
|
||||||
);
|
|
||||||
} else if ("cell-as-key".equals(nodeType)) {
|
|
||||||
node = new CellKeyNode(
|
|
||||||
reconstructTopic(o.getJSONObject("namespace"))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (o.has("columnName") && !o.isNull("columnName")) {
|
|
||||||
((CellNode) node).columnNames.add(o.getString("columnName"));
|
|
||||||
}
|
|
||||||
if (o.has("columnNames") && !o.isNull("columnNames")) {
|
|
||||||
JSONArray columnNames = o.getJSONArray("columnNames");
|
|
||||||
int count = columnNames.length();
|
|
||||||
|
|
||||||
for (int c = 0; c < count; c++) {
|
|
||||||
((CellNode) node).columnNames.add(columnNames.getString(c));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if ("topic".equals(nodeType)) {
|
|
||||||
node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic")));
|
|
||||||
} else if ("value".equals(nodeType)) {
|
|
||||||
node = new ValueNode(
|
|
||||||
o.get("value"),
|
|
||||||
o.getString("valueType"),
|
|
||||||
o.getString("lang")
|
|
||||||
);
|
|
||||||
} else if ("anonymous".equals(nodeType)) {
|
|
||||||
node = new AnonymousNode(reconstructType(o.getJSONObject("type")));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (node != null && node instanceof NodeWithLinks && o.has("links")) {
|
|
||||||
NodeWithLinks node2 = (NodeWithLinks) node;
|
|
||||||
|
|
||||||
JSONArray links = o.getJSONArray("links");
|
|
||||||
int linkCount = links.length();
|
|
||||||
|
|
||||||
for (int j = 0; j < linkCount; j++) {
|
|
||||||
JSONObject oLink = links.getJSONObject(j);
|
|
||||||
Condition condition = null;
|
|
||||||
|
|
||||||
if (oLink.has("condition") && !oLink.isNull("condition")) {
|
|
||||||
JSONObject oCondition = oLink.getJSONObject("condition");
|
|
||||||
if (oCondition.has("columnName") && !oCondition.isNull("columnName")) {
|
|
||||||
condition = new BooleanColumnCondition(oCondition.getString("columnName"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
node2.addLink(new Link(
|
|
||||||
reconstructProperty(oLink.getJSONObject("property")),
|
|
||||||
oLink.has("target") && !oLink.isNull("target") ?
|
|
||||||
reconstructNode(oLink.getJSONObject("target")) : null,
|
|
||||||
condition,
|
|
||||||
oLink.has("load") && !oLink.isNull("load") ?
|
|
||||||
oLink.getBoolean("load") : true
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return node;
|
|
||||||
}
|
|
||||||
|
|
||||||
static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException {
|
|
||||||
return new FreebaseProperty(
|
|
||||||
o.getString("id"),
|
|
||||||
o.getString("name")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
static protected FreebaseType reconstructType(JSONObject o) throws JSONException {
|
|
||||||
return new FreebaseType(
|
|
||||||
o.getString("id"),
|
|
||||||
o.getString("name")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException {
|
|
||||||
return new FreebaseTopic(
|
|
||||||
o.getString("id"),
|
|
||||||
o.getString("name")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options) throws JSONException {
|
|
||||||
writer.object();
|
|
||||||
writer.key("rootNodes"); writer.array();
|
|
||||||
|
|
||||||
for (Node node : _rootNodes) {
|
|
||||||
node.write(writer, options);
|
|
||||||
}
|
|
||||||
|
|
||||||
writer.endArray();
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
|
|
||||||
static public Protograph load(Project project, JSONObject obj) throws Exception {
|
|
||||||
return reconstruct(obj);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,29 +0,0 @@
|
|||||||
package com.google.refine.protograph;
|
|
||||||
|
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
public class ValueNode implements Node {
|
|
||||||
final public Object value;
|
|
||||||
final public String valueType;
|
|
||||||
final public String lang;
|
|
||||||
|
|
||||||
public ValueNode(Object value, String valueType, String lang) {
|
|
||||||
this.value = value;
|
|
||||||
this.valueType = valueType;
|
|
||||||
this.lang = lang;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void write(JSONWriter writer, Properties options)
|
|
||||||
throws JSONException {
|
|
||||||
|
|
||||||
writer.object();
|
|
||||||
writer.key("nodeType"); writer.value("value");
|
|
||||||
writer.key("value"); writer.value(value);
|
|
||||||
writer.key("valueType"); writer.value(valueType);
|
|
||||||
writer.key("lang"); writer.value(lang);
|
|
||||||
writer.endObject();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,346 +0,0 @@
|
|||||||
package com.google.refine.protograph.transpose;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.Writer;
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import org.json.JSONArray;
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONObject;
|
|
||||||
import org.json.JSONWriter;
|
|
||||||
|
|
||||||
import com.google.refine.model.Cell;
|
|
||||||
import com.google.refine.model.Recon;
|
|
||||||
import com.google.refine.protograph.AnonymousNode;
|
|
||||||
import com.google.refine.protograph.CellKeyNode;
|
|
||||||
import com.google.refine.protograph.CellNode;
|
|
||||||
import com.google.refine.protograph.CellTopicNode;
|
|
||||||
import com.google.refine.protograph.CellValueNode;
|
|
||||||
import com.google.refine.protograph.FreebaseProperty;
|
|
||||||
import com.google.refine.protograph.FreebaseTopicNode;
|
|
||||||
import com.google.refine.protograph.Link;
|
|
||||||
import com.google.refine.protograph.ValueNode;
|
|
||||||
import com.google.refine.util.JSONUtilities;
|
|
||||||
|
|
||||||
public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory {
|
|
||||||
protected Writer writer;
|
|
||||||
protected List<JSONObject> rootObjects = new LinkedList<JSONObject>();
|
|
||||||
|
|
||||||
private static final String TYPE = "type";
|
|
||||||
private static final String ID = "id";
|
|
||||||
private static final String NAME = "name";
|
|
||||||
private static final String CREATE = "create";
|
|
||||||
private static final String VALUE = "value";
|
|
||||||
private static final String CONNECT = "connect";
|
|
||||||
private static final String LANG = "lang";
|
|
||||||
|
|
||||||
public MqlwriteLikeTransposedNodeFactory(Writer writer) {
|
|
||||||
this.writer = writer;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected JSONArray getJSON() {
|
|
||||||
return new JSONArray(rootObjects);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void flush() throws IOException {
|
|
||||||
try {
|
|
||||||
JSONWriter jsonWriter = new JSONWriter(writer);
|
|
||||||
|
|
||||||
jsonWriter.array();
|
|
||||||
for (JSONObject obj : rootObjects) {
|
|
||||||
jsonWriter.value(obj);
|
|
||||||
}
|
|
||||||
jsonWriter.endArray();
|
|
||||||
|
|
||||||
} catch (JSONException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
writer.flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract protected class JsonTransposedNode implements TransposedNode {
|
|
||||||
abstract public Object getJSON();
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract protected class JsonObjectTransposedNode extends JsonTransposedNode {
|
|
||||||
abstract public JSONObject getJSONObject();
|
|
||||||
|
|
||||||
protected JSONObject obj;
|
|
||||||
|
|
||||||
public Object getJSON() {
|
|
||||||
return getJSONObject();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class AnonymousTransposedNode extends JsonObjectTransposedNode {
|
|
||||||
JsonObjectTransposedNode parent;
|
|
||||||
FreebaseProperty property;
|
|
||||||
AnonymousNode node;
|
|
||||||
|
|
||||||
protected AnonymousTransposedNode(
|
|
||||||
JsonObjectTransposedNode parent,
|
|
||||||
FreebaseProperty property,
|
|
||||||
AnonymousNode node
|
|
||||||
) {
|
|
||||||
this.parent = parent;
|
|
||||||
this.property = property;
|
|
||||||
this.node = node;
|
|
||||||
}
|
|
||||||
|
|
||||||
public JSONObject getJSONObject() {
|
|
||||||
if (obj == null) {
|
|
||||||
obj = new JSONObject();
|
|
||||||
try {
|
|
||||||
obj.put(TYPE, this.node.type.id);
|
|
||||||
obj.put(ID, (String) null);
|
|
||||||
obj.put(CREATE, "unconditional");
|
|
||||||
} catch (JSONException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
|
|
||||||
linkTransposedNodeJSON(obj, parent, property);
|
|
||||||
}
|
|
||||||
|
|
||||||
return obj;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class CellTopicTransposedNode extends JsonObjectTransposedNode {
|
|
||||||
protected CellTopicNode node;
|
|
||||||
protected Cell cell;
|
|
||||||
|
|
||||||
public CellTopicTransposedNode(CellTopicNode node, Cell cell) {
|
|
||||||
this.node = node;
|
|
||||||
this.cell = cell;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public JSONObject getJSONObject() {
|
|
||||||
if (obj == null) {
|
|
||||||
obj = new JSONObject();
|
|
||||||
try {
|
|
||||||
if (cell.recon != null &&
|
|
||||||
cell.recon.judgment == Recon.Judgment.Matched &&
|
|
||||||
cell.recon.match != null) {
|
|
||||||
obj.put(ID, cell.recon.match.id);
|
|
||||||
} else {
|
|
||||||
obj.put(ID, (String) null);
|
|
||||||
obj.put(NAME, cell.value.toString());
|
|
||||||
obj.put(TYPE, node.type.id);
|
|
||||||
obj.put(CREATE, "unless_exists");
|
|
||||||
}
|
|
||||||
} catch (JSONException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return obj;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class CellValueTransposedNode extends JsonTransposedNode {
|
|
||||||
protected JSONObject obj;
|
|
||||||
protected CellValueNode node;
|
|
||||||
protected Cell cell;
|
|
||||||
|
|
||||||
public CellValueTransposedNode(CellValueNode node, Cell cell) {
|
|
||||||
this.node = node;
|
|
||||||
this.cell = cell;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object getJSON() {
|
|
||||||
if (obj == null) {
|
|
||||||
obj = new JSONObject();
|
|
||||||
try {
|
|
||||||
JSONUtilities.putField(obj, VALUE, cell.value);
|
|
||||||
|
|
||||||
obj.put(TYPE, node.valueType);
|
|
||||||
if ("/type/text".equals(node.valueType)) {
|
|
||||||
obj.put(LANG, node.lang);
|
|
||||||
}
|
|
||||||
|
|
||||||
obj.put(CONNECT, "insert");
|
|
||||||
} catch (JSONException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return obj;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class CellKeyTransposedNode extends JsonTransposedNode {
|
|
||||||
protected JSONObject obj;
|
|
||||||
protected CellKeyNode node;
|
|
||||||
protected Cell cell;
|
|
||||||
|
|
||||||
public CellKeyTransposedNode(CellKeyNode node, Cell cell) {
|
|
||||||
this.node = node;
|
|
||||||
this.cell = cell;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object getJSON() {
|
|
||||||
if (obj == null) {
|
|
||||||
obj = new JSONObject();
|
|
||||||
try {
|
|
||||||
obj.put(VALUE, cell.value.toString());
|
|
||||||
|
|
||||||
JSONObject nsObj = new JSONObject();
|
|
||||||
nsObj.put(ID, node.namespace.id);
|
|
||||||
|
|
||||||
obj.put("namespace", nsObj);
|
|
||||||
obj.put(CONNECT, "insert");
|
|
||||||
} catch (JSONException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return obj;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class TopicTransposedNode extends JsonObjectTransposedNode {
|
|
||||||
protected FreebaseTopicNode node;
|
|
||||||
|
|
||||||
public TopicTransposedNode(FreebaseTopicNode node) {
|
|
||||||
this.node = node;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public JSONObject getJSONObject() {
|
|
||||||
if (obj == null) {
|
|
||||||
obj = new JSONObject();
|
|
||||||
try {
|
|
||||||
obj.put(ID, node.topic.id);
|
|
||||||
} catch (JSONException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return obj;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class ValueTransposedNode extends JsonTransposedNode {
|
|
||||||
protected JSONObject obj;
|
|
||||||
protected ValueNode node;
|
|
||||||
|
|
||||||
public ValueTransposedNode(ValueNode node) {
|
|
||||||
this.node = node;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object getJSON() {
|
|
||||||
if (obj == null) {
|
|
||||||
obj = new JSONObject();
|
|
||||||
try {
|
|
||||||
obj.put(VALUE, node.value);
|
|
||||||
obj.put(TYPE, node.valueType);
|
|
||||||
if ("/type/text".equals(node.valueType)) {
|
|
||||||
obj.put(LANG, node.lang);
|
|
||||||
}
|
|
||||||
|
|
||||||
obj.put(CONNECT, "insert");
|
|
||||||
} catch (JSONException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return obj;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
public TransposedNode transposeAnonymousNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
AnonymousNode node, int rowIndex) {
|
|
||||||
|
|
||||||
return new AnonymousTransposedNode(
|
|
||||||
parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null,
|
|
||||||
link != null ? link.property : null,
|
|
||||||
node
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransposedNode transposeCellNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
CellNode node,
|
|
||||||
int rowIndex,
|
|
||||||
int cellIndex,
|
|
||||||
Cell cell) {
|
|
||||||
|
|
||||||
JsonTransposedNode tnode = null;
|
|
||||||
if (node instanceof CellTopicNode) {
|
|
||||||
tnode = new CellTopicTransposedNode((CellTopicNode) node, cell);
|
|
||||||
} else if (node instanceof CellValueNode) {
|
|
||||||
tnode = new CellValueTransposedNode((CellValueNode) node, cell);
|
|
||||||
} else if (node instanceof CellKeyNode) {
|
|
||||||
tnode = new CellKeyTransposedNode((CellKeyNode) node, cell);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tnode != null) {
|
|
||||||
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
|
|
||||||
}
|
|
||||||
return tnode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransposedNode transposeTopicNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
FreebaseTopicNode node, int rowIndex) {
|
|
||||||
|
|
||||||
JsonTransposedNode tnode = new TopicTransposedNode(node);
|
|
||||||
|
|
||||||
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
|
|
||||||
|
|
||||||
return tnode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransposedNode transposeValueNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
ValueNode node, int rowIndex) {
|
|
||||||
|
|
||||||
JsonTransposedNode tnode = new ValueTransposedNode(node);
|
|
||||||
|
|
||||||
processTransposedNode(tnode, parentNode, link != null ? link.property : null);
|
|
||||||
|
|
||||||
return tnode;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void processTransposedNode(
|
|
||||||
JsonTransposedNode tnode,
|
|
||||||
TransposedNode parentNode,
|
|
||||||
FreebaseProperty property
|
|
||||||
) {
|
|
||||||
|
|
||||||
if (!(tnode instanceof AnonymousTransposedNode)) {
|
|
||||||
linkTransposedNodeJSON(tnode.getJSON(), parentNode, property);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void linkTransposedNodeJSON(
|
|
||||||
Object obj,
|
|
||||||
TransposedNode parentNode,
|
|
||||||
FreebaseProperty property
|
|
||||||
) {
|
|
||||||
|
|
||||||
if (parentNode == null) {
|
|
||||||
if (obj instanceof JSONObject) {
|
|
||||||
rootObjects.add((JSONObject) obj);
|
|
||||||
}
|
|
||||||
} else if (parentNode instanceof JsonTransposedNode) {
|
|
||||||
JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject();
|
|
||||||
|
|
||||||
try {
|
|
||||||
JSONArray a = null;
|
|
||||||
if (parentObj.has(property.id)) {
|
|
||||||
a = parentObj.getJSONArray(property.id);
|
|
||||||
} else {
|
|
||||||
a = new JSONArray();
|
|
||||||
parentObj.put(property.id, a);
|
|
||||||
}
|
|
||||||
|
|
||||||
a.put(a.length(), obj);
|
|
||||||
} catch (JSONException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,4 +0,0 @@
|
|||||||
package com.google.refine.protograph.transpose;
|
|
||||||
|
|
||||||
public interface TransposedNode {
|
|
||||||
}
|
|
||||||
@ -1,43 +0,0 @@
|
|||||||
package com.google.refine.protograph.transpose;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import com.google.refine.model.Cell;
|
|
||||||
import com.google.refine.protograph.AnonymousNode;
|
|
||||||
import com.google.refine.protograph.CellNode;
|
|
||||||
import com.google.refine.protograph.FreebaseTopicNode;
|
|
||||||
import com.google.refine.protograph.Link;
|
|
||||||
import com.google.refine.protograph.ValueNode;
|
|
||||||
|
|
||||||
public interface TransposedNodeFactory {
|
|
||||||
public TransposedNode transposeAnonymousNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
AnonymousNode node, int rowIndex
|
|
||||||
);
|
|
||||||
|
|
||||||
public TransposedNode transposeCellNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
CellNode node,
|
|
||||||
int rowIndex,
|
|
||||||
int cellIndex,
|
|
||||||
Cell cell
|
|
||||||
);
|
|
||||||
|
|
||||||
public TransposedNode transposeValueNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
ValueNode node,
|
|
||||||
int rowIndex
|
|
||||||
);
|
|
||||||
|
|
||||||
public TransposedNode transposeTopicNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
FreebaseTopicNode node,
|
|
||||||
int rowIndex
|
|
||||||
);
|
|
||||||
|
|
||||||
public void flush() throws IOException;
|
|
||||||
}
|
|
||||||
@ -1,222 +0,0 @@
|
|||||||
package com.google.refine.protograph.transpose;
|
|
||||||
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import com.google.refine.browsing.FilteredRows;
|
|
||||||
import com.google.refine.browsing.RowVisitor;
|
|
||||||
import com.google.refine.expr.ExpressionUtils;
|
|
||||||
import com.google.refine.model.Cell;
|
|
||||||
import com.google.refine.model.Column;
|
|
||||||
import com.google.refine.model.Project;
|
|
||||||
import com.google.refine.model.Row;
|
|
||||||
import com.google.refine.model.Recon.Judgment;
|
|
||||||
import com.google.refine.protograph.AnonymousNode;
|
|
||||||
import com.google.refine.protograph.CellNode;
|
|
||||||
import com.google.refine.protograph.CellTopicNode;
|
|
||||||
import com.google.refine.protograph.FreebaseTopicNode;
|
|
||||||
import com.google.refine.protograph.Link;
|
|
||||||
import com.google.refine.protograph.Node;
|
|
||||||
import com.google.refine.protograph.NodeWithLinks;
|
|
||||||
import com.google.refine.protograph.Protograph;
|
|
||||||
import com.google.refine.protograph.ValueNode;
|
|
||||||
|
|
||||||
public class Transposer {
|
|
||||||
static public void transpose(
|
|
||||||
Project project,
|
|
||||||
FilteredRows filteredRows,
|
|
||||||
Protograph protograph,
|
|
||||||
Node rootNode,
|
|
||||||
TransposedNodeFactory nodeFactory
|
|
||||||
) {
|
|
||||||
transpose(project, filteredRows, protograph, rootNode, nodeFactory, 20);
|
|
||||||
}
|
|
||||||
|
|
||||||
static public void transpose(
|
|
||||||
Project project,
|
|
||||||
FilteredRows filteredRows,
|
|
||||||
Protograph protograph,
|
|
||||||
Node rootNode,
|
|
||||||
TransposedNodeFactory nodeFactory,
|
|
||||||
int limit
|
|
||||||
) {
|
|
||||||
Context rootContext = new Context(rootNode, null, null, limit);
|
|
||||||
|
|
||||||
filteredRows.accept(project, new RowVisitor() {
|
|
||||||
Context rootContext;
|
|
||||||
Protograph protograph;
|
|
||||||
Node rootNode;
|
|
||||||
TransposedNodeFactory nodeFactory;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean visit(Project project, int rowIndex, Row row) {
|
|
||||||
if (rootContext.limit <= 0 || rootContext.count < rootContext.limit) {
|
|
||||||
descend(project, protograph, nodeFactory, rowIndex, row, rootNode, rootContext);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (rootContext.limit > 0 && rootContext.count > rootContext.limit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void start(Project project) {
|
|
||||||
// TODO Auto-generated method stub
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void end(Project project) {
|
|
||||||
// TODO Auto-generated method stub
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public RowVisitor init(
|
|
||||||
Context rootContext,
|
|
||||||
Protograph protograph,
|
|
||||||
Node rootNode,
|
|
||||||
TransposedNodeFactory nodeFactory
|
|
||||||
) {
|
|
||||||
this.rootContext = rootContext;
|
|
||||||
this.protograph = protograph;
|
|
||||||
this.rootNode = rootNode;
|
|
||||||
this.nodeFactory = nodeFactory;
|
|
||||||
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
}.init(rootContext, protograph, rootNode, nodeFactory));
|
|
||||||
}
|
|
||||||
|
|
||||||
static protected void descend(
|
|
||||||
Project project,
|
|
||||||
Protograph protograph,
|
|
||||||
TransposedNodeFactory nodeFactory,
|
|
||||||
int rowIndex,
|
|
||||||
Row row,
|
|
||||||
Node node,
|
|
||||||
Context context
|
|
||||||
) {
|
|
||||||
List<TransposedNode> tnodes = new LinkedList<TransposedNode>();
|
|
||||||
|
|
||||||
TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode;
|
|
||||||
Link link = context.parent == null ? null : context.link;
|
|
||||||
|
|
||||||
if (node instanceof CellNode) {
|
|
||||||
CellNode node2 = (CellNode) node;
|
|
||||||
for (String columnName : node2.columnNames) {
|
|
||||||
Column column = project.columnModel.getColumnByName(columnName);
|
|
||||||
if (column != null) {
|
|
||||||
int cellIndex = column.getCellIndex();
|
|
||||||
|
|
||||||
Cell cell = row.getCell(cellIndex);
|
|
||||||
if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
|
|
||||||
if (node2 instanceof CellTopicNode &&
|
|
||||||
(cell.recon == null || cell.recon.judgment == Judgment.None)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
context.count++;
|
|
||||||
if (context.limit > 0 && context.count > context.limit) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
tnodes.add(nodeFactory.transposeCellNode(
|
|
||||||
parentNode,
|
|
||||||
link,
|
|
||||||
node2,
|
|
||||||
rowIndex,
|
|
||||||
cellIndex,
|
|
||||||
cell
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (node instanceof AnonymousNode) {
|
|
||||||
tnodes.add(nodeFactory.transposeAnonymousNode(
|
|
||||||
parentNode,
|
|
||||||
link,
|
|
||||||
(AnonymousNode) node,
|
|
||||||
rowIndex
|
|
||||||
));
|
|
||||||
} else if (node instanceof FreebaseTopicNode) {
|
|
||||||
tnodes.add(nodeFactory.transposeTopicNode(
|
|
||||||
parentNode,
|
|
||||||
link,
|
|
||||||
(FreebaseTopicNode) node,
|
|
||||||
rowIndex
|
|
||||||
));
|
|
||||||
} else if (node instanceof ValueNode) {
|
|
||||||
tnodes.add(nodeFactory.transposeValueNode(
|
|
||||||
parentNode,
|
|
||||||
link,
|
|
||||||
(ValueNode) node,
|
|
||||||
rowIndex
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (node instanceof NodeWithLinks) {
|
|
||||||
NodeWithLinks node2 = (NodeWithLinks) node;
|
|
||||||
int linkCount = node2.getLinkCount();
|
|
||||||
|
|
||||||
for (int i = 0; i < linkCount; i++) {
|
|
||||||
Link link2 = node2.getLink(i);
|
|
||||||
if (link2.condition == null || link2.condition.test(project, rowIndex, row)) {
|
|
||||||
for (TransposedNode tnode : tnodes) {
|
|
||||||
context.transposedNode = tnode;
|
|
||||||
context.nullifySubContextNodes();
|
|
||||||
|
|
||||||
descend(
|
|
||||||
project,
|
|
||||||
protograph,
|
|
||||||
nodeFactory,
|
|
||||||
rowIndex,
|
|
||||||
row,
|
|
||||||
link2.getTarget(),
|
|
||||||
context.subContexts.get(i)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static class Context {
|
|
||||||
TransposedNode transposedNode;
|
|
||||||
List<Context> subContexts;
|
|
||||||
Context parent;
|
|
||||||
Link link;
|
|
||||||
int count;
|
|
||||||
int limit;
|
|
||||||
|
|
||||||
Context(Node node, Context parent, Link link, int limit) {
|
|
||||||
this.parent = parent;
|
|
||||||
this.link = link;
|
|
||||||
this.limit = limit;
|
|
||||||
|
|
||||||
if (node instanceof NodeWithLinks) {
|
|
||||||
NodeWithLinks node2 = (NodeWithLinks) node;
|
|
||||||
|
|
||||||
int subContextCount = node2.getLinkCount();
|
|
||||||
|
|
||||||
subContexts = new LinkedList<Context>();
|
|
||||||
for (int i = 0; i < subContextCount; i++) {
|
|
||||||
Link link2 = node2.getLink(i);
|
|
||||||
subContexts.add(
|
|
||||||
new Context(link2.getTarget(), this, link2, -1));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void nullifySubContextNodes() {
|
|
||||||
if (subContexts != null) {
|
|
||||||
for (Context context : subContexts) {
|
|
||||||
context.transposedNode = null;
|
|
||||||
context.nullifySubContextNodes();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,725 +0,0 @@
|
|||||||
package com.google.refine.protograph.transpose;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.Writer;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
import org.json.JSONArray;
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONObject;
|
|
||||||
|
|
||||||
import com.google.refine.model.Cell;
|
|
||||||
import com.google.refine.model.Column;
|
|
||||||
import com.google.refine.model.Project;
|
|
||||||
import com.google.refine.model.Recon;
|
|
||||||
import com.google.refine.model.Recon.Judgment;
|
|
||||||
import com.google.refine.model.recon.ReconConfig;
|
|
||||||
import com.google.refine.model.recon.StandardReconConfig;
|
|
||||||
import com.google.refine.protograph.AnonymousNode;
|
|
||||||
import com.google.refine.protograph.CellKeyNode;
|
|
||||||
import com.google.refine.protograph.CellNode;
|
|
||||||
import com.google.refine.protograph.CellTopicNode;
|
|
||||||
import com.google.refine.protograph.CellValueNode;
|
|
||||||
import com.google.refine.protograph.FreebaseProperty;
|
|
||||||
import com.google.refine.protograph.FreebaseTopic;
|
|
||||||
import com.google.refine.protograph.FreebaseTopicNode;
|
|
||||||
import com.google.refine.protograph.Link;
|
|
||||||
import com.google.refine.protograph.ValueNode;
|
|
||||||
|
|
||||||
public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory {
|
|
||||||
protected Project project;
|
|
||||||
|
|
||||||
protected boolean start = true;
|
|
||||||
protected Writer writer;
|
|
||||||
protected WritingTransposedNode lastRootNode;
|
|
||||||
protected Map<String, Long> varPool = new HashMap<String, Long>();
|
|
||||||
protected Map<Long, String> newTopicVars = new HashMap<Long, String>();
|
|
||||||
protected Set<Long> serializedRecons = new HashSet<Long>();
|
|
||||||
|
|
||||||
protected long contextID = 0;
|
|
||||||
protected int contextRowIndex;
|
|
||||||
protected int contextRefCount = 0;
|
|
||||||
protected JSONObject contextTreeRoot;
|
|
||||||
|
|
||||||
public TripleLoaderTransposedNodeFactory(Project project, Writer writer) {
|
|
||||||
this.project = project;
|
|
||||||
this.writer = writer;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void flush() throws IOException {
|
|
||||||
if (lastRootNode != null) {
|
|
||||||
lastRootNode.write(null, null, project, -1, -1, null);
|
|
||||||
lastRootNode = null;
|
|
||||||
|
|
||||||
writeContextTreeNode();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void writeLine(String line) {
|
|
||||||
try {
|
|
||||||
if (start) {
|
|
||||||
start = false;
|
|
||||||
} else {
|
|
||||||
writer.write('\n');
|
|
||||||
}
|
|
||||||
writer.write(line);
|
|
||||||
} catch (IOException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void writeRecon(
|
|
||||||
StringBuffer sb,
|
|
||||||
Project project,
|
|
||||||
int rowIndex,
|
|
||||||
int cellIndex,
|
|
||||||
Cell cell
|
|
||||||
) {
|
|
||||||
Recon recon = cell.recon;
|
|
||||||
|
|
||||||
sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\"");
|
|
||||||
contextRefCount++;
|
|
||||||
|
|
||||||
if (!serializedRecons.contains(recon.id)) {
|
|
||||||
serializedRecons.add(recon.id);
|
|
||||||
|
|
||||||
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
|
|
||||||
|
|
||||||
// qa:sample_group
|
|
||||||
{
|
|
||||||
StringBuffer sb2 = new StringBuffer();
|
|
||||||
|
|
||||||
sb2.append("{ \"s\" : \"rec");
|
|
||||||
sb2.append(Long.toString(recon.id));
|
|
||||||
sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : ");
|
|
||||||
sb2.append(JSONObject.quote(column.getName()));
|
|
||||||
sb2.append(", \"ignore\" : true }");
|
|
||||||
|
|
||||||
writeLine(sb2.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
// qa:recon_data
|
|
||||||
{
|
|
||||||
StringBuffer sb2 = new StringBuffer();
|
|
||||||
|
|
||||||
String s = cell.value instanceof String ? (String) cell.value : cell.value.toString();
|
|
||||||
|
|
||||||
sb2.append("{ \"s\" : \"rec");
|
|
||||||
sb2.append(Long.toString(recon.id));
|
|
||||||
sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { ");
|
|
||||||
|
|
||||||
sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry));
|
|
||||||
sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s));
|
|
||||||
sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName()));
|
|
||||||
sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service));
|
|
||||||
sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction));
|
|
||||||
sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize));
|
|
||||||
|
|
||||||
if (recon.judgment == Judgment.Matched) {
|
|
||||||
sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank));
|
|
||||||
sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id));
|
|
||||||
}
|
|
||||||
|
|
||||||
ReconConfig reconConfig = column.getReconConfig();
|
|
||||||
if (reconConfig != null && reconConfig instanceof StandardReconConfig) {
|
|
||||||
StandardReconConfig standardReconConfig = (StandardReconConfig) reconConfig;
|
|
||||||
sb2.append(", \"type\" : "); sb2.append(JSONObject.quote(standardReconConfig.typeID));
|
|
||||||
}
|
|
||||||
|
|
||||||
sb2.append(" } }");
|
|
||||||
|
|
||||||
writeLine(sb2.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void writeLine(
|
|
||||||
String subject, String predicate, Object object,
|
|
||||||
Project project,
|
|
||||||
int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
|
|
||||||
int objectRowIndex, int objectCellIndex, Cell objectCell,
|
|
||||||
boolean ignore
|
|
||||||
) {
|
|
||||||
if (subject != null && object != null) {
|
|
||||||
String s = object instanceof String ?
|
|
||||||
JSONObject.quote((String) object) : object.toString();
|
|
||||||
|
|
||||||
StringBuffer sb = new StringBuffer();
|
|
||||||
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
|
|
||||||
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
|
|
||||||
sb.append(", \"o\" : "); sb.append(s);
|
|
||||||
if (subjectCell != null || objectCell != null) {
|
|
||||||
sb.append(", \"meta\" : { ");
|
|
||||||
|
|
||||||
sb.append("\"recon\" : { ");
|
|
||||||
if (subjectCell != null) {
|
|
||||||
sb.append("\"s\" : ");
|
|
||||||
writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
|
|
||||||
}
|
|
||||||
if (objectCell != null) {
|
|
||||||
if (subjectCell != null) {
|
|
||||||
sb.append(", ");
|
|
||||||
}
|
|
||||||
sb.append("\"o\" : ");
|
|
||||||
writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell);
|
|
||||||
}
|
|
||||||
sb.append(" }");
|
|
||||||
|
|
||||||
sb.append(" }");
|
|
||||||
}
|
|
||||||
if (ignore) {
|
|
||||||
sb.append(", \"ignore\" : true");
|
|
||||||
}
|
|
||||||
sb.append(" }");
|
|
||||||
|
|
||||||
writeLine(sb.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void writeLine(
|
|
||||||
String subject, String predicate, Object object, String lang,
|
|
||||||
Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
|
|
||||||
boolean ignore
|
|
||||||
) {
|
|
||||||
if (subject != null && object != null) {
|
|
||||||
String s = object instanceof String ?
|
|
||||||
JSONObject.quote((String) object) : object.toString();
|
|
||||||
|
|
||||||
StringBuffer sb = new StringBuffer();
|
|
||||||
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
|
|
||||||
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
|
|
||||||
sb.append(", \"o\" : "); sb.append(s);
|
|
||||||
sb.append(", \"lang\" : "); sb.append(lang);
|
|
||||||
|
|
||||||
if (subjectCell != null) {
|
|
||||||
sb.append(", \"meta\" : { ");
|
|
||||||
sb.append("\"recon\" : { ");
|
|
||||||
sb.append("\"s\" : ");
|
|
||||||
writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
|
|
||||||
sb.append(" }");
|
|
||||||
sb.append(" }");
|
|
||||||
}
|
|
||||||
if (ignore) {
|
|
||||||
sb.append(", \"ignore\" : true");
|
|
||||||
}
|
|
||||||
sb.append(" }");
|
|
||||||
|
|
||||||
writeLine(sb.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract protected class WritingTransposedNode implements TransposedNode {
|
|
||||||
JSONObject jsonContextNode;
|
|
||||||
boolean load;
|
|
||||||
|
|
||||||
public Object write(
|
|
||||||
String subject, String predicate, Project project,
|
|
||||||
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
|
||||||
|
|
||||||
return internalWrite(
|
|
||||||
subject, predicate, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell);
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract public Object internalWrite(
|
|
||||||
String subject, String predicate, Project project,
|
|
||||||
int subjectRowIndex, int subjectCellIndex, Cell subjectCell);
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract protected class TransposedNodeWithChildren extends WritingTransposedNode {
|
|
||||||
public List<Link> links = new LinkedList<Link>();
|
|
||||||
public List<Integer> rowIndices = new LinkedList<Integer>();
|
|
||||||
public List<WritingTransposedNode> children = new LinkedList<WritingTransposedNode>();
|
|
||||||
|
|
||||||
protected void writeChildren(
|
|
||||||
String subject, Project project,
|
|
||||||
int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
|
||||||
|
|
||||||
for (int i = 0; i < children.size(); i++) {
|
|
||||||
WritingTransposedNode child = children.get(i);
|
|
||||||
Link link = links.get(i);
|
|
||||||
String predicate = link.property.id;
|
|
||||||
|
|
||||||
child.write(subject, predicate, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class AnonymousTransposedNode extends TransposedNodeWithChildren {
|
|
||||||
|
|
||||||
//protected AnonymousTransposedNode(AnonymousNode node) { }
|
|
||||||
|
|
||||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
|
||||||
if (children.size() == 0 || subject == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
StringBuffer sb = new StringBuffer();
|
|
||||||
sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
|
|
||||||
sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
|
|
||||||
sb.append(", \"o\" : { ");
|
|
||||||
|
|
||||||
StringBuffer sbRecon = new StringBuffer();
|
|
||||||
|
|
||||||
boolean first = true;
|
|
||||||
boolean firstRecon = true;
|
|
||||||
|
|
||||||
if (subjectCell.recon != null) {
|
|
||||||
sbRecon.append("\"s\" : ");
|
|
||||||
writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell);
|
|
||||||
|
|
||||||
firstRecon = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < children.size(); i++) {
|
|
||||||
WritingTransposedNode child = children.get(i);
|
|
||||||
Link link = links.get(i);
|
|
||||||
|
|
||||||
FreebaseProperty property = link.property;
|
|
||||||
|
|
||||||
Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null);
|
|
||||||
if (c != null) {
|
|
||||||
if (first) {
|
|
||||||
first = false;
|
|
||||||
} else {
|
|
||||||
sb.append(", ");
|
|
||||||
}
|
|
||||||
sb.append("\"" + property.id + "\": ");
|
|
||||||
sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (child instanceof CellTopicTransposedNode) {
|
|
||||||
CellTopicTransposedNode child2 = (CellTopicTransposedNode) child;
|
|
||||||
Recon recon = child2.cell.recon;
|
|
||||||
|
|
||||||
if (recon != null &&
|
|
||||||
(recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) {
|
|
||||||
|
|
||||||
if (firstRecon) {
|
|
||||||
firstRecon = false;
|
|
||||||
} else {
|
|
||||||
sbRecon.append(", ");
|
|
||||||
}
|
|
||||||
|
|
||||||
sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
|
|
||||||
|
|
||||||
writeRecon(sbRecon, project,
|
|
||||||
rowIndices.get(i), child2.cellIndex, child2.cell);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sb.append(" }, \"meta\" : { \"recon\" : { ");
|
|
||||||
sb.append(sbRecon.toString());
|
|
||||||
sb.append(" } } }");
|
|
||||||
|
|
||||||
writeLine(sb.toString());
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class CellTopicTransposedNode extends TransposedNodeWithChildren {
|
|
||||||
protected CellTopicNode node;
|
|
||||||
protected int rowIndex;
|
|
||||||
protected int cellIndex;
|
|
||||||
protected Cell cell;
|
|
||||||
|
|
||||||
public CellTopicTransposedNode(CellTopicNode node, int rowIndex, int cellIndex, Cell cell) {
|
|
||||||
this.node = node;
|
|
||||||
this.rowIndex = rowIndex;
|
|
||||||
this.cellIndex = cellIndex;
|
|
||||||
this.cell = cell;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
|
||||||
String id = null;
|
|
||||||
if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) {
|
|
||||||
int objectRowIndex = rowIndex;
|
|
||||||
int objectCellIndex = cellIndex;
|
|
||||||
Cell objectCell = cell;
|
|
||||||
|
|
||||||
if (cell.recon.judgment == Recon.Judgment.Matched) {
|
|
||||||
id = cell.recon.match.id;
|
|
||||||
|
|
||||||
} else if (cell.recon.judgment == Judgment.New) {
|
|
||||||
if (newTopicVars.containsKey(cell.recon.id)) {
|
|
||||||
id = newTopicVars.get(cell.recon.id);
|
|
||||||
} else {
|
|
||||||
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
|
|
||||||
String columnName = column.getName();
|
|
||||||
|
|
||||||
long var = 0;
|
|
||||||
if (varPool.containsKey(columnName)) {
|
|
||||||
var = varPool.get(columnName);
|
|
||||||
}
|
|
||||||
varPool.put(columnName, var + 1);
|
|
||||||
|
|
||||||
id = "$" + columnName.replaceAll("\\W+", "_") + "_" + var;
|
|
||||||
|
|
||||||
String typeID = node.type.id;
|
|
||||||
|
|
||||||
ReconConfig reconConfig = column.getReconConfig();
|
|
||||||
if (reconConfig instanceof StandardReconConfig) {
|
|
||||||
typeID = ((StandardReconConfig) reconConfig).typeID;
|
|
||||||
}
|
|
||||||
|
|
||||||
writeLine(id, "type", typeID, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load);
|
|
||||||
writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load);
|
|
||||||
|
|
||||||
if (cell.recon != null) {
|
|
||||||
newTopicVars.put(cell.recon.id, id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (subject != null) {
|
|
||||||
writeLine(subject, predicate, id, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
|
||||||
objectRowIndex, objectCellIndex, objectCell, !load);
|
|
||||||
}
|
|
||||||
|
|
||||||
writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell);
|
|
||||||
}
|
|
||||||
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class CellValueTransposedNode extends WritingTransposedNode {
|
|
||||||
protected JSONObject obj;
|
|
||||||
protected CellValueNode node;
|
|
||||||
protected int rowIndex;
|
|
||||||
protected int cellIndex;
|
|
||||||
protected Cell cell;
|
|
||||||
|
|
||||||
public CellValueTransposedNode(CellValueNode node, int rowIndex, int cellIndex, Cell cell) {
|
|
||||||
this.node = node;
|
|
||||||
this.rowIndex = rowIndex;
|
|
||||||
this.cellIndex = cellIndex;
|
|
||||||
this.cell = cell;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
|
||||||
if (subject != null) {
|
|
||||||
if ("/type/text".equals(node.lang)) {
|
|
||||||
writeLine(subject, predicate, cell.value, node.lang, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell, !load);
|
|
||||||
} else {
|
|
||||||
writeLine(subject, predicate, cell.value, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
|
||||||
-1, -1, null, !load);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return cell.value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class CellKeyTransposedNode extends WritingTransposedNode {
|
|
||||||
protected CellKeyNode node;
|
|
||||||
protected int rowIndex;
|
|
||||||
protected int cellIndex;
|
|
||||||
protected Cell cell;
|
|
||||||
|
|
||||||
public CellKeyTransposedNode(CellKeyNode node, int rowIndex, int cellIndex, Cell cell) {
|
|
||||||
this.node = node;
|
|
||||||
this.rowIndex = rowIndex;
|
|
||||||
this.cellIndex = cellIndex;
|
|
||||||
this.cell = cell;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
|
||||||
writeLine(subject, "key", node.namespace.id + "/" + cell.value, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
|
||||||
-1, -1, null, !load);
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class TopicTransposedNode extends TransposedNodeWithChildren {
|
|
||||||
protected FreebaseTopicNode node;
|
|
||||||
|
|
||||||
public TopicTransposedNode(FreebaseTopicNode node) {
|
|
||||||
this.node = node;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
|
||||||
writeLine(subject, predicate, node.topic.id, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
|
||||||
-1, -1, null, !load);
|
|
||||||
|
|
||||||
writeChildren(node.topic.id, project, -1, -1, null);
|
|
||||||
|
|
||||||
return node.topic.id;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected class ValueTransposedNode extends WritingTransposedNode {
|
|
||||||
protected ValueNode node;
|
|
||||||
|
|
||||||
public ValueTransposedNode(ValueNode node) {
|
|
||||||
this.node = node;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
|
|
||||||
if ("/type/text".equals(node.lang)) {
|
|
||||||
writeLine(subject, predicate, node.value, node.lang, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell, !load);
|
|
||||||
} else {
|
|
||||||
writeLine(subject, predicate, node.value, project,
|
|
||||||
subjectRowIndex, subjectCellIndex, subjectCell,
|
|
||||||
-1, -1, null, !load);
|
|
||||||
}
|
|
||||||
|
|
||||||
return node.value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransposedNode transposeAnonymousNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
AnonymousNode node, int rowIndex) {
|
|
||||||
|
|
||||||
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
|
|
||||||
WritingTransposedNode tnode = new AnonymousTransposedNode();
|
|
||||||
|
|
||||||
tnode.load =
|
|
||||||
(parentNode2 == null || parentNode2.load) &&
|
|
||||||
(link == null || link.load);
|
|
||||||
|
|
||||||
processTransposedNode(tnode, parentNode, link, rowIndex);
|
|
||||||
|
|
||||||
tnode.jsonContextNode = addJsonContext(
|
|
||||||
parentNode2 != null ? parentNode2.jsonContextNode : null,
|
|
||||||
link != null ? link.property.id : null,
|
|
||||||
null
|
|
||||||
);
|
|
||||||
|
|
||||||
return tnode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransposedNode transposeCellNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
CellNode node,
|
|
||||||
int rowIndex,
|
|
||||||
int cellIndex,
|
|
||||||
Cell cell) {
|
|
||||||
|
|
||||||
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
|
|
||||||
|
|
||||||
WritingTransposedNode tnode = null;
|
|
||||||
if (node instanceof CellTopicNode) {
|
|
||||||
if (cell.recon != null &&
|
|
||||||
(cell.recon.judgment == Judgment.Matched ||
|
|
||||||
cell.recon.judgment == Judgment.New)) {
|
|
||||||
|
|
||||||
tnode = new CellTopicTransposedNode(
|
|
||||||
(CellTopicNode) node, rowIndex, cellIndex, cell);
|
|
||||||
}
|
|
||||||
} else if (node instanceof CellValueNode) {
|
|
||||||
tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell);
|
|
||||||
} else if (node instanceof CellKeyNode) {
|
|
||||||
tnode = new CellKeyTransposedNode((CellKeyNode) node, rowIndex, cellIndex, cell);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tnode != null) {
|
|
||||||
tnode.load =
|
|
||||||
(parentNode2 == null || parentNode2.load) &&
|
|
||||||
(link == null || link.load);
|
|
||||||
|
|
||||||
processTransposedNode(tnode, parentNode, link, rowIndex);
|
|
||||||
|
|
||||||
tnode.jsonContextNode = addJsonContext(
|
|
||||||
parentNode2 != null ? parentNode2.jsonContextNode : null,
|
|
||||||
link != null ? link.property.id : null,
|
|
||||||
cell,
|
|
||||||
rowIndex
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return tnode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransposedNode transposeTopicNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
FreebaseTopicNode node,
|
|
||||||
int rowIndex) {
|
|
||||||
|
|
||||||
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
|
|
||||||
WritingTransposedNode tnode = new TopicTransposedNode(node);
|
|
||||||
|
|
||||||
tnode.load =
|
|
||||||
(parentNode2 == null || parentNode2.load) &&
|
|
||||||
(link == null || link.load);
|
|
||||||
|
|
||||||
processTransposedNode(tnode, parentNode, link, rowIndex);
|
|
||||||
|
|
||||||
tnode.jsonContextNode = addJsonContext(
|
|
||||||
parentNode2 != null ? parentNode2.jsonContextNode : null,
|
|
||||||
link != null ? link.property.id : null,
|
|
||||||
node.topic
|
|
||||||
);
|
|
||||||
|
|
||||||
return tnode;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TransposedNode transposeValueNode(
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
ValueNode node,
|
|
||||||
int rowIndex) {
|
|
||||||
|
|
||||||
WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
|
|
||||||
WritingTransposedNode tnode = new ValueTransposedNode(node);
|
|
||||||
|
|
||||||
tnode.load =
|
|
||||||
(parentNode2 == null || parentNode2.load) &&
|
|
||||||
(link == null || link.load);
|
|
||||||
|
|
||||||
processTransposedNode(tnode, parentNode, link, rowIndex);
|
|
||||||
|
|
||||||
tnode.jsonContextNode = addJsonContext(
|
|
||||||
parentNode2 != null ? parentNode2.jsonContextNode : null,
|
|
||||||
link != null ? link.property.id : null,
|
|
||||||
node.value
|
|
||||||
);
|
|
||||||
|
|
||||||
return tnode;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void processTransposedNode(
|
|
||||||
WritingTransposedNode tnode,
|
|
||||||
TransposedNode parentNode,
|
|
||||||
Link link,
|
|
||||||
int rowIndex
|
|
||||||
) {
|
|
||||||
if (parentNode != null) {
|
|
||||||
if (parentNode instanceof TransposedNodeWithChildren) {
|
|
||||||
TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode;
|
|
||||||
parentNode2.rowIndices.add(rowIndex);
|
|
||||||
parentNode2.children.add(tnode);
|
|
||||||
parentNode2.links.add(link);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
addRootNode(tnode, rowIndex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected JSONObject addJsonContext(JSONObject parent, String key, Object value) {
|
|
||||||
JSONObject o = new JSONObject();
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (value instanceof FreebaseTopic) {
|
|
||||||
FreebaseTopic topic = (FreebaseTopic) value;
|
|
||||||
o.put("id", topic.id);
|
|
||||||
o.put("name", topic.name);
|
|
||||||
} else {
|
|
||||||
o.put("v", value);
|
|
||||||
}
|
|
||||||
} catch (JSONException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
|
|
||||||
connectJsonContext(parent, o, key);
|
|
||||||
return o;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) {
|
|
||||||
JSONObject o = new JSONObject();
|
|
||||||
|
|
||||||
connectJsonContext(parent, o, key);
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (cell != null) {
|
|
||||||
o.put("v", cell.value);
|
|
||||||
if (cell.recon != null) {
|
|
||||||
o.put("recon", "rec" + cell.recon.id);
|
|
||||||
|
|
||||||
if (cell.recon.judgment == Judgment.Matched) {
|
|
||||||
o.put("id", cell.recon.match.id);
|
|
||||||
o.put("name", cell.recon.match.name);
|
|
||||||
}
|
|
||||||
|
|
||||||
// qa:display_context
|
|
||||||
{
|
|
||||||
StringBuffer sb2 = new StringBuffer();
|
|
||||||
|
|
||||||
sb2.append("{ \"ignore\" : true, \"s\" : \"rec");
|
|
||||||
sb2.append(Long.toString(cell.recon.id));
|
|
||||||
sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx");
|
|
||||||
sb2.append(Long.toString(contextID));
|
|
||||||
sb2.append("\", \"meta\" : { \"row\" : ");
|
|
||||||
sb2.append(Integer.toString(rowIndex));
|
|
||||||
sb2.append(" } }");
|
|
||||||
|
|
||||||
writeLine(sb2.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (JSONException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
|
|
||||||
return o;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void connectJsonContext(JSONObject parent, JSONObject o, String key) {
|
|
||||||
try {
|
|
||||||
if (parent == null) {
|
|
||||||
contextTreeRoot = o;
|
|
||||||
} else {
|
|
||||||
JSONArray a = null;
|
|
||||||
if (parent.has(key)) {
|
|
||||||
a = parent.getJSONArray(key);
|
|
||||||
} else {
|
|
||||||
a = new JSONArray();
|
|
||||||
parent.put(key, a);
|
|
||||||
}
|
|
||||||
|
|
||||||
a.put(o);
|
|
||||||
}
|
|
||||||
} catch (JSONException e) {
|
|
||||||
// ignore
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void addRootNode(WritingTransposedNode tnode, int rowIndex) {
|
|
||||||
if (lastRootNode != null) {
|
|
||||||
lastRootNode.write(null, null, project, -1, -1, null);
|
|
||||||
writeContextTreeNode();
|
|
||||||
}
|
|
||||||
lastRootNode = tnode;
|
|
||||||
|
|
||||||
contextTreeRoot = null;
|
|
||||||
contextRowIndex = rowIndex;
|
|
||||||
contextRefCount = 0;
|
|
||||||
contextID++;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void writeContextTreeNode() {
|
|
||||||
if (contextTreeRoot != null && contextRefCount > 0) {
|
|
||||||
StringBuffer sb = new StringBuffer();
|
|
||||||
|
|
||||||
sb.append("{ \"ignore\" : true, \"s\" : \"ctx");
|
|
||||||
sb.append(Long.toString(contextID));
|
|
||||||
sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : ");
|
|
||||||
sb.append(Integer.toString(contextRowIndex));
|
|
||||||
sb.append(", \"data\" : ");
|
|
||||||
sb.append(contextTreeRoot.toString());
|
|
||||||
sb.append(" } }");
|
|
||||||
|
|
||||||
writeLine(sb.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,239 +0,0 @@
|
|||||||
package com.google.refine.util;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import javax.servlet.http.HttpServletRequest;
|
|
||||||
|
|
||||||
import oauth.signpost.OAuthConsumer;
|
|
||||||
import oauth.signpost.exception.OAuthCommunicationException;
|
|
||||||
import oauth.signpost.exception.OAuthExpectationFailedException;
|
|
||||||
import oauth.signpost.exception.OAuthMessageSignerException;
|
|
||||||
|
|
||||||
import org.apache.http.Header;
|
|
||||||
import org.apache.http.HttpResponse;
|
|
||||||
import org.apache.http.NameValuePair;
|
|
||||||
import org.apache.http.client.ClientProtocolException;
|
|
||||||
import org.apache.http.client.HttpClient;
|
|
||||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
|
||||||
import org.apache.http.client.methods.HttpGet;
|
|
||||||
import org.apache.http.client.methods.HttpPost;
|
|
||||||
import org.apache.http.impl.client.DefaultHttpClient;
|
|
||||||
import org.apache.http.message.BasicNameValuePair;
|
|
||||||
import org.apache.http.params.CoreProtocolPNames;
|
|
||||||
import org.apache.http.util.EntityUtils;
|
|
||||||
import org.json.JSONException;
|
|
||||||
import org.json.JSONObject;
|
|
||||||
|
|
||||||
import com.google.refine.ProjectManager;
|
|
||||||
import com.google.refine.RefineServlet;
|
|
||||||
import com.google.refine.oauth.Credentials;
|
|
||||||
import com.google.refine.oauth.OAuthUtilities;
|
|
||||||
import com.google.refine.oauth.Provider;
|
|
||||||
|
|
||||||
public class FreebaseUtils {
|
|
||||||
|
|
||||||
static final public String FREEBASE_HOST = "www.freebase.com";
|
|
||||||
|
|
||||||
static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
|
|
||||||
|
|
||||||
static final private String AGENT_ID = "/en/google_refine";
|
|
||||||
|
|
||||||
static final private int SAMPLE_SIZE = 300;
|
|
||||||
static final private int JUDGES = 4;
|
|
||||||
|
|
||||||
private static String getUserInfoURL(String host) {
|
|
||||||
return "http://" + host + "/api/service/user_info";
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String getMQLWriteURL(String host) {
|
|
||||||
return "http://" + host + "/api/service/mqlwrite";
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String getMQLReadURL(String host) {
|
|
||||||
return "http://" + host + "/api/service/mqlread";
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String getUserAgent() {
|
|
||||||
return RefineServlet.FULLNAME;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String getUserInfo(Credentials credentials, Provider provider)
|
|
||||||
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
|
|
||||||
|
|
||||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
|
||||||
|
|
||||||
HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
|
|
||||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
|
||||||
|
|
||||||
// this is required by the Metaweb API to avoid XSS
|
|
||||||
httpRequest.setHeader("X-Requested-With", "1");
|
|
||||||
|
|
||||||
// sign the request with the oauth library
|
|
||||||
consumer.sign(httpRequest);
|
|
||||||
|
|
||||||
// execute the request
|
|
||||||
HttpClient httpClient = new DefaultHttpClient();
|
|
||||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
|
||||||
|
|
||||||
// return the results
|
|
||||||
return EntityUtils.toString(httpResponse.getEntity());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String getUserBadges(Provider provider, String user_id)
|
|
||||||
throws ClientProtocolException, IOException, JSONException {
|
|
||||||
|
|
||||||
String query = "{" +
|
|
||||||
"'id' : '" + user_id + "'," +
|
|
||||||
"'!/type/usergroup/member' : [{" +
|
|
||||||
"'id' : null," +
|
|
||||||
"'key' : [{" +
|
|
||||||
"'namespace' : null" +
|
|
||||||
"}]" +
|
|
||||||
"}]" +
|
|
||||||
"}".replace("'", "\"");
|
|
||||||
|
|
||||||
return mqlread(provider, query);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String mqlread(Provider provider, String query)
|
|
||||||
throws ClientProtocolException, IOException, JSONException {
|
|
||||||
|
|
||||||
JSONObject envelope = new JSONObject();
|
|
||||||
envelope.put("query", new JSONObject(query));
|
|
||||||
|
|
||||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
|
||||||
formparams.add(new BasicNameValuePair("query", envelope.toString()));
|
|
||||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
|
||||||
|
|
||||||
HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
|
|
||||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
|
||||||
httpRequest.setEntity(entity);
|
|
||||||
|
|
||||||
// this is required by the Metaweb API to avoid XSS
|
|
||||||
httpRequest.setHeader("X-Requested-With", "1");
|
|
||||||
|
|
||||||
// execute the request
|
|
||||||
HttpClient httpClient = new DefaultHttpClient();
|
|
||||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
|
||||||
|
|
||||||
// return the results
|
|
||||||
return EntityUtils.toString(httpResponse.getEntity());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String mqlwrite(Credentials credentials, Provider provider, String query)
|
|
||||||
throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
|
|
||||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
|
||||||
|
|
||||||
JSONObject envelope = new JSONObject();
|
|
||||||
envelope.put("query", new JSONObject(query));
|
|
||||||
|
|
||||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
|
||||||
formparams.add(new BasicNameValuePair("query", envelope.toString()));
|
|
||||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
|
||||||
|
|
||||||
HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
|
|
||||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
|
||||||
httpRequest.setEntity(entity);
|
|
||||||
|
|
||||||
// this is required by the Metaweb API to avoid XSS
|
|
||||||
httpRequest.setHeader("X-Requested-With", "1");
|
|
||||||
|
|
||||||
// sign the request with the oauth library
|
|
||||||
consumer.sign(httpRequest);
|
|
||||||
|
|
||||||
// execute the request
|
|
||||||
HttpClient httpClient = new DefaultHttpClient();
|
|
||||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
|
||||||
|
|
||||||
// return the results
|
|
||||||
return EntityUtils.toString(httpResponse.getEntity());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String getTweezersParams(int sample_size, int judges) {
|
|
||||||
String o = "{" +
|
|
||||||
"'sample_size':" + sample_size + "," +
|
|
||||||
"'votes':{" +
|
|
||||||
"'reconciled':" + judges + "," +
|
|
||||||
"'invalid':" + judges + "," +
|
|
||||||
"'new':" + judges + "," +
|
|
||||||
"'skip':" + (judges + 2) +
|
|
||||||
"}" +
|
|
||||||
"}";
|
|
||||||
return o.replace('\'', '"');
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String uploadTriples(
|
|
||||||
HttpServletRequest request,
|
|
||||||
String qa,
|
|
||||||
String source_name,
|
|
||||||
String source_id,
|
|
||||||
String mdo_id,
|
|
||||||
String triples
|
|
||||||
) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
|
|
||||||
|
|
||||||
Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
|
|
||||||
|
|
||||||
Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
|
|
||||||
|
|
||||||
JSONObject mdo_info = new JSONObject();
|
|
||||||
mdo_info.put("name", source_name);
|
|
||||||
if (source_id != null) {
|
|
||||||
mdo_info.put("info_source",source_id);
|
|
||||||
}
|
|
||||||
|
|
||||||
JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
|
|
||||||
if (user_info.has("username")) {
|
|
||||||
|
|
||||||
List<NameValuePair> formparams = new ArrayList<NameValuePair>();
|
|
||||||
formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
|
|
||||||
formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
|
|
||||||
formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
|
|
||||||
formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
|
|
||||||
formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
|
|
||||||
formparams.add(new BasicNameValuePair("graphport", "sandbox"));
|
|
||||||
formparams.add(new BasicNameValuePair("payload", triples));
|
|
||||||
formparams.add(new BasicNameValuePair("check_params", "false"));
|
|
||||||
if (mdo_id != null) {
|
|
||||||
formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
|
|
||||||
}
|
|
||||||
if (Boolean.parseBoolean(qa)) {
|
|
||||||
formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES)));
|
|
||||||
}
|
|
||||||
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
|
|
||||||
|
|
||||||
HttpPost httpRequest = new HttpPost(getFreeQUrl());
|
|
||||||
httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
|
|
||||||
httpRequest.setEntity(entity);
|
|
||||||
|
|
||||||
HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
|
|
||||||
surrogateRequest.setEntity(entity);
|
|
||||||
|
|
||||||
OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
|
|
||||||
|
|
||||||
consumer.sign(surrogateRequest);
|
|
||||||
|
|
||||||
Header[] h = surrogateRequest.getHeaders("Authorization");
|
|
||||||
if (h.length > 0) {
|
|
||||||
httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
|
|
||||||
} else {
|
|
||||||
throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
|
|
||||||
}
|
|
||||||
|
|
||||||
// execute the request
|
|
||||||
HttpClient httpClient = new DefaultHttpClient();
|
|
||||||
HttpResponse httpResponse = httpClient.execute(httpRequest);
|
|
||||||
|
|
||||||
// return the results
|
|
||||||
return EntityUtils.toString(httpResponse.getEntity());
|
|
||||||
} else {
|
|
||||||
throw new RuntimeException("Invalid credentials");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static public String getFreeQUrl() {
|
|
||||||
String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
|
|
||||||
return url != null ? url : FREEQ_URL;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue
Block a user