diff --git a/main/src/com/google/refine/commands/auth/AuthorizeCommand.java b/main/src/com/google/refine/commands/auth/AuthorizeCommand.java deleted file mode 100644 index 04a896b93..000000000 --- a/main/src/com/google/refine/commands/auth/AuthorizeCommand.java +++ /dev/null @@ -1,135 +0,0 @@ -package com.google.refine.commands.auth; - -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URI; -import java.net.URISyntaxException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import oauth.signpost.OAuthConsumer; -import oauth.signpost.OAuthProvider; - -import com.google.refine.commands.Command; -import com.google.refine.oauth.Credentials; -import com.google.refine.oauth.OAuthUtilities; -import com.google.refine.oauth.Provider; - -public class AuthorizeCommand extends Command { - - private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier"; - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - - // get the provider from the request - Provider provider = OAuthUtilities.getProvider(request); - - try { - - // see if the request comes with access credentials - Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); - - // prepare the continuation URL that the OAuth provider will redirect the user to - // (we need to make sure this URL points back to this code or the dance will never complete) - String callbackURL = getBaseURL(request,provider); - - if (access_credentials == null) { - // access credentials are not available so we need to check - // to see at what stage of the OAuth dance we are - - // get the request token credentials - Credentials request_credentials = Credentials.getCredentials(request, provider, Credentials.Type.REQUEST); - - OAuthConsumer consumer = OAuthUtilities.getConsumer(request_credentials, provider); - OAuthProvider pp = OAuthUtilities.getOAuthProvider(provider); - - if (request_credentials == null) { - // no credentials were found, so let's start the dance - - // get the request token - - String url = pp.retrieveRequestToken(consumer, callbackURL); - - request_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider); - - // and set them to that we can retrieve them later in the second part of the dance - Credentials.setCredentials(request, response, request_credentials, Credentials.Type.REQUEST, 3600); - - // now redirect the user to the Authorize URL where she can authenticate against the - // service provider and authorize us. - // The provider will bounce the user back here for us to continue the dance. - - response.sendRedirect(url); - } else { - // we are at the second stage of the dance, so we need need to obtain the access credentials now - - // if we got here, it means that the user performed a valid authentication against the - // service provider and authorized us, so now we can request more permanent credentials - // to the service provider and save those as well for later use. - - // this is set only for OAuth 1.0a - String verificationCode = request.getParameter(OAUTH_VERIFIER_PARAM); - - pp.retrieveAccessToken(consumer, verificationCode); - - access_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider); - - // no matter the result, we need to remove the request token - Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST); - - Credentials.setCredentials(request, response, access_credentials, Credentials.Type.ACCESS, 30 * 24 * 3600); - - finish(response); - } - } else { - finish(response); - } - } catch (Exception e) { - Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST); - Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS); - respondException(response, e); - } - } - - private void finish(HttpServletResponse response) throws IOException { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "text/html"); - - PrintWriter writer = response.getWriter(); - writer.write( - "" + - "" + - "" + - "" - ); - writer.flush(); - } - - private String getBaseURL(HttpServletRequest request, Provider provider) { - String host = request.getHeader("host"); - if (host == null) { - String referrer = request.getHeader("referer"); - if (referrer != null) { - URI url; - try { - url = new URI(referrer); - int port = url.getPort(); - host = url.getHost() + ((port > -1) ? ":" + url.getPort() : ""); - } catch (URISyntaxException e) { - throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL"); - } - } else { - throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL Google Refine is listening to."); - } - } - return "http://" + host + "/command/core/authorize/" + provider.getHost(); - } -} diff --git a/main/src/com/google/refine/commands/auth/CheckAuthorizationCommand.java b/main/src/com/google/refine/commands/auth/CheckAuthorizationCommand.java deleted file mode 100644 index e6dd3f3d9..000000000 --- a/main/src/com/google/refine/commands/auth/CheckAuthorizationCommand.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.google.refine.commands.auth; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.commands.Command; -import com.google.refine.oauth.Credentials; -import com.google.refine.oauth.OAuthUtilities; -import com.google.refine.oauth.Provider; -import com.google.refine.util.FreebaseUtils; - -public class CheckAuthorizationCommand extends Command { - - final static Logger logger = LoggerFactory.getLogger("check-authorization_command"); - - public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - - try { - Provider provider = OAuthUtilities.getProvider(request); - - // this cookie should not be there, but this is good hygiene practice - Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST); - - Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); - - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - if (access_credentials != null) { - String user_info = FreebaseUtils.getUserInfo(access_credentials, provider); - response.getWriter().write(user_info); - } else { - respond(response, "401 Unauthorized", "You don't have the right credentials"); - } - } catch (Exception e) { - logger.info("error",e); - respondException(response, e); - } - } - -} diff --git a/main/src/com/google/refine/commands/auth/DeAuthorizeCommand.java b/main/src/com/google/refine/commands/auth/DeAuthorizeCommand.java deleted file mode 100644 index 668f07c4b..000000000 --- a/main/src/com/google/refine/commands/auth/DeAuthorizeCommand.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.google.refine.commands.auth; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import com.google.refine.commands.Command; -import com.google.refine.oauth.Credentials; -import com.google.refine.oauth.OAuthUtilities; -import com.google.refine.oauth.Provider; - -public class DeAuthorizeCommand extends Command { - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - - try { - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - - Provider provider = OAuthUtilities.getProvider(request); - - Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS); - - respond(response, "200 OK", ""); - } catch (Exception e) { - respondException(response, e); - } - } -} diff --git a/main/src/com/google/refine/commands/auth/GetUserBadgesCommand.java b/main/src/com/google/refine/commands/auth/GetUserBadgesCommand.java deleted file mode 100644 index 66b2289a3..000000000 --- a/main/src/com/google/refine/commands/auth/GetUserBadgesCommand.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.google.refine.commands.auth; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.refine.commands.Command; -import com.google.refine.oauth.OAuthUtilities; -import com.google.refine.oauth.Provider; -import com.google.refine.util.FreebaseUtils; - -public class GetUserBadgesCommand extends Command { - - final static Logger logger = LoggerFactory.getLogger("get-version_command"); - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - - try { - Provider provider = OAuthUtilities.getProvider(request); - String user_id = request.getParameter("user_id"); - response.setCharacterEncoding("UTF-8"); - response.setHeader("Content-Type", "application/json"); - String user_badges = FreebaseUtils.getUserBadges(provider, user_id); - response.getWriter().write(user_badges); - } catch (Exception e) { - logger.info("error",e); - respondException(response, e); - } - } - -} diff --git a/main/src/com/google/refine/protograph/AnonymousNode.java b/main/src/com/google/refine/protograph/AnonymousNode.java deleted file mode 100644 index d489c7eec..000000000 --- a/main/src/com/google/refine/protograph/AnonymousNode.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.google.refine.protograph; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class AnonymousNode implements Node, NodeWithLinks { - final public FreebaseType type; - final public List links = new LinkedList(); - - public AnonymousNode(FreebaseType type) { - this.type = type; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("anonymous"); - writer.key("type"); type.write(writer, options); - if (links != null) { - writer.key("links"); writer.array(); - for (Link link : links) { - link.write(writer, options); - } - writer.endArray(); - } - writer.endObject(); - } - - public void addLink(Link link) { - links.add(link); - } - - public Link getLink(int index) { - return links.get(index); - } - - public int getLinkCount() { - return links.size(); - } -} diff --git a/main/src/com/google/refine/protograph/BooleanColumnCondition.java b/main/src/com/google/refine/protograph/BooleanColumnCondition.java deleted file mode 100644 index a70c012cf..000000000 --- a/main/src/com/google/refine/protograph/BooleanColumnCondition.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.google.refine.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.google.refine.model.Column; -import com.google.refine.model.Project; -import com.google.refine.model.Row; - - -public class BooleanColumnCondition implements Condition { - final public String columnName; - - public BooleanColumnCondition(String columnName) { - this.columnName = columnName; - } - - @Override - public boolean test(Project project, int rowIndex, Row row) { - Column column = project.columnModel.getColumnByName(columnName); - if (column != null) { - Object o = row.getCellValue(column.getCellIndex()); - if (o != null) { - if (o instanceof Boolean) { - return ((Boolean) o).booleanValue(); - } else { - return Boolean.parseBoolean(o.toString()); - } - } - } - return false; - } - - @Override - public void write(JSONWriter writer, Properties options) throws JSONException { - writer.object(); - writer.key("columnName"); writer.value(columnName); - writer.endObject(); - } -} diff --git a/main/src/com/google/refine/protograph/CellKeyNode.java b/main/src/com/google/refine/protograph/CellKeyNode.java deleted file mode 100644 index 5ce182d0b..000000000 --- a/main/src/com/google/refine/protograph/CellKeyNode.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.google.refine.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class CellKeyNode extends CellNode { - final public FreebaseTopic namespace; - - public CellKeyNode( - FreebaseTopic namespace - ) { - this.namespace = namespace; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("cell-as-key"); - - writer.key("columnNames"); - writer.array(); - for (String name : columnNames) { - writer.value(name); - } - writer.endArray(); - - writer.key("namespace"); namespace.write(writer, options); - writer.endObject(); - } -} diff --git a/main/src/com/google/refine/protograph/CellNode.java b/main/src/com/google/refine/protograph/CellNode.java deleted file mode 100644 index 1e42aed56..000000000 --- a/main/src/com/google/refine/protograph/CellNode.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.google.refine.protograph; - -import java.util.LinkedList; -import java.util.List; - -abstract public class CellNode implements Node { - final public List columnNames = new LinkedList(); -} diff --git a/main/src/com/google/refine/protograph/CellTopicNode.java b/main/src/com/google/refine/protograph/CellTopicNode.java deleted file mode 100644 index cb42ef8a4..000000000 --- a/main/src/com/google/refine/protograph/CellTopicNode.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.google.refine.protograph; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class CellTopicNode extends CellNode implements NodeWithLinks { - final public FreebaseType type; - final public List links = new LinkedList(); - - public CellTopicNode( - FreebaseType type - ) { - this.type = type; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("cell-as-topic"); - writer.key("columnNames"); - writer.array(); - for (String name : columnNames) { - writer.value(name); - } - writer.endArray(); - if (type != null) { - writer.key("type"); type.write(writer, options); - } - if (links != null) { - writer.key("links"); writer.array(); - for (Link link : links) { - link.write(writer, options); - } - writer.endArray(); - } - - writer.endObject(); - } - - public void addLink(Link link) { - links.add(link); - } - - public Link getLink(int index) { - return links.get(index); - } - - public int getLinkCount() { - return links.size(); - } -} diff --git a/main/src/com/google/refine/protograph/CellValueNode.java b/main/src/com/google/refine/protograph/CellValueNode.java deleted file mode 100644 index efd02bf20..000000000 --- a/main/src/com/google/refine/protograph/CellValueNode.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.google.refine.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class CellValueNode extends CellNode { - final public String valueType; - final public String lang; - - public CellValueNode( - String valueType, - String lang - ) { - this.valueType = valueType; - this.lang = lang; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("cell-as-value"); - writer.key("columnNames"); - writer.array(); - for (String name : columnNames) { - writer.value(name); - } - writer.endArray(); - writer.key("valueType"); writer.value(valueType); - writer.key("lang"); writer.value(lang); - writer.endObject(); - } - -} diff --git a/main/src/com/google/refine/protograph/Condition.java b/main/src/com/google/refine/protograph/Condition.java deleted file mode 100644 index 11f2f8d00..000000000 --- a/main/src/com/google/refine/protograph/Condition.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.google.refine.protograph; - -import com.google.refine.Jsonizable; -import com.google.refine.model.Project; -import com.google.refine.model.Row; - -public interface Condition extends Jsonizable { - public boolean test(Project project, int rowIndex, Row row); -} diff --git a/main/src/com/google/refine/protograph/FreebaseProperty.java b/main/src/com/google/refine/protograph/FreebaseProperty.java deleted file mode 100644 index 9a9110350..000000000 --- a/main/src/com/google/refine/protograph/FreebaseProperty.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.google.refine.protograph; - -public class FreebaseProperty extends FreebaseTopic { - //final protected FreebaseType _expectedType; - - public FreebaseProperty(String id, String name) { - super(id, name); - } -} diff --git a/main/src/com/google/refine/protograph/FreebaseTopic.java b/main/src/com/google/refine/protograph/FreebaseTopic.java deleted file mode 100644 index dd0bc5bba..000000000 --- a/main/src/com/google/refine/protograph/FreebaseTopic.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.google.refine.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.google.refine.Jsonizable; - -public class FreebaseTopic implements Jsonizable { - final public String id; - final public String name; - - public FreebaseTopic(String id, String name) { - this.id = id; - this.name = name; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(id); - writer.key("name"); writer.value(name); - writer.endObject(); - } - -} diff --git a/main/src/com/google/refine/protograph/FreebaseTopicNode.java b/main/src/com/google/refine/protograph/FreebaseTopicNode.java deleted file mode 100644 index 7d2774b9f..000000000 --- a/main/src/com/google/refine/protograph/FreebaseTopicNode.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.google.refine.protograph; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class FreebaseTopicNode implements Node, NodeWithLinks { - final public FreebaseTopic topic; - final public List links = new LinkedList(); - - public FreebaseTopicNode(FreebaseTopic topic) { - this.topic = topic; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("topic"); - writer.key("topic"); topic.write(writer, options); - if (links != null) { - writer.key("links"); writer.array(); - for (Link link : links) { - link.write(writer, options); - } - writer.endArray(); - } - - writer.endObject(); - } - - public void addLink(Link link) { - links.add(link); - } - - public Link getLink(int index) { - return links.get(index); - } - - public int getLinkCount() { - return links.size(); - } -} diff --git a/main/src/com/google/refine/protograph/FreebaseType.java b/main/src/com/google/refine/protograph/FreebaseType.java deleted file mode 100644 index 67282c71d..000000000 --- a/main/src/com/google/refine/protograph/FreebaseType.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.google.refine.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.google.refine.Jsonizable; - -public class FreebaseType extends FreebaseTopic implements Jsonizable { - public FreebaseType(String id, String name) { - super(id, name); - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("id"); writer.value(id); - writer.key("name"); writer.value(name); - writer.endObject(); - } - - static public FreebaseType load(JSONObject obj) throws Exception { - if (obj == null) { - return null; - } - - FreebaseType type = new FreebaseType( - obj.getString("id"), - obj.getString("name") - ); - return type; - } -} diff --git a/main/src/com/google/refine/protograph/Link.java b/main/src/com/google/refine/protograph/Link.java deleted file mode 100644 index fb0813352..000000000 --- a/main/src/com/google/refine/protograph/Link.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.google.refine.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -import com.google.refine.Jsonizable; - -public class Link implements Jsonizable { - final public FreebaseProperty property; - final public Node target; - final public Condition condition; - final public boolean load; - - public Link(FreebaseProperty property, Node target, Condition condition, boolean load) { - this.property = property; - this.target = target; - this.condition = condition; - this.load = load; - } - - public FreebaseProperty getProperty() { - return property; - } - - public Node getTarget() { - return target; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("property"); property.write(writer, options); - if (target != null) { - writer.key("target"); - target.write(writer, options); - } - if (condition != null) { - writer.key("condition"); - condition.write(writer, options); - } - writer.endObject(); - } - -} diff --git a/main/src/com/google/refine/protograph/Node.java b/main/src/com/google/refine/protograph/Node.java deleted file mode 100644 index 5ccd8b598..000000000 --- a/main/src/com/google/refine/protograph/Node.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.google.refine.protograph; - -import com.google.refine.Jsonizable; - -public interface Node extends Jsonizable { -} diff --git a/main/src/com/google/refine/protograph/NodeWithLinks.java b/main/src/com/google/refine/protograph/NodeWithLinks.java deleted file mode 100644 index ed32a3b73..000000000 --- a/main/src/com/google/refine/protograph/NodeWithLinks.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.google.refine.protograph; - -public interface NodeWithLinks { - public void addLink(Link link); - - public int getLinkCount(); - - public Link getLink(int index); -} diff --git a/main/src/com/google/refine/protograph/Protograph.java b/main/src/com/google/refine/protograph/Protograph.java deleted file mode 100644 index 4b6eada0c..000000000 --- a/main/src/com/google/refine/protograph/Protograph.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.google.refine.protograph; - -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.google.refine.model.OverlayModel; -import com.google.refine.model.Project; - -public class Protograph implements OverlayModel { - final protected List _rootNodes = new LinkedList(); - - public int getRootNodeCount() { - return _rootNodes.size(); - } - - public Node getRootNode(int index) { - return _rootNodes.get(index); - } - - @Override - public void onBeforeSave(Project project) { - } - - @Override - public void onAfterSave(Project project) { - } - - - @Override - public void dispose(Project project) { - } - - static public Protograph reconstruct(JSONObject o) throws JSONException { - Protograph g = new Protograph(); - - JSONArray rootNodes = o.getJSONArray("rootNodes"); - int count = rootNodes.length(); - - for (int i = 0; i < count; i++) { - JSONObject o2 = rootNodes.getJSONObject(i); - Node node = reconstructNode(o2); - if (node != null) { - g._rootNodes.add(node); - } - } - - return g; - } - - static protected Node reconstructNode(JSONObject o) throws JSONException { - Node node = null; - - String nodeType = o.getString("nodeType"); - if (nodeType.startsWith("cell-as-")) { - if ("cell-as-topic".equals(nodeType)) { - if (o.has("type")) { - node = new CellTopicNode( - reconstructType(o.getJSONObject("type")) - ); - } - } else if ("cell-as-value".equals(nodeType)) { - node = new CellValueNode( - o.getString("valueType"), - o.getString("lang") - ); - } else if ("cell-as-key".equals(nodeType)) { - node = new CellKeyNode( - reconstructTopic(o.getJSONObject("namespace")) - ); - } - - if (o.has("columnName") && !o.isNull("columnName")) { - ((CellNode) node).columnNames.add(o.getString("columnName")); - } - if (o.has("columnNames") && !o.isNull("columnNames")) { - JSONArray columnNames = o.getJSONArray("columnNames"); - int count = columnNames.length(); - - for (int c = 0; c < count; c++) { - ((CellNode) node).columnNames.add(columnNames.getString(c)); - } - } - } else if ("topic".equals(nodeType)) { - node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic"))); - } else if ("value".equals(nodeType)) { - node = new ValueNode( - o.get("value"), - o.getString("valueType"), - o.getString("lang") - ); - } else if ("anonymous".equals(nodeType)) { - node = new AnonymousNode(reconstructType(o.getJSONObject("type"))); - } - - if (node != null && node instanceof NodeWithLinks && o.has("links")) { - NodeWithLinks node2 = (NodeWithLinks) node; - - JSONArray links = o.getJSONArray("links"); - int linkCount = links.length(); - - for (int j = 0; j < linkCount; j++) { - JSONObject oLink = links.getJSONObject(j); - Condition condition = null; - - if (oLink.has("condition") && !oLink.isNull("condition")) { - JSONObject oCondition = oLink.getJSONObject("condition"); - if (oCondition.has("columnName") && !oCondition.isNull("columnName")) { - condition = new BooleanColumnCondition(oCondition.getString("columnName")); - } - } - - node2.addLink(new Link( - reconstructProperty(oLink.getJSONObject("property")), - oLink.has("target") && !oLink.isNull("target") ? - reconstructNode(oLink.getJSONObject("target")) : null, - condition, - oLink.has("load") && !oLink.isNull("load") ? - oLink.getBoolean("load") : true - )); - } - } - - return node; - } - - static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException { - return new FreebaseProperty( - o.getString("id"), - o.getString("name") - ); - } - - static protected FreebaseType reconstructType(JSONObject o) throws JSONException { - return new FreebaseType( - o.getString("id"), - o.getString("name") - ); - } - - static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException { - return new FreebaseTopic( - o.getString("id"), - o.getString("name") - ); - } - - public void write(JSONWriter writer, Properties options) throws JSONException { - writer.object(); - writer.key("rootNodes"); writer.array(); - - for (Node node : _rootNodes) { - node.write(writer, options); - } - - writer.endArray(); - writer.endObject(); - } - - static public Protograph load(Project project, JSONObject obj) throws Exception { - return reconstruct(obj); - } -} diff --git a/main/src/com/google/refine/protograph/ValueNode.java b/main/src/com/google/refine/protograph/ValueNode.java deleted file mode 100644 index ced705021..000000000 --- a/main/src/com/google/refine/protograph/ValueNode.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.google.refine.protograph; - -import java.util.Properties; - -import org.json.JSONException; -import org.json.JSONWriter; - -public class ValueNode implements Node { - final public Object value; - final public String valueType; - final public String lang; - - public ValueNode(Object value, String valueType, String lang) { - this.value = value; - this.valueType = valueType; - this.lang = lang; - } - - public void write(JSONWriter writer, Properties options) - throws JSONException { - - writer.object(); - writer.key("nodeType"); writer.value("value"); - writer.key("value"); writer.value(value); - writer.key("valueType"); writer.value(valueType); - writer.key("lang"); writer.value(lang); - writer.endObject(); - } -} diff --git a/main/src/com/google/refine/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java b/main/src/com/google/refine/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java deleted file mode 100644 index a03c743fd..000000000 --- a/main/src/com/google/refine/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java +++ /dev/null @@ -1,346 +0,0 @@ -package com.google.refine.protograph.transpose; - -import java.io.IOException; -import java.io.Writer; -import java.util.LinkedList; -import java.util.List; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONWriter; - -import com.google.refine.model.Cell; -import com.google.refine.model.Recon; -import com.google.refine.protograph.AnonymousNode; -import com.google.refine.protograph.CellKeyNode; -import com.google.refine.protograph.CellNode; -import com.google.refine.protograph.CellTopicNode; -import com.google.refine.protograph.CellValueNode; -import com.google.refine.protograph.FreebaseProperty; -import com.google.refine.protograph.FreebaseTopicNode; -import com.google.refine.protograph.Link; -import com.google.refine.protograph.ValueNode; -import com.google.refine.util.JSONUtilities; - -public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory { - protected Writer writer; - protected List rootObjects = new LinkedList(); - - private static final String TYPE = "type"; - private static final String ID = "id"; - private static final String NAME = "name"; - private static final String CREATE = "create"; - private static final String VALUE = "value"; - private static final String CONNECT = "connect"; - private static final String LANG = "lang"; - - public MqlwriteLikeTransposedNodeFactory(Writer writer) { - this.writer = writer; - } - - protected JSONArray getJSON() { - return new JSONArray(rootObjects); - } - - @Override - public void flush() throws IOException { - try { - JSONWriter jsonWriter = new JSONWriter(writer); - - jsonWriter.array(); - for (JSONObject obj : rootObjects) { - jsonWriter.value(obj); - } - jsonWriter.endArray(); - - } catch (JSONException e) { - e.printStackTrace(); - } - writer.flush(); - } - - abstract protected class JsonTransposedNode implements TransposedNode { - abstract public Object getJSON(); - } - - abstract protected class JsonObjectTransposedNode extends JsonTransposedNode { - abstract public JSONObject getJSONObject(); - - protected JSONObject obj; - - public Object getJSON() { - return getJSONObject(); - } - } - - protected class AnonymousTransposedNode extends JsonObjectTransposedNode { - JsonObjectTransposedNode parent; - FreebaseProperty property; - AnonymousNode node; - - protected AnonymousTransposedNode( - JsonObjectTransposedNode parent, - FreebaseProperty property, - AnonymousNode node - ) { - this.parent = parent; - this.property = property; - this.node = node; - } - - public JSONObject getJSONObject() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(TYPE, this.node.type.id); - obj.put(ID, (String) null); - obj.put(CREATE, "unconditional"); - } catch (JSONException e) { - e.printStackTrace(); - } - - linkTransposedNodeJSON(obj, parent, property); - } - - return obj; - } - } - - protected class CellTopicTransposedNode extends JsonObjectTransposedNode { - protected CellTopicNode node; - protected Cell cell; - - public CellTopicTransposedNode(CellTopicNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - @Override - public JSONObject getJSONObject() { - if (obj == null) { - obj = new JSONObject(); - try { - if (cell.recon != null && - cell.recon.judgment == Recon.Judgment.Matched && - cell.recon.match != null) { - obj.put(ID, cell.recon.match.id); - } else { - obj.put(ID, (String) null); - obj.put(NAME, cell.value.toString()); - obj.put(TYPE, node.type.id); - obj.put(CREATE, "unless_exists"); - } - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - - protected class CellValueTransposedNode extends JsonTransposedNode { - protected JSONObject obj; - protected CellValueNode node; - protected Cell cell; - - public CellValueTransposedNode(CellValueNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - public Object getJSON() { - if (obj == null) { - obj = new JSONObject(); - try { - JSONUtilities.putField(obj, VALUE, cell.value); - - obj.put(TYPE, node.valueType); - if ("/type/text".equals(node.valueType)) { - obj.put(LANG, node.lang); - } - - obj.put(CONNECT, "insert"); - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - - protected class CellKeyTransposedNode extends JsonTransposedNode { - protected JSONObject obj; - protected CellKeyNode node; - protected Cell cell; - - public CellKeyTransposedNode(CellKeyNode node, Cell cell) { - this.node = node; - this.cell = cell; - } - - public Object getJSON() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(VALUE, cell.value.toString()); - - JSONObject nsObj = new JSONObject(); - nsObj.put(ID, node.namespace.id); - - obj.put("namespace", nsObj); - obj.put(CONNECT, "insert"); - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - - protected class TopicTransposedNode extends JsonObjectTransposedNode { - protected FreebaseTopicNode node; - - public TopicTransposedNode(FreebaseTopicNode node) { - this.node = node; - } - - @Override - public JSONObject getJSONObject() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(ID, node.topic.id); - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - - protected class ValueTransposedNode extends JsonTransposedNode { - protected JSONObject obj; - protected ValueNode node; - - public ValueTransposedNode(ValueNode node) { - this.node = node; - } - - public Object getJSON() { - if (obj == null) { - obj = new JSONObject(); - try { - obj.put(VALUE, node.value); - obj.put(TYPE, node.valueType); - if ("/type/text".equals(node.valueType)) { - obj.put(LANG, node.lang); - } - - obj.put(CONNECT, "insert"); - } catch (JSONException e) { - e.printStackTrace(); - } - } - return obj; - } - } - public TransposedNode transposeAnonymousNode( - TransposedNode parentNode, - Link link, - AnonymousNode node, int rowIndex) { - - return new AnonymousTransposedNode( - parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null, - link != null ? link.property : null, - node - ); - } - - public TransposedNode transposeCellNode( - TransposedNode parentNode, - Link link, - CellNode node, - int rowIndex, - int cellIndex, - Cell cell) { - - JsonTransposedNode tnode = null; - if (node instanceof CellTopicNode) { - tnode = new CellTopicTransposedNode((CellTopicNode) node, cell); - } else if (node instanceof CellValueNode) { - tnode = new CellValueTransposedNode((CellValueNode) node, cell); - } else if (node instanceof CellKeyNode) { - tnode = new CellKeyTransposedNode((CellKeyNode) node, cell); - } - - if (tnode != null) { - processTransposedNode(tnode, parentNode, link != null ? link.property : null); - } - return tnode; - } - - public TransposedNode transposeTopicNode( - TransposedNode parentNode, - Link link, - FreebaseTopicNode node, int rowIndex) { - - JsonTransposedNode tnode = new TopicTransposedNode(node); - - processTransposedNode(tnode, parentNode, link != null ? link.property : null); - - return tnode; - } - - public TransposedNode transposeValueNode( - TransposedNode parentNode, - Link link, - ValueNode node, int rowIndex) { - - JsonTransposedNode tnode = new ValueTransposedNode(node); - - processTransposedNode(tnode, parentNode, link != null ? link.property : null); - - return tnode; - } - - protected void processTransposedNode( - JsonTransposedNode tnode, - TransposedNode parentNode, - FreebaseProperty property - ) { - - if (!(tnode instanceof AnonymousTransposedNode)) { - linkTransposedNodeJSON(tnode.getJSON(), parentNode, property); - } - } - - protected void linkTransposedNodeJSON( - Object obj, - TransposedNode parentNode, - FreebaseProperty property - ) { - - if (parentNode == null) { - if (obj instanceof JSONObject) { - rootObjects.add((JSONObject) obj); - } - } else if (parentNode instanceof JsonTransposedNode) { - JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject(); - - try { - JSONArray a = null; - if (parentObj.has(property.id)) { - a = parentObj.getJSONArray(property.id); - } else { - a = new JSONArray(); - parentObj.put(property.id, a); - } - - a.put(a.length(), obj); - } catch (JSONException e) { - e.printStackTrace(); - } - } - } -} diff --git a/main/src/com/google/refine/protograph/transpose/TransposedNode.java b/main/src/com/google/refine/protograph/transpose/TransposedNode.java deleted file mode 100644 index 437b5b9d8..000000000 --- a/main/src/com/google/refine/protograph/transpose/TransposedNode.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.google.refine.protograph.transpose; - -public interface TransposedNode { -} diff --git a/main/src/com/google/refine/protograph/transpose/TransposedNodeFactory.java b/main/src/com/google/refine/protograph/transpose/TransposedNodeFactory.java deleted file mode 100644 index 92716beba..000000000 --- a/main/src/com/google/refine/protograph/transpose/TransposedNodeFactory.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.google.refine.protograph.transpose; - -import java.io.IOException; - -import com.google.refine.model.Cell; -import com.google.refine.protograph.AnonymousNode; -import com.google.refine.protograph.CellNode; -import com.google.refine.protograph.FreebaseTopicNode; -import com.google.refine.protograph.Link; -import com.google.refine.protograph.ValueNode; - -public interface TransposedNodeFactory { - public TransposedNode transposeAnonymousNode( - TransposedNode parentNode, - Link link, - AnonymousNode node, int rowIndex - ); - - public TransposedNode transposeCellNode( - TransposedNode parentNode, - Link link, - CellNode node, - int rowIndex, - int cellIndex, - Cell cell - ); - - public TransposedNode transposeValueNode( - TransposedNode parentNode, - Link link, - ValueNode node, - int rowIndex - ); - - public TransposedNode transposeTopicNode( - TransposedNode parentNode, - Link link, - FreebaseTopicNode node, - int rowIndex - ); - - public void flush() throws IOException; -} diff --git a/main/src/com/google/refine/protograph/transpose/Transposer.java b/main/src/com/google/refine/protograph/transpose/Transposer.java deleted file mode 100644 index b6f288b4b..000000000 --- a/main/src/com/google/refine/protograph/transpose/Transposer.java +++ /dev/null @@ -1,222 +0,0 @@ -package com.google.refine.protograph.transpose; - -import java.util.LinkedList; -import java.util.List; - -import com.google.refine.browsing.FilteredRows; -import com.google.refine.browsing.RowVisitor; -import com.google.refine.expr.ExpressionUtils; -import com.google.refine.model.Cell; -import com.google.refine.model.Column; -import com.google.refine.model.Project; -import com.google.refine.model.Row; -import com.google.refine.model.Recon.Judgment; -import com.google.refine.protograph.AnonymousNode; -import com.google.refine.protograph.CellNode; -import com.google.refine.protograph.CellTopicNode; -import com.google.refine.protograph.FreebaseTopicNode; -import com.google.refine.protograph.Link; -import com.google.refine.protograph.Node; -import com.google.refine.protograph.NodeWithLinks; -import com.google.refine.protograph.Protograph; -import com.google.refine.protograph.ValueNode; - -public class Transposer { - static public void transpose( - Project project, - FilteredRows filteredRows, - Protograph protograph, - Node rootNode, - TransposedNodeFactory nodeFactory - ) { - transpose(project, filteredRows, protograph, rootNode, nodeFactory, 20); - } - - static public void transpose( - Project project, - FilteredRows filteredRows, - Protograph protograph, - Node rootNode, - TransposedNodeFactory nodeFactory, - int limit - ) { - Context rootContext = new Context(rootNode, null, null, limit); - - filteredRows.accept(project, new RowVisitor() { - Context rootContext; - Protograph protograph; - Node rootNode; - TransposedNodeFactory nodeFactory; - - @Override - public boolean visit(Project project, int rowIndex, Row row) { - if (rootContext.limit <= 0 || rootContext.count < rootContext.limit) { - descend(project, protograph, nodeFactory, rowIndex, row, rootNode, rootContext); - } - - if (rootContext.limit > 0 && rootContext.count > rootContext.limit) { - return true; - } - return false; - } - - @Override - public void start(Project project) { - // TODO Auto-generated method stub - - } - - @Override - public void end(Project project) { - // TODO Auto-generated method stub - - } - - public RowVisitor init( - Context rootContext, - Protograph protograph, - Node rootNode, - TransposedNodeFactory nodeFactory - ) { - this.rootContext = rootContext; - this.protograph = protograph; - this.rootNode = rootNode; - this.nodeFactory = nodeFactory; - - return this; - } - }.init(rootContext, protograph, rootNode, nodeFactory)); - } - - static protected void descend( - Project project, - Protograph protograph, - TransposedNodeFactory nodeFactory, - int rowIndex, - Row row, - Node node, - Context context - ) { - List tnodes = new LinkedList(); - - TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode; - Link link = context.parent == null ? null : context.link; - - if (node instanceof CellNode) { - CellNode node2 = (CellNode) node; - for (String columnName : node2.columnNames) { - Column column = project.columnModel.getColumnByName(columnName); - if (column != null) { - int cellIndex = column.getCellIndex(); - - Cell cell = row.getCell(cellIndex); - if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) { - if (node2 instanceof CellTopicNode && - (cell.recon == null || cell.recon.judgment == Judgment.None)) { - return; - } - - context.count++; - if (context.limit > 0 && context.count > context.limit) { - return; - } - - tnodes.add(nodeFactory.transposeCellNode( - parentNode, - link, - node2, - rowIndex, - cellIndex, - cell - )); - } - } - } - } else { - if (node instanceof AnonymousNode) { - tnodes.add(nodeFactory.transposeAnonymousNode( - parentNode, - link, - (AnonymousNode) node, - rowIndex - )); - } else if (node instanceof FreebaseTopicNode) { - tnodes.add(nodeFactory.transposeTopicNode( - parentNode, - link, - (FreebaseTopicNode) node, - rowIndex - )); - } else if (node instanceof ValueNode) { - tnodes.add(nodeFactory.transposeValueNode( - parentNode, - link, - (ValueNode) node, - rowIndex - )); - } - } - - if (node instanceof NodeWithLinks) { - NodeWithLinks node2 = (NodeWithLinks) node; - int linkCount = node2.getLinkCount(); - - for (int i = 0; i < linkCount; i++) { - Link link2 = node2.getLink(i); - if (link2.condition == null || link2.condition.test(project, rowIndex, row)) { - for (TransposedNode tnode : tnodes) { - context.transposedNode = tnode; - context.nullifySubContextNodes(); - - descend( - project, - protograph, - nodeFactory, - rowIndex, - row, - link2.getTarget(), - context.subContexts.get(i) - ); - } - } - } - } - } - - static class Context { - TransposedNode transposedNode; - List subContexts; - Context parent; - Link link; - int count; - int limit; - - Context(Node node, Context parent, Link link, int limit) { - this.parent = parent; - this.link = link; - this.limit = limit; - - if (node instanceof NodeWithLinks) { - NodeWithLinks node2 = (NodeWithLinks) node; - - int subContextCount = node2.getLinkCount(); - - subContexts = new LinkedList(); - for (int i = 0; i < subContextCount; i++) { - Link link2 = node2.getLink(i); - subContexts.add( - new Context(link2.getTarget(), this, link2, -1)); - } - } - } - - public void nullifySubContextNodes() { - if (subContexts != null) { - for (Context context : subContexts) { - context.transposedNode = null; - context.nullifySubContextNodes(); - } - } - } - } -} diff --git a/main/src/com/google/refine/protograph/transpose/TripleLoaderTransposedNodeFactory.java b/main/src/com/google/refine/protograph/transpose/TripleLoaderTransposedNodeFactory.java deleted file mode 100644 index 753528768..000000000 --- a/main/src/com/google/refine/protograph/transpose/TripleLoaderTransposedNodeFactory.java +++ /dev/null @@ -1,725 +0,0 @@ -package com.google.refine.protograph.transpose; - -import java.io.IOException; -import java.io.Writer; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import com.google.refine.model.Cell; -import com.google.refine.model.Column; -import com.google.refine.model.Project; -import com.google.refine.model.Recon; -import com.google.refine.model.Recon.Judgment; -import com.google.refine.model.recon.ReconConfig; -import com.google.refine.model.recon.StandardReconConfig; -import com.google.refine.protograph.AnonymousNode; -import com.google.refine.protograph.CellKeyNode; -import com.google.refine.protograph.CellNode; -import com.google.refine.protograph.CellTopicNode; -import com.google.refine.protograph.CellValueNode; -import com.google.refine.protograph.FreebaseProperty; -import com.google.refine.protograph.FreebaseTopic; -import com.google.refine.protograph.FreebaseTopicNode; -import com.google.refine.protograph.Link; -import com.google.refine.protograph.ValueNode; - -public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory { - protected Project project; - - protected boolean start = true; - protected Writer writer; - protected WritingTransposedNode lastRootNode; - protected Map varPool = new HashMap(); - protected Map newTopicVars = new HashMap(); - protected Set serializedRecons = new HashSet(); - - protected long contextID = 0; - protected int contextRowIndex; - protected int contextRefCount = 0; - protected JSONObject contextTreeRoot; - - public TripleLoaderTransposedNodeFactory(Project project, Writer writer) { - this.project = project; - this.writer = writer; - } - - @Override - public void flush() throws IOException { - if (lastRootNode != null) { - lastRootNode.write(null, null, project, -1, -1, null); - lastRootNode = null; - - writeContextTreeNode(); - } - } - - protected void writeLine(String line) { - try { - if (start) { - start = false; - } else { - writer.write('\n'); - } - writer.write(line); - } catch (IOException e) { - // ignore - } - } - - protected void writeRecon( - StringBuffer sb, - Project project, - int rowIndex, - int cellIndex, - Cell cell - ) { - Recon recon = cell.recon; - - sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\""); - contextRefCount++; - - if (!serializedRecons.contains(recon.id)) { - serializedRecons.add(recon.id); - - Column column = project.columnModel.getColumnByCellIndex(cellIndex); - - // qa:sample_group - { - StringBuffer sb2 = new StringBuffer(); - - sb2.append("{ \"s\" : \"rec"); - sb2.append(Long.toString(recon.id)); - sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : "); - sb2.append(JSONObject.quote(column.getName())); - sb2.append(", \"ignore\" : true }"); - - writeLine(sb2.toString()); - } - - // qa:recon_data - { - StringBuffer sb2 = new StringBuffer(); - - String s = cell.value instanceof String ? (String) cell.value : cell.value.toString(); - - sb2.append("{ \"s\" : \"rec"); - sb2.append(Long.toString(recon.id)); - sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { "); - - sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry)); - sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s)); - sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName())); - sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service)); - sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction)); - sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize)); - - if (recon.judgment == Judgment.Matched) { - sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank)); - sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id)); - } - - ReconConfig reconConfig = column.getReconConfig(); - if (reconConfig != null && reconConfig instanceof StandardReconConfig) { - StandardReconConfig standardReconConfig = (StandardReconConfig) reconConfig; - sb2.append(", \"type\" : "); sb2.append(JSONObject.quote(standardReconConfig.typeID)); - } - - sb2.append(" } }"); - - writeLine(sb2.toString()); - } - } - } - - protected void writeLine( - String subject, String predicate, Object object, - Project project, - int subjectRowIndex, int subjectCellIndex, Cell subjectCell, - int objectRowIndex, int objectCellIndex, Cell objectCell, - boolean ignore - ) { - if (subject != null && object != null) { - String s = object instanceof String ? - JSONObject.quote((String) object) : object.toString(); - - StringBuffer sb = new StringBuffer(); - sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"'); - sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"'); - sb.append(", \"o\" : "); sb.append(s); - if (subjectCell != null || objectCell != null) { - sb.append(", \"meta\" : { "); - - sb.append("\"recon\" : { "); - if (subjectCell != null) { - sb.append("\"s\" : "); - writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell); - } - if (objectCell != null) { - if (subjectCell != null) { - sb.append(", "); - } - sb.append("\"o\" : "); - writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell); - } - sb.append(" }"); - - sb.append(" }"); - } - if (ignore) { - sb.append(", \"ignore\" : true"); - } - sb.append(" }"); - - writeLine(sb.toString()); - } - } - - protected void writeLine( - String subject, String predicate, Object object, String lang, - Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell, - boolean ignore - ) { - if (subject != null && object != null) { - String s = object instanceof String ? - JSONObject.quote((String) object) : object.toString(); - - StringBuffer sb = new StringBuffer(); - sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"'); - sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"'); - sb.append(", \"o\" : "); sb.append(s); - sb.append(", \"lang\" : "); sb.append(lang); - - if (subjectCell != null) { - sb.append(", \"meta\" : { "); - sb.append("\"recon\" : { "); - sb.append("\"s\" : "); - writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell); - sb.append(" }"); - sb.append(" }"); - } - if (ignore) { - sb.append(", \"ignore\" : true"); - } - sb.append(" }"); - - writeLine(sb.toString()); - } - } - - abstract protected class WritingTransposedNode implements TransposedNode { - JSONObject jsonContextNode; - boolean load; - - public Object write( - String subject, String predicate, Project project, - int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { - - return internalWrite( - subject, predicate, project, - subjectRowIndex, subjectCellIndex, subjectCell); - } - - abstract public Object internalWrite( - String subject, String predicate, Project project, - int subjectRowIndex, int subjectCellIndex, Cell subjectCell); - } - - abstract protected class TransposedNodeWithChildren extends WritingTransposedNode { - public List links = new LinkedList(); - public List rowIndices = new LinkedList(); - public List children = new LinkedList(); - - protected void writeChildren( - String subject, Project project, - int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { - - for (int i = 0; i < children.size(); i++) { - WritingTransposedNode child = children.get(i); - Link link = links.get(i); - String predicate = link.property.id; - - child.write(subject, predicate, project, - subjectRowIndex, subjectCellIndex, subjectCell); - } - } - } - - protected class AnonymousTransposedNode extends TransposedNodeWithChildren { - - //protected AnonymousTransposedNode(AnonymousNode node) { } - - public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { - if (children.size() == 0 || subject == null) { - return null; - } - - StringBuffer sb = new StringBuffer(); - sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"'); - sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"'); - sb.append(", \"o\" : { "); - - StringBuffer sbRecon = new StringBuffer(); - - boolean first = true; - boolean firstRecon = true; - - if (subjectCell.recon != null) { - sbRecon.append("\"s\" : "); - writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell); - - firstRecon = false; - } - - for (int i = 0; i < children.size(); i++) { - WritingTransposedNode child = children.get(i); - Link link = links.get(i); - - FreebaseProperty property = link.property; - - Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null); - if (c != null) { - if (first) { - first = false; - } else { - sb.append(", "); - } - sb.append("\"" + property.id + "\": "); - sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString()); - } - - if (child instanceof CellTopicTransposedNode) { - CellTopicTransposedNode child2 = (CellTopicTransposedNode) child; - Recon recon = child2.cell.recon; - - if (recon != null && - (recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) { - - if (firstRecon) { - firstRecon = false; - } else { - sbRecon.append(", "); - } - - sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : "); - - writeRecon(sbRecon, project, - rowIndices.get(i), child2.cellIndex, child2.cell); - } - } - } - sb.append(" }, \"meta\" : { \"recon\" : { "); - sb.append(sbRecon.toString()); - sb.append(" } } }"); - - writeLine(sb.toString()); - - return null; - } - } - - protected class CellTopicTransposedNode extends TransposedNodeWithChildren { - protected CellTopicNode node; - protected int rowIndex; - protected int cellIndex; - protected Cell cell; - - public CellTopicTransposedNode(CellTopicNode node, int rowIndex, int cellIndex, Cell cell) { - this.node = node; - this.rowIndex = rowIndex; - this.cellIndex = cellIndex; - this.cell = cell; - } - - public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { - String id = null; - if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) { - int objectRowIndex = rowIndex; - int objectCellIndex = cellIndex; - Cell objectCell = cell; - - if (cell.recon.judgment == Recon.Judgment.Matched) { - id = cell.recon.match.id; - - } else if (cell.recon.judgment == Judgment.New) { - if (newTopicVars.containsKey(cell.recon.id)) { - id = newTopicVars.get(cell.recon.id); - } else { - Column column = project.columnModel.getColumnByCellIndex(cellIndex); - String columnName = column.getName(); - - long var = 0; - if (varPool.containsKey(columnName)) { - var = varPool.get(columnName); - } - varPool.put(columnName, var + 1); - - id = "$" + columnName.replaceAll("\\W+", "_") + "_" + var; - - String typeID = node.type.id; - - ReconConfig reconConfig = column.getReconConfig(); - if (reconConfig instanceof StandardReconConfig) { - typeID = ((StandardReconConfig) reconConfig).typeID; - } - - writeLine(id, "type", typeID, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load); - writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load); - - if (cell.recon != null) { - newTopicVars.put(cell.recon.id, id); - } - } - } else { - return null; - } - - if (subject != null) { - writeLine(subject, predicate, id, project, - subjectRowIndex, subjectCellIndex, subjectCell, - objectRowIndex, objectCellIndex, objectCell, !load); - } - - writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell); - } - - return id; - } - } - - protected class CellValueTransposedNode extends WritingTransposedNode { - protected JSONObject obj; - protected CellValueNode node; - protected int rowIndex; - protected int cellIndex; - protected Cell cell; - - public CellValueTransposedNode(CellValueNode node, int rowIndex, int cellIndex, Cell cell) { - this.node = node; - this.rowIndex = rowIndex; - this.cellIndex = cellIndex; - this.cell = cell; - } - - public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { - if (subject != null) { - if ("/type/text".equals(node.lang)) { - writeLine(subject, predicate, cell.value, node.lang, project, - subjectRowIndex, subjectCellIndex, subjectCell, !load); - } else { - writeLine(subject, predicate, cell.value, project, - subjectRowIndex, subjectCellIndex, subjectCell, - -1, -1, null, !load); - } - } - - return cell.value; - } - } - - protected class CellKeyTransposedNode extends WritingTransposedNode { - protected CellKeyNode node; - protected int rowIndex; - protected int cellIndex; - protected Cell cell; - - public CellKeyTransposedNode(CellKeyNode node, int rowIndex, int cellIndex, Cell cell) { - this.node = node; - this.rowIndex = rowIndex; - this.cellIndex = cellIndex; - this.cell = cell; - } - - public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { - writeLine(subject, "key", node.namespace.id + "/" + cell.value, project, - subjectRowIndex, subjectCellIndex, subjectCell, - -1, -1, null, !load); - - return null; - } - } - - protected class TopicTransposedNode extends TransposedNodeWithChildren { - protected FreebaseTopicNode node; - - public TopicTransposedNode(FreebaseTopicNode node) { - this.node = node; - } - - public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { - writeLine(subject, predicate, node.topic.id, project, - subjectRowIndex, subjectCellIndex, subjectCell, - -1, -1, null, !load); - - writeChildren(node.topic.id, project, -1, -1, null); - - return node.topic.id; - } - } - - protected class ValueTransposedNode extends WritingTransposedNode { - protected ValueNode node; - - public ValueTransposedNode(ValueNode node) { - this.node = node; - } - - public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { - if ("/type/text".equals(node.lang)) { - writeLine(subject, predicate, node.value, node.lang, project, - subjectRowIndex, subjectCellIndex, subjectCell, !load); - } else { - writeLine(subject, predicate, node.value, project, - subjectRowIndex, subjectCellIndex, subjectCell, - -1, -1, null, !load); - } - - return node.value; - } - } - - public TransposedNode transposeAnonymousNode( - TransposedNode parentNode, - Link link, - AnonymousNode node, int rowIndex) { - - WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode; - WritingTransposedNode tnode = new AnonymousTransposedNode(); - - tnode.load = - (parentNode2 == null || parentNode2.load) && - (link == null || link.load); - - processTransposedNode(tnode, parentNode, link, rowIndex); - - tnode.jsonContextNode = addJsonContext( - parentNode2 != null ? parentNode2.jsonContextNode : null, - link != null ? link.property.id : null, - null - ); - - return tnode; - } - - public TransposedNode transposeCellNode( - TransposedNode parentNode, - Link link, - CellNode node, - int rowIndex, - int cellIndex, - Cell cell) { - - WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode; - - WritingTransposedNode tnode = null; - if (node instanceof CellTopicNode) { - if (cell.recon != null && - (cell.recon.judgment == Judgment.Matched || - cell.recon.judgment == Judgment.New)) { - - tnode = new CellTopicTransposedNode( - (CellTopicNode) node, rowIndex, cellIndex, cell); - } - } else if (node instanceof CellValueNode) { - tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell); - } else if (node instanceof CellKeyNode) { - tnode = new CellKeyTransposedNode((CellKeyNode) node, rowIndex, cellIndex, cell); - } - - if (tnode != null) { - tnode.load = - (parentNode2 == null || parentNode2.load) && - (link == null || link.load); - - processTransposedNode(tnode, parentNode, link, rowIndex); - - tnode.jsonContextNode = addJsonContext( - parentNode2 != null ? parentNode2.jsonContextNode : null, - link != null ? link.property.id : null, - cell, - rowIndex - ); - } - return tnode; - } - - public TransposedNode transposeTopicNode( - TransposedNode parentNode, - Link link, - FreebaseTopicNode node, - int rowIndex) { - - WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode; - WritingTransposedNode tnode = new TopicTransposedNode(node); - - tnode.load = - (parentNode2 == null || parentNode2.load) && - (link == null || link.load); - - processTransposedNode(tnode, parentNode, link, rowIndex); - - tnode.jsonContextNode = addJsonContext( - parentNode2 != null ? parentNode2.jsonContextNode : null, - link != null ? link.property.id : null, - node.topic - ); - - return tnode; - } - - public TransposedNode transposeValueNode( - TransposedNode parentNode, - Link link, - ValueNode node, - int rowIndex) { - - WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode; - WritingTransposedNode tnode = new ValueTransposedNode(node); - - tnode.load = - (parentNode2 == null || parentNode2.load) && - (link == null || link.load); - - processTransposedNode(tnode, parentNode, link, rowIndex); - - tnode.jsonContextNode = addJsonContext( - parentNode2 != null ? parentNode2.jsonContextNode : null, - link != null ? link.property.id : null, - node.value - ); - - return tnode; - } - - protected void processTransposedNode( - WritingTransposedNode tnode, - TransposedNode parentNode, - Link link, - int rowIndex - ) { - if (parentNode != null) { - if (parentNode instanceof TransposedNodeWithChildren) { - TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode; - parentNode2.rowIndices.add(rowIndex); - parentNode2.children.add(tnode); - parentNode2.links.add(link); - } - } else { - addRootNode(tnode, rowIndex); - } - } - - protected JSONObject addJsonContext(JSONObject parent, String key, Object value) { - JSONObject o = new JSONObject(); - - try { - if (value instanceof FreebaseTopic) { - FreebaseTopic topic = (FreebaseTopic) value; - o.put("id", topic.id); - o.put("name", topic.name); - } else { - o.put("v", value); - } - } catch (JSONException e) { - // ignore - } - - connectJsonContext(parent, o, key); - return o; - } - - protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) { - JSONObject o = new JSONObject(); - - connectJsonContext(parent, o, key); - - try { - if (cell != null) { - o.put("v", cell.value); - if (cell.recon != null) { - o.put("recon", "rec" + cell.recon.id); - - if (cell.recon.judgment == Judgment.Matched) { - o.put("id", cell.recon.match.id); - o.put("name", cell.recon.match.name); - } - - // qa:display_context - { - StringBuffer sb2 = new StringBuffer(); - - sb2.append("{ \"ignore\" : true, \"s\" : \"rec"); - sb2.append(Long.toString(cell.recon.id)); - sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx"); - sb2.append(Long.toString(contextID)); - sb2.append("\", \"meta\" : { \"row\" : "); - sb2.append(Integer.toString(rowIndex)); - sb2.append(" } }"); - - writeLine(sb2.toString()); - } - } - } - } catch (JSONException e) { - // ignore - } - - return o; - } - - protected void connectJsonContext(JSONObject parent, JSONObject o, String key) { - try { - if (parent == null) { - contextTreeRoot = o; - } else { - JSONArray a = null; - if (parent.has(key)) { - a = parent.getJSONArray(key); - } else { - a = new JSONArray(); - parent.put(key, a); - } - - a.put(o); - } - } catch (JSONException e) { - // ignore - } - } - - protected void addRootNode(WritingTransposedNode tnode, int rowIndex) { - if (lastRootNode != null) { - lastRootNode.write(null, null, project, -1, -1, null); - writeContextTreeNode(); - } - lastRootNode = tnode; - - contextTreeRoot = null; - contextRowIndex = rowIndex; - contextRefCount = 0; - contextID++; - } - - protected void writeContextTreeNode() { - if (contextTreeRoot != null && contextRefCount > 0) { - StringBuffer sb = new StringBuffer(); - - sb.append("{ \"ignore\" : true, \"s\" : \"ctx"); - sb.append(Long.toString(contextID)); - sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : "); - sb.append(Integer.toString(contextRowIndex)); - sb.append(", \"data\" : "); - sb.append(contextTreeRoot.toString()); - sb.append(" } }"); - - writeLine(sb.toString()); - } - } -} diff --git a/main/src/com/google/refine/util/FreebaseUtils.java b/main/src/com/google/refine/util/FreebaseUtils.java deleted file mode 100644 index d7ec1766f..000000000 --- a/main/src/com/google/refine/util/FreebaseUtils.java +++ /dev/null @@ -1,239 +0,0 @@ -package com.google.refine.util; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import javax.servlet.http.HttpServletRequest; - -import oauth.signpost.OAuthConsumer; -import oauth.signpost.exception.OAuthCommunicationException; -import oauth.signpost.exception.OAuthExpectationFailedException; -import oauth.signpost.exception.OAuthMessageSignerException; - -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.HttpClient; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.params.CoreProtocolPNames; -import org.apache.http.util.EntityUtils; -import org.json.JSONException; -import org.json.JSONObject; - -import com.google.refine.ProjectManager; -import com.google.refine.RefineServlet; -import com.google.refine.oauth.Credentials; -import com.google.refine.oauth.OAuthUtilities; -import com.google.refine.oauth.Provider; - -public class FreebaseUtils { - - static final public String FREEBASE_HOST = "www.freebase.com"; - - static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine"; - - static final private String AGENT_ID = "/en/google_refine"; - - static final private int SAMPLE_SIZE = 300; - static final private int JUDGES = 4; - - private static String getUserInfoURL(String host) { - return "http://" + host + "/api/service/user_info"; - } - - private static String getMQLWriteURL(String host) { - return "http://" + host + "/api/service/mqlwrite"; - } - - private static String getMQLReadURL(String host) { - return "http://" + host + "/api/service/mqlread"; - } - - private static String getUserAgent() { - return RefineServlet.FULLNAME; - } - - public static String getUserInfo(Credentials credentials, Provider provider) - throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException { - - OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); - - HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); - - // this is required by the Metaweb API to avoid XSS - httpRequest.setHeader("X-Requested-With", "1"); - - // sign the request with the oauth library - consumer.sign(httpRequest); - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } - - public static String getUserBadges(Provider provider, String user_id) - throws ClientProtocolException, IOException, JSONException { - - String query = "{" + - "'id' : '" + user_id + "'," + - "'!/type/usergroup/member' : [{" + - "'id' : null," + - "'key' : [{" + - "'namespace' : null" + - "}]" + - "}]" + - "}".replace("'", "\""); - - return mqlread(provider, query); - } - - public static String mqlread(Provider provider, String query) - throws ClientProtocolException, IOException, JSONException { - - JSONObject envelope = new JSONObject(); - envelope.put("query", new JSONObject(query)); - - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("query", envelope.toString())); - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - - HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); - httpRequest.setEntity(entity); - - // this is required by the Metaweb API to avoid XSS - httpRequest.setHeader("X-Requested-With", "1"); - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } - - public static String mqlwrite(Credentials credentials, Provider provider, String query) - throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException { - OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); - - JSONObject envelope = new JSONObject(); - envelope.put("query", new JSONObject(query)); - - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("query", envelope.toString())); - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - - HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); - httpRequest.setEntity(entity); - - // this is required by the Metaweb API to avoid XSS - httpRequest.setHeader("X-Requested-With", "1"); - - // sign the request with the oauth library - consumer.sign(httpRequest); - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } - - private static String getTweezersParams(int sample_size, int judges) { - String o = "{" + - "'sample_size':" + sample_size + "," + - "'votes':{" + - "'reconciled':" + judges + "," + - "'invalid':" + judges + "," + - "'new':" + judges + "," + - "'skip':" + (judges + 2) + - "}" + - "}"; - return o.replace('\'', '"'); - } - - public static String uploadTriples( - HttpServletRequest request, - String qa, - String source_name, - String source_id, - String mdo_id, - String triples - ) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException { - - Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST); - - Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); - - JSONObject mdo_info = new JSONObject(); - mdo_info.put("name", source_name); - if (source_id != null) { - mdo_info.put("info_source",source_id); - } - - JSONObject user_info = new JSONObject(getUserInfo(credentials, provider)); - if (user_info.has("username")) { - - List formparams = new ArrayList(); - formparams.add(new BasicNameValuePair("user", user_info.getString("id"))); - formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE")); - formparams.add(new BasicNameValuePair("operator", user_info.getString("id"))); - formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID)); - formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString())); - formparams.add(new BasicNameValuePair("graphport", "sandbox")); - formparams.add(new BasicNameValuePair("payload", triples)); - formparams.add(new BasicNameValuePair("check_params", "false")); - if (mdo_id != null) { - formparams.add(new BasicNameValuePair("mdo_guid", mdo_id)); - } - if (Boolean.parseBoolean(qa)) { - formparams.add(new BasicNameValuePair("rabj", getTweezersParams(SAMPLE_SIZE,JUDGES))); - } - UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); - - HttpPost httpRequest = new HttpPost(getFreeQUrl()); - httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); - httpRequest.setEntity(entity); - - HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); - surrogateRequest.setEntity(entity); - - OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); - - consumer.sign(surrogateRequest); - - Header[] h = surrogateRequest.getHeaders("Authorization"); - if (h.length > 0) { - httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); - } else { - throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); - } - - // execute the request - HttpClient httpClient = new DefaultHttpClient(); - HttpResponse httpResponse = httpClient.execute(httpRequest); - - // return the results - return EntityUtils.toString(httpResponse.getEntity()); - } else { - throw new RuntimeException("Invalid credentials"); - } - } - - static public String getFreeQUrl() { - String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); - return url != null ? url : FREEQ_URL; - } -}