')
+ .addClass("schema-alignment-link-menu-section-last")
+ .html('
').appendTo(divSearch));
+
+ MenuSystem.showMenu(menu, function(){});
+ MenuSystem.positionMenuAboveBelow(menu, $(elmt));
+
+ var suggestOptions = {
+ type : '/type/property'
+ };
+ var sourceTypeID = this._parentUINode.getExpectedType();
+ if (sourceTypeID !== null) {
+ suggestOptions.ac_param = { schema: sourceTypeID };
+ }
+ input.suggestP(suggestOptions).bind("fb-select", function(e, data) { commitProperty(data); });
+
+ input[0].focus();
+};
+
+SchemaAlignmentDialog.UILink.prototype.getJSON = function() {
+ if ("property" in this._link && this._link.property !== null &&
+ "target" in this._link && this._link.target !== null) {
+
+ var targetJSON = this._targetUI.getJSON();
+ if (targetJSON !== null) {
+ var json = {
+ property: cloneDeep(this._link.property),
+ target: targetJSON
+ };
+ if (this._link.condition) {
+ json.condition = cloneDeep(this._link.condition);
+ }
+ return json;
+ }
+ }
+ return null;
+};
+
+SchemaAlignmentDialog.UILink.prototype._configureTarget = function() {
+ var self = this;
+ var dismissBusy = DialogSystem.showBusy();
+
+ $.getJSON(
+ "http://api.freebase.com/api/service/mqlread?query=" + JSON.stringify({
+ query: {
+ "id" : this._link.property.id,
+ "type" : "/type/property",
+ "expected_type" : {
+ "id" : null,
+ "name" : null,
+ "/freebase/type_hints/mediator" : null
+ }
+ }
+ }) + "&callback=?",
+ null,
+ function(o) {
+ dismissBusy();
+
+ if ("result" in o) {
+ var expected_type = o.result.expected_type;
+ self._link.target.type = {
+ id: expected_type.id,
+ name: expected_type.name
+ };
+ if (expected_type["/freebase/type_hints/mediator"] === true) {
+ self._link.target.nodeType = "anonymous";
+ } else if (expected_type.id == "/type/key") {
+ self._link.target.nodeType = "cell-as-key";
+ } else if (expected_type.id.match(/^\/type\//)) {
+ self._link.target.nodeType = "cell-as-value";
+ } else if (!("topic" in self._link.target)) {
+ self._link.target.nodeType = "cell-as-topic";
+ self._link.target.createForNoReconMatch = true;
+ }
+
+ self._targetUI.render();
+ }
+
+ self._renderMain();
+ self._dialog.preview();
+ },
+ "jsonp"
+ );
+};
diff --git a/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js b/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js
new file mode 100644
index 000000000..09d4fd750
--- /dev/null
+++ b/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js
@@ -0,0 +1,732 @@
+SchemaAlignmentDialog.UINode = function(dialog, node, table, options) {
+ this._dialog = dialog;
+ this._node = node;
+ this._options = options;
+
+ if ("columnName" in this._node) {
+ this._node.columnNames = [ this._node.columnName ];
+ delete this._node.columnName;
+ }
+
+ this._linkUIs = [];
+ this._detailsRendered = false;
+
+ this._tr = table.insertRow(table.rows.length);
+ this._tdMain = this._tr.insertCell(0);
+ this._tdToggle = this._tr.insertCell(1);
+ this._tdDetails = this._tr.insertCell(2);
+
+ $(this._tdMain).addClass("schema-alignment-node-main").attr("width", "250").addClass("padded");
+ $(this._tdToggle).addClass("schema-alignment-node-toggle").attr("width", "1%").addClass("padded").hide();
+ $(this._tdDetails).addClass("schema-alignment-node-details").attr("width", "90%").hide();
+
+ this._renderMain();
+
+ this._expanded = options.expanded;
+ if (this._isExpandable()) {
+ this._showExpandable();
+ }
+};
+
+SchemaAlignmentDialog.UINode.prototype.dispose = function() {
+ // nothing for now
+};
+
+SchemaAlignmentDialog.UINode.prototype.removeLink = function(linkUI) {
+ for (var i = this._linkUIs.length - 1; i >= 0; i--) {
+ if (this._linkUIs[i] === linkUI) {
+ this._linkUIs.splice(i, 1);
+ this._node.links.splice(i, 1);
+ break;
+ }
+ }
+};
+
+SchemaAlignmentDialog.UINode.prototype._isExpandable = function() {
+ return this._node.nodeType == "cell-as-topic" ||
+ this._node.nodeType == "anonymous" ||
+ this._node.nodeType == "topic";
+};
+
+SchemaAlignmentDialog.UINode.prototype.render = function() {
+ this._renderMain();
+ if (this._isExpandable()) {
+ this._showExpandable();
+ } else {
+ this._hideExpandable();
+ }
+};
+
+SchemaAlignmentDialog.UINode.prototype.getExpectedType = function() {
+ if ("type" in this._node) {
+ return this._node.type.id;
+ }
+ return null;
+};
+
+SchemaAlignmentDialog.UINode.prototype._renderMain = function() {
+ $(this._tdMain).empty();
+
+ var self = this;
+ var a = $('
')
+ .addClass("schema-alignment-node-tag")
+ .appendTo(this._tdMain)
+ .click(function(evt) {
+ self._showNodeConfigDialog();
+ });
+
+ if (this._node.nodeType == "cell-as-topic" ||
+ this._node.nodeType == "cell-as-value" ||
+ this._node.nodeType == "cell-as-key") {
+
+ if ("columnNames" in this._node) {
+ for (var c = 0; c < this._node.columnNames.length; c++) {
+ if (c > 0) {
+ $('
').text(", ").appendTo(a);
+ }
+
+ $('')
+ .text(this._node.columnNames[c])
+ .addClass("schema-alignment-node-column")
+ .appendTo(a);
+ }
+
+ $('').text(this._node.columnNames.length > 1 ? " cells" : " cell").appendTo(a);
+ } else {
+ a.html(this._options.mustBeCellTopic ? "Which column?" : "Configure...");
+ }
+ } else if (this._node.nodeType == "topic") {
+ if ("topic" in this._node) {
+ a.html(this._node.topic.name);
+ } else if ("id" in this._node) {
+ a.html(this._node.topic.id);
+ } else {
+ a.html("Which topic?");
+ }
+ } else if (this._node.nodeType == "value") {
+ if ("value" in this._node) {
+ a.html(this._node.value);
+ } else {
+ a.html("What value?");
+ }
+ } else if (this._node.nodeType == "anonymous") {
+ a.html("(anonymous)");
+ }
+};
+
+SchemaAlignmentDialog.UINode.prototype._showExpandable = function() {
+ $(this._tdToggle).show();
+ $(this._tdDetails).show();
+
+ if (this._detailsRendered) {
+ return;
+ }
+ this._detailsRendered = true;
+
+ this._collapsedDetailDiv = $('
').appendTo(this._tdDetails).addClass("padded").html("...");
+ this._expandedDetailDiv = $('
').appendTo(this._tdDetails).addClass("schema-alignment-detail-container");
+
+ this._renderDetails();
+
+ var self = this;
+ var show = function() {
+ if (self._expanded) {
+ self._collapsedDetailDiv.hide();
+ self._expandedDetailDiv.show();
+ } else {
+ self._collapsedDetailDiv.show();
+ self._expandedDetailDiv.hide();
+ }
+ };
+ show();
+
+ $(this._tdToggle).html(" ");
+ $('
').addClass("padded").appendTo(this._expandedDetailDiv);
+
+ $('
').addClass("dialog-header").text("Schema Skeleton Node").appendTo(frame);
+ var body = $('
').addClass("dialog-body").appendTo(frame);
+ var footer = $('
').addClass("dialog-footer").appendTo(frame);
+
+ /*--------------------------------------------------
+ * Body
+ *--------------------------------------------------
+ */
+ var literalTypeSelectHtml =
+ 'text ' +
+ 'int ' +
+ 'float ' +
+ 'double ' +
+ 'boolean ' +
+ 'date/time ';
+
+ var html = $(
+ '' +
+ '' +
+ '' +
+ '' +
+ ' ' +
+
+ '' +
+ '' +
+ ' ' +
+ ' ' +
+ '
'
+ ).appendTo(body);
+
+ var elmts = DOM.bind(html);
+
+ var tableColumns = $('')
+ .attr("cellspacing", "5")
+ .attr("cellpadding", "0")
+ .appendTo(elmts.divColumns)[0];
+
+ var columnMap = {};
+ if ("columnNames" in self._node) {
+ for (var i = 0; i < self._node.columnNames.length; i++) {
+ columnMap[self._node.columnNames[i]] = true;
+ }
+ }
+
+ var makeColumnChoice = function(column, columnIndex) {
+ var tr = tableColumns.insertRow(tableColumns.rows.length);
+
+ var radio = $('' + window.user.username + ' ');
+ }
+ if (typeof cont == 'function') cont();
+ },
+
+ signin : function(success, provider, width, height) {
+ var newwin = window.Sign.popup("/command/freebase-extension/authorize/" + provider, width, height);
+
+ if (newwin !== null) {
+ newwin.opener = window;
+ }
+
+ window.onauthorization = function() {
+ if (typeof success == 'undefined') {
+ window.location.reload();
+ } else {
+ $.ajax({
+ url: "/command/freebase-extension/check-authorization/" + provider,
+ dataType: "json",
+ success: function(data) {
+ window.user = data;
+ window.Sign.signintize(success);
+ }
+ });
+ }
+ };
+
+ if (window.focus && newwin !== null) {
+ newwin.focus();
+ }
+
+ return false;
+ },
+
+ signoutize : function(cont) {
+ $('.signedin').hide();
+ $('.signedout').show();
+ if (typeof cont == 'function') cont();
+ },
+
+ signout : function(success,provider) {
+ $.ajax({
+ url: "/command/freebase-extension/deauthorize/" + provider,
+ success: function() {
+ if (typeof success == 'undefined') {
+ window.location.reload();
+ } else {
+ window.Sign.signoutize(success);
+ }
+ }
+ });
+ }
+ };
+ }
+
\ No newline at end of file
diff --git a/extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less b/extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less
new file mode 100644
index 000000000..a10d37926
--- /dev/null
+++ b/extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less
@@ -0,0 +1,38 @@
+@import-less url("../theme.less");
+
+.extend-data-preview-dialog .suggested-property-container {
+ border: 1px solid #aaa;
+ padding: 5px;
+ overflow: auto;
+ height: 400px;
+}
+
+.extend-data-preview-dialog .suggested-property {
+ padding: 5px;
+}
+
+.extend-data-preview-dialog input.property-suggest {
+ display: block;
+ padding: 2%;
+ width: 96%;
+}
+
+.extend-data-preview-dialog .preview-container {
+ border: 1px solid #aaa;
+ overflow: auto;
+}
+
+.extend-data-preview-dialog .preview-container table {
+ border-collapse: collapse;
+}
+
+.extend-data-preview-dialog .preview-container td, .extend-data-preview-dialog .preview-container th {
+ padding: 3px 5px;
+ border-bottom: 1px solid #ddd;
+ border-right: 1px solid #ddd;
+}
+
+.extend-data-preview-dialog .preview-container th img {
+ vertical-align: top;
+ margin-left: 5px;
+}
\ No newline at end of file
diff --git a/extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less b/extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less
new file mode 100644
index 000000000..d4a681204
--- /dev/null
+++ b/extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less
@@ -0,0 +1,60 @@
+@import-less url("../theme.less");
+
+.freebase-loading-tripleloader-data {
+ height: 400px;
+ width: 99%;
+ overflow: auto;
+ border: 1px solid #aaa;
+ white-space: pre;
+ padding: 0.3em 0.5em 0.5em 0.5em;
+ font-family: monospace;
+}
+
+.freebase-loading-tripleloader-message {
+ height: 500px;
+ overflow: auto;
+ border: 1px solid #aaa;
+ padding: 0.3em 0.5em 0.5em 0.5em;
+}
+
+.freebase-loading-tripleloader-message h2 {
+ text-align: center;
+ font-size: 150%;
+ margin-top: 3em;
+ font-weight: normal;
+}
+
+.freebase-loading-tripleloader-message h2 span {
+ font-size; 130%;
+ font-weight: bold;
+}
+
+.freebase-loading-tripleloader-message h4 {
+ text-align: center;
+ font-size: 120%;
+ margin-top: 2em;
+ font-weight: normal;
+}
+
+.freebase-loading-tripleloader-message a {
+ font-size: 120%;
+ font-weight: bold;
+}
+
+.freebase-loading-authorization {
+ margin: 0em 2em;
+ padding: 0.6em 0.8em;
+ border: 1px solid #ccc;
+ background-color: #fff;
+ -moz-border-radius: 0.8em;
+ -webkit-border-radius: 0.8em;
+}
+
+.freebase-loading-tripleloader-info {
+ margin-bottom: 0.5em;
+}
+
+.freebase-loading-tripleloader-info textarea {
+ width: 99%;
+ height: 5em;
+}
\ No newline at end of file
diff --git a/extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less b/extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less
new file mode 100644
index 000000000..fcd9de427
--- /dev/null
+++ b/extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less
@@ -0,0 +1,114 @@
+@import-less url("../theme.less");
+
+.schema-alignment-dialog-canvas {
+ height: 400px;
+ overflow: auto;
+ border: 1px solid #aaa;
+ padding: 10px;
+ background: white;
+}
+
+table.schema-alignment-table-layout {
+ border-collapse: collapse;
+ margin: 0px;
+ padding: 0px;
+ margin-bottom: 1em;
+}
+
+table.schema-alignment-table-layout .padded {
+ padding: 3px 6px;
+}
+
+div.schema-alignment-detail-container {
+ border-left: 3px solid #eee;
+}
+
+td.schema-alignment-node-main, td.schema-alignment-link-main {
+ white-space: pre;
+ width: 300px;
+}
+
+td.schema-alignment-node-toggle, td.schema-alignment-link-toggle {
+ white-space: pre;
+ width: 1%;
+}
+
+td.schema-alignment-node-toggle img, td.schema-alignment-link-toggle img {
+ cursor: pointer;
+ vertical-align: middle;
+ padding: 2px;
+}
+
+td.schema-alignment-node-details, td.schema-alignment-link-details {
+ width: 90%;
+}
+
+a.schema-alignment-node-tag {
+ padding: 3px 6px;
+ background: #ddd;
+ text-decoration: none;
+ color: black;
+ -moz-border-radius: 10px;
+}
+a.schema-alignment-node-tag:hover {
+ background: #888;
+ color: white;
+}
+.schema-alignment-node-column {
+ font-weight: bold;
+}
+
+a.schema-alignment-link-tag {
+ padding: 3px 6px;
+ text-decoration: none;
+ color: black;
+}
+a.schema-alignment-link-tag:hover {
+ color: #88f;
+}
+
+div.schema-alignment-dialog-preview {
+ height: 400px;
+ overflow: auto;
+ border: 1px solid #aaa;
+ background: white;
+ padding: 10px;
+ white-space: pre;
+ font-family: monospace;
+ font-size: 9pt;
+}
+
+/*--------------------------------------------------
+ * Node dialog
+ *--------------------------------------------------
+ */
+
+.schema-align-node-dialog-node-type {
+ padding: 0.25em;
+ background: #ddf;
+ -moz-border-radius: 0.5em;
+}
+.schema-align-node-dialog-node-type input {
+ vertical-align: text-bottom;
+}
+.schema-alignment-node-dialog-column-list {
+ height: 300px;
+ width: 200px;
+ overflow: auto;
+ border: 1px solid #ddd;
+}
+
+/*--------------------------------------------------
+ * Link dialog
+ *--------------------------------------------------
+ */
+
+.schema-alignment-link-menu-section {
+ padding: 8px;
+ border-bottom: 1px solid #ddd;
+ margin-bottom: 3px;
+}
+
+.schema-alignment-link-menu-section-last {
+ padding: 8px;
+}
diff --git a/extensions/freebase/module/styles/theme.less b/extensions/freebase/module/styles/theme.less
new file mode 100644
index 000000000..35db2d57b
--- /dev/null
+++ b/extensions/freebase/module/styles/theme.less
@@ -0,0 +1 @@
+@import-less url("../../../../main/webapp/modules/core/styles/theme.less");
diff --git a/extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java b/extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java
new file mode 100644
index 000000000..cef56a307
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java
@@ -0,0 +1,9 @@
+package com.google.refine.freebase;
+
+public class FreebaseProperty extends FreebaseTopic {
+ //final protected FreebaseType _expectedType;
+
+ public FreebaseProperty(String id, String name) {
+ super(id, name);
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java b/extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java
new file mode 100644
index 000000000..01f3f1d06
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java
@@ -0,0 +1,28 @@
+package com.google.refine.freebase;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.refine.Jsonizable;
+
+public class FreebaseTopic implements Jsonizable {
+ final public String id;
+ final public String name;
+
+ public FreebaseTopic(String id, String name) {
+ this.id = id;
+ this.name = name;
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("id"); writer.value(id);
+ writer.key("name"); writer.value(name);
+ writer.endObject();
+ }
+
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/FreebaseType.java b/extensions/freebase/src/com/google/refine/freebase/FreebaseType.java
new file mode 100644
index 000000000..c77f16677
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/FreebaseType.java
@@ -0,0 +1,36 @@
+package com.google.refine.freebase;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.Jsonizable;
+
+public class FreebaseType extends FreebaseTopic implements Jsonizable {
+ public FreebaseType(String id, String name) {
+ super(id, name);
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("id"); writer.value(id);
+ writer.key("name"); writer.value(name);
+ writer.endObject();
+ }
+
+ static public FreebaseType load(JSONObject obj) throws Exception {
+ if (obj == null) {
+ return null;
+ }
+
+ FreebaseType type = new FreebaseType(
+ obj.getString("id"),
+ obj.getString("name")
+ );
+ return type;
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java b/extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java
new file mode 100644
index 000000000..63990b090
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java
@@ -0,0 +1,75 @@
+package com.google.refine.freebase;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.util.Properties;
+
+import com.google.refine.browsing.Engine;
+import com.google.refine.exporters.Exporter;
+import com.google.refine.model.Project;
+import com.google.refine.freebase.protograph.Protograph;
+import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
+import com.google.refine.freebase.protograph.transpose.TransposedNodeFactory;
+import com.google.refine.freebase.protograph.transpose.Transposer;
+import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory;
+
+abstract public class ProtographTransposeExporter implements Exporter {
+ final protected String _contentType;
+
+ public ProtographTransposeExporter(String contentType) {
+ _contentType = contentType;
+ }
+
+ public String getContentType() {
+ return "application/x-unknown";
+ }
+
+ public boolean takeWriter() {
+ return true;
+ }
+
+ public void export(Project project, Properties options, Engine engine,
+ OutputStream outputStream) throws IOException {
+ throw new RuntimeException("Not implemented");
+ }
+
+ public void export(Project project, Properties options, Engine engine,
+ Writer writer) throws IOException {
+
+ Protograph protograph = (Protograph) project.overlayModels.get("freebaseProtograph");
+ if (protograph != null) {
+ TransposedNodeFactory nodeFactory = createNodeFactory(project, writer);
+
+ Transposer.transpose(project, engine.getAllFilteredRows(),
+ protograph, protograph.getRootNode(0), nodeFactory, -1);
+
+ nodeFactory.flush();
+ }
+ }
+
+ abstract protected TransposedNodeFactory createNodeFactory(Project project, Writer writer);
+
+ static public class TripleLoaderExporter extends ProtographTransposeExporter {
+ public TripleLoaderExporter() {
+ super("application/x-unknown");
+ }
+
+ @Override
+ protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) {
+ return new TripleLoaderTransposedNodeFactory(project, writer);
+ }
+ }
+
+ static public class MqlwriteLikeExporter extends ProtographTransposeExporter {
+ public MqlwriteLikeExporter() {
+ super("application/x-unknown");
+ }
+
+ @Override
+ protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) {
+ return new MqlwriteLikeTransposedNodeFactory(writer);
+ }
+ }
+
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java
new file mode 100644
index 000000000..f739bb56b
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java
@@ -0,0 +1,32 @@
+package com.google.refine.freebase.commands;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.json.JSONObject;
+
+import com.google.refine.commands.EngineDependentCommand;
+import com.google.refine.model.AbstractOperation;
+import com.google.refine.model.Project;
+import com.google.refine.freebase.operations.ExtendDataOperation;
+import com.google.refine.util.ParsingUtilities;
+
+public class ExtendDataCommand extends EngineDependentCommand {
+ @Override
+ protected AbstractOperation createOperation(Project project,
+ HttpServletRequest request, JSONObject engineConfig) throws Exception {
+
+ String baseColumnName = request.getParameter("baseColumnName");
+ int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex"));
+
+ String jsonString = request.getParameter("extension");
+ JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString);
+
+ return new ExtendDataOperation(
+ engineConfig,
+ baseColumnName,
+ extension,
+ columnInsertIndex
+ );
+ }
+
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java
new file mode 100644
index 000000000..8ddd0ce3d
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java
@@ -0,0 +1,36 @@
+package com.google.refine.freebase.commands;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.google.refine.ProjectManager;
+import com.google.refine.commands.Command;
+import com.google.refine.model.AbstractOperation;
+import com.google.refine.model.Project;
+import com.google.refine.freebase.operations.ImportQADataOperation;
+import com.google.refine.process.Process;
+
+public class ImportQADataCommand extends Command {
+ @Override
+ public void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ ProjectManager.singleton.setBusy(true);
+ try {
+ Project project = getProject(request);
+
+ AbstractOperation op = new ImportQADataOperation();
+ Process process = op.createProcess(project, new Properties());
+
+ performProcessAndRespond(request, response, project, process);
+ } catch (Exception e) {
+ respondException(response, e);
+ } finally {
+ ProjectManager.singleton.setBusy(false);
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java
new file mode 100644
index 000000000..e05a004ab
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java
@@ -0,0 +1,31 @@
+package com.google.refine.freebase.commands;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.google.refine.commands.Command;
+import com.google.refine.freebase.util.FreebaseUtils;
+import com.google.refine.oauth.OAuthUtilities;
+import com.google.refine.oauth.Provider;
+
+public class MQLReadCommand extends Command {
+
+ @Override
+ public void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ try {
+ Provider provider = OAuthUtilities.getProvider(request);
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "application/json");
+ String query = request.getParameter("query");
+ String result = FreebaseUtils.mqlread(provider,query);
+ response.getWriter().write(result);
+ } catch (Exception e) {
+ respondException(response, e);
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java
new file mode 100644
index 000000000..0ce475a52
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java
@@ -0,0 +1,40 @@
+package com.google.refine.freebase.commands;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.google.refine.commands.Command;
+import com.google.refine.freebase.util.FreebaseUtils;
+import com.google.refine.oauth.Credentials;
+import com.google.refine.oauth.OAuthUtilities;
+import com.google.refine.oauth.Provider;
+
+public class MQLWriteCommand extends Command {
+
+ @Override
+ public void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ try {
+ Provider provider = OAuthUtilities.getProvider(request);
+
+ Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
+
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "application/json");
+
+ if (access_credentials != null) {
+ String query = request.getParameter("query");
+ String result = FreebaseUtils.mqlwrite(access_credentials, provider, query);
+ response.getWriter().write(result);
+ } else {
+ respond(response, "401 Unauthorized", "You don't have the right credentials");
+ }
+ } catch (Exception e) {
+ respondException(response, e);
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java
new file mode 100644
index 000000000..a3d7ade24
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java
@@ -0,0 +1,158 @@
+package com.google.refine.freebase.commands;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.commands.Command;
+import com.google.refine.freebase.util.FreebaseDataExtensionJob;
+import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo;
+import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Project;
+import com.google.refine.model.ReconCandidate;
+import com.google.refine.model.Row;
+import com.google.refine.util.ParsingUtilities;
+
+public class PreviewExtendDataCommand extends Command {
+
+ @Override
+ public void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ try {
+ Project project = getProject(request);
+ String columnName = request.getParameter("columnName");
+
+ String rowIndicesString = request.getParameter("rowIndices");
+ if (rowIndicesString == null) {
+ respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }");
+ return;
+ }
+
+ String jsonString = request.getParameter("extension");
+ JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
+
+ JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString);
+ int length = rowIndices.length();
+ int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex();
+
+ List topicNames = new ArrayList();
+ List topicIds = new ArrayList();
+ Set ids = new HashSet();
+ for (int i = 0; i < length; i++) {
+ int rowIndex = rowIndices.getInt(i);
+ if (rowIndex >= 0 && rowIndex < project.rows.size()) {
+ Row row = project.rows.get(rowIndex);
+ Cell cell = row.getCell(cellIndex);
+ if (cell != null && cell.recon != null && cell.recon.match != null) {
+ topicNames.add(cell.recon.match.name);
+ topicIds.add(cell.recon.match.id);
+ ids.add(cell.recon.match.id);
+ } else {
+ topicNames.add(null);
+ topicIds.add(null);
+ ids.add(null);
+ }
+ }
+ }
+
+ Map reconCandidateMap = new HashMap();
+ FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json);
+ Map map = job.extend(ids, reconCandidateMap);
+
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "application/json");
+
+ JSONWriter writer = new JSONWriter(response.getWriter());
+ writer.object();
+ writer.key("code"); writer.value("ok");
+ writer.key("columns");
+ writer.array();
+ for (ColumnInfo info : job.columns) {
+ writer.object();
+ writer.key("names");
+ writer.array();
+ for (String name : info.names) {
+ writer.value(name);
+ }
+ writer.endArray();
+ writer.key("path");
+ writer.array();
+ for (String id : info.path) {
+ writer.value(id);
+ }
+ writer.endArray();
+ writer.endObject();
+ }
+ writer.endArray();
+
+ writer.key("rows");
+ writer.array();
+ for (int r = 0; r < topicNames.size(); r++) {
+ String id = topicIds.get(r);
+ String topicName = topicNames.get(r);
+
+ if (id != null && map.containsKey(id)) {
+ DataExtension ext = map.get(id);
+ boolean first = true;
+
+ if (ext.data.length > 0) {
+ for (Object[] row : ext.data) {
+ writer.array();
+ if (first) {
+ writer.value(topicName);
+ first = false;
+ } else {
+ writer.value(null);
+ }
+
+ for (Object cell : row) {
+ if (cell != null && cell instanceof ReconCandidate) {
+ ReconCandidate rc = (ReconCandidate) cell;
+ writer.object();
+ writer.key("id"); writer.value(rc.id);
+ writer.key("name"); writer.value(rc.name);
+ writer.endObject();
+ } else {
+ writer.value(cell);
+ }
+ }
+
+ writer.endArray();
+ }
+ continue;
+ }
+ }
+
+ writer.array();
+ if (id != null) {
+ writer.object();
+ writer.key("id"); writer.value(id);
+ writer.key("name"); writer.value(topicName);
+ writer.endObject();
+ } else {
+ writer.value("");
+ }
+ writer.endArray();
+ }
+ writer.endArray();
+
+ writer.endObject();
+ } catch (Exception e) {
+ respondException(response, e);
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java
new file mode 100644
index 000000000..c9eddd4ea
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java
@@ -0,0 +1,71 @@
+package com.google.refine.freebase.commands;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONObject;
+
+import com.google.refine.browsing.Engine;
+import com.google.refine.browsing.FilteredRows;
+import com.google.refine.commands.Command;
+import com.google.refine.freebase.protograph.Protograph;
+import com.google.refine.model.Project;
+import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory;
+import com.google.refine.freebase.protograph.transpose.Transposer;
+import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory;
+import com.google.refine.util.ParsingUtilities;
+
+public class PreviewProtographCommand extends Command {
+ @Override
+ public void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ try {
+ Project project = getProject(request);
+ Engine engine = getEngine(request, project);
+ FilteredRows filteredRows = engine.getAllFilteredRows();
+
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "application/json");
+
+ String jsonString = request.getParameter("protograph");
+ JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
+ Protograph protograph = Protograph.reconstruct(json);
+
+ StringBuffer sb = new StringBuffer(2048);
+ sb.append("{ ");
+
+ {
+ StringWriter stringWriter = new StringWriter();
+ TripleLoaderTransposedNodeFactory nodeFactory = new TripleLoaderTransposedNodeFactory(project, stringWriter);
+
+ Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
+ nodeFactory.flush();
+
+ sb.append("\"tripleloader\" : ");
+ sb.append(JSONObject.quote(stringWriter.toString()));
+ }
+
+ {
+ StringWriter stringWriter = new StringWriter();
+ MqlwriteLikeTransposedNodeFactory nodeFactory = new MqlwriteLikeTransposedNodeFactory(stringWriter);
+
+ Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory);
+ nodeFactory.flush();
+
+ sb.append(", \"mqllike\" : ");
+ sb.append(stringWriter.toString());
+ }
+
+ sb.append(" }");
+
+ respond(response, sb.toString());
+ } catch (Exception e) {
+ respondException(response, e);
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java
new file mode 100644
index 000000000..4382de096
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java
@@ -0,0 +1,40 @@
+package com.google.refine.freebase.commands;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONObject;
+
+import com.google.refine.commands.Command;
+import com.google.refine.freebase.protograph.Protograph;
+import com.google.refine.model.AbstractOperation;
+import com.google.refine.model.Project;
+import com.google.refine.freebase.operations.SaveProtographOperation;
+import com.google.refine.process.Process;
+import com.google.refine.util.ParsingUtilities;
+
+public class SaveProtographCommand extends Command {
+ @Override
+ public void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ try {
+ Project project = getProject(request);
+
+ String jsonString = request.getParameter("protograph");
+ JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
+ Protograph protograph = Protograph.reconstruct(json);
+
+ AbstractOperation op = new SaveProtographOperation(protograph);
+ Process process = op.createProcess(project, new Properties());
+
+ performProcessAndRespond(request, response, project, process);
+ } catch (Exception e) {
+ respondException(response, e);
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java
new file mode 100644
index 000000000..180c1a5e8
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java
@@ -0,0 +1,89 @@
+package com.google.refine.freebase.commands;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.net.URL;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import com.google.refine.ProjectManager;
+import com.google.refine.browsing.Engine;
+import com.google.refine.commands.Command;
+import com.google.refine.freebase.ProtographTransposeExporter.TripleLoaderExporter;
+import com.google.refine.freebase.util.FreebaseUtils;
+import com.google.refine.model.Project;
+import com.google.refine.preference.PreferenceStore;
+import com.google.refine.util.ParsingUtilities;
+
+public class UploadDataCommand extends Command {
+ final static public String s_dataLoadJobIDPref = "freebase.load.jobID";
+ final static public String s_dataLoadJobNamePref = "freebase.load.jobName";
+
+ @Override
+ public void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ ProjectManager.singleton.setBusy(true);
+ try {
+ Project project = getProject(request);
+ Engine engine = getEngine(request, project);
+ PreferenceStore preferenceStore = project.getMetadata().getPreferenceStore();
+
+ TripleLoaderExporter exporter = new TripleLoaderExporter();
+ StringWriter triples = new StringWriter(10 * 1024 * 1024);
+ exporter.export(project, new Properties(), engine, triples);
+
+ String source_name = request.getParameter("source_name");
+ String source_id = request.getParameter("source_id");
+ String qa = request.getParameter("qa");
+ String mdo_id = null;
+
+ preferenceStore.put(s_dataLoadJobNamePref, source_name);
+
+ try {
+ Integer jobID = (Integer) preferenceStore.get(s_dataLoadJobIDPref);
+ if (jobID != null) {
+ URL url = new URL("http://refinery.freebaseapps.com/job_id_to_mdo?job=" + jobID);
+ String s = ParsingUtilities.inputStreamToString(url.openConnection().getInputStream());
+
+ if (!s.equals("null")) {
+ mdo_id = s;
+ }
+ }
+ } catch (Exception e) {
+ // ignore
+ }
+
+ String uploadResponse = FreebaseUtils.uploadTriples(
+ request, qa, source_name, source_id, mdo_id, triples.toString()
+ );
+
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "application/json");
+
+ try {
+ JSONObject obj = new JSONObject(uploadResponse);
+ if (obj.has("result") && !obj.isNull("result")) {
+ JSONObject result = obj.getJSONObject("result");
+ if (result.has("job_id") && !result.isNull("job_id")) {
+ Integer jobID = result.getInt("job_id");
+ project.getMetadata().getPreferenceStore().put(s_dataLoadJobIDPref, jobID);
+ }
+ }
+ response.getWriter().write(uploadResponse);
+ } catch (JSONException e) {
+ respond(response,"500 Error", uploadResponse);
+ }
+ } catch (Exception e) {
+ respondException(response, e);
+ } finally {
+ ProjectManager.singleton.setBusy(false);
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/AuthorizeCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/AuthorizeCommand.java
new file mode 100644
index 000000000..6940c193e
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/auth/AuthorizeCommand.java
@@ -0,0 +1,135 @@
+package com.google.refine.freebase.commands.auth;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import oauth.signpost.OAuthConsumer;
+import oauth.signpost.OAuthProvider;
+
+import com.google.refine.commands.Command;
+import com.google.refine.oauth.Credentials;
+import com.google.refine.oauth.OAuthUtilities;
+import com.google.refine.oauth.Provider;
+
+public class AuthorizeCommand extends Command {
+
+ private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier";
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+
+ // get the provider from the request
+ Provider provider = OAuthUtilities.getProvider(request);
+
+ try {
+
+ // see if the request comes with access credentials
+ Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
+
+ // prepare the continuation URL that the OAuth provider will redirect the user to
+ // (we need to make sure this URL points back to this code or the dance will never complete)
+ String callbackURL = getBaseURL(request,provider);
+
+ if (access_credentials == null) {
+ // access credentials are not available so we need to check
+ // to see at what stage of the OAuth dance we are
+
+ // get the request token credentials
+ Credentials request_credentials = Credentials.getCredentials(request, provider, Credentials.Type.REQUEST);
+
+ OAuthConsumer consumer = OAuthUtilities.getConsumer(request_credentials, provider);
+ OAuthProvider pp = OAuthUtilities.getOAuthProvider(provider);
+
+ if (request_credentials == null) {
+ // no credentials were found, so let's start the dance
+
+ // get the request token
+
+ String url = pp.retrieveRequestToken(consumer, callbackURL);
+
+ request_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
+
+ // and set them to that we can retrieve them later in the second part of the dance
+ Credentials.setCredentials(request, response, request_credentials, Credentials.Type.REQUEST, 3600);
+
+ // now redirect the user to the Authorize URL where she can authenticate against the
+ // service provider and authorize us.
+ // The provider will bounce the user back here for us to continue the dance.
+
+ response.sendRedirect(url);
+ } else {
+ // we are at the second stage of the dance, so we need need to obtain the access credentials now
+
+ // if we got here, it means that the user performed a valid authentication against the
+ // service provider and authorized us, so now we can request more permanent credentials
+ // to the service provider and save those as well for later use.
+
+ // this is set only for OAuth 1.0a
+ String verificationCode = request.getParameter(OAUTH_VERIFIER_PARAM);
+
+ pp.retrieveAccessToken(consumer, verificationCode);
+
+ access_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider);
+
+ // no matter the result, we need to remove the request token
+ Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
+
+ Credentials.setCredentials(request, response, access_credentials, Credentials.Type.ACCESS, 30 * 24 * 3600);
+
+ finish(response);
+ }
+ } else {
+ finish(response);
+ }
+ } catch (Exception e) {
+ Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
+ Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
+ respondException(response, e);
+ }
+ }
+
+ private void finish(HttpServletResponse response) throws IOException {
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "text/html");
+
+ PrintWriter writer = response.getWriter();
+ writer.write(
+ "" +
+ "" +
+ "" +
+ ""
+ );
+ writer.flush();
+ }
+
+ private String getBaseURL(HttpServletRequest request, Provider provider) {
+ String host = request.getHeader("host");
+ if (host == null) {
+ String referrer = request.getHeader("referer");
+ if (referrer != null) {
+ URI url;
+ try {
+ url = new URI(referrer);
+ int port = url.getPort();
+ host = url.getHost() + ((port > -1) ? ":" + url.getPort() : "");
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL");
+ }
+ } else {
+ throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL Google Refine is listening to.");
+ }
+ }
+ return "http://" + host + "/command/freebase-extension/authorize/" + provider.getHost();
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java
new file mode 100644
index 000000000..9ee8f8d51
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java
@@ -0,0 +1,47 @@
+package com.google.refine.freebase.commands.auth;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.refine.commands.Command;
+import com.google.refine.freebase.util.FreebaseUtils;
+import com.google.refine.oauth.Credentials;
+import com.google.refine.oauth.OAuthUtilities;
+import com.google.refine.oauth.Provider;
+
+public class CheckAuthorizationCommand extends Command {
+
+ final static Logger logger = LoggerFactory.getLogger("check-authorization_command");
+
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+
+ try {
+ Provider provider = OAuthUtilities.getProvider(request);
+
+ // this cookie should not be there, but this is good hygiene practice
+ Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST);
+
+ Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
+
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "application/json");
+
+ if (access_credentials != null) {
+ String user_info = FreebaseUtils.getUserInfo(access_credentials, provider);
+ response.getWriter().write(user_info);
+ } else {
+ respond(response, "401 Unauthorized", "You don't have the right credentials");
+ }
+ } catch (Exception e) {
+ logger.info("error",e);
+ respondException(response, e);
+ }
+ }
+
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/DeAuthorizeCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/DeAuthorizeCommand.java
new file mode 100644
index 000000000..f0f82bdeb
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/auth/DeAuthorizeCommand.java
@@ -0,0 +1,32 @@
+package com.google.refine.freebase.commands.auth;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.google.refine.commands.Command;
+import com.google.refine.oauth.Credentials;
+import com.google.refine.oauth.OAuthUtilities;
+import com.google.refine.oauth.Provider;
+
+public class DeAuthorizeCommand extends Command {
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+
+ try {
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "application/json");
+
+ Provider provider = OAuthUtilities.getProvider(request);
+
+ Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS);
+
+ respond(response, "200 OK", "");
+ } catch (Exception e) {
+ respondException(response, e);
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java
new file mode 100644
index 000000000..a3a1221c6
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java
@@ -0,0 +1,37 @@
+package com.google.refine.freebase.commands.auth;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.refine.commands.Command;
+import com.google.refine.freebase.util.FreebaseUtils;
+import com.google.refine.oauth.OAuthUtilities;
+import com.google.refine.oauth.Provider;
+
+public class GetUserBadgesCommand extends Command {
+
+ final static Logger logger = LoggerFactory.getLogger("get-version_command");
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+
+ try {
+ Provider provider = OAuthUtilities.getProvider(request);
+ String user_id = request.getParameter("user_id");
+ response.setCharacterEncoding("UTF-8");
+ response.setHeader("Content-Type", "application/json");
+ String user_badges = FreebaseUtils.getUserBadges(provider, user_id);
+ response.getWriter().write(user_badges);
+ } catch (Exception e) {
+ logger.info("error",e);
+ respondException(response, e);
+ }
+ }
+
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java b/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java
new file mode 100644
index 000000000..b57ebbe75
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java
@@ -0,0 +1,431 @@
+package com.google.refine.freebase.model.changes;
+
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.io.Serializable;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.FreebaseType;
+import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
+import com.google.refine.history.Change;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Column;
+import com.google.refine.model.ModelException;
+import com.google.refine.model.Project;
+import com.google.refine.model.Recon;
+import com.google.refine.model.ReconCandidate;
+import com.google.refine.model.ReconStats;
+import com.google.refine.model.Row;
+import com.google.refine.model.Recon.Judgment;
+import com.google.refine.freebase.model.recon.DataExtensionReconConfig;
+import com.google.refine.util.ParsingUtilities;
+import com.google.refine.util.Pool;
+
+public class DataExtensionChange implements Change {
+ final protected String _baseColumnName;
+ final protected int _columnInsertIndex;
+
+ final protected List _columnNames;
+ final protected List _columnTypes;
+
+ final protected List _rowIndices;
+ final protected List _dataExtensions;
+
+ protected long _historyEntryID;
+ protected int _firstNewCellIndex = -1;
+ protected List _oldRows;
+ protected List _newRows;
+
+ public DataExtensionChange(
+ String baseColumnName,
+ int columnInsertIndex,
+ List columnNames,
+ List columnTypes,
+ List rowIndices,
+ List dataExtensions,
+ long historyEntryID
+ ) {
+ _baseColumnName = baseColumnName;
+ _columnInsertIndex = columnInsertIndex;
+
+ _columnNames = columnNames;
+ _columnTypes = columnTypes;
+
+ _rowIndices = rowIndices;
+ _dataExtensions = dataExtensions;
+
+ _historyEntryID = historyEntryID;
+ }
+
+ protected DataExtensionChange(
+ String baseColumnName,
+ int columnInsertIndex,
+
+ List columnNames,
+ List columnTypes,
+
+ List rowIndices,
+ List dataExtensions,
+ int firstNewCellIndex,
+ List oldRows,
+ List newRows
+ ) {
+ _baseColumnName = baseColumnName;
+ _columnInsertIndex = columnInsertIndex;
+
+ _columnNames = columnNames;
+ _columnTypes = columnTypes;
+
+ _rowIndices = rowIndices;
+ _dataExtensions = dataExtensions;
+
+ _firstNewCellIndex = firstNewCellIndex;
+ _oldRows = oldRows;
+ _newRows = newRows;
+ }
+
+ public void apply(Project project) {
+ synchronized (project) {
+ if (_firstNewCellIndex < 0) {
+ _firstNewCellIndex = project.columnModel.allocateNewCellIndex();
+ for (int i = 1; i < _columnNames.size(); i++) {
+ project.columnModel.allocateNewCellIndex();
+ }
+
+ _oldRows = new ArrayList(project.rows);
+
+ _newRows = new ArrayList(project.rows.size());
+
+ int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex();
+ int keyCellIndex = project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex();
+ int index = 0;
+
+ int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
+ DataExtension dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
+
+ index++;
+
+ Map reconMap = new HashMap();
+
+ for (int r = 0; r < _oldRows.size(); r++) {
+ Row oldRow = _oldRows.get(r);
+ if (r < rowIndex) {
+ _newRows.add(oldRow.dup());
+ continue;
+ }
+
+ if (dataExtension == null || dataExtension.data.length == 0) {
+ _newRows.add(oldRow);
+ } else {
+ Row firstNewRow = oldRow.dup();
+ extendRow(firstNewRow, dataExtension, 0, reconMap);
+ _newRows.add(firstNewRow);
+
+ int r2 = r + 1;
+ for (int subR = 1; subR < dataExtension.data.length; subR++) {
+ if (r2 < project.rows.size()) {
+ Row oldRow2 = project.rows.get(r2);
+ if (oldRow2.isCellBlank(cellIndex) &&
+ oldRow2.isCellBlank(keyCellIndex)) {
+
+ Row newRow = oldRow2.dup();
+ extendRow(newRow, dataExtension, subR, reconMap);
+
+ _newRows.add(newRow);
+ r2++;
+
+ continue;
+ }
+ }
+
+ Row newRow = new Row(cellIndex + _columnNames.size());
+ extendRow(newRow, dataExtension, subR, reconMap);
+
+ _newRows.add(newRow);
+ }
+
+ r = r2 - 1; // r will be incremented by the for loop anyway
+ }
+
+ rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
+ dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
+ index++;
+ }
+ }
+
+ project.rows.clear();
+ project.rows.addAll(_newRows);
+
+ for (int i = 0; i < _columnNames.size(); i++) {
+ String name = _columnNames.get(i);
+ int cellIndex = _firstNewCellIndex + i;
+
+ Column column = new Column(cellIndex, name);
+ column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i)));
+ column.setReconStats(ReconStats.create(project, cellIndex));
+
+ try {
+ project.columnModel.addColumn(_columnInsertIndex + i, column, true);
+
+ // the column might have been renamed to avoid collision
+ _columnNames.set(i, column.getName());
+ } catch (ModelException e) {
+ // won't get here since we set the avoid collision flag
+ }
+ }
+
+ project.update();
+ }
+ }
+
+ protected void extendRow(
+ Row row,
+ DataExtension dataExtension,
+ int extensionRowIndex,
+ Map reconMap
+ ) {
+ Object[] values = dataExtension.data[extensionRowIndex];
+ for (int c = 0; c < values.length; c++) {
+ Object value = values[c];
+ Cell cell = null;
+
+ if (value instanceof ReconCandidate) {
+ ReconCandidate rc = (ReconCandidate) value;
+ Recon recon;
+ if (reconMap.containsKey(rc.id)) {
+ recon = reconMap.get(rc.id);
+ } else {
+ recon = Recon.makeFreebaseRecon(_historyEntryID);
+ recon.addCandidate(rc);
+ recon.service = "mql";
+ recon.match = rc;
+ recon.matchRank = 0;
+ recon.judgment = Judgment.Matched;
+ recon.judgmentAction = "auto";
+ recon.judgmentBatchSize = 1;
+
+ reconMap.put(rc.id, recon);
+ }
+ cell = new Cell(rc.name, recon);
+ } else {
+ cell = new Cell((Serializable) value, null);
+ }
+
+ row.setCell(_firstNewCellIndex + c, cell);
+ }
+ }
+
+ public void revert(Project project) {
+ synchronized (project) {
+ project.rows.clear();
+ project.rows.addAll(_oldRows);
+
+ for (int i = 0; i < _columnNames.size(); i++) {
+ project.columnModel.columns.remove(_columnInsertIndex);
+ }
+
+ project.update();
+ }
+ }
+
+ public void save(Writer writer, Properties options) throws IOException {
+ writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n');
+ writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n');
+ writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
+ for (String name : _columnNames) {
+ writer.write(name); writer.write('\n');
+ }
+ writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n');
+ for (FreebaseType type : _columnTypes) {
+ try {
+ JSONWriter jsonWriter = new JSONWriter(writer);
+
+ type.write(jsonWriter, options);
+ } catch (JSONException e) {
+ // ???
+ }
+ writer.write('\n');
+ }
+ writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
+ for (Integer rowIndex : _rowIndices) {
+ writer.write(rowIndex.toString()); writer.write('\n');
+ }
+ writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
+ for (DataExtension dataExtension : _dataExtensions) {
+ if (dataExtension == null) {
+ writer.write('\n');
+ continue;
+ }
+
+ writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
+
+ for (Object[] values : dataExtension.data) {
+ for (Object value : values) {
+ if (value == null) {
+ writer.write("null");
+ } else if (value instanceof ReconCandidate) {
+ try {
+ JSONWriter jsonWriter = new JSONWriter(writer);
+ ((ReconCandidate) value).write(jsonWriter, options);
+ } catch (JSONException e) {
+ // ???
+ }
+ } else if (value instanceof String) {
+ writer.write(JSONObject.quote((String) value));
+ } else {
+ writer.write(value.toString());
+ }
+ writer.write('\n');
+ }
+ }
+ }
+
+ writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
+
+ writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
+ for (Row row : _newRows) {
+ row.save(writer, options);
+ writer.write('\n');
+ }
+ writer.write("oldRowCount="); writer.write(Integer.toString(_oldRows.size())); writer.write('\n');
+ for (Row row : _oldRows) {
+ row.save(writer, options);
+ writer.write('\n');
+ }
+ writer.write("/ec/\n"); // end of change marker
+ }
+
+ static public Change load(LineNumberReader reader, Pool pool) throws Exception {
+ String baseColumnName = null;
+ int columnInsertIndex = -1;
+
+ List columnNames = null;
+ List columnTypes = null;
+
+ List rowIndices = null;
+ List dataExtensions = null;
+
+ List oldRows = null;
+ List newRows = null;
+
+ int firstNewCellIndex = -1;
+
+ String line;
+ while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
+ int equal = line.indexOf('=');
+ CharSequence field = line.subSequence(0, equal);
+ String value = line.substring(equal + 1);
+
+ if ("baseColumnName".equals(field)) {
+ baseColumnName = value;
+ } else if ("columnInsertIndex".equals(field)) {
+ columnInsertIndex = Integer.parseInt(value);
+ } else if ("firstNewCellIndex".equals(field)) {
+ firstNewCellIndex = Integer.parseInt(value);
+ } else if ("rowIndexCount".equals(field)) {
+ int count = Integer.parseInt(value);
+
+ rowIndices = new ArrayList(count);
+ for (int i = 0; i < count; i++) {
+ line = reader.readLine();
+ if (line != null) {
+ rowIndices.add(Integer.parseInt(line));
+ }
+ }
+ } else if ("columnNameCount".equals(field)) {
+ int count = Integer.parseInt(value);
+
+ columnNames = new ArrayList(count);
+ for (int i = 0; i < count; i++) {
+ line = reader.readLine();
+ if (line != null) {
+ columnNames.add(line);
+ }
+ }
+ } else if ("columnTypeCount".equals(field)) {
+ int count = Integer.parseInt(value);
+
+ columnTypes = new ArrayList(count);
+ for (int i = 0; i < count; i++) {
+ line = reader.readLine();
+ columnTypes.add(FreebaseType.load(ParsingUtilities.evaluateJsonStringToObject(line)));
+ }
+ } else if ("dataExtensionCount".equals(field)) {
+ int count = Integer.parseInt(value);
+
+ dataExtensions = new ArrayList(count);
+ for (int i = 0; i < count; i++) {
+ line = reader.readLine();
+
+ if (line == null) continue;
+
+ if (line.length() == 0) {
+ dataExtensions.add(null);
+ continue;
+ }
+
+ int rowCount = Integer.parseInt(line);
+ Object[][] data = new Object[rowCount][];
+
+ for (int r = 0; r < rowCount; r++) {
+ Object[] row = new Object[columnNames.size()];
+ for (int c = 0; c < columnNames.size(); c++) {
+ line = reader.readLine();
+
+ row[c] = ReconCandidate.loadStreaming(line);
+ }
+
+ data[r] = row;
+ }
+
+ dataExtensions.add(new DataExtension(data));
+ }
+ } else if ("oldRowCount".equals(field)) {
+ int count = Integer.parseInt(value);
+
+ oldRows = new ArrayList(count);
+ for (int i = 0; i < count; i++) {
+ line = reader.readLine();
+ if (line != null) {
+ oldRows.add(Row.load(line, pool));
+ }
+ }
+ } else if ("newRowCount".equals(field)) {
+ int count = Integer.parseInt(value);
+
+ newRows = new ArrayList(count);
+ for (int i = 0; i < count; i++) {
+ line = reader.readLine();
+ if (line != null) {
+ newRows.add(Row.load(line, pool));
+ }
+ }
+ }
+
+ }
+
+ DataExtensionChange change = new DataExtensionChange(
+ baseColumnName,
+ columnInsertIndex,
+ columnNames,
+ columnTypes,
+ rowIndices,
+ dataExtensions,
+ firstNewCellIndex,
+ oldRows,
+ newRows
+ );
+
+
+ return change;
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java
new file mode 100644
index 000000000..57a78112f
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java
@@ -0,0 +1,66 @@
+package com.google.refine.freebase.model.recon;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.FreebaseType;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Project;
+import com.google.refine.model.Recon;
+import com.google.refine.model.Row;
+import com.google.refine.model.recon.ReconConfig;
+import com.google.refine.model.recon.ReconJob;
+import com.google.refine.freebase.model.recon.StrictReconConfig;
+
+public class DataExtensionReconConfig extends StrictReconConfig {
+ final public FreebaseType type;
+
+ private final static String WARN = "Not implemented";
+
+ static public ReconConfig reconstruct(JSONObject obj) throws Exception {
+ JSONObject type = obj.getJSONObject("type");
+
+ return new DataExtensionReconConfig(
+ new FreebaseType(
+ type.getString("id"),
+ type.getString("name")
+ )
+ );
+ }
+
+ public DataExtensionReconConfig(FreebaseType type) {
+ this.type = type;
+ }
+
+ @Override
+ public ReconJob createJob(Project project, int rowIndex, Row row,
+ String columnName, Cell cell) {
+ throw new RuntimeException(WARN);
+ }
+
+ @Override
+ public int getBatchSize() {
+ throw new RuntimeException(WARN);
+ }
+
+ public void write(JSONWriter writer, Properties options) throws JSONException {
+ writer.object();
+ writer.key("mode"); writer.value("extend");
+ writer.key("type"); type.write(writer, options);
+ writer.endObject();
+ }
+
+ @Override
+ public List batchRecon(List jobs, long historyEntryID) {
+ throw new RuntimeException(WARN);
+ }
+
+ @Override
+ public String getBriefDescription(Project project, String columnName) {
+ throw new RuntimeException(WARN);
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
new file mode 100644
index 000000000..726d1129e
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java
@@ -0,0 +1,176 @@
+package com.google.refine.freebase.model.recon;
+
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.model.Cell;
+import com.google.refine.model.Project;
+import com.google.refine.model.Recon;
+import com.google.refine.model.ReconCandidate;
+import com.google.refine.model.Row;
+import com.google.refine.model.Recon.Judgment;
+import com.google.refine.model.recon.ReconConfig;
+import com.google.refine.model.recon.ReconJob;
+import com.google.refine.util.ParsingUtilities;
+
+public class GuidBasedReconConfig extends StrictReconConfig {
+ static public ReconConfig reconstruct(JSONObject obj) throws Exception {
+ return new GuidBasedReconConfig();
+ }
+
+ public GuidBasedReconConfig() {
+ }
+
+ static protected class GuidBasedReconJob extends ReconJob {
+ String guid;
+
+ public int getKey() {
+ return guid.hashCode();
+ }
+ }
+
+ @Override
+ public ReconJob createJob(Project project, int rowIndex, Row row,
+ String columnName, Cell cell) {
+
+ GuidBasedReconJob job = new GuidBasedReconJob();
+ String s = cell.value.toString();
+
+ if (s.startsWith("/guid/")) {
+ s = "#" + s.substring(6);
+ } else if (!s.startsWith("#")) {
+ s = "#" + s;
+ }
+
+ job.guid = s;
+
+ return job;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return 10;
+ }
+
+ @Override
+ public String getBriefDescription(Project project, String columnName) {
+ return "Reconcile cells in column " + columnName + " as Freebase IDs";
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("mode"); writer.value("strict");
+ writer.key("match"); writer.value("id");
+ writer.endObject();
+ }
+
+ @Override
+ public List batchRecon(List jobs, long historyEntryID) {
+ List recons = new ArrayList(jobs.size());
+ Map guidToRecon = new HashMap();
+
+ try {
+ String query = null;
+ {
+ StringWriter stringWriter = new StringWriter();
+ JSONWriter jsonWriter = new JSONWriter(stringWriter);
+
+ jsonWriter.object();
+ jsonWriter.key("query");
+ jsonWriter.array();
+ jsonWriter.object();
+
+ jsonWriter.key("id"); jsonWriter.value(null);
+ jsonWriter.key("name"); jsonWriter.value(null);
+ jsonWriter.key("guid"); jsonWriter.value(null);
+ jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
+
+ jsonWriter.key("guid|=");
+ jsonWriter.array();
+ for (ReconJob job : jobs) {
+ jsonWriter.value(((GuidBasedReconJob) job).guid);
+ }
+ jsonWriter.endArray();
+
+ jsonWriter.endObject();
+ jsonWriter.endArray();
+ jsonWriter.endObject();
+
+ query = stringWriter.toString();
+ }
+
+ StringBuffer sb = new StringBuffer(1024);
+ sb.append(s_mqlreadService);
+ sb.append("?query=");
+ sb.append(ParsingUtilities.encode(query));
+
+ URL url = new URL(sb.toString());
+ URLConnection connection = url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.connect();
+
+ InputStream is = connection.getInputStream();
+ try {
+ String s = ParsingUtilities.inputStreamToString(is);
+ JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
+ JSONArray results = o.getJSONArray("result");
+ int count = results.length();
+
+ for (int i = 0; i < count; i++) {
+ JSONObject result = results.getJSONObject(i);
+
+ String guid = result.getString("guid");
+
+ JSONArray types = result.getJSONArray("type");
+ String[] typeIDs = new String[types.length()];
+ for (int j = 0; j < typeIDs.length; j++) {
+ typeIDs[j] = types.getString(j);
+ }
+
+ ReconCandidate candidate = new ReconCandidate(
+ result.getString("id"),
+ result.getString("name"),
+ typeIDs,
+ 100
+ );
+
+ Recon recon = Recon.makeFreebaseRecon(historyEntryID);
+ recon.addCandidate(candidate);
+ recon.service = "mql";
+ recon.judgment = Judgment.Matched;
+ recon.judgmentAction = "auto";
+ recon.match = candidate;
+ recon.matchRank = 0;
+
+ guidToRecon.put(guid, recon);
+ }
+ } finally {
+ is.close();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ for (int i = 0; i < jobs.size(); i++) {
+ String guid = ((GuidBasedReconJob) jobs.get(i)).guid;
+ Recon recon = guidToRecon.get(guid);
+ recons.add(recon);
+ }
+
+ return recons;
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
new file mode 100644
index 000000000..6955f5ac1
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java
@@ -0,0 +1,181 @@
+package com.google.refine.freebase.model.recon;
+
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.model.Cell;
+import com.google.refine.model.Project;
+import com.google.refine.model.Recon;
+import com.google.refine.model.ReconCandidate;
+import com.google.refine.model.Row;
+import com.google.refine.model.Recon.Judgment;
+import com.google.refine.model.recon.ReconConfig;
+import com.google.refine.model.recon.ReconJob;
+import com.google.refine.util.ParsingUtilities;
+
+public class IdBasedReconConfig extends StrictReconConfig {
+ static public ReconConfig reconstruct(JSONObject obj) throws Exception {
+ return new IdBasedReconConfig();
+ }
+
+ public IdBasedReconConfig() {
+ }
+
+ static protected class IdBasedReconJob extends ReconJob {
+ String id;
+
+ public int getKey() {
+ return id.hashCode();
+ }
+ }
+
+ @Override
+ public ReconJob createJob(Project project, int rowIndex, Row row,
+ String columnName, Cell cell) {
+
+ IdBasedReconJob job = new IdBasedReconJob();
+ String s = cell.value.toString();
+
+ if (!s.startsWith("/")) {
+ if (s.startsWith("92")) {
+ s = "/guid/" + s;
+ } else if (!s.contains("/")){
+ s = "/en/" + s;
+ } else {
+ s = "/" + s;
+ }
+ }
+
+ job.id = s;
+
+ return job;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return 10;
+ }
+
+ @Override
+ public String getBriefDescription(Project project, String columnName) {
+ return "Reconcile cells in column " + columnName + " as Freebase IDs";
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("mode"); writer.value("strict");
+ writer.key("match"); writer.value("id");
+ writer.endObject();
+ }
+
+ @Override
+ public List batchRecon(List jobs, long historyEntryID) {
+ List recons = new ArrayList(jobs.size());
+ Map idToRecon = new HashMap();
+
+ try {
+ String query = null;
+ {
+ StringWriter stringWriter = new StringWriter();
+ JSONWriter jsonWriter = new JSONWriter(stringWriter);
+
+ jsonWriter.object();
+ jsonWriter.key("query");
+ jsonWriter.array();
+ jsonWriter.object();
+
+ jsonWriter.key("id"); jsonWriter.value(null);
+ jsonWriter.key("name"); jsonWriter.value(null);
+ jsonWriter.key("guid"); jsonWriter.value(null);
+ jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
+
+ jsonWriter.key("id|=");
+ jsonWriter.array();
+ for (ReconJob job : jobs) {
+ jsonWriter.value(((IdBasedReconJob) job).id);
+ }
+ jsonWriter.endArray();
+
+ jsonWriter.endObject();
+ jsonWriter.endArray();
+ jsonWriter.endObject();
+
+ query = stringWriter.toString();
+ }
+
+ StringBuffer sb = new StringBuffer(1024);
+ sb.append(s_mqlreadService);
+ sb.append("?query=");
+ sb.append(ParsingUtilities.encode(query));
+
+ URL url = new URL(sb.toString());
+ URLConnection connection = url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.connect();
+
+ InputStream is = connection.getInputStream();
+ try {
+ String s = ParsingUtilities.inputStreamToString(is);
+ JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
+ JSONArray results = o.getJSONArray("result");
+ int count = results.length();
+
+ for (int i = 0; i < count; i++) {
+ JSONObject result = results.getJSONObject(i);
+
+ String id = result.getString("id");
+
+ JSONArray types = result.getJSONArray("type");
+ String[] typeIDs = new String[types.length()];
+ for (int j = 0; j < typeIDs.length; j++) {
+ typeIDs[j] = types.getString(j);
+ }
+
+ ReconCandidate candidate = new ReconCandidate(
+ id,
+ result.getString("name"),
+ typeIDs,
+ 100
+ );
+
+ Recon recon = Recon.makeFreebaseRecon(historyEntryID);
+ recon.addCandidate(candidate);
+ recon.service = "mql";
+ recon.judgment = Judgment.Matched;
+ recon.judgmentAction = "auto";
+ recon.match = candidate;
+ recon.matchRank = 0;
+
+ idToRecon.put(id, recon);
+ }
+ } finally {
+ is.close();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ for (int i = 0; i < jobs.size(); i++) {
+ String id = ((IdBasedReconJob) jobs.get(i)).id;
+ Recon recon = idToRecon.get(id);
+ recons.add(recon);
+ }
+
+ return recons;
+ }
+
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
new file mode 100644
index 000000000..7c0d14d96
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java
@@ -0,0 +1,195 @@
+package com.google.refine.freebase.model.recon;
+
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.FreebaseTopic;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Project;
+import com.google.refine.model.Recon;
+import com.google.refine.model.ReconCandidate;
+import com.google.refine.model.Row;
+import com.google.refine.model.Recon.Judgment;
+import com.google.refine.model.recon.ReconConfig;
+import com.google.refine.model.recon.ReconJob;
+import com.google.refine.util.ParsingUtilities;
+
+public class KeyBasedReconConfig extends StrictReconConfig {
+ final public FreebaseTopic namespace;
+
+ static public ReconConfig reconstruct(JSONObject obj) throws Exception {
+ JSONObject ns = obj.getJSONObject("namespace");
+
+ return new KeyBasedReconConfig(
+ new FreebaseTopic(
+ ns.getString("id"),
+ ns.getString("name")
+ )
+ );
+ }
+
+ public KeyBasedReconConfig(FreebaseTopic namespace) {
+ this.namespace = namespace;
+ }
+
+ static protected class KeyBasedReconJob extends ReconJob {
+ String key;
+
+ public int getKey() {
+ return key.hashCode();
+ }
+ }
+
+ @Override
+ public ReconJob createJob(Project project, int rowIndex, Row row,
+ String columnName, Cell cell) {
+
+ KeyBasedReconJob job = new KeyBasedReconJob();
+
+ job.key = cell.value.toString().replace(' ', '_');
+
+ return job;
+ }
+
+ @Override
+ public int getBatchSize() {
+ return 10;
+ }
+
+ @Override
+ public String getBriefDescription(Project project, String columnName) {
+ return "Reconcile cells in column " + columnName + " to topics with keys in namespace " + namespace.id;
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("mode"); writer.value("strict");
+ writer.key("match"); writer.value("key");
+ writer.key("namespace"); namespace.write(writer, options);
+ writer.endObject();
+ }
+
+ @Override
+ public List batchRecon(List jobs, long historyEntryID) {
+ List recons = new ArrayList(jobs.size());
+ Map keyToRecon = new HashMap();
+
+ try {
+ String query = null;
+ {
+ StringWriter stringWriter = new StringWriter();
+ JSONWriter jsonWriter = new JSONWriter(stringWriter);
+
+ jsonWriter.object();
+ jsonWriter.key("query");
+ jsonWriter.array();
+ jsonWriter.object();
+
+ jsonWriter.key("id"); jsonWriter.value(null);
+ jsonWriter.key("name"); jsonWriter.value(null);
+ jsonWriter.key("guid"); jsonWriter.value(null);
+ jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray();
+
+ jsonWriter.key("key");
+ jsonWriter.array();
+ jsonWriter.object();
+
+ jsonWriter.key("namespace");
+ jsonWriter.object();
+ jsonWriter.key("id"); jsonWriter.value(namespace.id);
+ jsonWriter.endObject();
+
+ jsonWriter.key("value"); jsonWriter.value(null);
+ jsonWriter.key("value|=");
+ jsonWriter.array();
+ for (ReconJob job : jobs) {
+ jsonWriter.value(((KeyBasedReconJob) job).key);
+ }
+ jsonWriter.endArray();
+
+ jsonWriter.endObject();
+ jsonWriter.endArray();
+
+ jsonWriter.endObject();
+ jsonWriter.endArray();
+ jsonWriter.endObject();
+
+ query = stringWriter.toString();
+ }
+
+ StringBuffer sb = new StringBuffer(1024);
+ sb.append(s_mqlreadService);
+ sb.append("?query=");
+ sb.append(ParsingUtilities.encode(query));
+
+ URL url = new URL(sb.toString());
+ URLConnection connection = url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.connect();
+
+ InputStream is = connection.getInputStream();
+ try {
+ String s = ParsingUtilities.inputStreamToString(is);
+ JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
+ JSONArray results = o.getJSONArray("result");
+ int count = results.length();
+
+ for (int i = 0; i < count; i++) {
+ JSONObject result = results.getJSONObject(i);
+
+ String key = result.getJSONArray("key").getJSONObject(0).getString("value");
+
+ JSONArray types = result.getJSONArray("type");
+ String[] typeIDs = new String[types.length()];
+ for (int j = 0; j < typeIDs.length; j++) {
+ typeIDs[j] = types.getString(j);
+ }
+
+ ReconCandidate candidate = new ReconCandidate(
+ result.getString("id"),
+ result.getString("name"),
+ typeIDs,
+ 100
+ );
+
+ Recon recon = Recon.makeFreebaseRecon(historyEntryID);
+ recon.addCandidate(candidate);
+ recon.service = "mql";
+ recon.judgment = Judgment.Matched;
+ recon.judgmentAction = "auto";
+ recon.match = candidate;
+ recon.matchRank = 0;
+
+ keyToRecon.put(key, recon);
+ }
+ } finally {
+ is.close();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ for (int i = 0; i < jobs.size(); i++) {
+ String key = ((KeyBasedReconJob) jobs.get(i)).key;
+ Recon recon = keyToRecon.get(key);
+ recons.add(recon);
+ }
+
+ return recons;
+ }
+
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
new file mode 100644
index 000000000..e15a1b7d5
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java
@@ -0,0 +1,21 @@
+package com.google.refine.freebase.model.recon;
+
+import org.json.JSONObject;
+
+import com.google.refine.model.recon.ReconConfig;
+
+abstract public class StrictReconConfig extends ReconConfig {
+ final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread";
+
+ static public ReconConfig reconstruct(JSONObject obj) throws Exception {
+ String match = obj.getString("match");
+ if ("key".equals(match)) {
+ return KeyBasedReconConfig.reconstruct(obj);
+ } else if ("id".equals(match)) {
+ return IdBasedReconConfig.reconstruct(obj);
+ } else if ("guid".equals(match)) {
+ return GuidBasedReconConfig.reconstruct(obj);
+ }
+ return null;
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java
new file mode 100644
index 000000000..abadfd707
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java
@@ -0,0 +1,37 @@
+package com.google.refine.freebase.oauth;
+
+import oauth.signpost.OAuthConsumer;
+
+import com.google.refine.oauth.OAuthUtilities;
+import com.google.refine.oauth.Provider;
+import com.google.refine.freebase.util.FreebaseUtils;
+
+public class FreebaseProvider extends Provider {
+ static private final String[] FREEBASE_OAUTH_INFO = { "#9202a8c04000641f80000000185352db" , "4561ee02279e6f04ebd88a1557e4292489380adf"};
+
+ static public void register() {
+ OAuthUtilities.registerOAuthProvider(new FreebaseProvider(FreebaseUtils.FREEBASE_HOST), FREEBASE_OAUTH_INFO);
+ }
+
+ public FreebaseProvider(String host) {
+ super(host);
+ }
+
+ public String getRequestTokenServiceURL() {
+ return "https://" + host + "/api/oauth/request_token";
+ }
+
+ public String getAccessTokenServiceURL() {
+ return "https://" + host + "/api/oauth/access_token";
+ }
+
+ public String getUserAuthorizationURL() {
+ return "https://" + host + "/signin/app";
+ }
+
+
+ @Override
+ public OAuthConsumer createConsumer(String consumerKey, String consumerSecret) {
+ return new FreebaseTimeCommonsHttpOAuthConsumer(consumerKey, consumerSecret);
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java
new file mode 100644
index 000000000..d0ebe1029
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java
@@ -0,0 +1,69 @@
+package com.google.refine.freebase.oauth;
+
+import java.io.IOException;
+
+import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FreebaseTimeCommonsHttpOAuthConsumer extends CommonsHttpOAuthConsumer {
+
+ final static Logger logger = LoggerFactory.getLogger("oauth");
+
+ private static final long serialVersionUID = -4139931605235255279L;
+
+ private static final int SOCKET_TIMEOUT = 3000;
+ private static final int CONNECTION_TIMEOUT = 3000;
+
+ private static final String TIMER_URL = "http://refinery.freebaseapps.com/time";
+
+ public FreebaseTimeCommonsHttpOAuthConsumer(String consumerKey, String consumerSecret) {
+ super(consumerKey, consumerSecret);
+ }
+
+ /**
+ * It might be that the user's computer's clock is not synchronized enough with the Freebase servers
+ * and this might result in Freebase thinking that it was under a replay attack.
+ * To avoid this problem we get the timestamp directly from acre that we know is synchronized.
+ *
+ * NOTE: this call is potentially vulnerable to a man-in-the-middle (MITM) attack, but the same
+ * could be said if we used an NTP client.
+ */
+ protected String generateTimestamp() {
+
+ long time = -1;
+
+ try {
+ HttpParams httpParams = new BasicHttpParams();
+ HttpConnectionParams.setSoTimeout(httpParams, SOCKET_TIMEOUT);
+ HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT);
+ HttpClient httpClient = new DefaultHttpClient(httpParams);
+ HttpGet httpget = new HttpGet(TIMER_URL);
+ HttpResponse response = httpClient.execute(httpget);
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ time = Long.parseLong(EntityUtils.toString(entity),10);
+ logger.debug("Got remote timestamp {}", time);
+ }
+ } catch (IOException e) {
+ logger.warn("Error obtaining the synchronized remote timestamp, defaulting to the local one",e);
+ }
+
+ if (time == -1) {
+ time = System.currentTimeMillis();
+ }
+
+ return Long.toString(time / 1000L);
+ }
+
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java
new file mode 100644
index 000000000..e1374d11e
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java
@@ -0,0 +1,275 @@
+package com.google.refine.freebase.operations;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.browsing.Engine;
+import com.google.refine.browsing.FilteredRows;
+import com.google.refine.browsing.RowVisitor;
+import com.google.refine.freebase.FreebaseType;
+import com.google.refine.freebase.util.FreebaseDataExtensionJob;
+import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo;
+import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension;
+import com.google.refine.history.HistoryEntry;
+import com.google.refine.model.AbstractOperation;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Column;
+import com.google.refine.model.Project;
+import com.google.refine.model.ReconCandidate;
+import com.google.refine.model.Row;
+import com.google.refine.model.changes.CellAtRow;
+import com.google.refine.freebase.model.changes.DataExtensionChange;
+import com.google.refine.operations.EngineDependentOperation;
+import com.google.refine.operations.OperationRegistry;
+import com.google.refine.process.LongRunningProcess;
+import com.google.refine.process.Process;
+
+public class ExtendDataOperation extends EngineDependentOperation {
+ final protected String _baseColumnName;
+ final protected JSONObject _extension;
+ final protected int _columnInsertIndex;
+
+ static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
+ JSONObject engineConfig = obj.getJSONObject("engineConfig");
+
+ return new ExtendDataOperation(
+ engineConfig,
+ obj.getString("baseColumnName"),
+ obj.getJSONObject("extension"),
+ obj.getInt("columnInsertIndex")
+ );
+ }
+
+ public ExtendDataOperation(
+ JSONObject engineConfig,
+ String baseColumnName,
+ JSONObject extension,
+ int columnInsertIndex
+ ) {
+ super(engineConfig);
+
+ _baseColumnName = baseColumnName;
+ _extension = extension;
+ _columnInsertIndex = columnInsertIndex;
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
+ writer.key("description"); writer.value(getBriefDescription(null));
+ writer.key("engineConfig"); writer.value(getEngineConfig());
+ writer.key("columnInsertIndex"); writer.value(_columnInsertIndex);
+ writer.key("baseColumnName"); writer.value(_baseColumnName);
+ writer.key("extension"); writer.value(_extension);
+ writer.endObject();
+ }
+
+ protected String getBriefDescription(Project project) {
+ return "Extend data at index " + _columnInsertIndex +
+ " based on column " + _baseColumnName;
+ }
+
+ protected String createDescription(Column column, List cellsAtRows) {
+ return "Extend data at index " + _columnInsertIndex +
+ " based on column " + column.getName() +
+ " by filling " + cellsAtRows.size();
+ }
+
+ public Process createProcess(Project project, Properties options) throws Exception {
+ return new ExtendDataProcess(
+ project,
+ getEngineConfig(),
+ getBriefDescription(null)
+ );
+ }
+
+ public class ExtendDataProcess extends LongRunningProcess implements Runnable {
+ final protected Project _project;
+ final protected JSONObject _engineConfig;
+ final protected long _historyEntryID;
+ protected int _cellIndex;
+ protected FreebaseDataExtensionJob _job;
+
+ public ExtendDataProcess(
+ Project project,
+ JSONObject engineConfig,
+ String description
+ ) throws JSONException {
+ super(description);
+ _project = project;
+ _engineConfig = engineConfig;
+ _historyEntryID = HistoryEntry.allocateID();
+
+ _job = new FreebaseDataExtensionJob(_extension);
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("id"); writer.value(hashCode());
+ writer.key("description"); writer.value(_description);
+ writer.key("immediate"); writer.value(false);
+ writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done"));
+ writer.key("progress"); writer.value(_progress);
+ writer.endObject();
+ }
+
+ protected Runnable getRunnable() {
+ return this;
+ }
+
+ protected void populateRowsWithMatches(List rowIndices) throws Exception {
+ Engine engine = new Engine(_project);
+ engine.initializeFromJSON(_engineConfig);
+
+ Column column = _project.columnModel.getColumnByName(_baseColumnName);
+ if (column == null) {
+ throw new Exception("No column named " + _baseColumnName);
+ }
+
+ _cellIndex = column.getCellIndex();
+
+ FilteredRows filteredRows = engine.getAllFilteredRows();
+ filteredRows.accept(_project, new RowVisitor() {
+ List _rowIndices;
+
+ public RowVisitor init(List rowIndices) {
+ _rowIndices = rowIndices;
+ return this;
+ }
+
+ @Override
+ public void start(Project project) {
+ // nothing to do
+ }
+
+ @Override
+ public void end(Project project) {
+ // nothing to do
+ }
+
+ public boolean visit(Project project, int rowIndex, Row row) {
+ Cell cell = row.getCell(_cellIndex);
+ if (cell != null && cell.recon != null && cell.recon.match != null) {
+ _rowIndices.add(rowIndex);
+ }
+
+ return false;
+ }
+ }.init(rowIndices));
+ }
+
+ protected int extendRows(
+ List rowIndices,
+ List dataExtensions,
+ int from,
+ int limit,
+ Map reconCandidateMap
+ ) {
+ Set ids = new HashSet();
+
+ int end;
+ for (end = from; end < limit && ids.size() < 10; end++) {
+ int index = rowIndices.get(end);
+ Row row = _project.rows.get(index);
+ Cell cell = row.getCell(_cellIndex);
+
+ ids.add(cell.recon.match.id);
+ }
+
+ Map map = null;
+ try {
+ map = _job.extend(ids, reconCandidateMap);
+ } catch (Exception e) {
+ map = new HashMap();
+ }
+
+ for (int i = from; i < end; i++) {
+ int index = rowIndices.get(i);
+ Row row = _project.rows.get(index);
+ Cell cell = row.getCell(_cellIndex);
+ String guid = cell.recon.match.id;
+
+ if (map.containsKey(guid)) {
+ dataExtensions.add(map.get(guid));
+ } else {
+ dataExtensions.add(null);
+ }
+ }
+
+ return end;
+ }
+
+ public void run() {
+ List rowIndices = new ArrayList();
+ List dataExtensions = new ArrayList();
+
+ try {
+ populateRowsWithMatches(rowIndices);
+ } catch (Exception e2) {
+ // TODO : Not sure what to do here?
+ e2.printStackTrace();
+ }
+
+ int start = 0;
+ Map reconCandidateMap = new HashMap();
+
+ while (start < rowIndices.size()) {
+ int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
+ start = end;
+
+ _progress = end * 100 / rowIndices.size();
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ if (_canceled) {
+ break;
+ }
+ }
+ }
+
+ if (!_canceled) {
+ List columnNames = new ArrayList();
+ for (ColumnInfo info : _job.columns) {
+ columnNames.add(StringUtils.join(info.names, " - "));
+ }
+
+ List columnTypes = new ArrayList();
+ for (ColumnInfo info : _job.columns) {
+ columnTypes.add(info.expectedType);
+ }
+
+ HistoryEntry historyEntry = new HistoryEntry(
+ _historyEntryID,
+ _project,
+ _description,
+ ExtendDataOperation.this,
+ new DataExtensionChange(
+ _baseColumnName,
+ _columnInsertIndex,
+ columnNames,
+ columnTypes,
+ rowIndices,
+ dataExtensions,
+ _historyEntryID)
+ );
+
+ _project.history.addEntry(historyEntry);
+ _project.processManager.onDoneProcess(this);
+ }
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java
new file mode 100644
index 000000000..c10d29317
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java
@@ -0,0 +1,106 @@
+package com.google.refine.freebase.operations;
+
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.commands.UploadDataCommand;
+import com.google.refine.history.HistoryEntry;
+import com.google.refine.model.AbstractOperation;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Project;
+import com.google.refine.model.Recon;
+import com.google.refine.model.Row;
+import com.google.refine.model.changes.MassReconChange;
+import com.google.refine.operations.OperationRegistry;
+import com.google.refine.util.ParsingUtilities;
+
+public class ImportQADataOperation extends AbstractOperation {
+ static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
+ return new ImportQADataOperation();
+ }
+
+ public ImportQADataOperation() {
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
+ writer.key("description"); writer.value(getBriefDescription(null));
+ writer.endObject();
+ }
+
+ @Override
+ protected String getBriefDescription(Project project) {
+ return "Import QA DAta";
+ }
+
+ @Override
+ protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
+ Integer jobID = (Integer) project.getMetadata().getPreferenceStore().get(UploadDataCommand.s_dataLoadJobIDPref);
+ if (jobID == null) {
+ throw new InternalError("Project is not associated with any data loading job.");
+ }
+
+ Map reconIDToResult = new HashMap();
+
+ URL url = new URL("http://refinery.freebaseapps.com/get_answers/" + jobID);
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setReadTimeout(30000); // 30 seconds
+
+ LineNumberReader reader = new LineNumberReader(new InputStreamReader(conn.getInputStream()));
+ try {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(line);
+ long reconID = Long.parseLong(obj.getString("recon_id").substring(3));
+
+ reconIDToResult.put(reconID, obj.getString("result"));
+ }
+ } finally {
+ reader.close();
+ }
+
+ Map oldRecons = new HashMap();
+ Map newRecons = new HashMap();
+
+ for (int r = 0; r < project.rows.size(); r++) {
+ Row row = project.rows.get(r);
+
+ for (int c = 0; c < row.cells.size(); c++) {
+ Cell cell = row.cells.get(c);
+ if (cell != null && cell.recon != null) {
+ Recon oldRecon = cell.recon;
+
+ if (reconIDToResult.containsKey(oldRecon.id)) {
+ Recon newRecon = oldRecon.dup();
+ newRecon.setFeature(Recon.Feature_qaResult, reconIDToResult.get(oldRecon.id));
+
+ reconIDToResult.remove(oldRecon.id);
+
+ oldRecons.put(oldRecon.id, oldRecon);
+ newRecons.put(oldRecon.id, newRecon);
+ }
+ }
+ }
+ }
+
+ return new HistoryEntry(
+ historyEntryID,
+ project,
+ getBriefDescription(project),
+ this,
+ new MassReconChange(newRecons, oldRecons)
+ );
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java
new file mode 100644
index 000000000..5b7a41f0d
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java
@@ -0,0 +1,121 @@
+package com.google.refine.freebase.operations;
+
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.io.Writer;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.protograph.Protograph;
+import com.google.refine.history.Change;
+import com.google.refine.history.HistoryEntry;
+import com.google.refine.model.AbstractOperation;
+import com.google.refine.model.Project;
+import com.google.refine.operations.OperationRegistry;
+import com.google.refine.util.ParsingUtilities;
+import com.google.refine.util.Pool;
+
+public class SaveProtographOperation extends AbstractOperation {
+ final protected Protograph _protograph;
+
+ static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
+ return new SaveProtographOperation(
+ Protograph.reconstruct(obj.getJSONObject("protograph"))
+ );
+ }
+
+ public SaveProtographOperation(Protograph protograph) {
+ _protograph = protograph;
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
+ writer.key("description"); writer.value(getBriefDescription());
+ writer.key("protograph"); _protograph.write(writer, options);
+ writer.endObject();
+ }
+
+ protected String getBriefDescription() {
+ return "Save schema alignment skeleton";
+ }
+
+ @Override
+ protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
+ Change change = new ProtographChange(_protograph);
+
+ return new HistoryEntry(historyEntryID, project, getBriefDescription(), SaveProtographOperation.this, change);
+ }
+
+ static public class ProtographChange implements Change {
+ final protected Protograph _newProtograph;
+ protected Protograph _oldProtograph;
+
+ public ProtographChange(Protograph protograph) {
+ _newProtograph = protograph;
+ }
+
+ public void apply(Project project) {
+ synchronized (project) {
+ _oldProtograph = (Protograph) project.overlayModels.get("freebaseProtograph");
+
+ project.overlayModels.put("freebaseProtograph", _newProtograph);
+ }
+ }
+
+ public void revert(Project project) {
+ synchronized (project) {
+ if (_oldProtograph == null) {
+ project.overlayModels.remove("freebaseProtograph");
+ } else {
+ project.overlayModels.put("freebaseProtograph", _oldProtograph);
+ }
+ }
+ }
+
+ public void save(Writer writer, Properties options) throws IOException {
+ writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n');
+ writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n');
+ writer.write("/ec/\n"); // end of change marker
+ }
+
+ static public Change load(LineNumberReader reader, Pool pool) throws Exception {
+ Protograph oldProtograph = null;
+ Protograph newProtograph = null;
+
+ String line;
+ while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
+ int equal = line.indexOf('=');
+ CharSequence field = line.subSequence(0, equal);
+ String value = line.substring(equal + 1);
+
+ if ("oldProtograph".equals(field) && value.length() > 0) {
+ oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
+ } else if ("newProtograph".equals(field) && value.length() > 0) {
+ newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
+ }
+ }
+
+ ProtographChange change = new ProtographChange(newProtograph);
+ change._oldProtograph = oldProtograph;
+
+ return change;
+ }
+
+ static protected void writeProtograph(Protograph p, Writer writer) throws IOException {
+ if (p != null) {
+ JSONWriter jsonWriter = new JSONWriter(writer);
+ try {
+ p.write(jsonWriter, new Properties());
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java
new file mode 100644
index 000000000..5a0ed007c
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java
@@ -0,0 +1,47 @@
+package com.google.refine.freebase.protograph;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.FreebaseType;
+
+public class AnonymousNode implements Node, NodeWithLinks {
+ final public FreebaseType type;
+ final public List columnNames = new LinkedList();
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java
new file mode 100644
index 000000000..4a8923a5d
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java
@@ -0,0 +1,58 @@
+package com.google.refine.freebase.protograph;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.FreebaseType;
+
+public class CellTopicNode extends CellNode implements NodeWithLinks {
+ final public FreebaseType type;
+ final public List _rootNodes = new LinkedList();
+
+ public int getRootNodeCount() {
+ return _rootNodes.size();
+ }
+
+ public Node getRootNode(int index) {
+ return _rootNodes.get(index);
+ }
+
+ @Override
+ public void onBeforeSave(Project project) {
+ }
+
+ @Override
+ public void onAfterSave(Project project) {
+ }
+
+
+ @Override
+ public void dispose(Project project) {
+ }
+
+ static public Protograph reconstruct(JSONObject o) throws JSONException {
+ Protograph g = new Protograph();
+
+ JSONArray rootNodes = o.getJSONArray("rootNodes");
+ int count = rootNodes.length();
+
+ for (int i = 0; i < count; i++) {
+ JSONObject o2 = rootNodes.getJSONObject(i);
+ Node node = reconstructNode(o2);
+ if (node != null) {
+ g._rootNodes.add(node);
+ }
+ }
+
+ return g;
+ }
+
+ static protected Node reconstructNode(JSONObject o) throws JSONException {
+ Node node = null;
+
+ String nodeType = o.getString("nodeType");
+ if (nodeType.startsWith("cell-as-")) {
+ if ("cell-as-topic".equals(nodeType)) {
+ if (o.has("type")) {
+ node = new CellTopicNode(
+ reconstructType(o.getJSONObject("type"))
+ );
+ }
+ } else if ("cell-as-value".equals(nodeType)) {
+ node = new CellValueNode(
+ o.getString("valueType"),
+ o.getString("lang")
+ );
+ } else if ("cell-as-key".equals(nodeType)) {
+ node = new CellKeyNode(
+ reconstructTopic(o.getJSONObject("namespace"))
+ );
+ }
+
+ if (o.has("columnName") && !o.isNull("columnName")) {
+ ((CellNode) node).columnNames.add(o.getString("columnName"));
+ }
+ if (o.has("columnNames") && !o.isNull("columnNames")) {
+ JSONArray columnNames = o.getJSONArray("columnNames");
+ int count = columnNames.length();
+
+ for (int c = 0; c < count; c++) {
+ ((CellNode) node).columnNames.add(columnNames.getString(c));
+ }
+ }
+ } else if ("topic".equals(nodeType)) {
+ node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic")));
+ } else if ("value".equals(nodeType)) {
+ node = new ValueNode(
+ o.get("value"),
+ o.getString("valueType"),
+ o.getString("lang")
+ );
+ } else if ("anonymous".equals(nodeType)) {
+ node = new AnonymousNode(reconstructType(o.getJSONObject("type")));
+ }
+
+ if (node != null && node instanceof NodeWithLinks && o.has("links")) {
+ NodeWithLinks node2 = (NodeWithLinks) node;
+
+ JSONArray links = o.getJSONArray("links");
+ int linkCount = links.length();
+
+ for (int j = 0; j < linkCount; j++) {
+ JSONObject oLink = links.getJSONObject(j);
+ Condition condition = null;
+
+ if (oLink.has("condition") && !oLink.isNull("condition")) {
+ JSONObject oCondition = oLink.getJSONObject("condition");
+ if (oCondition.has("columnName") && !oCondition.isNull("columnName")) {
+ condition = new BooleanColumnCondition(oCondition.getString("columnName"));
+ }
+ }
+
+ node2.addLink(new Link(
+ reconstructProperty(oLink.getJSONObject("property")),
+ oLink.has("target") && !oLink.isNull("target") ?
+ reconstructNode(oLink.getJSONObject("target")) : null,
+ condition,
+ oLink.has("load") && !oLink.isNull("load") ?
+ oLink.getBoolean("load") : true
+ ));
+ }
+ }
+
+ return node;
+ }
+
+ static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException {
+ return new FreebaseProperty(
+ o.getString("id"),
+ o.getString("name")
+ );
+ }
+
+ static protected FreebaseType reconstructType(JSONObject o) throws JSONException {
+ return new FreebaseType(
+ o.getString("id"),
+ o.getString("name")
+ );
+ }
+
+ static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException {
+ return new FreebaseTopic(
+ o.getString("id"),
+ o.getString("name")
+ );
+ }
+
+ public void write(JSONWriter writer, Properties options) throws JSONException {
+ writer.object();
+ writer.key("rootNodes"); writer.array();
+
+ for (Node node : _rootNodes) {
+ node.write(writer, options);
+ }
+
+ writer.endArray();
+ writer.endObject();
+ }
+
+ static public Protograph load(Project project, JSONObject obj) throws Exception {
+ return reconstruct(obj);
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java
new file mode 100644
index 000000000..aa5296ec1
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java
@@ -0,0 +1,29 @@
+package com.google.refine.freebase.protograph;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+public class ValueNode implements Node {
+ final public Object value;
+ final public String valueType;
+ final public String lang;
+
+ public ValueNode(Object value, String valueType, String lang) {
+ this.value = value;
+ this.valueType = valueType;
+ this.lang = lang;
+ }
+
+ public void write(JSONWriter writer, Properties options)
+ throws JSONException {
+
+ writer.object();
+ writer.key("nodeType"); writer.value("value");
+ writer.key("value"); writer.value(value);
+ writer.key("valueType"); writer.value(valueType);
+ writer.key("lang"); writer.value(lang);
+ writer.endObject();
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java
new file mode 100644
index 000000000..0d89e0ce3
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java
@@ -0,0 +1,346 @@
+package com.google.refine.freebase.protograph.transpose;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.FreebaseProperty;
+import com.google.refine.freebase.protograph.AnonymousNode;
+import com.google.refine.freebase.protograph.CellKeyNode;
+import com.google.refine.freebase.protograph.CellNode;
+import com.google.refine.freebase.protograph.CellTopicNode;
+import com.google.refine.freebase.protograph.CellValueNode;
+import com.google.refine.freebase.protograph.FreebaseTopicNode;
+import com.google.refine.freebase.protograph.Link;
+import com.google.refine.freebase.protograph.ValueNode;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Recon;
+import com.google.refine.util.JSONUtilities;
+
+public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory {
+ protected Writer writer;
+ protected List rootObjects = new LinkedList();
+
+ private static final String TYPE = "type";
+ private static final String ID = "id";
+ private static final String NAME = "name";
+ private static final String CREATE = "create";
+ private static final String VALUE = "value";
+ private static final String CONNECT = "connect";
+ private static final String LANG = "lang";
+
+ public MqlwriteLikeTransposedNodeFactory(Writer writer) {
+ this.writer = writer;
+ }
+
+ protected JSONArray getJSON() {
+ return new JSONArray(rootObjects);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ try {
+ JSONWriter jsonWriter = new JSONWriter(writer);
+
+ jsonWriter.array();
+ for (JSONObject obj : rootObjects) {
+ jsonWriter.value(obj);
+ }
+ jsonWriter.endArray();
+
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ writer.flush();
+ }
+
+ abstract protected class JsonTransposedNode implements TransposedNode {
+ abstract public Object getJSON();
+ }
+
+ abstract protected class JsonObjectTransposedNode extends JsonTransposedNode {
+ abstract public JSONObject getJSONObject();
+
+ protected JSONObject obj;
+
+ public Object getJSON() {
+ return getJSONObject();
+ }
+ }
+
+ protected class AnonymousTransposedNode extends JsonObjectTransposedNode {
+ JsonObjectTransposedNode parent;
+ FreebaseProperty property;
+ AnonymousNode node;
+
+ protected AnonymousTransposedNode(
+ JsonObjectTransposedNode parent,
+ FreebaseProperty property,
+ AnonymousNode node
+ ) {
+ this.parent = parent;
+ this.property = property;
+ this.node = node;
+ }
+
+ public JSONObject getJSONObject() {
+ if (obj == null) {
+ obj = new JSONObject();
+ try {
+ obj.put(TYPE, this.node.type.id);
+ obj.put(ID, (String) null);
+ obj.put(CREATE, "unconditional");
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+
+ linkTransposedNodeJSON(obj, parent, property);
+ }
+
+ return obj;
+ }
+ }
+
+ protected class CellTopicTransposedNode extends JsonObjectTransposedNode {
+ protected CellTopicNode node;
+ protected Cell cell;
+
+ public CellTopicTransposedNode(CellTopicNode node, Cell cell) {
+ this.node = node;
+ this.cell = cell;
+ }
+
+ @Override
+ public JSONObject getJSONObject() {
+ if (obj == null) {
+ obj = new JSONObject();
+ try {
+ if (cell.recon != null &&
+ cell.recon.judgment == Recon.Judgment.Matched &&
+ cell.recon.match != null) {
+ obj.put(ID, cell.recon.match.id);
+ } else {
+ obj.put(ID, (String) null);
+ obj.put(NAME, cell.value.toString());
+ obj.put(TYPE, node.type.id);
+ obj.put(CREATE, "unless_exists");
+ }
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ return obj;
+ }
+ }
+
+ protected class CellValueTransposedNode extends JsonTransposedNode {
+ protected JSONObject obj;
+ protected CellValueNode node;
+ protected Cell cell;
+
+ public CellValueTransposedNode(CellValueNode node, Cell cell) {
+ this.node = node;
+ this.cell = cell;
+ }
+
+ public Object getJSON() {
+ if (obj == null) {
+ obj = new JSONObject();
+ try {
+ JSONUtilities.putField(obj, VALUE, cell.value);
+
+ obj.put(TYPE, node.valueType);
+ if ("/type/text".equals(node.valueType)) {
+ obj.put(LANG, node.lang);
+ }
+
+ obj.put(CONNECT, "insert");
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ return obj;
+ }
+ }
+
+ protected class CellKeyTransposedNode extends JsonTransposedNode {
+ protected JSONObject obj;
+ protected CellKeyNode node;
+ protected Cell cell;
+
+ public CellKeyTransposedNode(CellKeyNode node, Cell cell) {
+ this.node = node;
+ this.cell = cell;
+ }
+
+ public Object getJSON() {
+ if (obj == null) {
+ obj = new JSONObject();
+ try {
+ obj.put(VALUE, cell.value.toString());
+
+ JSONObject nsObj = new JSONObject();
+ nsObj.put(ID, node.namespace.id);
+
+ obj.put("namespace", nsObj);
+ obj.put(CONNECT, "insert");
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ return obj;
+ }
+ }
+
+ protected class TopicTransposedNode extends JsonObjectTransposedNode {
+ protected FreebaseTopicNode node;
+
+ public TopicTransposedNode(FreebaseTopicNode node) {
+ this.node = node;
+ }
+
+ @Override
+ public JSONObject getJSONObject() {
+ if (obj == null) {
+ obj = new JSONObject();
+ try {
+ obj.put(ID, node.topic.id);
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ return obj;
+ }
+ }
+
+ protected class ValueTransposedNode extends JsonTransposedNode {
+ protected JSONObject obj;
+ protected ValueNode node;
+
+ public ValueTransposedNode(ValueNode node) {
+ this.node = node;
+ }
+
+ public Object getJSON() {
+ if (obj == null) {
+ obj = new JSONObject();
+ try {
+ obj.put(VALUE, node.value);
+ obj.put(TYPE, node.valueType);
+ if ("/type/text".equals(node.valueType)) {
+ obj.put(LANG, node.lang);
+ }
+
+ obj.put(CONNECT, "insert");
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ return obj;
+ }
+ }
+ public TransposedNode transposeAnonymousNode(
+ TransposedNode parentNode,
+ Link link,
+ AnonymousNode node, int rowIndex) {
+
+ return new AnonymousTransposedNode(
+ parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null,
+ link != null ? link.property : null,
+ node
+ );
+ }
+
+ public TransposedNode transposeCellNode(
+ TransposedNode parentNode,
+ Link link,
+ CellNode node,
+ int rowIndex,
+ int cellIndex,
+ Cell cell) {
+
+ JsonTransposedNode tnode = null;
+ if (node instanceof CellTopicNode) {
+ tnode = new CellTopicTransposedNode((CellTopicNode) node, cell);
+ } else if (node instanceof CellValueNode) {
+ tnode = new CellValueTransposedNode((CellValueNode) node, cell);
+ } else if (node instanceof CellKeyNode) {
+ tnode = new CellKeyTransposedNode((CellKeyNode) node, cell);
+ }
+
+ if (tnode != null) {
+ processTransposedNode(tnode, parentNode, link != null ? link.property : null);
+ }
+ return tnode;
+ }
+
+ public TransposedNode transposeTopicNode(
+ TransposedNode parentNode,
+ Link link,
+ FreebaseTopicNode node, int rowIndex) {
+
+ JsonTransposedNode tnode = new TopicTransposedNode(node);
+
+ processTransposedNode(tnode, parentNode, link != null ? link.property : null);
+
+ return tnode;
+ }
+
+ public TransposedNode transposeValueNode(
+ TransposedNode parentNode,
+ Link link,
+ ValueNode node, int rowIndex) {
+
+ JsonTransposedNode tnode = new ValueTransposedNode(node);
+
+ processTransposedNode(tnode, parentNode, link != null ? link.property : null);
+
+ return tnode;
+ }
+
+ protected void processTransposedNode(
+ JsonTransposedNode tnode,
+ TransposedNode parentNode,
+ FreebaseProperty property
+ ) {
+
+ if (!(tnode instanceof AnonymousTransposedNode)) {
+ linkTransposedNodeJSON(tnode.getJSON(), parentNode, property);
+ }
+ }
+
+ protected void linkTransposedNodeJSON(
+ Object obj,
+ TransposedNode parentNode,
+ FreebaseProperty property
+ ) {
+
+ if (parentNode == null) {
+ if (obj instanceof JSONObject) {
+ rootObjects.add((JSONObject) obj);
+ }
+ } else if (parentNode instanceof JsonTransposedNode) {
+ JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject();
+
+ try {
+ JSONArray a = null;
+ if (parentObj.has(property.id)) {
+ a = parentObj.getJSONArray(property.id);
+ } else {
+ a = new JSONArray();
+ parentObj.put(property.id, a);
+ }
+
+ a.put(a.length(), obj);
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java
new file mode 100644
index 000000000..043d865af
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java
@@ -0,0 +1,4 @@
+package com.google.refine.freebase.protograph.transpose;
+
+public interface TransposedNode {
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java
new file mode 100644
index 000000000..f0813d25c
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java
@@ -0,0 +1,43 @@
+package com.google.refine.freebase.protograph.transpose;
+
+import java.io.IOException;
+
+import com.google.refine.freebase.protograph.AnonymousNode;
+import com.google.refine.freebase.protograph.CellNode;
+import com.google.refine.freebase.protograph.FreebaseTopicNode;
+import com.google.refine.freebase.protograph.Link;
+import com.google.refine.freebase.protograph.ValueNode;
+import com.google.refine.model.Cell;
+
+public interface TransposedNodeFactory {
+ public TransposedNode transposeAnonymousNode(
+ TransposedNode parentNode,
+ Link link,
+ AnonymousNode node, int rowIndex
+ );
+
+ public TransposedNode transposeCellNode(
+ TransposedNode parentNode,
+ Link link,
+ CellNode node,
+ int rowIndex,
+ int cellIndex,
+ Cell cell
+ );
+
+ public TransposedNode transposeValueNode(
+ TransposedNode parentNode,
+ Link link,
+ ValueNode node,
+ int rowIndex
+ );
+
+ public TransposedNode transposeTopicNode(
+ TransposedNode parentNode,
+ Link link,
+ FreebaseTopicNode node,
+ int rowIndex
+ );
+
+ public void flush() throws IOException;
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java
new file mode 100644
index 000000000..66ef0364b
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java
@@ -0,0 +1,222 @@
+package com.google.refine.freebase.protograph.transpose;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.refine.browsing.FilteredRows;
+import com.google.refine.browsing.RowVisitor;
+import com.google.refine.expr.ExpressionUtils;
+import com.google.refine.freebase.protograph.AnonymousNode;
+import com.google.refine.freebase.protograph.CellNode;
+import com.google.refine.freebase.protograph.CellTopicNode;
+import com.google.refine.freebase.protograph.FreebaseTopicNode;
+import com.google.refine.freebase.protograph.Link;
+import com.google.refine.freebase.protograph.Node;
+import com.google.refine.freebase.protograph.NodeWithLinks;
+import com.google.refine.freebase.protograph.Protograph;
+import com.google.refine.freebase.protograph.ValueNode;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Column;
+import com.google.refine.model.Project;
+import com.google.refine.model.Row;
+import com.google.refine.model.Recon.Judgment;
+
+public class Transposer {
+ static public void transpose(
+ Project project,
+ FilteredRows filteredRows,
+ Protograph protograph,
+ Node rootNode,
+ TransposedNodeFactory nodeFactory
+ ) {
+ transpose(project, filteredRows, protograph, rootNode, nodeFactory, 20);
+ }
+
+ static public void transpose(
+ Project project,
+ FilteredRows filteredRows,
+ Protograph protograph,
+ Node rootNode,
+ TransposedNodeFactory nodeFactory,
+ int limit
+ ) {
+ Context rootContext = new Context(rootNode, null, null, limit);
+
+ filteredRows.accept(project, new RowVisitor() {
+ Context rootContext;
+ Protograph protograph;
+ Node rootNode;
+ TransposedNodeFactory nodeFactory;
+
+ @Override
+ public boolean visit(Project project, int rowIndex, Row row) {
+ if (rootContext.limit <= 0 || rootContext.count < rootContext.limit) {
+ descend(project, protograph, nodeFactory, rowIndex, row, rootNode, rootContext);
+ }
+
+ if (rootContext.limit > 0 && rootContext.count > rootContext.limit) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void start(Project project) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void end(Project project) {
+ // TODO Auto-generated method stub
+
+ }
+
+ public RowVisitor init(
+ Context rootContext,
+ Protograph protograph,
+ Node rootNode,
+ TransposedNodeFactory nodeFactory
+ ) {
+ this.rootContext = rootContext;
+ this.protograph = protograph;
+ this.rootNode = rootNode;
+ this.nodeFactory = nodeFactory;
+
+ return this;
+ }
+ }.init(rootContext, protograph, rootNode, nodeFactory));
+ }
+
+ static protected void descend(
+ Project project,
+ Protograph protograph,
+ TransposedNodeFactory nodeFactory,
+ int rowIndex,
+ Row row,
+ Node node,
+ Context context
+ ) {
+ List tnodes = new LinkedList();
+
+ TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode;
+ Link link = context.parent == null ? null : context.link;
+
+ if (node instanceof CellNode) {
+ CellNode node2 = (CellNode) node;
+ for (String columnName : node2.columnNames) {
+ Column column = project.columnModel.getColumnByName(columnName);
+ if (column != null) {
+ int cellIndex = column.getCellIndex();
+
+ Cell cell = row.getCell(cellIndex);
+ if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
+ if (node2 instanceof CellTopicNode &&
+ (cell.recon == null || cell.recon.judgment == Judgment.None)) {
+ return;
+ }
+
+ context.count++;
+ if (context.limit > 0 && context.count > context.limit) {
+ return;
+ }
+
+ tnodes.add(nodeFactory.transposeCellNode(
+ parentNode,
+ link,
+ node2,
+ rowIndex,
+ cellIndex,
+ cell
+ ));
+ }
+ }
+ }
+ } else {
+ if (node instanceof AnonymousNode) {
+ tnodes.add(nodeFactory.transposeAnonymousNode(
+ parentNode,
+ link,
+ (AnonymousNode) node,
+ rowIndex
+ ));
+ } else if (node instanceof FreebaseTopicNode) {
+ tnodes.add(nodeFactory.transposeTopicNode(
+ parentNode,
+ link,
+ (FreebaseTopicNode) node,
+ rowIndex
+ ));
+ } else if (node instanceof ValueNode) {
+ tnodes.add(nodeFactory.transposeValueNode(
+ parentNode,
+ link,
+ (ValueNode) node,
+ rowIndex
+ ));
+ }
+ }
+
+ if (node instanceof NodeWithLinks) {
+ NodeWithLinks node2 = (NodeWithLinks) node;
+ int linkCount = node2.getLinkCount();
+
+ for (int i = 0; i < linkCount; i++) {
+ Link link2 = node2.getLink(i);
+ if (link2.condition == null || link2.condition.test(project, rowIndex, row)) {
+ for (TransposedNode tnode : tnodes) {
+ context.transposedNode = tnode;
+ context.nullifySubContextNodes();
+
+ descend(
+ project,
+ protograph,
+ nodeFactory,
+ rowIndex,
+ row,
+ link2.getTarget(),
+ context.subContexts.get(i)
+ );
+ }
+ }
+ }
+ }
+ }
+
+ static class Context {
+ TransposedNode transposedNode;
+ List subContexts;
+ Context parent;
+ Link link;
+ int count;
+ int limit;
+
+ Context(Node node, Context parent, Link link, int limit) {
+ this.parent = parent;
+ this.link = link;
+ this.limit = limit;
+
+ if (node instanceof NodeWithLinks) {
+ NodeWithLinks node2 = (NodeWithLinks) node;
+
+ int subContextCount = node2.getLinkCount();
+
+ subContexts = new LinkedList();
+ for (int i = 0; i < subContextCount; i++) {
+ Link link2 = node2.getLink(i);
+ subContexts.add(
+ new Context(link2.getTarget(), this, link2, -1));
+ }
+ }
+ }
+
+ public void nullifySubContextNodes() {
+ if (subContexts != null) {
+ for (Context context : subContexts) {
+ context.transposedNode = null;
+ context.nullifySubContextNodes();
+ }
+ }
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java
new file mode 100644
index 000000000..748e15c13
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java
@@ -0,0 +1,725 @@
+package com.google.refine.freebase.protograph.transpose;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import com.google.refine.freebase.FreebaseProperty;
+import com.google.refine.freebase.FreebaseTopic;
+import com.google.refine.freebase.protograph.AnonymousNode;
+import com.google.refine.freebase.protograph.CellKeyNode;
+import com.google.refine.freebase.protograph.CellNode;
+import com.google.refine.freebase.protograph.CellTopicNode;
+import com.google.refine.freebase.protograph.CellValueNode;
+import com.google.refine.freebase.protograph.FreebaseTopicNode;
+import com.google.refine.freebase.protograph.Link;
+import com.google.refine.freebase.protograph.ValueNode;
+import com.google.refine.model.Cell;
+import com.google.refine.model.Column;
+import com.google.refine.model.Project;
+import com.google.refine.model.Recon;
+import com.google.refine.model.Recon.Judgment;
+import com.google.refine.model.recon.ReconConfig;
+import com.google.refine.model.recon.StandardReconConfig;
+
+public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory {
+ protected Project project;
+
+ protected boolean start = true;
+ protected Writer writer;
+ protected WritingTransposedNode lastRootNode;
+ protected Map varPool = new HashMap();
+ protected Map newTopicVars = new HashMap();
+ protected Set serializedRecons = new HashSet();
+
+ protected long contextID = 0;
+ protected int contextRowIndex;
+ protected int contextRefCount = 0;
+ protected JSONObject contextTreeRoot;
+
+ public TripleLoaderTransposedNodeFactory(Project project, Writer writer) {
+ this.project = project;
+ this.writer = writer;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (lastRootNode != null) {
+ lastRootNode.write(null, null, project, -1, -1, null);
+ lastRootNode = null;
+
+ writeContextTreeNode();
+ }
+ }
+
+ protected void writeLine(String line) {
+ try {
+ if (start) {
+ start = false;
+ } else {
+ writer.write('\n');
+ }
+ writer.write(line);
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+
+ protected void writeRecon(
+ StringBuffer sb,
+ Project project,
+ int rowIndex,
+ int cellIndex,
+ Cell cell
+ ) {
+ Recon recon = cell.recon;
+
+ sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\"");
+ contextRefCount++;
+
+ if (!serializedRecons.contains(recon.id)) {
+ serializedRecons.add(recon.id);
+
+ Column column = project.columnModel.getColumnByCellIndex(cellIndex);
+
+ // qa:sample_group
+ {
+ StringBuffer sb2 = new StringBuffer();
+
+ sb2.append("{ \"s\" : \"rec");
+ sb2.append(Long.toString(recon.id));
+ sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : ");
+ sb2.append(JSONObject.quote(column.getName()));
+ sb2.append(", \"ignore\" : true }");
+
+ writeLine(sb2.toString());
+ }
+
+ // qa:recon_data
+ {
+ StringBuffer sb2 = new StringBuffer();
+
+ String s = cell.value instanceof String ? (String) cell.value : cell.value.toString();
+
+ sb2.append("{ \"s\" : \"rec");
+ sb2.append(Long.toString(recon.id));
+ sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { ");
+
+ sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry));
+ sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s));
+ sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName()));
+ sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service));
+ sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction));
+ sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize));
+
+ if (recon.judgment == Judgment.Matched) {
+ sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank));
+ sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id));
+ }
+
+ ReconConfig reconConfig = column.getReconConfig();
+ if (reconConfig != null && reconConfig instanceof StandardReconConfig) {
+ StandardReconConfig standardReconConfig = (StandardReconConfig) reconConfig;
+ sb2.append(", \"type\" : "); sb2.append(JSONObject.quote(standardReconConfig.typeID));
+ }
+
+ sb2.append(" } }");
+
+ writeLine(sb2.toString());
+ }
+ }
+ }
+
+ protected void writeLine(
+ String subject, String predicate, Object object,
+ Project project,
+ int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
+ int objectRowIndex, int objectCellIndex, Cell objectCell,
+ boolean ignore
+ ) {
+ if (subject != null && object != null) {
+ String s = object instanceof String ?
+ JSONObject.quote((String) object) : object.toString();
+
+ StringBuffer sb = new StringBuffer();
+ sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
+ sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
+ sb.append(", \"o\" : "); sb.append(s);
+ if (subjectCell != null || objectCell != null) {
+ sb.append(", \"meta\" : { ");
+
+ sb.append("\"recon\" : { ");
+ if (subjectCell != null) {
+ sb.append("\"s\" : ");
+ writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
+ }
+ if (objectCell != null) {
+ if (subjectCell != null) {
+ sb.append(", ");
+ }
+ sb.append("\"o\" : ");
+ writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell);
+ }
+ sb.append(" }");
+
+ sb.append(" }");
+ }
+ if (ignore) {
+ sb.append(", \"ignore\" : true");
+ }
+ sb.append(" }");
+
+ writeLine(sb.toString());
+ }
+ }
+
+ protected void writeLine(
+ String subject, String predicate, Object object, String lang,
+ Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell,
+ boolean ignore
+ ) {
+ if (subject != null && object != null) {
+ String s = object instanceof String ?
+ JSONObject.quote((String) object) : object.toString();
+
+ StringBuffer sb = new StringBuffer();
+ sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
+ sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
+ sb.append(", \"o\" : "); sb.append(s);
+ sb.append(", \"lang\" : "); sb.append(lang);
+
+ if (subjectCell != null) {
+ sb.append(", \"meta\" : { ");
+ sb.append("\"recon\" : { ");
+ sb.append("\"s\" : ");
+ writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell);
+ sb.append(" }");
+ sb.append(" }");
+ }
+ if (ignore) {
+ sb.append(", \"ignore\" : true");
+ }
+ sb.append(" }");
+
+ writeLine(sb.toString());
+ }
+ }
+
+ abstract protected class WritingTransposedNode implements TransposedNode {
+ JSONObject jsonContextNode;
+ boolean load;
+
+ public Object write(
+ String subject, String predicate, Project project,
+ int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+
+ return internalWrite(
+ subject, predicate, project,
+ subjectRowIndex, subjectCellIndex, subjectCell);
+ }
+
+ abstract public Object internalWrite(
+ String subject, String predicate, Project project,
+ int subjectRowIndex, int subjectCellIndex, Cell subjectCell);
+ }
+
+ abstract protected class TransposedNodeWithChildren extends WritingTransposedNode {
+ public List rowIndices = new LinkedList();
+ public List children = new LinkedList();
+
+ protected void writeChildren(
+ String subject, Project project,
+ int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+
+ for (int i = 0; i < children.size(); i++) {
+ WritingTransposedNode child = children.get(i);
+ Link link = links.get(i);
+ String predicate = link.property.id;
+
+ child.write(subject, predicate, project,
+ subjectRowIndex, subjectCellIndex, subjectCell);
+ }
+ }
+ }
+
+ protected class AnonymousTransposedNode extends TransposedNodeWithChildren {
+
+ //protected AnonymousTransposedNode(AnonymousNode node) { }
+
+ public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+ if (children.size() == 0 || subject == null) {
+ return null;
+ }
+
+ StringBuffer sb = new StringBuffer();
+ sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"');
+ sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"');
+ sb.append(", \"o\" : { ");
+
+ StringBuffer sbRecon = new StringBuffer();
+
+ boolean first = true;
+ boolean firstRecon = true;
+
+ if (subjectCell.recon != null) {
+ sbRecon.append("\"s\" : ");
+ writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell);
+
+ firstRecon = false;
+ }
+
+ for (int i = 0; i < children.size(); i++) {
+ WritingTransposedNode child = children.get(i);
+ Link link = links.get(i);
+
+ FreebaseProperty property = link.property;
+
+ Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null);
+ if (c != null) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(", ");
+ }
+ sb.append("\"" + property.id + "\": ");
+ sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString());
+ }
+
+ if (child instanceof CellTopicTransposedNode) {
+ CellTopicTransposedNode child2 = (CellTopicTransposedNode) child;
+ Recon recon = child2.cell.recon;
+
+ if (recon != null &&
+ (recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) {
+
+ if (firstRecon) {
+ firstRecon = false;
+ } else {
+ sbRecon.append(", ");
+ }
+
+ sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : ");
+
+ writeRecon(sbRecon, project,
+ rowIndices.get(i), child2.cellIndex, child2.cell);
+ }
+ }
+ }
+ sb.append(" }, \"meta\" : { \"recon\" : { ");
+ sb.append(sbRecon.toString());
+ sb.append(" } } }");
+
+ writeLine(sb.toString());
+
+ return null;
+ }
+ }
+
+ protected class CellTopicTransposedNode extends TransposedNodeWithChildren {
+ protected CellTopicNode node;
+ protected int rowIndex;
+ protected int cellIndex;
+ protected Cell cell;
+
+ public CellTopicTransposedNode(CellTopicNode node, int rowIndex, int cellIndex, Cell cell) {
+ this.node = node;
+ this.rowIndex = rowIndex;
+ this.cellIndex = cellIndex;
+ this.cell = cell;
+ }
+
+ public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+ String id = null;
+ if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) {
+ int objectRowIndex = rowIndex;
+ int objectCellIndex = cellIndex;
+ Cell objectCell = cell;
+
+ if (cell.recon.judgment == Recon.Judgment.Matched) {
+ id = cell.recon.match.id;
+
+ } else if (cell.recon.judgment == Judgment.New) {
+ if (newTopicVars.containsKey(cell.recon.id)) {
+ id = newTopicVars.get(cell.recon.id);
+ } else {
+ Column column = project.columnModel.getColumnByCellIndex(cellIndex);
+ String columnName = column.getName();
+
+ long var = 0;
+ if (varPool.containsKey(columnName)) {
+ var = varPool.get(columnName);
+ }
+ varPool.put(columnName, var + 1);
+
+ id = "$" + columnName.replaceAll("\\W+", "_") + "_" + var;
+
+ String typeID = node.type.id;
+
+ ReconConfig reconConfig = column.getReconConfig();
+ if (reconConfig instanceof StandardReconConfig) {
+ typeID = ((StandardReconConfig) reconConfig).typeID;
+ }
+
+ writeLine(id, "type", typeID, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load);
+ writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load);
+
+ if (cell.recon != null) {
+ newTopicVars.put(cell.recon.id, id);
+ }
+ }
+ } else {
+ return null;
+ }
+
+ if (subject != null) {
+ writeLine(subject, predicate, id, project,
+ subjectRowIndex, subjectCellIndex, subjectCell,
+ objectRowIndex, objectCellIndex, objectCell, !load);
+ }
+
+ writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell);
+ }
+
+ return id;
+ }
+ }
+
+ protected class CellValueTransposedNode extends WritingTransposedNode {
+ protected JSONObject obj;
+ protected CellValueNode node;
+ protected int rowIndex;
+ protected int cellIndex;
+ protected Cell cell;
+
+ public CellValueTransposedNode(CellValueNode node, int rowIndex, int cellIndex, Cell cell) {
+ this.node = node;
+ this.rowIndex = rowIndex;
+ this.cellIndex = cellIndex;
+ this.cell = cell;
+ }
+
+ public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+ if (subject != null) {
+ if ("/type/text".equals(node.lang)) {
+ writeLine(subject, predicate, cell.value, node.lang, project,
+ subjectRowIndex, subjectCellIndex, subjectCell, !load);
+ } else {
+ writeLine(subject, predicate, cell.value, project,
+ subjectRowIndex, subjectCellIndex, subjectCell,
+ -1, -1, null, !load);
+ }
+ }
+
+ return cell.value;
+ }
+ }
+
+ protected class CellKeyTransposedNode extends WritingTransposedNode {
+ protected CellKeyNode node;
+ protected int rowIndex;
+ protected int cellIndex;
+ protected Cell cell;
+
+ public CellKeyTransposedNode(CellKeyNode node, int rowIndex, int cellIndex, Cell cell) {
+ this.node = node;
+ this.rowIndex = rowIndex;
+ this.cellIndex = cellIndex;
+ this.cell = cell;
+ }
+
+ public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+ writeLine(subject, "key", node.namespace.id + "/" + cell.value, project,
+ subjectRowIndex, subjectCellIndex, subjectCell,
+ -1, -1, null, !load);
+
+ return null;
+ }
+ }
+
+ protected class TopicTransposedNode extends TransposedNodeWithChildren {
+ protected FreebaseTopicNode node;
+
+ public TopicTransposedNode(FreebaseTopicNode node) {
+ this.node = node;
+ }
+
+ public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+ writeLine(subject, predicate, node.topic.id, project,
+ subjectRowIndex, subjectCellIndex, subjectCell,
+ -1, -1, null, !load);
+
+ writeChildren(node.topic.id, project, -1, -1, null);
+
+ return node.topic.id;
+ }
+ }
+
+ protected class ValueTransposedNode extends WritingTransposedNode {
+ protected ValueNode node;
+
+ public ValueTransposedNode(ValueNode node) {
+ this.node = node;
+ }
+
+ public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) {
+ if ("/type/text".equals(node.lang)) {
+ writeLine(subject, predicate, node.value, node.lang, project,
+ subjectRowIndex, subjectCellIndex, subjectCell, !load);
+ } else {
+ writeLine(subject, predicate, node.value, project,
+ subjectRowIndex, subjectCellIndex, subjectCell,
+ -1, -1, null, !load);
+ }
+
+ return node.value;
+ }
+ }
+
+ public TransposedNode transposeAnonymousNode(
+ TransposedNode parentNode,
+ Link link,
+ AnonymousNode node, int rowIndex) {
+
+ WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
+ WritingTransposedNode tnode = new AnonymousTransposedNode();
+
+ tnode.load =
+ (parentNode2 == null || parentNode2.load) &&
+ (link == null || link.load);
+
+ processTransposedNode(tnode, parentNode, link, rowIndex);
+
+ tnode.jsonContextNode = addJsonContext(
+ parentNode2 != null ? parentNode2.jsonContextNode : null,
+ link != null ? link.property.id : null,
+ null
+ );
+
+ return tnode;
+ }
+
+ public TransposedNode transposeCellNode(
+ TransposedNode parentNode,
+ Link link,
+ CellNode node,
+ int rowIndex,
+ int cellIndex,
+ Cell cell) {
+
+ WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
+
+ WritingTransposedNode tnode = null;
+ if (node instanceof CellTopicNode) {
+ if (cell.recon != null &&
+ (cell.recon.judgment == Judgment.Matched ||
+ cell.recon.judgment == Judgment.New)) {
+
+ tnode = new CellTopicTransposedNode(
+ (CellTopicNode) node, rowIndex, cellIndex, cell);
+ }
+ } else if (node instanceof CellValueNode) {
+ tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell);
+ } else if (node instanceof CellKeyNode) {
+ tnode = new CellKeyTransposedNode((CellKeyNode) node, rowIndex, cellIndex, cell);
+ }
+
+ if (tnode != null) {
+ tnode.load =
+ (parentNode2 == null || parentNode2.load) &&
+ (link == null || link.load);
+
+ processTransposedNode(tnode, parentNode, link, rowIndex);
+
+ tnode.jsonContextNode = addJsonContext(
+ parentNode2 != null ? parentNode2.jsonContextNode : null,
+ link != null ? link.property.id : null,
+ cell,
+ rowIndex
+ );
+ }
+ return tnode;
+ }
+
+ public TransposedNode transposeTopicNode(
+ TransposedNode parentNode,
+ Link link,
+ FreebaseTopicNode node,
+ int rowIndex) {
+
+ WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
+ WritingTransposedNode tnode = new TopicTransposedNode(node);
+
+ tnode.load =
+ (parentNode2 == null || parentNode2.load) &&
+ (link == null || link.load);
+
+ processTransposedNode(tnode, parentNode, link, rowIndex);
+
+ tnode.jsonContextNode = addJsonContext(
+ parentNode2 != null ? parentNode2.jsonContextNode : null,
+ link != null ? link.property.id : null,
+ node.topic
+ );
+
+ return tnode;
+ }
+
+ public TransposedNode transposeValueNode(
+ TransposedNode parentNode,
+ Link link,
+ ValueNode node,
+ int rowIndex) {
+
+ WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode;
+ WritingTransposedNode tnode = new ValueTransposedNode(node);
+
+ tnode.load =
+ (parentNode2 == null || parentNode2.load) &&
+ (link == null || link.load);
+
+ processTransposedNode(tnode, parentNode, link, rowIndex);
+
+ tnode.jsonContextNode = addJsonContext(
+ parentNode2 != null ? parentNode2.jsonContextNode : null,
+ link != null ? link.property.id : null,
+ node.value
+ );
+
+ return tnode;
+ }
+
+ protected void processTransposedNode(
+ WritingTransposedNode tnode,
+ TransposedNode parentNode,
+ Link link,
+ int rowIndex
+ ) {
+ if (parentNode != null) {
+ if (parentNode instanceof TransposedNodeWithChildren) {
+ TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode;
+ parentNode2.rowIndices.add(rowIndex);
+ parentNode2.children.add(tnode);
+ parentNode2.links.add(link);
+ }
+ } else {
+ addRootNode(tnode, rowIndex);
+ }
+ }
+
+ protected JSONObject addJsonContext(JSONObject parent, String key, Object value) {
+ JSONObject o = new JSONObject();
+
+ try {
+ if (value instanceof FreebaseTopic) {
+ FreebaseTopic topic = (FreebaseTopic) value;
+ o.put("id", topic.id);
+ o.put("name", topic.name);
+ } else {
+ o.put("v", value);
+ }
+ } catch (JSONException e) {
+ // ignore
+ }
+
+ connectJsonContext(parent, o, key);
+ return o;
+ }
+
+ protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) {
+ JSONObject o = new JSONObject();
+
+ connectJsonContext(parent, o, key);
+
+ try {
+ if (cell != null) {
+ o.put("v", cell.value);
+ if (cell.recon != null) {
+ o.put("recon", "rec" + cell.recon.id);
+
+ if (cell.recon.judgment == Judgment.Matched) {
+ o.put("id", cell.recon.match.id);
+ o.put("name", cell.recon.match.name);
+ }
+
+ // qa:display_context
+ {
+ StringBuffer sb2 = new StringBuffer();
+
+ sb2.append("{ \"ignore\" : true, \"s\" : \"rec");
+ sb2.append(Long.toString(cell.recon.id));
+ sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx");
+ sb2.append(Long.toString(contextID));
+ sb2.append("\", \"meta\" : { \"row\" : ");
+ sb2.append(Integer.toString(rowIndex));
+ sb2.append(" } }");
+
+ writeLine(sb2.toString());
+ }
+ }
+ }
+ } catch (JSONException e) {
+ // ignore
+ }
+
+ return o;
+ }
+
+ protected void connectJsonContext(JSONObject parent, JSONObject o, String key) {
+ try {
+ if (parent == null) {
+ contextTreeRoot = o;
+ } else {
+ JSONArray a = null;
+ if (parent.has(key)) {
+ a = parent.getJSONArray(key);
+ } else {
+ a = new JSONArray();
+ parent.put(key, a);
+ }
+
+ a.put(o);
+ }
+ } catch (JSONException e) {
+ // ignore
+ }
+ }
+
+ protected void addRootNode(WritingTransposedNode tnode, int rowIndex) {
+ if (lastRootNode != null) {
+ lastRootNode.write(null, null, project, -1, -1, null);
+ writeContextTreeNode();
+ }
+ lastRootNode = tnode;
+
+ contextTreeRoot = null;
+ contextRowIndex = rowIndex;
+ contextRefCount = 0;
+ contextID++;
+ }
+
+ protected void writeContextTreeNode() {
+ if (contextTreeRoot != null && contextRefCount > 0) {
+ StringBuffer sb = new StringBuffer();
+
+ sb.append("{ \"ignore\" : true, \"s\" : \"ctx");
+ sb.append(Long.toString(contextID));
+ sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : ");
+ sb.append(Integer.toString(contextRowIndex));
+ sb.append(", \"data\" : ");
+ sb.append(contextTreeRoot.toString());
+ sb.append(" } }");
+
+ writeLine(sb.toString());
+ }
+ }
+}
diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java
new file mode 100644
index 000000000..aa2459974
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java
@@ -0,0 +1,417 @@
+/**
+ *
+ */
+package com.google.refine.freebase.util;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.freebase.FreebaseType;
+import com.google.refine.util.JSONUtilities;
+import com.google.refine.util.ParsingUtilities;
+import com.google.refine.model.ReconCandidate;
+
+public class FreebaseDataExtensionJob {
+ static public class DataExtension {
+ final public Object[][] data;
+
+ public DataExtension(Object[][] data) {
+ this.data = data;
+ }
+ }
+
+ static public class ColumnInfo {
+ final public List names;
+ final public List path;
+ final public FreebaseType expectedType;
+
+ protected ColumnInfo(List names, List path, FreebaseType expectedType) {
+ this.names = names;
+ this.path = path;
+ this.expectedType = expectedType;
+ }
+ }
+
+ final public JSONObject extension;
+ final public int columnCount;
+ final public List columns = new ArrayList();
+
+ public FreebaseDataExtensionJob(JSONObject obj) throws JSONException {
+ this.extension = obj;
+ this.columnCount = (obj.has("properties") && !obj.isNull("properties")) ?
+ countColumns(obj.getJSONArray("properties"), columns, new ArrayList(), new ArrayList()) : 0;
+ }
+
+ public Map extend(
+ Set ids,
+ Map reconCandidateMap
+ ) throws Exception {
+ StringWriter writer = new StringWriter();
+ formulateQuery(ids, extension, writer);
+
+ String query = writer.toString();
+ InputStream is = doMqlRead(query);
+ try {
+ String s = ParsingUtilities.inputStreamToString(is);
+ JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
+
+ Map map = new HashMap();
+ if (o.has("result")) {
+ JSONArray a = o.getJSONArray("result");
+ int l = a.length();
+
+ for (int i = 0; i < l; i++) {
+ JSONObject o2 = a.getJSONObject(i);
+ String id = o2.getString("id");
+ FreebaseDataExtensionJob.DataExtension ext = collectResult(o2, reconCandidateMap);
+
+ if (ext != null) {
+ map.put(id, ext);
+ }
+ }
+ }
+
+ return map;
+ } finally {
+ is.close();
+ }
+ }
+
+ protected FreebaseDataExtensionJob.DataExtension collectResult(
+ JSONObject obj,
+ Map reconCandidateMap
+ ) throws JSONException {
+ List rows = new ArrayList();
+
+ collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap);
+
+ Object[][] data = new Object[rows.size()][columnCount];
+ rows.toArray(data);
+
+ return new DataExtension(data);
+ }
+
+ protected void storeCell(
+ List rows,
+ int row,
+ int col,
+ Object value,
+ Map reconCandidateMap
+ ) {
+ while (row >= rows.size()) {
+ rows.add(new Object[columnCount]);
+ }
+ rows.get(row)[col] = value;
+ }
+
+ protected void storeCell(
+ List rows,
+ int row,
+ int col,
+ JSONObject obj,
+ Map reconCandidateMap
+ ) throws JSONException {
+ String id = obj.getString("id");
+ ReconCandidate rc;
+ if (reconCandidateMap.containsKey(id)) {
+ rc = reconCandidateMap.get(id);
+ } else {
+ rc = new ReconCandidate(
+ obj.getString("id"),
+ obj.getString("name"),
+ JSONUtilities.getStringArray(obj, "type"),
+ 100
+ );
+
+ reconCandidateMap.put(id, rc);
+ }
+
+ storeCell(rows, row, col, rc, reconCandidateMap);
+ }
+
+ protected int[] collectResult(
+ List rows,
+ JSONObject extNode,
+ JSONObject resultNode,
+ int startRowIndex,
+ int startColumnIndex,
+ Map reconCandidateMap
+ ) throws JSONException {
+ String propertyID = extNode.getString("id");
+ String expectedTypeID = extNode.getJSONObject("expected").getString("id");
+
+ JSONArray a = resultNode != null && resultNode.has(propertyID) && !resultNode.isNull(propertyID) ?
+ resultNode.getJSONArray(propertyID) : null;
+
+ if (expectedTypeID.startsWith("/type/")) {
+ if (a != null) {
+ int l = a.length();
+ for (int r = 0; r < l; r++) {
+ Object o = a.isNull(r) ? null : a.get(r);
+ if (o instanceof Serializable) {
+ storeCell(rows, startRowIndex++, startColumnIndex, o, reconCandidateMap);
+ }
+ }
+ }
+
+ // note that we still take up a column even if we don't have any data
+ return new int[] { startRowIndex, startColumnIndex + 1 };
+ } else {
+ boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties"));
+ boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included"));
+
+ if (a != null && a.length() > 0) {
+ int maxColIndex = startColumnIndex;
+
+ int l = a.length();
+ for (int r = 0; r < l; r++) {
+ Object v = a.isNull(r) ? null : a.get(r);
+ JSONObject o = v != null && v instanceof JSONObject ? (JSONObject) v : null;
+
+ int startColumnIndex2 = startColumnIndex;
+ int startRowIndex2 = startRowIndex;
+
+ if (isOwnColumn) {
+ if (o != null) {
+ storeCell(rows, startRowIndex2++, startColumnIndex2++, o, reconCandidateMap);
+ } else {
+ storeCell(rows, startRowIndex2++, startColumnIndex2++, v, reconCandidateMap);
+ }
+ }
+
+ if (hasSubProperties && o != null) {
+ int[] rowcol = collectResult(
+ rows,
+ extNode.getJSONArray("properties"),
+ o,
+ startRowIndex,
+ startColumnIndex2,
+ reconCandidateMap
+ );
+
+ startRowIndex2 = rowcol[0];
+ startColumnIndex2 = rowcol[1];
+ }
+
+ startRowIndex = startRowIndex2;
+ maxColIndex = Math.max(maxColIndex, startColumnIndex2);
+ }
+
+ return new int[] { startRowIndex, maxColIndex };
+ } else {
+ return new int[] {
+ startRowIndex,
+ startColumnIndex + countColumns(extNode, null, new ArrayList(), new ArrayList())
+ };
+ }
+ }
+ }
+
+ protected int[] collectResult(
+ List rows,
+ JSONArray subProperties,
+ JSONObject resultNode,
+ int startRowIndex,
+ int startColumnIndex,
+ Map reconCandidateMap
+ ) throws JSONException {
+ int maxStartRowIndex = startRowIndex;
+
+ int k = subProperties.length();
+ for (int c = 0; c < k; c++) {
+ int[] rowcol = collectResult(
+ rows,
+ subProperties.getJSONObject(c),
+ resultNode,
+ startRowIndex,
+ startColumnIndex,
+ reconCandidateMap
+ );
+
+ maxStartRowIndex = Math.max(maxStartRowIndex, rowcol[0]);
+ startColumnIndex = rowcol[1];
+ }
+
+ return new int[] { maxStartRowIndex, startColumnIndex };
+ }
+
+
+ static protected InputStream doMqlRead(String query) throws IOException {
+ URL url = new URL("http://api.freebase.com/api/service/mqlread");
+
+ URLConnection connection = url.openConnection();
+ connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+ connection.setConnectTimeout(5000);
+ connection.setDoOutput(true);
+
+ DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
+ try {
+ String body = "extended=1&query=" + ParsingUtilities.encode(query);
+
+ dos.writeBytes(body);
+ } finally {
+ dos.flush();
+ dos.close();
+ }
+
+ connection.connect();
+
+ return connection.getInputStream();
+ }
+
+ static protected void formulateQuery(Set ids, JSONObject node, Writer writer) throws JSONException {
+ JSONWriter jsonWriter = new JSONWriter(writer);
+
+ jsonWriter.object();
+ jsonWriter.key("query");
+ jsonWriter.array();
+ jsonWriter.object();
+
+ jsonWriter.key("id"); jsonWriter.value(null);
+ jsonWriter.key("id|=");
+ jsonWriter.array();
+ for (String id : ids) {
+ if (id != null) {
+ jsonWriter.value(id);
+ }
+ }
+ jsonWriter.endArray();
+
+ formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
+
+ jsonWriter.endObject();
+ jsonWriter.endArray();
+ jsonWriter.endObject();
+ }
+
+ static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
+ String propertyID = node.getString("id");
+ String expectedTypeID = node.getJSONObject("expected").getString("id");
+
+ writer.key(propertyID);
+ writer.array();
+ {
+ if (!expectedTypeID.startsWith("/type/")) { // not literal
+ writer.object();
+ writer.key("optional"); writer.value(true);
+
+ boolean hasLimit = false;
+ if (node.has("constraints") && !node.isNull("constraints")) {
+ JSONObject constraints = node.getJSONObject("constraints");
+
+ String[] names = JSONObject.getNames(constraints);
+ for (String name : names) {
+ Object value = constraints.get(name);
+ if (name.equals("limit")) {
+ hasLimit = true;
+ }
+
+ if (!name.contains(":") &&
+ !name.equals("limit") &&
+ !name.equals("optional") &&
+ !name.equals("count") &&
+ !name.equals("estimate-count") &&
+ !name.equals("sort") &&
+ !name.equals("return")) {
+
+ if (name.startsWith("!")) {
+ name = "!c:" + name.substring(1);
+ } else {
+ name = "c:" + name;
+ }
+ }
+ writer.key(name);
+ writer.value(value);
+ }
+ }
+ if (!hasLimit) {
+ writer.key("limit"); writer.value(10);
+ }
+
+ {
+ boolean hasSubProperties = (node.has("properties") && !node.isNull("properties"));
+
+ if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
+ writer.key("name"); writer.value(null);
+ writer.key("id"); writer.value(null);
+ writer.key("type"); writer.array(); writer.endArray();
+ }
+
+ if (hasSubProperties) {
+ formulateQueryNode(node.getJSONArray("properties"), writer);
+ }
+ }
+ writer.endObject();
+ }
+ }
+ writer.endArray();
+ }
+
+ static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
+ int l = propertiesA.length();
+
+ for (int i = 0; i < l; i++) {
+ formulateQueryNode(propertiesA.getJSONObject(i), writer);
+ }
+ }
+
+ static protected int countColumns(JSONObject obj, List columns, List names, List path) throws JSONException {
+ String name = obj.getString("name");
+
+ List names2 = null;
+ List path2 = null;
+ if (columns != null) {
+ names2 = new ArrayList(names);
+ names2.add(name);
+
+ path2 = new ArrayList(path);
+ path2.add(obj.getString("id"));
+ }
+
+ if (obj.has("properties") && !obj.isNull("properties")) {
+ boolean included = (obj.has("included") && obj.getBoolean("included"));
+ if (included && columns != null) {
+ JSONObject expected = obj.getJSONObject("expected");
+
+ columns.add(new ColumnInfo(names2, path2,
+ new FreebaseType(expected.getString("id"), expected.getString("name"))));
+ }
+
+ return (included ? 1 : 0) +
+ countColumns(obj.getJSONArray("properties"), columns, names2, path2);
+ } else {
+ if (columns != null) {
+ JSONObject expected = obj.getJSONObject("expected");
+
+ columns.add(new ColumnInfo(names2, path2,
+ new FreebaseType(expected.getString("id"), expected.getString("name"))));
+ }
+ return 1;
+ }
+ }
+
+ static protected int countColumns(JSONArray a, List columns, List names, List path) throws JSONException {
+ int c = 0;
+ int l = a.length();
+ for (int i = 0; i < l; i++) {
+ c += countColumns(a.getJSONObject(i), columns, names, path);
+ }
+ return c;
+ }
+}
\ No newline at end of file
diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
new file mode 100644
index 000000000..739c5e3c8
--- /dev/null
+++ b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java
@@ -0,0 +1,223 @@
+package com.google.refine.freebase.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.servlet.http.HttpServletRequest;
+
+import oauth.signpost.OAuthConsumer;
+import oauth.signpost.exception.OAuthCommunicationException;
+import oauth.signpost.exception.OAuthExpectationFailedException;
+import oauth.signpost.exception.OAuthMessageSignerException;
+
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.params.CoreProtocolPNames;
+import org.apache.http.util.EntityUtils;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import com.google.refine.ProjectManager;
+import com.google.refine.RefineServlet;
+import com.google.refine.oauth.Credentials;
+import com.google.refine.oauth.OAuthUtilities;
+import com.google.refine.oauth.Provider;
+
+public class FreebaseUtils {
+
+ static final public String FREEBASE_HOST = "www.freebase.com";
+
+ static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine";
+
+ static final private String AGENT_ID = "/en/google_refine";
+
+ private static String getUserInfoURL(String host) {
+ return "http://" + host + "/api/service/user_info";
+ }
+
+ private static String getMQLWriteURL(String host) {
+ return "http://" + host + "/api/service/mqlwrite";
+ }
+
+ private static String getMQLReadURL(String host) {
+ return "http://" + host + "/api/service/mqlread";
+ }
+
+ private static String getUserAgent() {
+ return RefineServlet.FULLNAME;
+ }
+
+ public static String getUserInfo(Credentials credentials, Provider provider)
+ throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException {
+
+ OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
+
+ HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost()));
+ httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
+
+ // this is required by the Metaweb API to avoid XSS
+ httpRequest.setHeader("X-Requested-With", "1");
+
+ // sign the request with the oauth library
+ consumer.sign(httpRequest);
+
+ // execute the request
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpResponse httpResponse = httpClient.execute(httpRequest);
+
+ // return the results
+ return EntityUtils.toString(httpResponse.getEntity());
+ }
+
+ public static String getUserBadges(Provider provider, String user_id)
+ throws ClientProtocolException, IOException, JSONException {
+
+ String query = "{" +
+ "'id' : '" + user_id + "'," +
+ "'!/type/usergroup/member' : [{" +
+ "'id' : null," +
+ "'key' : [{" +
+ "'namespace' : null" +
+ "}]" +
+ "}]" +
+ "}".replace("'", "\"");
+
+ return mqlread(provider, query);
+ }
+
+ public static String mqlread(Provider provider, String query)
+ throws ClientProtocolException, IOException, JSONException {
+
+ JSONObject envelope = new JSONObject();
+ envelope.put("query", new JSONObject(query));
+
+ List formparams = new ArrayList();
+ formparams.add(new BasicNameValuePair("query", envelope.toString()));
+ UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
+
+ HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost()));
+ httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
+ httpRequest.setEntity(entity);
+
+ // this is required by the Metaweb API to avoid XSS
+ httpRequest.setHeader("X-Requested-With", "1");
+
+ // execute the request
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpResponse httpResponse = httpClient.execute(httpRequest);
+
+ // return the results
+ return EntityUtils.toString(httpResponse.getEntity());
+ }
+
+ public static String mqlwrite(Credentials credentials, Provider provider, String query)
+ throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException {
+ OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
+
+ JSONObject envelope = new JSONObject();
+ envelope.put("query", new JSONObject(query));
+
+ List formparams = new ArrayList();
+ formparams.add(new BasicNameValuePair("query", envelope.toString()));
+ UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
+
+ HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost()));
+ httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
+ httpRequest.setEntity(entity);
+
+ // this is required by the Metaweb API to avoid XSS
+ httpRequest.setHeader("X-Requested-With", "1");
+
+ // sign the request with the oauth library
+ consumer.sign(httpRequest);
+
+ // execute the request
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpResponse httpResponse = httpClient.execute(httpRequest);
+
+ // return the results
+ return EntityUtils.toString(httpResponse.getEntity());
+ }
+
+ public static String uploadTriples(
+ HttpServletRequest request,
+ String qa,
+ String source_name,
+ String source_id,
+ String mdo_id,
+ String triples
+ ) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException {
+
+ Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST);
+
+ Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS);
+
+ JSONObject mdo_info = new JSONObject();
+ mdo_info.put("name", source_name);
+ if (source_id != null) {
+ mdo_info.put("info_source",source_id);
+ }
+
+ JSONObject user_info = new JSONObject(getUserInfo(credentials, provider));
+ if (user_info.has("username")) {
+
+ List formparams = new ArrayList();
+ formparams.add(new BasicNameValuePair("user", user_info.getString("id")));
+ formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE"));
+ formparams.add(new BasicNameValuePair("operator", user_info.getString("id")));
+ formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID));
+ formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString()));
+ formparams.add(new BasicNameValuePair("graphport", "sandbox"));
+ formparams.add(new BasicNameValuePair("payload", triples));
+ formparams.add(new BasicNameValuePair("check_params", "false"));
+ if (mdo_id != null) {
+ formparams.add(new BasicNameValuePair("mdo_guid", mdo_id));
+ }
+ if (Boolean.parseBoolean(qa)) {
+ formparams.add(new BasicNameValuePair("rabj", "true"));
+ }
+ UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
+
+ HttpPost httpRequest = new HttpPost(getFreeQUrl());
+ httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent());
+ httpRequest.setEntity(entity);
+
+ HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST));
+ surrogateRequest.setEntity(entity);
+
+ OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider);
+
+ consumer.sign(surrogateRequest);
+
+ Header[] h = surrogateRequest.getHeaders("Authorization");
+ if (h.length > 0) {
+ httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue());
+ } else {
+ throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request");
+ }
+
+ // execute the request
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpResponse httpResponse = httpClient.execute(httpRequest);
+
+ // return the results
+ return EntityUtils.toString(httpResponse.getEntity());
+ } else {
+ throw new RuntimeException("Invalid credentials");
+ }
+ }
+
+ static public String getFreeQUrl() {
+ String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
+ return url != null ? url : FREEQ_URL;
+ }
+}
