From be17d0b3b46574996d3ea11361c5839cf39bf31f Mon Sep 17 00:00:00 2001 From: David Huynh Date: Fri, 8 Oct 2010 01:39:38 +0000 Subject: [PATCH] Extracted out Freebase related functionalities. Another check-in will remove that code from the core module git-svn-id: http://google-refine.googlecode.com/svn/trunk@1451 7d457c2a-affb-35e4-300a-418c747d4874 --- extensions/freebase/.classpath | 16 + extensions/freebase/.project | 17 + extensions/freebase/build.xml | 71 ++ .../freebase/module/MOD-INF/controller.js | 79 ++ .../freebase/module/MOD-INF/module.properties | 4 + .../dialogs/extend-data-preview-dialog.html | 26 + .../dialogs/extend-data-preview-dialog.js | 363 +++++++++ .../dialogs/freebase-loading-dialog.html | 19 + .../dialogs/freebase-loading-dialog.js | 225 ++++++ .../dialogs/schema-alignment/dialog.js | 290 +++++++ .../dialogs/schema-alignment/ui-link.js | 348 +++++++++ .../dialogs/schema-alignment/ui-node.js | 732 ++++++++++++++++++ .../freebase/module/scripts/extension.js | 118 +++ .../freebase/module/scripts/util/freebase.js | 30 + .../freebase/module/scripts/util/sign.js | 120 +++ .../dialogs/extend-data-preview-dialog.less | 38 + .../dialogs/freebase-loading-dialog.less | 60 ++ .../dialogs/schema-alignment-dialog.less | 114 +++ extensions/freebase/module/styles/theme.less | 1 + .../refine/freebase/FreebaseProperty.java | 9 + .../google/refine/freebase/FreebaseTopic.java | 28 + .../google/refine/freebase/FreebaseType.java | 36 + .../freebase/ProtographTransposeExporter.java | 75 ++ .../freebase/commands/ExtendDataCommand.java | 32 + .../commands/ImportQADataCommand.java | 36 + .../freebase/commands/MQLReadCommand.java | 31 + .../freebase/commands/MQLWriteCommand.java | 40 + .../commands/PreviewExtendDataCommand.java | 158 ++++ .../commands/PreviewProtographCommand.java | 71 ++ .../commands/SaveProtographCommand.java | 40 + .../freebase/commands/UploadDataCommand.java | 89 +++ .../commands/auth/AuthorizeCommand.java | 135 ++++ .../auth/CheckAuthorizationCommand.java | 47 ++ .../commands/auth/DeAuthorizeCommand.java | 32 + .../commands/auth/GetUserBadgesCommand.java | 37 + .../model/changes/DataExtensionChange.java | 431 +++++++++++ .../model/recon/DataExtensionReconConfig.java | 66 ++ .../model/recon/GuidBasedReconConfig.java | 176 +++++ .../model/recon/IdBasedReconConfig.java | 181 +++++ .../model/recon/KeyBasedReconConfig.java | 195 +++++ .../model/recon/StrictReconConfig.java | 21 + .../freebase/oauth/FreebaseProvider.java | 37 + .../FreebaseTimeCommonsHttpOAuthConsumer.java | 69 ++ .../operations/ExtendDataOperation.java | 275 +++++++ .../operations/ImportQADataOperation.java | 106 +++ .../operations/SaveProtographOperation.java | 121 +++ .../freebase/protograph/AnonymousNode.java | 47 ++ .../protograph/BooleanColumnCondition.java | 42 + .../freebase/protograph/CellKeyNode.java | 35 + .../refine/freebase/protograph/CellNode.java | 8 + .../freebase/protograph/CellTopicNode.java | 58 ++ .../freebase/protograph/CellValueNode.java | 36 + .../refine/freebase/protograph/Condition.java | 9 + .../protograph/FreebaseTopicNode.java | 48 ++ .../refine/freebase/protograph/Link.java | 48 ++ .../refine/freebase/protograph/Node.java | 6 + .../freebase/protograph/NodeWithLinks.java | 9 + .../freebase/protograph/Protograph.java | 171 ++++ .../refine/freebase/protograph/ValueNode.java | 29 + .../MqlwriteLikeTransposedNodeFactory.java | 346 +++++++++ .../protograph/transpose/TransposedNode.java | 4 + .../transpose/TransposedNodeFactory.java | 43 + .../protograph/transpose/Transposer.java | 222 ++++++ .../TripleLoaderTransposedNodeFactory.java | 725 +++++++++++++++++ .../util/FreebaseDataExtensionJob.java | 417 ++++++++++ .../refine/freebase/util/FreebaseUtils.java | 223 ++++++ 66 files changed, 7771 insertions(+) create mode 100644 extensions/freebase/.classpath create mode 100644 extensions/freebase/.project create mode 100644 extensions/freebase/build.xml create mode 100644 extensions/freebase/module/MOD-INF/controller.js create mode 100644 extensions/freebase/module/MOD-INF/module.properties create mode 100644 extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.html create mode 100644 extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.js create mode 100644 extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.html create mode 100644 extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.js create mode 100644 extensions/freebase/module/scripts/dialogs/schema-alignment/dialog.js create mode 100644 extensions/freebase/module/scripts/dialogs/schema-alignment/ui-link.js create mode 100644 extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js create mode 100644 extensions/freebase/module/scripts/extension.js create mode 100644 extensions/freebase/module/scripts/util/freebase.js create mode 100644 extensions/freebase/module/scripts/util/sign.js create mode 100644 extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less create mode 100644 extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less create mode 100644 extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less create mode 100644 extensions/freebase/module/styles/theme.less create mode 100644 extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/FreebaseType.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/auth/AuthorizeCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/auth/DeAuthorizeCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/BooleanColumnCondition.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/CellKeyNode.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/CellNode.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/CellValueNode.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/Condition.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/FreebaseTopicNode.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/Link.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/Node.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/NodeWithLinks.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/Protograph.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java create mode 100644 extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java diff --git a/extensions/freebase/.classpath b/extensions/freebase/.classpath new file mode 100644 index 000000000..1571227e2 --- /dev/null +++ b/extensions/freebase/.classpath @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/extensions/freebase/.project b/extensions/freebase/.project new file mode 100644 index 000000000..68c1fa144 --- /dev/null +++ b/extensions/freebase/.project @@ -0,0 +1,17 @@ + + + grefine-freebase + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/extensions/freebase/build.xml b/extensions/freebase/build.xml new file mode 100644 index 000000000..3eec7bb35 --- /dev/null +++ b/extensions/freebase/build.xml @@ -0,0 +1,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/extensions/freebase/module/MOD-INF/controller.js b/extensions/freebase/module/MOD-INF/controller.js new file mode 100644 index 000000000..ccddcff31 --- /dev/null +++ b/extensions/freebase/module/MOD-INF/controller.js @@ -0,0 +1,79 @@ +function init() { + Packages.com.google.refine.freebase.oauth.FreebaseProvider.register(); + + var RS = Packages.com.google.refine.RefineServlet; + RS.registerClassMapping( + "com.google.refine.model.changes.DataExtensionChange", + "com.google.refine.freebase.model.changes.DataExtensionChange"); + RS.registerClassMapping( + "com.google.refine.operations.SaveProtographOperation$ProtographChange", + "com.google.refine.freebase.operations.SaveProtographOperation$ProtographChange"); + + // TODO(dfhuynh): Temporary hack until we know how the core module can resolve our module's classes + RS.cacheClass(Packages.com.google.refine.freebase.model.changes.DataExtensionChange); + RS.cacheClass(Packages.com.google.refine.freebase.operations.SaveProtographOperation$ProtographChange); + + RS.registerCommand(module, "extend-data", new Packages.com.google.refine.freebase.commands.ExtendDataCommand()); + RS.registerCommand(module, "preview-extend-data", new Packages.com.google.refine.freebase.commands.PreviewExtendDataCommand()); + + RS.registerCommand(module, "preview-protograph", new Packages.com.google.refine.freebase.commands.PreviewProtographCommand()); + RS.registerCommand(module, "save-protograph", new Packages.com.google.refine.freebase.commands.SaveProtographCommand()); + + RS.registerCommand(module, "check-authorization", new Packages.com.google.refine.freebase.commands.auth.CheckAuthorizationCommand()); + RS.registerCommand(module, "authorize", new Packages.com.google.refine.freebase.commands.auth.AuthorizeCommand()); + RS.registerCommand(module, "deauthorize", new Packages.com.google.refine.freebase.commands.auth.DeAuthorizeCommand()); + RS.registerCommand(module, "user-badges", new Packages.com.google.refine.freebase.commands.auth.GetUserBadgesCommand()); + + RS.registerCommand(module, "upload-data", new Packages.com.google.refine.freebase.commands.UploadDataCommand()); + RS.registerCommand(module, "import-qa-data", new Packages.com.google.refine.freebase.commands.ImportQADataCommand()); + RS.registerCommand(module, "mqlread", new Packages.com.google.refine.freebase.commands.MQLReadCommand()); + RS.registerCommand(module, "mqlwrite", new Packages.com.google.refine.freebase.commands.MQLWriteCommand()); + + var OR = Packages.com.google.refine.operations.OperationRegistry; + + OR.registerOperation(module, "extend-data", Packages.com.google.refine.freebase.operations.ExtendDataOperation); + OR.registerOperation(module, "import-qa-data", Packages.com.google.refine.freebase.operations.ImportQADataOperation); + OR.registerOperation(module, "save-protograph", Packages.com.google.refine.freebase.operations.SaveProtographOperation); // for backward compatibility + OR.registerOperation(module, "save-schema-alignment-skeleton", Packages.com.google.refine.freebase.operations.SaveProtographOperation); + + var RC = Packages.com.google.refine.model.recon.ReconConfig; + + RC.registerReconConfig(module, "strict", Packages.com.google.refine.freebase.model.recon.StrictReconConfig); + RC.registerReconConfig(module, "extend", Packages.com.google.refine.freebase.model.recon.DataExtensionReconConfig); + + var ER = Packages.com.google.refine.exporters.ExporterRegistry; + + ER.registerExporter("tripleloader", new Packages.com.google.refine.freebase.ProtographTransposeExporter.TripleLoaderExporter()); + ER.registerExporter("mqlwrite", new Packages.com.google.refine.freebase.ProtographTransposeExporter.MqlwriteLikeExporter()); + + Packages.com.google.refine.model.Project. + registerOverlayModel("freebaseProtograph", Packages.com.google.refine.freebase.protograph.Protograph); + + ClientSideResourceManager.addPaths( + "project/scripts", + module, + [ + "scripts/extension.js", + + "scripts/util/sign.js", + "scripts/util/freebase.js", + + "scripts/dialogs/freebase-loading-dialog.js", + "scripts/dialogs/extend-data-preview-dialog.js", + + "scripts/dialogs/schema-alignment/dialog.js", + "scripts/dialogs/schema-alignment/ui-node.js", + "scripts/dialogs/schema-alignment/ui-link.js" + ] + ); + + ClientSideResourceManager.addPaths( + "project/styles", + module, + [ + "styles/dialogs/freebase-loading-dialog.less", + "styles/dialogs/extend-data-preview-dialog.less", + "styles/dialogs/schema-alignment-dialog.less" + ] + ); +} diff --git a/extensions/freebase/module/MOD-INF/module.properties b/extensions/freebase/module/MOD-INF/module.properties new file mode 100644 index 000000000..fa86dc45f --- /dev/null +++ b/extensions/freebase/module/MOD-INF/module.properties @@ -0,0 +1,4 @@ +name = freebase-extension +description = Google Refine Freebase Extension +templating = false +requires = core diff --git a/extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.html b/extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.html new file mode 100644 index 000000000..c303e5eb1 --- /dev/null +++ b/extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.html @@ -0,0 +1,26 @@ +
+
+
+
+ + + + + + + + + + + + + + + +
Add PropertyPreview
Suggested Properties
+
+
+ + +
+
\ No newline at end of file diff --git a/extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.js b/extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.js new file mode 100644 index 000000000..e92565e65 --- /dev/null +++ b/extensions/freebase/module/scripts/dialogs/extend-data-preview-dialog.js @@ -0,0 +1,363 @@ +function ExtendDataPreviewDialog(column, columnIndex, rowIndices, onDone) { + this._column = column; + this._columnIndex = columnIndex; + this._rowIndices = rowIndices; + this._onDone = onDone; + this._extension = { properties: [] }; + + var self = this; + this._dialog = $(DOM.loadHTML("freebase-extension", "scripts/dialogs/extend-data-preview-dialog.html")); + this._elmts = DOM.bind(this._dialog); + this._elmts.dialogHeader.text("Add Columns from Freebase Based on Column " + column.name); + this._elmts.resetButton.click(function() { + self._extension.properties = []; + self._update(); + }); + + this._elmts.okButton.click(function() { + if (self._extension.properties.length === 0) { + alert("Please add some properties first."); + } else { + DialogSystem.dismissUntil(self._level - 1); + self._onDone(self._extension); + } + }); + this._elmts.cancelButton.click(function() { + DialogSystem.dismissUntil(self._level - 1); + }); + + var dismissBusy = DialogSystem.showBusy(); + var type = "reconConfig" in column && "type" in column.reconConfig ? column.reconConfig.type.id : "/common/topic"; + + ExtendDataPreviewDialog.getAllProperties(type, function(properties) { + dismissBusy(); + self._show(properties); + }); +} + +ExtendDataPreviewDialog.getAllProperties = function(typeID, onDone) { + var done = false; + + $.getJSON( + Refine.refineHelperService + "/get_properties_of_type?type=" + typeID + "&callback=?", + null, + function(data) { + if (done) return; + done = true; + + var allProperties = []; + for (var i = 0; i < data.properties.length; i++) { + var property = data.properties[i]; + var property2 = { + id: property.id, + name: property.name + }; + if ("id2" in property) { + property2.expected = property.schema2; + property2.properties = [{ + id: property.id2, + name: property.name2, + expected: property.expects + }]; + } else { + property2.expected = property.expects; + } + allProperties.push(property2); + } + allProperties.sort(function(a, b) { return a.name.localeCompare(b.name); }); + + onDone(allProperties); + } + ); + + window.setTimeout(function() { + if (done) return; + + done = true; + onDone([]); + }, 7000); // time to give up? +}; + +ExtendDataPreviewDialog.prototype._show = function(properties) { + this._level = DialogSystem.showDialog(this._dialog); + + var n = this._elmts.suggestedPropertyContainer.offset().top + + this._elmts.suggestedPropertyContainer.outerHeight(true) - + this._elmts.addPropertyInput.offset().top; + + this._elmts.previewContainer.height(Math.floor(n)); + + var self = this; + var container = this._elmts.suggestedPropertyContainer; + var renderSuggestedProperty = function(property) { + var label = ("properties" in property) ? (property.name + " » " + property.properties[0].name) : property.name; + var div = $('
').addClass("suggested-property").appendTo(container); + + $('') + .attr("href", "javascript:{}") + .html(label) + .appendTo(div) + .click(function() { + self._addProperty(property); + }); + }; + for (var i = 0; i < properties.length; i++) { + renderSuggestedProperty(properties[i]); + } + + var suggestConfig = { + type: '/type/property' + }; + if ("reconConfig" in this._column && "type" in this._column.reconConfig) { + suggestConfig.ac_param = { schema: this._column.reconConfig.type.id }; + } + + this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) { + var expected = data.expected_type; + self._addProperty({ + id : data.id, + name: data.name, + expected: { + id: expected.id, + name: expected.name + } + }); + }); +}; + +ExtendDataPreviewDialog.prototype._update = function() { + this._elmts.previewContainer.empty().text("Querying Freebase ..."); + + var self = this; + var params = { + project: theProject.id, + columnName: this._column.name + }; + + $.post( + "/command/freebase-extension/preview-extend-data?" + $.param(params), + { + rowIndices: JSON.stringify(this._rowIndices), + extension: JSON.stringify(this._extension) + }, + function(data) { + self._renderPreview(data); + }, + "json" + ); +}; + +ExtendDataPreviewDialog.prototype._addProperty = function(p) { + var addSeveralToList = function(properties, oldProperties) { + for (var i = 0; i < properties.length; i++) { + addToList(properties[i], oldProperties); + } + }; + var addToList = function(property, oldProperties) { + for (var i = 0; i < oldProperties.length; i++) { + var oldProperty = oldProperties[i]; + if (oldProperty.id == property.id) { + if ("included" in property) { + oldProperty.included = "included" in oldProperty ? + (oldProperty.included || property.included) : + property.included; + } + + if ("properties" in property) { + if ("properties" in oldProperty) { + addSeveralToList(property.properties, oldProperty.properties); + } else { + oldProperty.properties = property.properties; + } + } + return; + } + } + + oldProperties.push(property); + }; + + addToList(p, this._extension.properties); + + this._update(); +}; + +ExtendDataPreviewDialog.prototype._renderPreview = function(data) { + var self = this; + var container = this._elmts.previewContainer.empty(); + if (data.code == "error") { + container.text("Error."); + return; + } + + var table = $('')[0]; + var trHead = table.insertRow(table.rows.length); + $('
').appendTo(trHead).text(this._column.name); + + var renderColumnHeader = function(column) { + var th = $('').appendTo(trHead); + + $('').html(column.names.join(" » ")).appendTo(th); + $('
').appendTo(th); + + $('') + .text("remove") + .addClass("action") + .attr("title", "Remove this column") + .click(function() { + self._removeProperty(column.path); + }).appendTo(th); + + $('') + .text("constrain") + .addClass("action") + .attr("title", "Add constraints to this column") + .click(function() { + self._constrainProperty(column.path); + }).appendTo(th); + }; + for (var c = 0; c < data.columns.length; c++) { + renderColumnHeader(data.columns[c]); + } + + for (var r = 0; r < data.rows.length; r++) { + var tr = table.insertRow(table.rows.length); + var row = data.rows[r]; + + for (var c = 0; c < row.length; c++) { + var td = tr.insertCell(tr.cells.length); + var cell = row[c]; + if (cell !== null) { + if ($.isPlainObject(cell)) { + $('').attr("href", "http://www.freebase.com/view" + cell.id).text(cell.name).appendTo(td); + } else { + $('').text(cell).appendTo(td); + } + } + } + } + + container.append(table); +}; + +ExtendDataPreviewDialog.prototype._removeProperty = function(path) { + var removeFromList = function(path, index, properties) { + var id = path[index]; + + for (var i = properties.length - 1; i >= 0; i--) { + var property = properties[i]; + if (property.id == id) { + if (index === path.length - 1) { + if ("included" in property) { + delete property.included; + } + } else if ("properties" in property && property.properties.length > 0) { + removeFromList(path, index + 1, property.properties); + } + + if (!("properties" in property) || property.properties.length === 0) { + properties.splice(i, 1); + } + + return; + } + } + }; + + removeFromList(path, 0, this._extension.properties); + + this._update(); +}; + +ExtendDataPreviewDialog.prototype._findProperty = function(path) { + var find = function(path, index, properties) { + var id = path[index]; + + for (var i = properties.length - 1; i >= 0; i--) { + var property = properties[i]; + if (property.id == id) { + if (index === path.length - 1) { + return property; + } else if ("properties" in property && property.properties.length > 0) { + return find(path, index + 1, property.properties); + } + break; + } + } + + return null; + }; + + return find(path, 0, this._extension.properties); +}; + +ExtendDataPreviewDialog.prototype._constrainProperty = function(path) { + var self = this; + var property = this._findProperty(path); + + var frame = DialogSystem.createDialog(); + frame.width("500px"); + + var header = $('
').addClass("dialog-header").text("Constrain " + path.join(" > ")).appendTo(frame); + var body = $('
').addClass("dialog-body").appendTo(frame); + var footer = $('
').addClass("dialog-footer").appendTo(frame); + + body.html( + '
' + + '' + + '' + + '
' + + 'Enter MQL query constraints as JSON' + + '
' + + '' + + '
' + ); + var bodyElmts = DOM.bind(body); + + if ("constraints" in property) { + bodyElmts.textarea[0].value = JSON.stringify(property.constraints, null, 2); + } else { + bodyElmts.textarea[0].value = JSON.stringify({ "limit" : 10 }, null, 2); + } + + footer.html( + '' + + '' + ); + var footerElmts = DOM.bind(footer); + + var level = DialogSystem.showDialog(frame); + var dismiss = function() { + DialogSystem.dismissUntil(level - 1); + }; + + footerElmts.cancelButton.click(dismiss); + footerElmts.okButton.click(function() { + try { + var o = JSON.parse(bodyElmts.textarea[0].value); + if (o === undefined) { + alert("Please ensure that the JSON you enter is valid."); + return; + } + + if ($.isArray(o) && o.length == 1) { + o = o[0]; + } + if (!$.isPlainObject(o)) { + alert("The JSON you enter must be an object, that is, it is of this form { ... }."); + return; + } + + property.constraints = o; + + dismiss(); + + self._update(); + } catch (e) { + //console.log(e); + } + }); + + bodyElmts.textarea.focus(); +}; + diff --git a/extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.html b/extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.html new file mode 100644 index 000000000..3a0c9015c --- /dev/null +++ b/extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.html @@ -0,0 +1,19 @@ +
+
Load Data into Freebase
+
+
+
+ + + + +
+ + +
+ 		+ Ready to perform QA on this data? + +
+
+
\ No newline at end of file diff --git a/extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.js b/extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.js new file mode 100644 index 000000000..cf9e885e4 --- /dev/null +++ b/extensions/freebase/module/scripts/dialogs/freebase-loading-dialog.js @@ -0,0 +1,225 @@ +function FreebaseLoadingDialog() { + this._createDialog(); + this._signedin = false; +} + +FreebaseLoadingDialog.prototype._createDialog = function() { + var self = this; + var dialog = $(DOM.loadHTML("freebase-extension", "scripts/dialogs/freebase-loading-dialog.html")); + this._elmts = DOM.bind(dialog); + this._elmts.cancelButton.click(function() { self._dismiss(); }); + + var provider = "www.freebase.com"; + var authorization = this._elmts.authorization; + var loadButton = this._elmts.loadButton; + + var check_authorization = function(cont) { + $.get("/command/freebase-extension/check-authorization/" + provider, function(data) { + if ("status" in data && data.code == "/api/status/ok") { + authorization.html('Signed in as: ' + data.username + ' | Sign Out').show(); + DOM.bind(authorization).signout.click(function() { + self._signedin = false; + loadButton.attr("disabled","disabled"); + $("#freebase-loading-graph-selector-freebase").attr("disabled","disabled").button("refresh"); + Sign.signout(check_authorization,provider); + }); + loadButton.unbind().click(function() { + self._load(); + }); + self._signedin = true; + $("#freebase-loading-source-name").keyup(); + if (typeof cont == "function") cont(data); + } else { + authorization.html('Sign into Freebase to enable loading').show(); + DOM.bind(authorization).signin.click(function() { + Sign.signin(function() { + check_authorization(cont); + },provider); + }); + } + },"json"); + }; + + var make_topic = function(new_topic_id, topic_type, cont) { + var mql_query = [{ + "create": "unless_exists", + "name": new_topic_id, + "a:type": topic_type, + "b:type": "/common/topic", + "id": null, + "guid": null + }]; + + $.post("/command/freebase-extension/mqlwrite/" + provider, + { "query" : JSON.stringify(mql_query) }, + function(data) { + if ("status" in data && data.code == "/api/status/ok") { + self._elmts.source_id.val(data.result[0].id); + if (typeof cont == "function") cont(); + } else { + self._show_error("Error creating new topic", data); + } + }, + "json" + ); + }; + + var show_triples = function(cont) { + $.post( + "/command/freebase-extension/preview-protograph?" + $.param({ project: theProject.id }), + { + protograph: JSON.stringify(theProject.overlayModels.freebaseProtograph || {}), + engine: JSON.stringify(ui.browsingEngine.getJSON()) + }, + function(data) { + var body = self._elmts.dialogBody; + if ("tripleloader" in data) { + body.html( + '
' + + '' + + '' + + '
Name this data load ¬ required
' + + '
Source ID ¬ optional
' + + '
' + + '
' + data.tripleloader + '
' + ); + self._elmts = DOM.bind(dialog); + + self._elmts.source_name.keyup(function() { + if (self._signedin && $(this).val() != "") { + loadButton.removeAttr("disabled"); + } else { + loadButton.attr("disabled","disabled"); + } + }); + + self._elmts.source_id.suggest({ + "type": "/dataworld/information_source", + "suggest_new": "Click here to add a new information source" + }).bind("fb-select", function(e, data) { + self._elmts.source_id.val(data.id); + }).bind("fb-select-new", function(e, val) { + make_topic(val, "/dataworld/information_source"); + }); + + $.getJSON( + "/command/core/get-preference?" + $.param({ project: theProject.id, name: "freebase.load.jobName" }), + null, + function(data) { + if (data.value != null) { + self._elmts.source_name[0].value = data.value; + } + } + ); + + if (typeof cont == "function") cont(); + } else { + body.html( + '
'+ + '

This dataset has no triples

' + + '

Have you aligned it with the Freebase schemas yet?

' + + '
' + ); + self._elmts = DOM.bind(dialog); + self._end(); + } + self._level = DialogSystem.showDialog(dialog); + }, + "json" + ); + }; + + show_triples(check_authorization); +}; + +FreebaseLoadingDialog.prototype._load = function() { + var self = this; + var qa = self._elmts.qaCheckbox.is(':checked'); + + var get_refinery_url = function(url) { + return "http://refinery.freebaseapps.com/load/" + url.split("/").slice(-1)[0]; + }; + + var doLoad = function() { + var dismissBusy = DialogSystem.showBusy(); + + $.post("/command/freebase-extension/upload-data", + { + project: theProject.id, + "qa" : qa, + "engine" : JSON.stringify(ui.browsingEngine.getJSON()), + "source_name" : self._elmts.source_name.val(), + "source_id" : self._elmts.source_id.val() + }, + function(data) { + dismissBusy(); + + var body = self._elmts.dialogBody; + if ("status" in data && typeof data.status == "object" && "code" in data.status && data.status.code == 200) { + body.html( + '
' + + '

' + data.result.added + ' triples successfully scheduled for loading

' + + '

Follow the loading progress in the Freebase Refinery

' + + '
' + ); + self._end(); + } else { + self._show_error("Error loading data",data); + } + }, + "json" + ); + }; + + if (qa) { + var dialog = $( + '
' + + '
Are you sure this data is ready to be tested for upload into Freebase?
' + + '
' + ).dialog({ + resizable: false, + width: 400, + height: 230, + modal: true, + buttons: { + 'yes, I\'m sure': function() { + $(this).dialog('close'); + doLoad(); + }, + 'hmm, not really': function() { + $(this).dialog('close'); + } + } + }); + } else { + doLoad(); + } +}; + +FreebaseLoadingDialog.prototype._dismiss = function() { + DialogSystem.dismissUntil(this._level - 1); +}; + +FreebaseLoadingDialog.prototype._show_error = function(msg, error) { + console.log(error); + var self = this; + var body = self._elmts.dialogBody; + body.html( + '
' + + '

' + msg + '

' + + (('message' in error) ? '

' + error.message + '

' : '
' + JSON.stringify(error, null, 2) + '
') + + (('stack' in error) ? '
' + error.stack.replace(/\\n/g,'\n').replace(/\\t/g,'\t') + '
' : "") + + '
' + ); + this._end(); +}; + +FreebaseLoadingDialog.prototype._end = function() { + var self = this; + self._elmts.loadButton.text("Close").removeAttr("disabled").unbind().click(function() { + self._dismiss(); + }); + self._elmts.cancelButton.hide(); + self._elmts.qaCheckboxContainer.hide(); + self._elmts.authorization.hide(); +}; \ No newline at end of file diff --git a/extensions/freebase/module/scripts/dialogs/schema-alignment/dialog.js b/extensions/freebase/module/scripts/dialogs/schema-alignment/dialog.js new file mode 100644 index 000000000..d8b871a97 --- /dev/null +++ b/extensions/freebase/module/scripts/dialogs/schema-alignment/dialog.js @@ -0,0 +1,290 @@ +var SchemaAlignment = {}; + +SchemaAlignment.autoAlign = function() { + var protograph = {}; + + var columns = theProject.columnModel.columns; + + var typedCandidates = []; + var candidates = []; + + for (var c = 0; c < columns.length; c++) { + var column = columns[c]; + var typed = (column.reconConfig) && + ReconciliationManager.isFreebaseId(column.reconConfig.identifierSpace) && + ReconciliationManager.isFreebaseId(column.reconConfig.schemaSpace); + + var candidate = { + status: "unbound", + typed: typed, + index: c, + column: column + }; + + candidates.push(candidate); + if (typed) { + typedCandidates.push(candidate); + } + } + + if (typedCandidates.length > 0) { + + } else { + var queries = {}; + for (var i = 0; i < candidates.length; i++) { + var candidate = candidates[i]; + var name = SchemaAlignment._cleanName(candidate.column.name); + var key = "t" + i + ":search"; + queries[key] = { + "query" : name, + "limit" : 10, + "type" : "/type/type,/type/property", + "type_strict" : "any" + }; + } + + SchemaAlignment._batchSearch(queries, function(result) { + console.log(result); + }); + } +}; + +SchemaAlignment._batchSearch = function(queries, onDone) { + var keys = []; + for (var n in queries) { + if (queries.hasOwnProperty(n)) { + keys.push(n); + } + } + + var result = {}; + var args = []; + var makeBatch = function(keyBatch) { + var batch = {}; + for (var k = 0; k < keyBatch.length; k++) { + var key = keyBatch[k]; + batch[key] = queries[key]; + } + + args.push("http://api.freebase.com/api/service/search?" + + $.param({ "queries" : JSON.stringify(batch) }) + "&callback=?"); + + args.push(null); // no data + args.push(function(data) { + for (var k = 0; k < keyBatch.length; k++) { + var key = keyBatch[k]; + result[key] = data[key]; + } + }); + }; + + for (var i = 0; i < keys.length; i += 10) { + makeBatch(keys.slice(i, i + 10)); + } + + args.push(function() { + onDone(result); + }); + + Ajax.chainGetJSON.apply(null, args); +}; + +SchemaAlignment._cleanName = function(s) { + return s.replace(/\W/g, " ").replace(/\s+/g, " ").toLowerCase(); +}; + +SchemaAlignment.createNewRootNode = function() { + var rootNode = null; + var links = []; + var columns = theProject.columnModel.columns; + for (var i = 0; i < columns.length; i++) { + var column = columns[i]; + var target = { + nodeType: "cell-as-topic", + columnName: column.name, + createForNoReconMatch: true + }; + if ((column.reconConfig) && + ReconciliationManager.isFreebaseId(column.reconConfig.identifierSpace) && + ReconciliationManager.isFreebaseId(column.reconConfig.schemaSpace) && + (column.reconConfig.type)) { + + target.type = { + id: column.reconConfig.type.id, + name: column.reconConfig.type.name + }; + } + + if (column.name == theProject.columnModel.keyColumnName) { + rootNode = target; + } else { + links.push({ + property: null, + target: target + }); + } + } + + rootNode = rootNode || { nodeType: "cell-as-topic" }; + rootNode.links = links; + + return rootNode; +}; + +function SchemaAlignmentDialog(protograph, onDone) { + this._onDone = onDone; + this._originalProtograph = protograph || { rootNodes: [] }; + this._protograph = cloneDeep(this._originalProtograph); // this is what can be munched on + + if (!this._protograph.rootNodes.length) { + this._protograph.rootNodes.push(SchemaAlignment.createNewRootNode()); + } + + this._nodeUIs = []; + this._createDialog(); + this.preview(); +} + +SchemaAlignmentDialog.prototype._createDialog = function() { + var self = this; + var frame = DialogSystem.createDialog(); + + frame.width("1000px"); + + var header = $('
').addClass("dialog-header").text("Schema Alignment").appendTo(frame); + var body = $('
').addClass("dialog-body").appendTo(frame); + var footer = $('
').addClass("dialog-footer").appendTo(frame); + + this._constructFooter(footer); + this._constructBody(body); + + this._level = DialogSystem.showDialog(frame); + + this._renderBody(body); +}; + +SchemaAlignmentDialog.prototype._constructFooter = function(footer) { + var self = this; + + $('').html("  OK  ").click(function() { + var protograph = self.getJSON(); + + Refine.postProcess( + "freebase-extension", + "save-protograph", + {}, + { protograph: JSON.stringify(protograph) }, + {}, + { + onDone: function() { + DialogSystem.dismissUntil(self._level - 1); + theProject.overlayModels.freebaseProtograph = protograph; + } + } + ); + }).appendTo(footer); + + $('').text("Cancel").click(function() { + DialogSystem.dismissUntil(self._level - 1); + }).appendTo(footer); +}; + +SchemaAlignmentDialog.prototype._constructBody = function(body) { + $('

' + + 'The schema alignment skeleton below specifies how the graph-shaped data that will get generated ' + + 'from your grid-shaped data and written into Freebase. The cells in each record of your data will ' + + 'get placed into nodes within the skeleton. Configure the skeleton by specifying which ' + + 'column to substitute into which node. A node can also be an automatically generated ' + + 'anonymous node, or it can be an explicit value or topic that is the same for all records.' + + '

').appendTo(body); + + $( + '
' + + '' + + '
' + + '
' + + '
' + + '' + + '' + + '
' + ).appendTo(body); +}; + +SchemaAlignmentDialog.prototype._renderBody = function(body) { + var self = this; + + $("#schema-alignment-tabs").tabs(); + $("#schema-alignment-tabs-preview-mqllike").css("display", ""); + $("#schema-alignment-tabs-preview-tripleloader").css("display", ""); + + this._canvas = $(".schema-alignment-dialog-canvas"); + this._nodeTable = $('
').addClass("schema-alignment-table-layout").appendTo(this._canvas)[0]; + + for (var i = 0; i < this._protograph.rootNodes.length; i++) { + this._nodeUIs.push(new SchemaAlignmentDialog.UINode( + this, + this._protograph.rootNodes[i], + this._nodeTable, + { + expanded: true, + mustBeCellTopic: true + } + )); + } + + this._previewPanes = $(".schema-alignment-dialog-preview"); +}; + +SchemaAlignmentDialog.prototype.getJSON = function() { + var rootNodes = []; + for (var i = 0; i < this._nodeUIs.length; i++) { + var node = this._nodeUIs[i].getJSON(); + if (node !== null) { + rootNodes.push(node); + } + } + + return { + rootNodes: rootNodes + }; +}; + +SchemaAlignmentDialog.prototype.preview = function() { + var self = this; + + this._previewPanes.empty(); + + var protograph = this.getJSON(); + $.post( + "/command/freebase-extension/preview-protograph?" + $.param({ project: theProject.id }), + { protograph: JSON.stringify(protograph), engine: JSON.stringify(ui.browsingEngine.getJSON()) }, + function(data) { + if ("mqllike" in data) { + $(self._previewPanes[0]).text(JSON.stringify(data.mqllike, null, 2)); + } + if ("tripleloader" in data) { + $(self._previewPanes[1]).text(data.tripleloader); + } + }, + "json" + ); +}; + +SchemaAlignmentDialog._findColumn = function(cellIndex) { + var columns = theProject.columnModel.columns; + for (var i = 0; i < columns.length; i++) { + var column = columns[i]; + if (column.cellIndex == cellIndex) { + return column; + } + } + return null; +}; diff --git a/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-link.js b/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-link.js new file mode 100644 index 000000000..cb04c18ba --- /dev/null +++ b/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-link.js @@ -0,0 +1,348 @@ +SchemaAlignmentDialog.UILink = function(dialog, link, table, options, parentUINode) { + this._dialog = dialog; + this._link = link; + this._options = options; + this._parentUINode = parentUINode; + + // Make sure target node is there + this._link.target = this._link.target || { nodeType: "cell-as-value" } + + this._tr = table.insertRow(table.rows.length); + this._tdMain = this._tr.insertCell(0); + this._tdToggle = this._tr.insertCell(1); + this._tdDetails = this._tr.insertCell(2); + + $(this._tdMain).addClass("schema-alignment-link-main").attr("width", "250").addClass("padded"); + $(this._tdToggle).addClass("schema-alignment-link-toggle").attr("width", "1%").addClass("padded"); + $(this._tdDetails).addClass("schema-alignment-link-details").attr("width", "90%"); + + this._collapsedDetailDiv = $('
').appendTo(this._tdDetails).addClass("padded").html("..."); + this._expandedDetailDiv = $('
').appendTo(this._tdDetails).addClass("schema-alignment-detail-container"); + var self = this; + var show = function() { + if (self._options.expanded) { + self._collapsedDetailDiv.hide(); + self._expandedDetailDiv.show(); + } else { + self._collapsedDetailDiv.show(); + self._expandedDetailDiv.hide(); + } + }; + show(); + + $(this._tdToggle).html(" "); + $('') + .attr("src", this._options.expanded ? "images/expanded.png" : "images/collapsed.png") + .appendTo(this._tdToggle) + .click(function() { + self._options.expanded = !self._options.expanded; + + $(this).attr("src", self._options.expanded ? "images/expanded.png" : "images/collapsed.png"); + + show(); + }); + + this._renderMain(); + this._renderDetails(); +}; + +SchemaAlignmentDialog.UILink.prototype._renderMain = function() { + $(this._tdMain).empty(); + + var label = this._link.property !== null ? + (this._link.property.id + ((this._link.condition) ? " [?]" : "")) : + "property?"; + + var self = this; + + $('') + .attr("title", "remove property") + .attr("src", "images/close.png") + .css("cursor", "pointer") + .prependTo(this._tdMain) + .click(function() { + window.setTimeout(function() { + self._parentUINode.removeLink(self); + self._tr.parentNode.removeChild(self._tr); + self._dialog.preview(); + }, 100); + }); + + var a = $('') + .addClass("schema-alignment-link-tag") + .html(label) + .appendTo(this._tdMain) + .click(function(evt) { + self._startEditProperty(this); + }); + + $('').attr("src", "images/arrow-start.png").prependTo(a); + $('').attr("src", "images/arrow-end.png").appendTo(a); +}; + +SchemaAlignmentDialog.UILink.prototype._renderDetails = function() { + if (this._targetUI) { + this._targetUI.dispose(); + } + if (this._tableDetails) { + this._tableDetails.remove(); + } + + this._tableDetails = $('
').addClass("schema-alignment-table-layout").appendTo(this._expandedDetailDiv); + this._targetUI = new SchemaAlignmentDialog.UINode( + this._dialog, + this._link.target, + this._tableDetails[0], + { expanded: "links" in this._link.target && this._link.target.links.length > 0 }); +}; + +SchemaAlignmentDialog.UILink.prototype._startEditProperty = function(elmt) { + var sourceTypeID = this._parentUINode.getExpectedType(); + var targetNode = this._targetUI._node; + var targetTypeID = "type" in targetNode && targetNode.type !== null ? targetNode.type.id : null; + var targetTypeName = "columnNames" in targetNode ? targetNode.columnNames[0] : null; + + if (sourceTypeID !== null) { + var self = this; + var dismissBusy = DialogSystem.showBusy(); + + var instanceCount = 0; + var outgoing = []; + var incoming = []; + + var onDone = function(properties) { + dismissBusy(); + + self._showPropertySuggestPopup( + elmt, + properties + ); + }; + + SchemaAlignmentDialog.UILink._getPropertiesOfType( + sourceTypeID, + targetTypeID, + targetTypeName, + onDone + ); + } else { + this._showPropertySuggestPopup(elmt, []); + } +}; + +SchemaAlignmentDialog.UILink._getPropertiesOfType = function(typeID, targetTypeID, targetTypeName, onDone) { + var done = false; + + var params = { + "type" : typeID + }; + if (targetTypeID !== null) { + params.expects = targetTypeID; + } else if (targetTypeName !== null) { + params.expects = targetTypeName; + } + + $.getJSON( + Refine.refineHelperService + "/get_properties_of_type?" + $.param(params) + "&callback=?", + null, + function(data) { + if (done) return; + + done = true; + onDone(data.properties || []); + } + ); + + window.setTimeout(function() { + if (done) return; + + done = true; + onDone([]); + }, 7000); // time to give up? +}; + +SchemaAlignmentDialog.UILink.prototype._showPropertySuggestPopup = function(elmt, suggestions) { + self = this; + + var menu = MenuSystem.createMenu().width("350px"); + + var commitProperty = function(p) { + window.setTimeout(function() { MenuSystem.dismissAll(); }, 100); + + if ("id2" in p) { + // self._targetUI.dispose(); + self._link.property = { + id: p.id, + name: p.name + }; + self._link.target = { + nodeType: "anonymous", + links: [{ + property: { + id: p.id2, + name: p.name2 + }, + target: self._link.target + }] + }; + + self._renderDetails(); + } else { + self._link.property = { + id: p.id, + name: p.name + }; + } + + var conditionColumnName = conditionalSelect[0].value; + if (conditionColumnName != "") { + self._link.condition = { columnName: conditionColumnName }; + } else { + delete self._link.condition; + } + + self._configureTarget(); + }; + + var divConditional = $('
') + .addClass("schema-alignment-link-menu-section") + .html("Assert link when 'true' is found in column
").appendTo(menu); + + var conditionalSelect = $('').appendTo($('
').appendTo(divSearch)); + + MenuSystem.showMenu(menu, function(){}); + MenuSystem.positionMenuAboveBelow(menu, $(elmt)); + + var suggestOptions = { + type : '/type/property' + }; + var sourceTypeID = this._parentUINode.getExpectedType(); + if (sourceTypeID !== null) { + suggestOptions.ac_param = { schema: sourceTypeID }; + } + input.suggestP(suggestOptions).bind("fb-select", function(e, data) { commitProperty(data); }); + + input[0].focus(); +}; + +SchemaAlignmentDialog.UILink.prototype.getJSON = function() { + if ("property" in this._link && this._link.property !== null && + "target" in this._link && this._link.target !== null) { + + var targetJSON = this._targetUI.getJSON(); + if (targetJSON !== null) { + var json = { + property: cloneDeep(this._link.property), + target: targetJSON + }; + if (this._link.condition) { + json.condition = cloneDeep(this._link.condition); + } + return json; + } + } + return null; +}; + +SchemaAlignmentDialog.UILink.prototype._configureTarget = function() { + var self = this; + var dismissBusy = DialogSystem.showBusy(); + + $.getJSON( + "http://api.freebase.com/api/service/mqlread?query=" + JSON.stringify({ + query: { + "id" : this._link.property.id, + "type" : "/type/property", + "expected_type" : { + "id" : null, + "name" : null, + "/freebase/type_hints/mediator" : null + } + } + }) + "&callback=?", + null, + function(o) { + dismissBusy(); + + if ("result" in o) { + var expected_type = o.result.expected_type; + self._link.target.type = { + id: expected_type.id, + name: expected_type.name + }; + if (expected_type["/freebase/type_hints/mediator"] === true) { + self._link.target.nodeType = "anonymous"; + } else if (expected_type.id == "/type/key") { + self._link.target.nodeType = "cell-as-key"; + } else if (expected_type.id.match(/^\/type\//)) { + self._link.target.nodeType = "cell-as-value"; + } else if (!("topic" in self._link.target)) { + self._link.target.nodeType = "cell-as-topic"; + self._link.target.createForNoReconMatch = true; + } + + self._targetUI.render(); + } + + self._renderMain(); + self._dialog.preview(); + }, + "jsonp" + ); +}; diff --git a/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js b/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js new file mode 100644 index 000000000..09d4fd750 --- /dev/null +++ b/extensions/freebase/module/scripts/dialogs/schema-alignment/ui-node.js @@ -0,0 +1,732 @@ +SchemaAlignmentDialog.UINode = function(dialog, node, table, options) { + this._dialog = dialog; + this._node = node; + this._options = options; + + if ("columnName" in this._node) { + this._node.columnNames = [ this._node.columnName ]; + delete this._node.columnName; + } + + this._linkUIs = []; + this._detailsRendered = false; + + this._tr = table.insertRow(table.rows.length); + this._tdMain = this._tr.insertCell(0); + this._tdToggle = this._tr.insertCell(1); + this._tdDetails = this._tr.insertCell(2); + + $(this._tdMain).addClass("schema-alignment-node-main").attr("width", "250").addClass("padded"); + $(this._tdToggle).addClass("schema-alignment-node-toggle").attr("width", "1%").addClass("padded").hide(); + $(this._tdDetails).addClass("schema-alignment-node-details").attr("width", "90%").hide(); + + this._renderMain(); + + this._expanded = options.expanded; + if (this._isExpandable()) { + this._showExpandable(); + } +}; + +SchemaAlignmentDialog.UINode.prototype.dispose = function() { + // nothing for now +}; + +SchemaAlignmentDialog.UINode.prototype.removeLink = function(linkUI) { + for (var i = this._linkUIs.length - 1; i >= 0; i--) { + if (this._linkUIs[i] === linkUI) { + this._linkUIs.splice(i, 1); + this._node.links.splice(i, 1); + break; + } + } +}; + +SchemaAlignmentDialog.UINode.prototype._isExpandable = function() { + return this._node.nodeType == "cell-as-topic" || + this._node.nodeType == "anonymous" || + this._node.nodeType == "topic"; +}; + +SchemaAlignmentDialog.UINode.prototype.render = function() { + this._renderMain(); + if (this._isExpandable()) { + this._showExpandable(); + } else { + this._hideExpandable(); + } +}; + +SchemaAlignmentDialog.UINode.prototype.getExpectedType = function() { + if ("type" in this._node) { + return this._node.type.id; + } + return null; +}; + +SchemaAlignmentDialog.UINode.prototype._renderMain = function() { + $(this._tdMain).empty(); + + var self = this; + var a = $('') + .addClass("schema-alignment-node-tag") + .appendTo(this._tdMain) + .click(function(evt) { + self._showNodeConfigDialog(); + }); + + if (this._node.nodeType == "cell-as-topic" || + this._node.nodeType == "cell-as-value" || + this._node.nodeType == "cell-as-key") { + + if ("columnNames" in this._node) { + for (var c = 0; c < this._node.columnNames.length; c++) { + if (c > 0) { + $('').text(", ").appendTo(a); + } + + $('') + .text(this._node.columnNames[c]) + .addClass("schema-alignment-node-column") + .appendTo(a); + } + + $('').text(this._node.columnNames.length > 1 ? " cells" : " cell").appendTo(a); + } else { + a.html(this._options.mustBeCellTopic ? "Which column?" : "Configure..."); + } + } else if (this._node.nodeType == "topic") { + if ("topic" in this._node) { + a.html(this._node.topic.name); + } else if ("id" in this._node) { + a.html(this._node.topic.id); + } else { + a.html("Which topic?"); + } + } else if (this._node.nodeType == "value") { + if ("value" in this._node) { + a.html(this._node.value); + } else { + a.html("What value?"); + } + } else if (this._node.nodeType == "anonymous") { + a.html("(anonymous)"); + } +}; + +SchemaAlignmentDialog.UINode.prototype._showExpandable = function() { + $(this._tdToggle).show(); + $(this._tdDetails).show(); + + if (this._detailsRendered) { + return; + } + this._detailsRendered = true; + + this._collapsedDetailDiv = $('
').appendTo(this._tdDetails).addClass("padded").html("..."); + this._expandedDetailDiv = $('
').appendTo(this._tdDetails).addClass("schema-alignment-detail-container"); + + this._renderDetails(); + + var self = this; + var show = function() { + if (self._expanded) { + self._collapsedDetailDiv.hide(); + self._expandedDetailDiv.show(); + } else { + self._collapsedDetailDiv.show(); + self._expandedDetailDiv.hide(); + } + }; + show(); + + $(this._tdToggle).html(" "); + $('') + .attr("src", this._expanded ? "images/expanded.png" : "images/collapsed.png") + .appendTo(this._tdToggle) + .click(function() { + self._expanded = !self._expanded; + + $(this).attr("src", self._expanded ? "images/expanded.png" : "images/collapsed.png"); + + show(); + }); +}; + +SchemaAlignmentDialog.UINode.prototype._hideExpandable = function() { + $(this._tdToggle).hide(); + $(this._tdDetails).hide(); +}; + +SchemaAlignmentDialog.UINode.prototype._renderDetails = function() { + var self = this; + + this._tableLinks = $('
').addClass("schema-alignment-table-layout").appendTo(this._expandedDetailDiv)[0]; + + if ("links" in this._node && this._node.links !== null) { + for (var i = 0; i < this._node.links.length; i++) { + this._linkUIs.push(new SchemaAlignmentDialog.UILink( + this._dialog, + this._node.links[i], + this._tableLinks, + { expanded: true }, + this + )); + } + } + + var divFooter = $('
').addClass("padded").appendTo(this._expandedDetailDiv); + + $('') + .addClass("action") + .text("add property") + .appendTo(divFooter) + .click(function() { + var newLink = { + property: null, + target: { + nodeType: "cell-as-value" + } + }; + self._linkUIs.push(new SchemaAlignmentDialog.UILink( + self._dialog, + newLink, + self._tableLinks, + { + expanded: true, + mustBeCellTopic: false + }, + self + )); + }); +}; + +SchemaAlignmentDialog.UINode.prototype._showColumnPopupMenu = function(elmt) { + self = this; + + var menu = []; + + if (!this._options.mustBeCellTopic) { + menu.push({ + label: "Anonymous Node", + click: function() { + self._node.nodeType = "anonymous"; + self._showExpandable(); + self._renderMain(); + } + }); + menu.push({ + label: "Freebase Topic", + click: function() { + self._node.nodeType = "topic"; + self._hideExpandable(); + self._renderMain(); + } + }); + menu.push({ + label: "Value", + click: function() { + self._node.nodeType = "value"; + self._hideExpandable(); + self._renderMain(); + } + }); + menu.push({}); // separator + } + + var columns = theProject.columnModel.columns; + var createColumnMenuItem = function(index) { + menu.push({ + label: columns[index].name, + click: function() { + self._node.nodeType = "cell-as-topic"; + self._node.columnNames = [ columns[index].name ]; + self._showExpandable(); + self._renderMain(); + } + }); + }; + for (var i = 0; i < columns.length; i++) { + createColumnMenuItem(i); + } + + MenuSystem.createAndShowStandardMenu(menu, elmt); +}; + +SchemaAlignmentDialog.UINode.prototype._showNodeConfigDialog = function() { + var self = this; + var frame = DialogSystem.createDialog(); + + frame.width("800px"); + + var header = $('
').addClass("dialog-header").text("Schema Skeleton Node").appendTo(frame); + var body = $('
').addClass("dialog-body").appendTo(frame); + var footer = $('
').addClass("dialog-footer").appendTo(frame); + + /*-------------------------------------------------- + * Body + *-------------------------------------------------- + */ + var literalTypeSelectHtml = + '' + + '' + + '' + + '' + + '' + + ''; + + var html = $( + '' + + '' + + '' + + + '' + + '' + + '
' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '
' + + '
' + + ' Set to Cell in Column' + + '
' + + '
' + + '' + + '' + + '' + + '' + + '' + + '
' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '
The cell\'s content is used ...
to specify a Freebase topic, as reconciled
If not reconciled, create new topic named by the cell\'s content, and assign it a type
Type:
as a literal value
Literal type
Language (for text)
as a key in a namespace
Namespace
' + + '
' + + '
' + + '		' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '
' + + '
' + + ' Generate an anonymous graph node' + + '
' + + '
Assign a type to the node 
' + + '
' + + ' Use one existing Freebase topic' + + '
' + + '
Topic
' + + '
' + + ' Use a literal value' + + '
' + + '
Value
Value type
Language
' + + '
' + ).appendTo(body); + + var elmts = DOM.bind(html); + + var tableColumns = $('
') + .attr("cellspacing", "5") + .attr("cellpadding", "0") + .appendTo(elmts.divColumns)[0]; + + var columnMap = {}; + if ("columnNames" in self._node) { + for (var i = 0; i < self._node.columnNames.length; i++) { + columnMap[self._node.columnNames[i]] = true; + } + } + + var makeColumnChoice = function(column, columnIndex) { + var tr = tableColumns.insertRow(tableColumns.rows.length); + + var radio = $('') + .attr("type", "checkbox") + .attr("value", column.name) + .attr("name", "schema-align-node-dialog-column") + .appendTo(tr.insertCell(0)) + .click(function() { + elmts.radioNodeTypeCellAs[0].checked = true; + + if ("reconConfig" in column) { + var typeID = column.reconConfig.type.id; + var typeName = column.reconConfig.type.name; + + elmts.cellAsTopicNodeTypeInput[0].value = typeName; + elmts.cellAsTopicNodeTypeInput.data("data.suggest", { "id" : typeID, "name" : typeName }); + elmts.radioNodeTypeCellAsTopicCreate[0].checked = true; + elmts.radioNodeTypeCellAsTopic[0].checked = true; + } + }); + + if (column.name in columnMap) { + radio.attr("checked", "true"); + } + + $('').text(column.name).appendTo(tr.insertCell(1)); + }; + var columns = theProject.columnModel.columns; + for (var i = 0; i < columns.length; i++) { + makeColumnChoice(columns[i], i); + } + + elmts.anonymousNodeTypeInput + .bind("focus", function() { elmts.radioNodeTypeAnonymous[0].checked = true; }) + .suggestT({ type: "/type/type" }); + + elmts.topicNodeTypeInput + .bind("focus", function() { elmts.radioNodeTypeTopic[0].checked = true; }) + .suggest({}); + + elmts.valueNodeTypeValueInput + .bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; }); + elmts.valueNodeTypeValueTypeSelect + .bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; }); + elmts.valueNodeTypeLanguageInput + .bind("focus", function() { elmts.radioNodeTypeValue[0].checked = true; }) + .suggest({ type: "/type/lang" }); + + elmts.radioNodeTypeCellAsTopicCreate + .click(function() { + elmts.radioNodeTypeCellAs[0].checked = true; + elmts.radioNodeTypeCellAsTopic[0].checked = true; + }); + elmts.cellAsTopicNodeTypeInput + .bind("focus", function() { + elmts.radioNodeTypeCellAs[0].checked = true; + elmts.radioNodeTypeCellAsTopic[0].checked = true; + }) + .suggestT({ type: "/type/type" }); + + elmts.cellAsValueTypeSelect + .bind("focus", function() { + elmts.radioNodeTypeCellAs[0].checked = true; + elmts.radioNodeTypeCellAsValue[0].checked = true; + }); + elmts.cellAsValueLanguageInput + .bind("focus", function() { + elmts.radioNodeTypeCellAs[0].checked = true; + elmts.radioNodeTypeCellAsValue[0].checked = true; + }) + .suggest({ type: "/type/lang" }); + + elmts.cellAsKeyInput + .bind("focus", function() { + elmts.radioNodeTypeCellAs[0].checked = true; + elmts.radioNodeTypeCellAsKey[0].checked = true; + }) + .suggest({ type: "/type/namespace" }); + + elmts.radioNodeTypeCellAsTopic[0].checked = true; // just make sure some subtype is selected + if (this._node.nodeType.match(/^cell-as-/)) { + elmts.radioNodeTypeCellAs[0].checked = true; + if (this._node.nodeType == "cell-as-topic") { + elmts.radioNodeTypeCellAsTopic[0].checked = true; + } else if (this._node.nodeType == "cell-as-value") { + elmts.radioNodeTypeCellAsValue[0].checked = true; + } else if (this._node.nodeType == "cell-as-key") { + elmts.radioNodeTypeCellAsKey[0].checked = true; + } + } else if (this._node.nodeType == "anonymous") { + elmts.radioNodeTypeAnonymous[0].checked = true; + } else if (this._node.nodeType == "topic") { + elmts.radioNodeTypeTopic[0].checked = true; + } else if (this._node.nodeType == "value") { + elmts.radioNodeTypeValue[0].checked = true; + } + + if ("type" in this._node) { + elmts.anonymousNodeTypeInput[0].value = this._node.type.name; + elmts.anonymousNodeTypeInput.data("data.suggest", this._node.type); + + elmts.cellAsTopicNodeTypeInput[0].value = this._node.type.name; + elmts.cellAsTopicNodeTypeInput.data("data.suggest", this._node.type); + } + if ("topic" in this._node) { + elmts.topicNodeTypeInput[0].value = this._node.topic.name; + elmts.topicNodeTypeInput.data("data.suggest", this._node.topic); + } + if ("namespace" in this._node) { + elmts.cellAsKeyInput[0].value = this._node.namespace.name; + elmts.cellAsKeyInput.data("data.suggest", this._node.namespace); + } + if ("createForNoReconMatch" in this._node) { + elmts.radioNodeTypeCellAsTopicCreate[0].checked = this._node.createForNoReconMatch; + } + if ("lang" in this._node) { + elmts.valueNodeTypeLanguageInput[0].value = this._node.lang; + elmts.valueNodeTypeLanguageInput.data("data.suggest", { id: this._node.lang }); + + elmts.cellAsValueLanguageInput[0].value = this._node.lang; + elmts.cellAsValueLanguageInput.data("data.suggest", { id: this._node.lang }); + } + if ("valueType" in this._node) { + elmts.valueNodeTypeValueTypeSelect[0].value = this._node.valueType; + elmts.cellAsValueTypeSelect[0].value = this._node.valueType; + } + + /*-------------------------------------------------- + * Footer + *-------------------------------------------------- + */ + + var getResultJSON = function() { + var node = { + nodeType: $("input[name='schema-align-node-dialog-node-type']:checked")[0].value + }; + if (node.nodeType == "cell-as") { + node.nodeType = $("input[name='schema-align-node-dialog-node-subtype']:checked")[0].value; + node.columnNames = $("input[name='schema-align-node-dialog-column']:checked").map(function() { + return this.getAttribute("value"); + }).get(); + + if (node.columnNames.length == 0) { + alert("You must select at least one column."); + return null; + } + + if (node.nodeType == "cell-as-topic") { + node.createForNoReconMatch = elmts.radioNodeTypeCellAsTopicCreate[0].checked; + + var t = elmts.cellAsTopicNodeTypeInput.data("data.suggest"); + if (!(t) && node.createForNoReconMatch) { + alert("For creating a new graph node, you need to specify a type for it."); + return null; + } + node.type = { + id: t.id, + name: t.name + }; + } else if (node.nodeType == "cell-as-value") { + node.valueType = elmts.cellAsValueTypeSelect[0].value; + + if (node.valueType == "/type/text") { + var l = elmts.cellAsValueLanguageInput.data("data.suggest"); + node.lang = (l) ? l.id : "/lang/en"; + } + } else if (node.nodeType == "cell-as-key") { + var t = elmts.cellAsKeyInput.data("data.suggest"); + if (!(t)) { + alert("Please specify the namespace."); + return null; + } + node.namespace = { + id: t.id, + name: t.name + }; + } + } else if (node.nodeType == "anonymous") { + var t = elmts.anonymousNodeTypeInput.data("data.suggest"); + if (!(t)) { + alert("For generating an anonymous graph node, you need to specify a type for it."); + return null; + } + node.type = { + id: t.id, + name: t.name + }; + } else if (node.nodeType == "topic") { + var t = elmts.topicNodeTypeInput.data("data.suggest"); + if (!(t)) { + alert("Please specify which existing Freebase topic to use."); + return null; + } + node.topic = { + id: t.id, + name: t.name + }; + } else if (node.nodeType == "value") { + node.value = $.trim(elmts.valueNodeTypeValueInput[0].value); + if (!node.value.length) { + alert("Please specify the value to use."); + return null; + } + node.valueType = elmts.valueNodeTypeValueTypeSelect[0].value; + + if (node.valueType == "/type/text") { + var l = elmts.valueNodeTypeLanguageInput.data("data.suggest"); + node.lang = (l) ? l.id : "/lang/en"; + } + } + + return node; + }; + + $('').html("  OK  ").click(function() { + var node = getResultJSON(); + if (node !== null) { + DialogSystem.dismissUntil(level - 1); + + self._node = node; + self.render(); + self._dialog.preview(); + } + }).appendTo(footer); + + $('').text("Cancel").click(function() { + DialogSystem.dismissUntil(level - 1); + }).appendTo(footer); + + var level = DialogSystem.showDialog(frame); +}; + +SchemaAlignmentDialog.UINode.prototype.getJSON = function() { + var result = null; + var getLinks = false; + + if (this._node.nodeType.match(/^cell-as-/)) { + if (!("columnNames" in this._node) || !this._node.columnNames) { + return null; + } + + if (this._node.nodeType == "cell-as-topic") { + result = { + nodeType: this._node.nodeType, + columnNames: this._node.columnNames, + type: "type" in this._node ? cloneDeep(this._node.type) : { "id" : "/common/topic", "name" : "Topic", "cvt" : false }, + createForNoReconMatch: "createForNoReconMatch" in this._node ? this._node.createForNoReconMatch : true + }; + getLinks = true; + } else if (this._node.nodeType == "cell-as-value") { + result = { + nodeType: this._node.nodeType, + columnNames: this._node.columnNames, + valueType: "valueType" in this._node ? this._node.valueType : "/type/text", + lang: "lang" in this._node ? this._node.lang : "/lang/en" + }; + } else if (this._node.nodeType == "cell-as-key") { + if (!("namespace" in this._node) || !this._node.namespace) { + return null; + } + result = { + nodeType: this._node.nodeType, + columnNames: this._node.columnNames, + namespace: cloneDeep(this._node.namespace) + }; + } + } else if (this._node.nodeType == "topic") { + if (!("topic" in this._node) || !this._node.topic) { + return null; + } + result = { + nodeType: this._node.nodeType, + topic: cloneDeep(this._node.topic) + }; + getLinks = true; + } else if (this._node.nodeType == "value") { + if (!("value" in this._node) || !this._node.value) { + return null; + } + result = { + nodeType: this._node.nodeType, + value: this._node.value, + valueType: "valueType" in this._node ? this._node.valueType : "/type/text", + lang: "lang" in this._node ? this._node.lang : "/lang/en" + }; + } else if (this._node.nodeType == "anonymous") { + if (!("type" in this._node) || !this._node.type) { + return null; + } + result = { + nodeType: this._node.nodeType, + type: cloneDeep(this._node.type) + }; + getLinks = true; + } + + if (!result) { + return null; + } + if (getLinks) { + var links = []; + for (var i = 0; i < this._linkUIs.length; i++) { + var link = this._linkUIs[i].getJSON(); + if (link !== null) { + links.push(link); + } + } + result.links = links; + } + + return result; +}; + diff --git a/extensions/freebase/module/scripts/extension.js b/extensions/freebase/module/scripts/extension.js new file mode 100644 index 000000000..7d82636be --- /dev/null +++ b/extensions/freebase/module/scripts/extension.js @@ -0,0 +1,118 @@ +var FreebaseExtension = { handlers: {} }; + +FreebaseExtension.handlers.editSchemaAlignment = function(reset) { + new SchemaAlignmentDialog( + reset ? null : theProject.overlayModels.freebaseProtograph, function(newProtograph) {}); +}; + +FreebaseExtension.handlers.loadIntoFreebase = function() { + new FreebaseLoadingDialog(); +}; + +FreebaseExtension.handlers.browseToDataLoad = function() { + // The form has to be created as part of the click handler. If you create it + // inside the getJSON success handler, it won't work. + + var form = document.createElement("form"); + $(form) + .css("display", "none") + .attr("method", "GET") + .attr("target", "dataload"); + + document.body.appendChild(form); + var w = window.open("about:blank", "dataload"); + + $.getJSON( + "/command/core/get-preference?" + $.param({ project: theProject.id, name: "freebase.load.jobID" }), + null, + function(data) { + if (data.value == null) { + alert("You have not tried to load the data in this project into Freebase yet."); + } else { + $(form).attr("action", "http://refinery.freebaseapps.com/load/" + data.value); + form.submit(); + w.focus(); + } + document.body.removeChild(form); + } + ); +}; + +FreebaseExtension.handlers.importQAData = function() { + Refine.postProcess( + "freebase-extension", + "import-qa-data", + {}, + {}, + { cellsChanged: true } + ); +}; + +ExtensionBar.addExtensionMenu({ + "id" : "freebase", + "label" : "Freebase", + "submenu" : [ + { + "id" : "freebase/schema-alignment", + label: "Edit Schema Aligment Skeleton ...", + click: function() { FreebaseExtension.handlers.editSchemaAlignment(false); } + }, + { + "id" : "freebase/reset-schema-alignment", + label: "Reset Schema Alignment Skeleton ...", + click: function() { FreebaseExtension.handlers.editSchemaAlignment(true); } + }, + {}, + { + "id" : "freebase/load-info-freebase", + label: "Load into Freebase ...", + click: function() { FreebaseExtension.handlers.loadIntoFreebase(); } + }, + { + "id" : "freebase/browse-load", + label: "Browse to Data Load ...", + click: function() { FreebaseExtension.handlers.browseToDataLoad(); } + }, + { + "id" : "freebase/import-qa-data", + label: "Import QA Data", + click: function() { FreebaseExtension.handlers.importQAData(); } + } + ] +}); + +DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) { + var columnIndex = Refine.columnNameToColumnIndex(column.name); + var doAddColumnFromFreebase = function() { + var o = DataTableView.sampleVisibleRows(column); + new ExtendDataPreviewDialog( + column, + columnIndex, + o.rowIndices, + function(extension) { + Refine.postProcess( + "freebase-extension", + "extend-data", + { + baseColumnName: column.name, + columnInsertIndex: columnIndex + 1 + }, + { + extension: JSON.stringify(extension) + }, + { rowsChanged: true, modelsChanged: true } + ); + } + ); + }; + + MenuSystem.insertAfter( + menu, + [ "core/edit-column", "core/add-column-by-fetching-urls" ], + { + id: "freebase/add-columns-from-freebase", + label: "Add Columns From Freebase ...", + click: doAddColumnFromFreebase + } + ); +}); diff --git a/extensions/freebase/module/scripts/util/freebase.js b/extensions/freebase/module/scripts/util/freebase.js new file mode 100644 index 000000000..bddfd0871 --- /dev/null +++ b/extensions/freebase/module/scripts/util/freebase.js @@ -0,0 +1,30 @@ +var Freebase = {}; + +Freebase.mqlread = function(query, options, onDone) { + var params = {}; + var queryEnv = { + "query": query + }; + + if (options) { + for (var n in options) { + if (options.hasOwnProperty(n)) { + var v = options[n]; + if (typeof v != "string") { + v = JSON.stringify(v); + } + + queryEnv[n] = v; + } + } + } + + params.query = JSON.stringify(queryEnv); + + $.getJSON( + "http://api.freebase.com/api/service/mqlread?" + $.param(params) + "&callback=?", + null, + onDone, + "jsonp" + ); +}; \ No newline at end of file diff --git a/extensions/freebase/module/scripts/util/sign.js b/extensions/freebase/module/scripts/util/sign.js new file mode 100644 index 000000000..99f6176fd --- /dev/null +++ b/extensions/freebase/module/scripts/util/sign.js @@ -0,0 +1,120 @@ + if (typeof window.Sign == 'undefined') { + window.Sign = { + window_position: function() { + var position = {}; + + if (typeof(window.innerWidth) == 'number') { + // Non-IE + position.width = window.outerWidth; + position.height = window.outerHeight; + position.top = window.screenY; + position.left = window.screenX; + } else if (document.documentElement && (document.documentElement.clientWidth || document.documentElement.clientHeight)) { + // IE 6+ in 'standards compliant mode' + position.width = document.body.clientWidth; + position.height = document.body.clientHeight; + position.top = window.screenTop; + position.left = window.screenLeft; + } + + return position; + }, + + popup : function(url, width, height, windowname) { + width = width || 700; + height = height || 500; + + var pos = window.Sign.window_position(); + var left = Math.floor((pos.width - width) / 2) + pos.left; + var top = Math.floor((pos.height - height) / 2) + pos.top; + + // Chrome might fix this bug, but until then add some padding + // to the height of the popup for the urlbar + var is_chrome = /chrome/.test(navigator.userAgent.toLowerCase()); + if (is_chrome) { + height += 50; + } + + var params = { + width: width, + height: height, + top: top, + left: left, + directories: 'no', + location: 'no', + menubar: 'no', + resizable: 'no', + scrollbars: 'yes', + status: 'no', + toolbar: 'no' + }; + + var params_list = []; + for (var key in params) { + if (params.hasOwnProperty(key)) { + params_list.push(key + "=" + params[key]); + } + } + + return window.open(url, windowname || "", params_list.join(",")); + }, + + signintize : function(cont) { + $('.signedin').show(); + $('.signedout').hide(); + if (window.user) { + $('.user').html('' + window.user.username + ''); + } + if (typeof cont == 'function') cont(); + }, + + signin : function(success, provider, width, height) { + var newwin = window.Sign.popup("/command/freebase-extension/authorize/" + provider, width, height); + + if (newwin !== null) { + newwin.opener = window; + } + + window.onauthorization = function() { + if (typeof success == 'undefined') { + window.location.reload(); + } else { + $.ajax({ + url: "/command/freebase-extension/check-authorization/" + provider, + dataType: "json", + success: function(data) { + window.user = data; + window.Sign.signintize(success); + } + }); + } + }; + + if (window.focus && newwin !== null) { + newwin.focus(); + } + + return false; + }, + + signoutize : function(cont) { + $('.signedin').hide(); + $('.signedout').show(); + if (typeof cont == 'function') cont(); + }, + + signout : function(success,provider) { + $.ajax({ + url: "/command/freebase-extension/deauthorize/" + provider, + success: function() { + if (typeof success == 'undefined') { + window.location.reload(); + } else { + window.Sign.signoutize(success); + } + } + }); + } + }; + } + \ No newline at end of file diff --git a/extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less b/extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less new file mode 100644 index 000000000..a10d37926 --- /dev/null +++ b/extensions/freebase/module/styles/dialogs/extend-data-preview-dialog.less @@ -0,0 +1,38 @@ +@import-less url("../theme.less"); + +.extend-data-preview-dialog .suggested-property-container { + border: 1px solid #aaa; + padding: 5px; + overflow: auto; + height: 400px; +} + +.extend-data-preview-dialog .suggested-property { + padding: 5px; +} + +.extend-data-preview-dialog input.property-suggest { + display: block; + padding: 2%; + width: 96%; +} + +.extend-data-preview-dialog .preview-container { + border: 1px solid #aaa; + overflow: auto; +} + +.extend-data-preview-dialog .preview-container table { + border-collapse: collapse; +} + +.extend-data-preview-dialog .preview-container td, .extend-data-preview-dialog .preview-container th { + padding: 3px 5px; + border-bottom: 1px solid #ddd; + border-right: 1px solid #ddd; +} + +.extend-data-preview-dialog .preview-container th img { + vertical-align: top; + margin-left: 5px; +} \ No newline at end of file diff --git a/extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less b/extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less new file mode 100644 index 000000000..d4a681204 --- /dev/null +++ b/extensions/freebase/module/styles/dialogs/freebase-loading-dialog.less @@ -0,0 +1,60 @@ +@import-less url("../theme.less"); + +.freebase-loading-tripleloader-data { + height: 400px; + width: 99%; + overflow: auto; + border: 1px solid #aaa; + white-space: pre; + padding: 0.3em 0.5em 0.5em 0.5em; + font-family: monospace; +} + +.freebase-loading-tripleloader-message { + height: 500px; + overflow: auto; + border: 1px solid #aaa; + padding: 0.3em 0.5em 0.5em 0.5em; +} + +.freebase-loading-tripleloader-message h2 { + text-align: center; + font-size: 150%; + margin-top: 3em; + font-weight: normal; +} + +.freebase-loading-tripleloader-message h2 span { + font-size; 130%; + font-weight: bold; +} + +.freebase-loading-tripleloader-message h4 { + text-align: center; + font-size: 120%; + margin-top: 2em; + font-weight: normal; +} + +.freebase-loading-tripleloader-message a { + font-size: 120%; + font-weight: bold; +} + +.freebase-loading-authorization { + margin: 0em 2em; + padding: 0.6em 0.8em; + border: 1px solid #ccc; + background-color: #fff; + -moz-border-radius: 0.8em; + -webkit-border-radius: 0.8em; +} + +.freebase-loading-tripleloader-info { + margin-bottom: 0.5em; +} + +.freebase-loading-tripleloader-info textarea { + width: 99%; + height: 5em; +} \ No newline at end of file diff --git a/extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less b/extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less new file mode 100644 index 000000000..fcd9de427 --- /dev/null +++ b/extensions/freebase/module/styles/dialogs/schema-alignment-dialog.less @@ -0,0 +1,114 @@ +@import-less url("../theme.less"); + +.schema-alignment-dialog-canvas { + height: 400px; + overflow: auto; + border: 1px solid #aaa; + padding: 10px; + background: white; +} + +table.schema-alignment-table-layout { + border-collapse: collapse; + margin: 0px; + padding: 0px; + margin-bottom: 1em; +} + +table.schema-alignment-table-layout .padded { + padding: 3px 6px; +} + +div.schema-alignment-detail-container { + border-left: 3px solid #eee; +} + +td.schema-alignment-node-main, td.schema-alignment-link-main { + white-space: pre; + width: 300px; +} + +td.schema-alignment-node-toggle, td.schema-alignment-link-toggle { + white-space: pre; + width: 1%; +} + +td.schema-alignment-node-toggle img, td.schema-alignment-link-toggle img { + cursor: pointer; + vertical-align: middle; + padding: 2px; +} + +td.schema-alignment-node-details, td.schema-alignment-link-details { + width: 90%; +} + +a.schema-alignment-node-tag { + padding: 3px 6px; + background: #ddd; + text-decoration: none; + color: black; + -moz-border-radius: 10px; +} +a.schema-alignment-node-tag:hover { + background: #888; + color: white; +} +.schema-alignment-node-column { + font-weight: bold; +} + +a.schema-alignment-link-tag { + padding: 3px 6px; + text-decoration: none; + color: black; +} +a.schema-alignment-link-tag:hover { + color: #88f; +} + +div.schema-alignment-dialog-preview { + height: 400px; + overflow: auto; + border: 1px solid #aaa; + background: white; + padding: 10px; + white-space: pre; + font-family: monospace; + font-size: 9pt; +} + +/*-------------------------------------------------- + * Node dialog + *-------------------------------------------------- + */ + +.schema-align-node-dialog-node-type { + padding: 0.25em; + background: #ddf; + -moz-border-radius: 0.5em; +} +.schema-align-node-dialog-node-type input { + vertical-align: text-bottom; +} +.schema-alignment-node-dialog-column-list { + height: 300px; + width: 200px; + overflow: auto; + border: 1px solid #ddd; +} + +/*-------------------------------------------------- + * Link dialog + *-------------------------------------------------- + */ + +.schema-alignment-link-menu-section { + padding: 8px; + border-bottom: 1px solid #ddd; + margin-bottom: 3px; +} + +.schema-alignment-link-menu-section-last { + padding: 8px; +} diff --git a/extensions/freebase/module/styles/theme.less b/extensions/freebase/module/styles/theme.less new file mode 100644 index 000000000..35db2d57b --- /dev/null +++ b/extensions/freebase/module/styles/theme.less @@ -0,0 +1 @@ +@import-less url("../../../../main/webapp/modules/core/styles/theme.less"); diff --git a/extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java b/extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java new file mode 100644 index 000000000..cef56a307 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/FreebaseProperty.java @@ -0,0 +1,9 @@ +package com.google.refine.freebase; + +public class FreebaseProperty extends FreebaseTopic { + //final protected FreebaseType _expectedType; + + public FreebaseProperty(String id, String name) { + super(id, name); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java b/extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java new file mode 100644 index 000000000..01f3f1d06 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/FreebaseTopic.java @@ -0,0 +1,28 @@ +package com.google.refine.freebase; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.refine.Jsonizable; + +public class FreebaseTopic implements Jsonizable { + final public String id; + final public String name; + + public FreebaseTopic(String id, String name) { + this.id = id; + this.name = name; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(id); + writer.key("name"); writer.value(name); + writer.endObject(); + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/FreebaseType.java b/extensions/freebase/src/com/google/refine/freebase/FreebaseType.java new file mode 100644 index 000000000..c77f16677 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/FreebaseType.java @@ -0,0 +1,36 @@ +package com.google.refine.freebase; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.Jsonizable; + +public class FreebaseType extends FreebaseTopic implements Jsonizable { + public FreebaseType(String id, String name) { + super(id, name); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(id); + writer.key("name"); writer.value(name); + writer.endObject(); + } + + static public FreebaseType load(JSONObject obj) throws Exception { + if (obj == null) { + return null; + } + + FreebaseType type = new FreebaseType( + obj.getString("id"), + obj.getString("name") + ); + return type; + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java b/extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java new file mode 100644 index 000000000..63990b090 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/ProtographTransposeExporter.java @@ -0,0 +1,75 @@ +package com.google.refine.freebase; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.util.Properties; + +import com.google.refine.browsing.Engine; +import com.google.refine.exporters.Exporter; +import com.google.refine.model.Project; +import com.google.refine.freebase.protograph.Protograph; +import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory; +import com.google.refine.freebase.protograph.transpose.TransposedNodeFactory; +import com.google.refine.freebase.protograph.transpose.Transposer; +import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory; + +abstract public class ProtographTransposeExporter implements Exporter { + final protected String _contentType; + + public ProtographTransposeExporter(String contentType) { + _contentType = contentType; + } + + public String getContentType() { + return "application/x-unknown"; + } + + public boolean takeWriter() { + return true; + } + + public void export(Project project, Properties options, Engine engine, + OutputStream outputStream) throws IOException { + throw new RuntimeException("Not implemented"); + } + + public void export(Project project, Properties options, Engine engine, + Writer writer) throws IOException { + + Protograph protograph = (Protograph) project.overlayModels.get("freebaseProtograph"); + if (protograph != null) { + TransposedNodeFactory nodeFactory = createNodeFactory(project, writer); + + Transposer.transpose(project, engine.getAllFilteredRows(), + protograph, protograph.getRootNode(0), nodeFactory, -1); + + nodeFactory.flush(); + } + } + + abstract protected TransposedNodeFactory createNodeFactory(Project project, Writer writer); + + static public class TripleLoaderExporter extends ProtographTransposeExporter { + public TripleLoaderExporter() { + super("application/x-unknown"); + } + + @Override + protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) { + return new TripleLoaderTransposedNodeFactory(project, writer); + } + } + + static public class MqlwriteLikeExporter extends ProtographTransposeExporter { + public MqlwriteLikeExporter() { + super("application/x-unknown"); + } + + @Override + protected TransposedNodeFactory createNodeFactory(Project project, Writer writer) { + return new MqlwriteLikeTransposedNodeFactory(writer); + } + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java new file mode 100644 index 000000000..f739bb56b --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/ExtendDataCommand.java @@ -0,0 +1,32 @@ +package com.google.refine.freebase.commands; + +import javax.servlet.http.HttpServletRequest; + +import org.json.JSONObject; + +import com.google.refine.commands.EngineDependentCommand; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Project; +import com.google.refine.freebase.operations.ExtendDataOperation; +import com.google.refine.util.ParsingUtilities; + +public class ExtendDataCommand extends EngineDependentCommand { + @Override + protected AbstractOperation createOperation(Project project, + HttpServletRequest request, JSONObject engineConfig) throws Exception { + + String baseColumnName = request.getParameter("baseColumnName"); + int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex")); + + String jsonString = request.getParameter("extension"); + JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString); + + return new ExtendDataOperation( + engineConfig, + baseColumnName, + extension, + columnInsertIndex + ); + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java new file mode 100644 index 000000000..8ddd0ce3d --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/ImportQADataCommand.java @@ -0,0 +1,36 @@ +package com.google.refine.freebase.commands; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.google.refine.ProjectManager; +import com.google.refine.commands.Command; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Project; +import com.google.refine.freebase.operations.ImportQADataOperation; +import com.google.refine.process.Process; + +public class ImportQADataCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + ProjectManager.singleton.setBusy(true); + try { + Project project = getProject(request); + + AbstractOperation op = new ImportQADataOperation(); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } finally { + ProjectManager.singleton.setBusy(false); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java new file mode 100644 index 000000000..e05a004ab --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/MQLReadCommand.java @@ -0,0 +1,31 @@ +package com.google.refine.freebase.commands; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.google.refine.commands.Command; +import com.google.refine.freebase.util.FreebaseUtils; +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; + +public class MQLReadCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Provider provider = OAuthUtilities.getProvider(request); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + String query = request.getParameter("query"); + String result = FreebaseUtils.mqlread(provider,query); + response.getWriter().write(result); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java new file mode 100644 index 000000000..0ce475a52 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/MQLWriteCommand.java @@ -0,0 +1,40 @@ +package com.google.refine.freebase.commands; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.google.refine.commands.Command; +import com.google.refine.freebase.util.FreebaseUtils; +import com.google.refine.oauth.Credentials; +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; + +public class MQLWriteCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Provider provider = OAuthUtilities.getProvider(request); + + Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + if (access_credentials != null) { + String query = request.getParameter("query"); + String result = FreebaseUtils.mqlwrite(access_credentials, provider, query); + response.getWriter().write(result); + } else { + respond(response, "401 Unauthorized", "You don't have the right credentials"); + } + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java new file mode 100644 index 000000000..a3d7ade24 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewExtendDataCommand.java @@ -0,0 +1,158 @@ +package com.google.refine.freebase.commands; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.commands.Command; +import com.google.refine.freebase.util.FreebaseDataExtensionJob; +import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo; +import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension; +import com.google.refine.model.Cell; +import com.google.refine.model.Project; +import com.google.refine.model.ReconCandidate; +import com.google.refine.model.Row; +import com.google.refine.util.ParsingUtilities; + +public class PreviewExtendDataCommand extends Command { + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + String columnName = request.getParameter("columnName"); + + String rowIndicesString = request.getParameter("rowIndices"); + if (rowIndicesString == null) { + respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }"); + return; + } + + String jsonString = request.getParameter("extension"); + JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); + + JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString); + int length = rowIndices.length(); + int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex(); + + List topicNames = new ArrayList(); + List topicIds = new ArrayList(); + Set ids = new HashSet(); + for (int i = 0; i < length; i++) { + int rowIndex = rowIndices.getInt(i); + if (rowIndex >= 0 && rowIndex < project.rows.size()) { + Row row = project.rows.get(rowIndex); + Cell cell = row.getCell(cellIndex); + if (cell != null && cell.recon != null && cell.recon.match != null) { + topicNames.add(cell.recon.match.name); + topicIds.add(cell.recon.match.id); + ids.add(cell.recon.match.id); + } else { + topicNames.add(null); + topicIds.add(null); + ids.add(null); + } + } + } + + Map reconCandidateMap = new HashMap(); + FreebaseDataExtensionJob job = new FreebaseDataExtensionJob(json); + Map map = job.extend(ids, reconCandidateMap); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + JSONWriter writer = new JSONWriter(response.getWriter()); + writer.object(); + writer.key("code"); writer.value("ok"); + writer.key("columns"); + writer.array(); + for (ColumnInfo info : job.columns) { + writer.object(); + writer.key("names"); + writer.array(); + for (String name : info.names) { + writer.value(name); + } + writer.endArray(); + writer.key("path"); + writer.array(); + for (String id : info.path) { + writer.value(id); + } + writer.endArray(); + writer.endObject(); + } + writer.endArray(); + + writer.key("rows"); + writer.array(); + for (int r = 0; r < topicNames.size(); r++) { + String id = topicIds.get(r); + String topicName = topicNames.get(r); + + if (id != null && map.containsKey(id)) { + DataExtension ext = map.get(id); + boolean first = true; + + if (ext.data.length > 0) { + for (Object[] row : ext.data) { + writer.array(); + if (first) { + writer.value(topicName); + first = false; + } else { + writer.value(null); + } + + for (Object cell : row) { + if (cell != null && cell instanceof ReconCandidate) { + ReconCandidate rc = (ReconCandidate) cell; + writer.object(); + writer.key("id"); writer.value(rc.id); + writer.key("name"); writer.value(rc.name); + writer.endObject(); + } else { + writer.value(cell); + } + } + + writer.endArray(); + } + continue; + } + } + + writer.array(); + if (id != null) { + writer.object(); + writer.key("id"); writer.value(id); + writer.key("name"); writer.value(topicName); + writer.endObject(); + } else { + writer.value(""); + } + writer.endArray(); + } + writer.endArray(); + + writer.endObject(); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java new file mode 100644 index 000000000..c9eddd4ea --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/PreviewProtographCommand.java @@ -0,0 +1,71 @@ +package com.google.refine.freebase.commands; + +import java.io.IOException; +import java.io.StringWriter; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; + +import com.google.refine.browsing.Engine; +import com.google.refine.browsing.FilteredRows; +import com.google.refine.commands.Command; +import com.google.refine.freebase.protograph.Protograph; +import com.google.refine.model.Project; +import com.google.refine.freebase.protograph.transpose.MqlwriteLikeTransposedNodeFactory; +import com.google.refine.freebase.protograph.transpose.Transposer; +import com.google.refine.freebase.protograph.transpose.TripleLoaderTransposedNodeFactory; +import com.google.refine.util.ParsingUtilities; + +public class PreviewProtographCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + Engine engine = getEngine(request, project); + FilteredRows filteredRows = engine.getAllFilteredRows(); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + String jsonString = request.getParameter("protograph"); + JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); + Protograph protograph = Protograph.reconstruct(json); + + StringBuffer sb = new StringBuffer(2048); + sb.append("{ "); + + { + StringWriter stringWriter = new StringWriter(); + TripleLoaderTransposedNodeFactory nodeFactory = new TripleLoaderTransposedNodeFactory(project, stringWriter); + + Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory); + nodeFactory.flush(); + + sb.append("\"tripleloader\" : "); + sb.append(JSONObject.quote(stringWriter.toString())); + } + + { + StringWriter stringWriter = new StringWriter(); + MqlwriteLikeTransposedNodeFactory nodeFactory = new MqlwriteLikeTransposedNodeFactory(stringWriter); + + Transposer.transpose(project, filteredRows, protograph, protograph.getRootNode(0), nodeFactory); + nodeFactory.flush(); + + sb.append(", \"mqllike\" : "); + sb.append(stringWriter.toString()); + } + + sb.append(" }"); + + respond(response, sb.toString()); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java new file mode 100644 index 000000000..4382de096 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/SaveProtographCommand.java @@ -0,0 +1,40 @@ +package com.google.refine.freebase.commands; + +import java.io.IOException; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; + +import com.google.refine.commands.Command; +import com.google.refine.freebase.protograph.Protograph; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Project; +import com.google.refine.freebase.operations.SaveProtographOperation; +import com.google.refine.process.Process; +import com.google.refine.util.ParsingUtilities; + +public class SaveProtographCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + + String jsonString = request.getParameter("protograph"); + JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); + Protograph protograph = Protograph.reconstruct(json); + + AbstractOperation op = new SaveProtographOperation(protograph); + Process process = op.createProcess(project, new Properties()); + + performProcessAndRespond(request, response, project, process); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java new file mode 100644 index 000000000..180c1a5e8 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/UploadDataCommand.java @@ -0,0 +1,89 @@ +package com.google.refine.freebase.commands; + +import java.io.IOException; +import java.io.StringWriter; +import java.net.URL; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONException; +import org.json.JSONObject; + +import com.google.refine.ProjectManager; +import com.google.refine.browsing.Engine; +import com.google.refine.commands.Command; +import com.google.refine.freebase.ProtographTransposeExporter.TripleLoaderExporter; +import com.google.refine.freebase.util.FreebaseUtils; +import com.google.refine.model.Project; +import com.google.refine.preference.PreferenceStore; +import com.google.refine.util.ParsingUtilities; + +public class UploadDataCommand extends Command { + final static public String s_dataLoadJobIDPref = "freebase.load.jobID"; + final static public String s_dataLoadJobNamePref = "freebase.load.jobName"; + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + ProjectManager.singleton.setBusy(true); + try { + Project project = getProject(request); + Engine engine = getEngine(request, project); + PreferenceStore preferenceStore = project.getMetadata().getPreferenceStore(); + + TripleLoaderExporter exporter = new TripleLoaderExporter(); + StringWriter triples = new StringWriter(10 * 1024 * 1024); + exporter.export(project, new Properties(), engine, triples); + + String source_name = request.getParameter("source_name"); + String source_id = request.getParameter("source_id"); + String qa = request.getParameter("qa"); + String mdo_id = null; + + preferenceStore.put(s_dataLoadJobNamePref, source_name); + + try { + Integer jobID = (Integer) preferenceStore.get(s_dataLoadJobIDPref); + if (jobID != null) { + URL url = new URL("http://refinery.freebaseapps.com/job_id_to_mdo?job=" + jobID); + String s = ParsingUtilities.inputStreamToString(url.openConnection().getInputStream()); + + if (!s.equals("null")) { + mdo_id = s; + } + } + } catch (Exception e) { + // ignore + } + + String uploadResponse = FreebaseUtils.uploadTriples( + request, qa, source_name, source_id, mdo_id, triples.toString() + ); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + try { + JSONObject obj = new JSONObject(uploadResponse); + if (obj.has("result") && !obj.isNull("result")) { + JSONObject result = obj.getJSONObject("result"); + if (result.has("job_id") && !result.isNull("job_id")) { + Integer jobID = result.getInt("job_id"); + project.getMetadata().getPreferenceStore().put(s_dataLoadJobIDPref, jobID); + } + } + response.getWriter().write(uploadResponse); + } catch (JSONException e) { + respond(response,"500 Error", uploadResponse); + } + } catch (Exception e) { + respondException(response, e); + } finally { + ProjectManager.singleton.setBusy(false); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/AuthorizeCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/AuthorizeCommand.java new file mode 100644 index 000000000..6940c193e --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/auth/AuthorizeCommand.java @@ -0,0 +1,135 @@ +package com.google.refine.freebase.commands.auth; + +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URI; +import java.net.URISyntaxException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import oauth.signpost.OAuthConsumer; +import oauth.signpost.OAuthProvider; + +import com.google.refine.commands.Command; +import com.google.refine.oauth.Credentials; +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; + +public class AuthorizeCommand extends Command { + + private static final String OAUTH_VERIFIER_PARAM = "oauth_verifier"; + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + + // get the provider from the request + Provider provider = OAuthUtilities.getProvider(request); + + try { + + // see if the request comes with access credentials + Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); + + // prepare the continuation URL that the OAuth provider will redirect the user to + // (we need to make sure this URL points back to this code or the dance will never complete) + String callbackURL = getBaseURL(request,provider); + + if (access_credentials == null) { + // access credentials are not available so we need to check + // to see at what stage of the OAuth dance we are + + // get the request token credentials + Credentials request_credentials = Credentials.getCredentials(request, provider, Credentials.Type.REQUEST); + + OAuthConsumer consumer = OAuthUtilities.getConsumer(request_credentials, provider); + OAuthProvider pp = OAuthUtilities.getOAuthProvider(provider); + + if (request_credentials == null) { + // no credentials were found, so let's start the dance + + // get the request token + + String url = pp.retrieveRequestToken(consumer, callbackURL); + + request_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider); + + // and set them to that we can retrieve them later in the second part of the dance + Credentials.setCredentials(request, response, request_credentials, Credentials.Type.REQUEST, 3600); + + // now redirect the user to the Authorize URL where she can authenticate against the + // service provider and authorize us. + // The provider will bounce the user back here for us to continue the dance. + + response.sendRedirect(url); + } else { + // we are at the second stage of the dance, so we need need to obtain the access credentials now + + // if we got here, it means that the user performed a valid authentication against the + // service provider and authorized us, so now we can request more permanent credentials + // to the service provider and save those as well for later use. + + // this is set only for OAuth 1.0a + String verificationCode = request.getParameter(OAUTH_VERIFIER_PARAM); + + pp.retrieveAccessToken(consumer, verificationCode); + + access_credentials = new Credentials(consumer.getToken(), consumer.getTokenSecret(), provider); + + // no matter the result, we need to remove the request token + Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST); + + Credentials.setCredentials(request, response, access_credentials, Credentials.Type.ACCESS, 30 * 24 * 3600); + + finish(response); + } + } else { + finish(response); + } + } catch (Exception e) { + Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST); + Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS); + respondException(response, e); + } + } + + private void finish(HttpServletResponse response) throws IOException { + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "text/html"); + + PrintWriter writer = response.getWriter(); + writer.write( + "" + + "" + + "" + + "" + ); + writer.flush(); + } + + private String getBaseURL(HttpServletRequest request, Provider provider) { + String host = request.getHeader("host"); + if (host == null) { + String referrer = request.getHeader("referer"); + if (referrer != null) { + URI url; + try { + url = new URI(referrer); + int port = url.getPort(); + host = url.getHost() + ((port > -1) ? ":" + url.getPort() : ""); + } catch (URISyntaxException e) { + throw new RuntimeException("referrer '" + referrer + "' can't be parsed as a URL"); + } + } else { + throw new RuntimeException("neither the 'host' nor 'referer' headers were present in the HTTP response, I can't determine what URL Google Refine is listening to."); + } + } + return "http://" + host + "/command/freebase-extension/authorize/" + provider.getHost(); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java new file mode 100644 index 000000000..9ee8f8d51 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/auth/CheckAuthorizationCommand.java @@ -0,0 +1,47 @@ +package com.google.refine.freebase.commands.auth; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.commands.Command; +import com.google.refine.freebase.util.FreebaseUtils; +import com.google.refine.oauth.Credentials; +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; + +public class CheckAuthorizationCommand extends Command { + + final static Logger logger = LoggerFactory.getLogger("check-authorization_command"); + + public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + + try { + Provider provider = OAuthUtilities.getProvider(request); + + // this cookie should not be there, but this is good hygiene practice + Credentials.deleteCredentials(request, response, provider, Credentials.Type.REQUEST); + + Credentials access_credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + if (access_credentials != null) { + String user_info = FreebaseUtils.getUserInfo(access_credentials, provider); + response.getWriter().write(user_info); + } else { + respond(response, "401 Unauthorized", "You don't have the right credentials"); + } + } catch (Exception e) { + logger.info("error",e); + respondException(response, e); + } + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/DeAuthorizeCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/DeAuthorizeCommand.java new file mode 100644 index 000000000..f0f82bdeb --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/auth/DeAuthorizeCommand.java @@ -0,0 +1,32 @@ +package com.google.refine.freebase.commands.auth; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.google.refine.commands.Command; +import com.google.refine.oauth.Credentials; +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; + +public class DeAuthorizeCommand extends Command { + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + + try { + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + Provider provider = OAuthUtilities.getProvider(request); + + Credentials.deleteCredentials(request, response, provider, Credentials.Type.ACCESS); + + respond(response, "200 OK", ""); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java b/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java new file mode 100644 index 000000000..a3a1221c6 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/commands/auth/GetUserBadgesCommand.java @@ -0,0 +1,37 @@ +package com.google.refine.freebase.commands.auth; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.commands.Command; +import com.google.refine.freebase.util.FreebaseUtils; +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; + +public class GetUserBadgesCommand extends Command { + + final static Logger logger = LoggerFactory.getLogger("get-version_command"); + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + + try { + Provider provider = OAuthUtilities.getProvider(request); + String user_id = request.getParameter("user_id"); + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + String user_badges = FreebaseUtils.getUserBadges(provider, user_id); + response.getWriter().write(user_badges); + } catch (Exception e) { + logger.info("error",e); + respondException(response, e); + } + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java b/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java new file mode 100644 index 000000000..b57ebbe75 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/model/changes/DataExtensionChange.java @@ -0,0 +1,431 @@ +package com.google.refine.freebase.model.changes; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Serializable; +import java.io.Writer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseType; +import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension; +import com.google.refine.history.Change; +import com.google.refine.model.Cell; +import com.google.refine.model.Column; +import com.google.refine.model.ModelException; +import com.google.refine.model.Project; +import com.google.refine.model.Recon; +import com.google.refine.model.ReconCandidate; +import com.google.refine.model.ReconStats; +import com.google.refine.model.Row; +import com.google.refine.model.Recon.Judgment; +import com.google.refine.freebase.model.recon.DataExtensionReconConfig; +import com.google.refine.util.ParsingUtilities; +import com.google.refine.util.Pool; + +public class DataExtensionChange implements Change { + final protected String _baseColumnName; + final protected int _columnInsertIndex; + + final protected List _columnNames; + final protected List _columnTypes; + + final protected List _rowIndices; + final protected List _dataExtensions; + + protected long _historyEntryID; + protected int _firstNewCellIndex = -1; + protected List _oldRows; + protected List _newRows; + + public DataExtensionChange( + String baseColumnName, + int columnInsertIndex, + List columnNames, + List columnTypes, + List rowIndices, + List dataExtensions, + long historyEntryID + ) { + _baseColumnName = baseColumnName; + _columnInsertIndex = columnInsertIndex; + + _columnNames = columnNames; + _columnTypes = columnTypes; + + _rowIndices = rowIndices; + _dataExtensions = dataExtensions; + + _historyEntryID = historyEntryID; + } + + protected DataExtensionChange( + String baseColumnName, + int columnInsertIndex, + + List columnNames, + List columnTypes, + + List rowIndices, + List dataExtensions, + int firstNewCellIndex, + List oldRows, + List newRows + ) { + _baseColumnName = baseColumnName; + _columnInsertIndex = columnInsertIndex; + + _columnNames = columnNames; + _columnTypes = columnTypes; + + _rowIndices = rowIndices; + _dataExtensions = dataExtensions; + + _firstNewCellIndex = firstNewCellIndex; + _oldRows = oldRows; + _newRows = newRows; + } + + public void apply(Project project) { + synchronized (project) { + if (_firstNewCellIndex < 0) { + _firstNewCellIndex = project.columnModel.allocateNewCellIndex(); + for (int i = 1; i < _columnNames.size(); i++) { + project.columnModel.allocateNewCellIndex(); + } + + _oldRows = new ArrayList(project.rows); + + _newRows = new ArrayList(project.rows.size()); + + int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex(); + int keyCellIndex = project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex(); + int index = 0; + + int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size(); + DataExtension dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null; + + index++; + + Map reconMap = new HashMap(); + + for (int r = 0; r < _oldRows.size(); r++) { + Row oldRow = _oldRows.get(r); + if (r < rowIndex) { + _newRows.add(oldRow.dup()); + continue; + } + + if (dataExtension == null || dataExtension.data.length == 0) { + _newRows.add(oldRow); + } else { + Row firstNewRow = oldRow.dup(); + extendRow(firstNewRow, dataExtension, 0, reconMap); + _newRows.add(firstNewRow); + + int r2 = r + 1; + for (int subR = 1; subR < dataExtension.data.length; subR++) { + if (r2 < project.rows.size()) { + Row oldRow2 = project.rows.get(r2); + if (oldRow2.isCellBlank(cellIndex) && + oldRow2.isCellBlank(keyCellIndex)) { + + Row newRow = oldRow2.dup(); + extendRow(newRow, dataExtension, subR, reconMap); + + _newRows.add(newRow); + r2++; + + continue; + } + } + + Row newRow = new Row(cellIndex + _columnNames.size()); + extendRow(newRow, dataExtension, subR, reconMap); + + _newRows.add(newRow); + } + + r = r2 - 1; // r will be incremented by the for loop anyway + } + + rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size(); + dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null; + index++; + } + } + + project.rows.clear(); + project.rows.addAll(_newRows); + + for (int i = 0; i < _columnNames.size(); i++) { + String name = _columnNames.get(i); + int cellIndex = _firstNewCellIndex + i; + + Column column = new Column(cellIndex, name); + column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i))); + column.setReconStats(ReconStats.create(project, cellIndex)); + + try { + project.columnModel.addColumn(_columnInsertIndex + i, column, true); + + // the column might have been renamed to avoid collision + _columnNames.set(i, column.getName()); + } catch (ModelException e) { + // won't get here since we set the avoid collision flag + } + } + + project.update(); + } + } + + protected void extendRow( + Row row, + DataExtension dataExtension, + int extensionRowIndex, + Map reconMap + ) { + Object[] values = dataExtension.data[extensionRowIndex]; + for (int c = 0; c < values.length; c++) { + Object value = values[c]; + Cell cell = null; + + if (value instanceof ReconCandidate) { + ReconCandidate rc = (ReconCandidate) value; + Recon recon; + if (reconMap.containsKey(rc.id)) { + recon = reconMap.get(rc.id); + } else { + recon = Recon.makeFreebaseRecon(_historyEntryID); + recon.addCandidate(rc); + recon.service = "mql"; + recon.match = rc; + recon.matchRank = 0; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.judgmentBatchSize = 1; + + reconMap.put(rc.id, recon); + } + cell = new Cell(rc.name, recon); + } else { + cell = new Cell((Serializable) value, null); + } + + row.setCell(_firstNewCellIndex + c, cell); + } + } + + public void revert(Project project) { + synchronized (project) { + project.rows.clear(); + project.rows.addAll(_oldRows); + + for (int i = 0; i < _columnNames.size(); i++) { + project.columnModel.columns.remove(_columnInsertIndex); + } + + project.update(); + } + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n'); + writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n'); + writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n'); + for (String name : _columnNames) { + writer.write(name); writer.write('\n'); + } + writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n'); + for (FreebaseType type : _columnTypes) { + try { + JSONWriter jsonWriter = new JSONWriter(writer); + + type.write(jsonWriter, options); + } catch (JSONException e) { + // ??? + } + writer.write('\n'); + } + writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n'); + for (Integer rowIndex : _rowIndices) { + writer.write(rowIndex.toString()); writer.write('\n'); + } + writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n'); + for (DataExtension dataExtension : _dataExtensions) { + if (dataExtension == null) { + writer.write('\n'); + continue; + } + + writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n'); + + for (Object[] values : dataExtension.data) { + for (Object value : values) { + if (value == null) { + writer.write("null"); + } else if (value instanceof ReconCandidate) { + try { + JSONWriter jsonWriter = new JSONWriter(writer); + ((ReconCandidate) value).write(jsonWriter, options); + } catch (JSONException e) { + // ??? + } + } else if (value instanceof String) { + writer.write(JSONObject.quote((String) value)); + } else { + writer.write(value.toString()); + } + writer.write('\n'); + } + } + } + + writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n'); + + writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n'); + for (Row row : _newRows) { + row.save(writer, options); + writer.write('\n'); + } + writer.write("oldRowCount="); writer.write(Integer.toString(_oldRows.size())); writer.write('\n'); + for (Row row : _oldRows) { + row.save(writer, options); + writer.write('\n'); + } + writer.write("/ec/\n"); // end of change marker + } + + static public Change load(LineNumberReader reader, Pool pool) throws Exception { + String baseColumnName = null; + int columnInsertIndex = -1; + + List columnNames = null; + List columnTypes = null; + + List rowIndices = null; + List dataExtensions = null; + + List oldRows = null; + List newRows = null; + + int firstNewCellIndex = -1; + + String line; + while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("baseColumnName".equals(field)) { + baseColumnName = value; + } else if ("columnInsertIndex".equals(field)) { + columnInsertIndex = Integer.parseInt(value); + } else if ("firstNewCellIndex".equals(field)) { + firstNewCellIndex = Integer.parseInt(value); + } else if ("rowIndexCount".equals(field)) { + int count = Integer.parseInt(value); + + rowIndices = new ArrayList(count); + for (int i = 0; i < count; i++) { + line = reader.readLine(); + if (line != null) { + rowIndices.add(Integer.parseInt(line)); + } + } + } else if ("columnNameCount".equals(field)) { + int count = Integer.parseInt(value); + + columnNames = new ArrayList(count); + for (int i = 0; i < count; i++) { + line = reader.readLine(); + if (line != null) { + columnNames.add(line); + } + } + } else if ("columnTypeCount".equals(field)) { + int count = Integer.parseInt(value); + + columnTypes = new ArrayList(count); + for (int i = 0; i < count; i++) { + line = reader.readLine(); + columnTypes.add(FreebaseType.load(ParsingUtilities.evaluateJsonStringToObject(line))); + } + } else if ("dataExtensionCount".equals(field)) { + int count = Integer.parseInt(value); + + dataExtensions = new ArrayList(count); + for (int i = 0; i < count; i++) { + line = reader.readLine(); + + if (line == null) continue; + + if (line.length() == 0) { + dataExtensions.add(null); + continue; + } + + int rowCount = Integer.parseInt(line); + Object[][] data = new Object[rowCount][]; + + for (int r = 0; r < rowCount; r++) { + Object[] row = new Object[columnNames.size()]; + for (int c = 0; c < columnNames.size(); c++) { + line = reader.readLine(); + + row[c] = ReconCandidate.loadStreaming(line); + } + + data[r] = row; + } + + dataExtensions.add(new DataExtension(data)); + } + } else if ("oldRowCount".equals(field)) { + int count = Integer.parseInt(value); + + oldRows = new ArrayList(count); + for (int i = 0; i < count; i++) { + line = reader.readLine(); + if (line != null) { + oldRows.add(Row.load(line, pool)); + } + } + } else if ("newRowCount".equals(field)) { + int count = Integer.parseInt(value); + + newRows = new ArrayList(count); + for (int i = 0; i < count; i++) { + line = reader.readLine(); + if (line != null) { + newRows.add(Row.load(line, pool)); + } + } + } + + } + + DataExtensionChange change = new DataExtensionChange( + baseColumnName, + columnInsertIndex, + columnNames, + columnTypes, + rowIndices, + dataExtensions, + firstNewCellIndex, + oldRows, + newRows + ); + + + return change; + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java new file mode 100644 index 000000000..57a78112f --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/DataExtensionReconConfig.java @@ -0,0 +1,66 @@ +package com.google.refine.freebase.model.recon; + +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseType; +import com.google.refine.model.Cell; +import com.google.refine.model.Project; +import com.google.refine.model.Recon; +import com.google.refine.model.Row; +import com.google.refine.model.recon.ReconConfig; +import com.google.refine.model.recon.ReconJob; +import com.google.refine.freebase.model.recon.StrictReconConfig; + +public class DataExtensionReconConfig extends StrictReconConfig { + final public FreebaseType type; + + private final static String WARN = "Not implemented"; + + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + JSONObject type = obj.getJSONObject("type"); + + return new DataExtensionReconConfig( + new FreebaseType( + type.getString("id"), + type.getString("name") + ) + ); + } + + public DataExtensionReconConfig(FreebaseType type) { + this.type = type; + } + + @Override + public ReconJob createJob(Project project, int rowIndex, Row row, + String columnName, Cell cell) { + throw new RuntimeException(WARN); + } + + @Override + public int getBatchSize() { + throw new RuntimeException(WARN); + } + + public void write(JSONWriter writer, Properties options) throws JSONException { + writer.object(); + writer.key("mode"); writer.value("extend"); + writer.key("type"); type.write(writer, options); + writer.endObject(); + } + + @Override + public List batchRecon(List jobs, long historyEntryID) { + throw new RuntimeException(WARN); + } + + @Override + public String getBriefDescription(Project project, String columnName) { + throw new RuntimeException(WARN); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java new file mode 100644 index 000000000..726d1129e --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/GuidBasedReconConfig.java @@ -0,0 +1,176 @@ +package com.google.refine.freebase.model.recon; + +import java.io.InputStream; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.model.Cell; +import com.google.refine.model.Project; +import com.google.refine.model.Recon; +import com.google.refine.model.ReconCandidate; +import com.google.refine.model.Row; +import com.google.refine.model.Recon.Judgment; +import com.google.refine.model.recon.ReconConfig; +import com.google.refine.model.recon.ReconJob; +import com.google.refine.util.ParsingUtilities; + +public class GuidBasedReconConfig extends StrictReconConfig { + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + return new GuidBasedReconConfig(); + } + + public GuidBasedReconConfig() { + } + + static protected class GuidBasedReconJob extends ReconJob { + String guid; + + public int getKey() { + return guid.hashCode(); + } + } + + @Override + public ReconJob createJob(Project project, int rowIndex, Row row, + String columnName, Cell cell) { + + GuidBasedReconJob job = new GuidBasedReconJob(); + String s = cell.value.toString(); + + if (s.startsWith("/guid/")) { + s = "#" + s.substring(6); + } else if (!s.startsWith("#")) { + s = "#" + s; + } + + job.guid = s; + + return job; + } + + @Override + public int getBatchSize() { + return 10; + } + + @Override + public String getBriefDescription(Project project, String columnName) { + return "Reconcile cells in column " + columnName + " as Freebase IDs"; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("mode"); writer.value("strict"); + writer.key("match"); writer.value("id"); + writer.endObject(); + } + + @Override + public List batchRecon(List jobs, long historyEntryID) { + List recons = new ArrayList(jobs.size()); + Map guidToRecon = new HashMap(); + + try { + String query = null; + { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.object(); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("guid|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((GuidBasedReconJob) job).guid); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); + + query = stringWriter.toString(); + } + + StringBuffer sb = new StringBuffer(1024); + sb.append(s_mqlreadService); + sb.append("?query="); + sb.append(ParsingUtilities.encode(query)); + + URL url = new URL(sb.toString()); + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + JSONArray results = o.getJSONArray("result"); + int count = results.length(); + + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); + + String guid = result.getString("guid"); + + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + result.getString("id"), + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + guidToRecon.put(guid, recon); + } + } finally { + is.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + for (int i = 0; i < jobs.size(); i++) { + String guid = ((GuidBasedReconJob) jobs.get(i)).guid; + Recon recon = guidToRecon.get(guid); + recons.add(recon); + } + + return recons; + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java new file mode 100644 index 000000000..6955f5ac1 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/IdBasedReconConfig.java @@ -0,0 +1,181 @@ +package com.google.refine.freebase.model.recon; + +import java.io.InputStream; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.model.Cell; +import com.google.refine.model.Project; +import com.google.refine.model.Recon; +import com.google.refine.model.ReconCandidate; +import com.google.refine.model.Row; +import com.google.refine.model.Recon.Judgment; +import com.google.refine.model.recon.ReconConfig; +import com.google.refine.model.recon.ReconJob; +import com.google.refine.util.ParsingUtilities; + +public class IdBasedReconConfig extends StrictReconConfig { + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + return new IdBasedReconConfig(); + } + + public IdBasedReconConfig() { + } + + static protected class IdBasedReconJob extends ReconJob { + String id; + + public int getKey() { + return id.hashCode(); + } + } + + @Override + public ReconJob createJob(Project project, int rowIndex, Row row, + String columnName, Cell cell) { + + IdBasedReconJob job = new IdBasedReconJob(); + String s = cell.value.toString(); + + if (!s.startsWith("/")) { + if (s.startsWith("92")) { + s = "/guid/" + s; + } else if (!s.contains("/")){ + s = "/en/" + s; + } else { + s = "/" + s; + } + } + + job.id = s; + + return job; + } + + @Override + public int getBatchSize() { + return 10; + } + + @Override + public String getBriefDescription(Project project, String columnName) { + return "Reconcile cells in column " + columnName + " as Freebase IDs"; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("mode"); writer.value("strict"); + writer.key("match"); writer.value("id"); + writer.endObject(); + } + + @Override + public List batchRecon(List jobs, long historyEntryID) { + List recons = new ArrayList(jobs.size()); + Map idToRecon = new HashMap(); + + try { + String query = null; + { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.object(); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("id|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((IdBasedReconJob) job).id); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); + + query = stringWriter.toString(); + } + + StringBuffer sb = new StringBuffer(1024); + sb.append(s_mqlreadService); + sb.append("?query="); + sb.append(ParsingUtilities.encode(query)); + + URL url = new URL(sb.toString()); + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + JSONArray results = o.getJSONArray("result"); + int count = results.length(); + + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); + + String id = result.getString("id"); + + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + id, + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + idToRecon.put(id, recon); + } + } finally { + is.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + for (int i = 0; i < jobs.size(); i++) { + String id = ((IdBasedReconJob) jobs.get(i)).id; + Recon recon = idToRecon.get(id); + recons.add(recon); + } + + return recons; + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java new file mode 100644 index 000000000..7c0d14d96 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/KeyBasedReconConfig.java @@ -0,0 +1,195 @@ +package com.google.refine.freebase.model.recon; + +import java.io.InputStream; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseTopic; +import com.google.refine.model.Cell; +import com.google.refine.model.Project; +import com.google.refine.model.Recon; +import com.google.refine.model.ReconCandidate; +import com.google.refine.model.Row; +import com.google.refine.model.Recon.Judgment; +import com.google.refine.model.recon.ReconConfig; +import com.google.refine.model.recon.ReconJob; +import com.google.refine.util.ParsingUtilities; + +public class KeyBasedReconConfig extends StrictReconConfig { + final public FreebaseTopic namespace; + + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + JSONObject ns = obj.getJSONObject("namespace"); + + return new KeyBasedReconConfig( + new FreebaseTopic( + ns.getString("id"), + ns.getString("name") + ) + ); + } + + public KeyBasedReconConfig(FreebaseTopic namespace) { + this.namespace = namespace; + } + + static protected class KeyBasedReconJob extends ReconJob { + String key; + + public int getKey() { + return key.hashCode(); + } + } + + @Override + public ReconJob createJob(Project project, int rowIndex, Row row, + String columnName, Cell cell) { + + KeyBasedReconJob job = new KeyBasedReconJob(); + + job.key = cell.value.toString().replace(' ', '_'); + + return job; + } + + @Override + public int getBatchSize() { + return 10; + } + + @Override + public String getBriefDescription(Project project, String columnName) { + return "Reconcile cells in column " + columnName + " to topics with keys in namespace " + namespace.id; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("mode"); writer.value("strict"); + writer.key("match"); writer.value("key"); + writer.key("namespace"); namespace.write(writer, options); + writer.endObject(); + } + + @Override + public List batchRecon(List jobs, long historyEntryID) { + List recons = new ArrayList(jobs.size()); + Map keyToRecon = new HashMap(); + + try { + String query = null; + { + StringWriter stringWriter = new StringWriter(); + JSONWriter jsonWriter = new JSONWriter(stringWriter); + + jsonWriter.object(); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("name"); jsonWriter.value(null); + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("type"); jsonWriter.array(); jsonWriter.endArray(); + + jsonWriter.key("key"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("namespace"); + jsonWriter.object(); + jsonWriter.key("id"); jsonWriter.value(namespace.id); + jsonWriter.endObject(); + + jsonWriter.key("value"); jsonWriter.value(null); + jsonWriter.key("value|="); + jsonWriter.array(); + for (ReconJob job : jobs) { + jsonWriter.value(((KeyBasedReconJob) job).key); + } + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); + + query = stringWriter.toString(); + } + + StringBuffer sb = new StringBuffer(1024); + sb.append(s_mqlreadService); + sb.append("?query="); + sb.append(ParsingUtilities.encode(query)); + + URL url = new URL(sb.toString()); + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(5000); + connection.connect(); + + InputStream is = connection.getInputStream(); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + JSONArray results = o.getJSONArray("result"); + int count = results.length(); + + for (int i = 0; i < count; i++) { + JSONObject result = results.getJSONObject(i); + + String key = result.getJSONArray("key").getJSONObject(0).getString("value"); + + JSONArray types = result.getJSONArray("type"); + String[] typeIDs = new String[types.length()]; + for (int j = 0; j < typeIDs.length; j++) { + typeIDs[j] = types.getString(j); + } + + ReconCandidate candidate = new ReconCandidate( + result.getString("id"), + result.getString("name"), + typeIDs, + 100 + ); + + Recon recon = Recon.makeFreebaseRecon(historyEntryID); + recon.addCandidate(candidate); + recon.service = "mql"; + recon.judgment = Judgment.Matched; + recon.judgmentAction = "auto"; + recon.match = candidate; + recon.matchRank = 0; + + keyToRecon.put(key, recon); + } + } finally { + is.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } + + for (int i = 0; i < jobs.size(); i++) { + String key = ((KeyBasedReconJob) jobs.get(i)).key; + Recon recon = keyToRecon.get(key); + recons.add(recon); + } + + return recons; + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java new file mode 100644 index 000000000..e15a1b7d5 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/model/recon/StrictReconConfig.java @@ -0,0 +1,21 @@ +package com.google.refine.freebase.model.recon; + +import org.json.JSONObject; + +import com.google.refine.model.recon.ReconConfig; + +abstract public class StrictReconConfig extends ReconConfig { + final static protected String s_mqlreadService = "http://api.freebase.com/api/service/mqlread"; + + static public ReconConfig reconstruct(JSONObject obj) throws Exception { + String match = obj.getString("match"); + if ("key".equals(match)) { + return KeyBasedReconConfig.reconstruct(obj); + } else if ("id".equals(match)) { + return IdBasedReconConfig.reconstruct(obj); + } else if ("guid".equals(match)) { + return GuidBasedReconConfig.reconstruct(obj); + } + return null; + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java new file mode 100644 index 000000000..abadfd707 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseProvider.java @@ -0,0 +1,37 @@ +package com.google.refine.freebase.oauth; + +import oauth.signpost.OAuthConsumer; + +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; +import com.google.refine.freebase.util.FreebaseUtils; + +public class FreebaseProvider extends Provider { + static private final String[] FREEBASE_OAUTH_INFO = { "#9202a8c04000641f80000000185352db" , "4561ee02279e6f04ebd88a1557e4292489380adf"}; + + static public void register() { + OAuthUtilities.registerOAuthProvider(new FreebaseProvider(FreebaseUtils.FREEBASE_HOST), FREEBASE_OAUTH_INFO); + } + + public FreebaseProvider(String host) { + super(host); + } + + public String getRequestTokenServiceURL() { + return "https://" + host + "/api/oauth/request_token"; + } + + public String getAccessTokenServiceURL() { + return "https://" + host + "/api/oauth/access_token"; + } + + public String getUserAuthorizationURL() { + return "https://" + host + "/signin/app"; + } + + + @Override + public OAuthConsumer createConsumer(String consumerKey, String consumerSecret) { + return new FreebaseTimeCommonsHttpOAuthConsumer(consumerKey, consumerSecret); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java new file mode 100644 index 000000000..d0ebe1029 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/oauth/FreebaseTimeCommonsHttpOAuthConsumer.java @@ -0,0 +1,69 @@ +package com.google.refine.freebase.oauth; + +import java.io.IOException; + +import oauth.signpost.commonshttp.CommonsHttpOAuthConsumer; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.params.BasicHttpParams; +import org.apache.http.params.HttpConnectionParams; +import org.apache.http.params.HttpParams; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FreebaseTimeCommonsHttpOAuthConsumer extends CommonsHttpOAuthConsumer { + + final static Logger logger = LoggerFactory.getLogger("oauth"); + + private static final long serialVersionUID = -4139931605235255279L; + + private static final int SOCKET_TIMEOUT = 3000; + private static final int CONNECTION_TIMEOUT = 3000; + + private static final String TIMER_URL = "http://refinery.freebaseapps.com/time"; + + public FreebaseTimeCommonsHttpOAuthConsumer(String consumerKey, String consumerSecret) { + super(consumerKey, consumerSecret); + } + + /** + * It might be that the user's computer's clock is not synchronized enough with the Freebase servers + * and this might result in Freebase thinking that it was under a replay attack. + * To avoid this problem we get the timestamp directly from acre that we know is synchronized. + * + * NOTE: this call is potentially vulnerable to a man-in-the-middle (MITM) attack, but the same + * could be said if we used an NTP client. + */ + protected String generateTimestamp() { + + long time = -1; + + try { + HttpParams httpParams = new BasicHttpParams(); + HttpConnectionParams.setSoTimeout(httpParams, SOCKET_TIMEOUT); + HttpConnectionParams.setConnectionTimeout(httpParams, CONNECTION_TIMEOUT); + HttpClient httpClient = new DefaultHttpClient(httpParams); + HttpGet httpget = new HttpGet(TIMER_URL); + HttpResponse response = httpClient.execute(httpget); + HttpEntity entity = response.getEntity(); + if (entity != null) { + time = Long.parseLong(EntityUtils.toString(entity),10); + logger.debug("Got remote timestamp {}", time); + } + } catch (IOException e) { + logger.warn("Error obtaining the synchronized remote timestamp, defaulting to the local one",e); + } + + if (time == -1) { + time = System.currentTimeMillis(); + } + + return Long.toString(time / 1000L); + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java new file mode 100644 index 000000000..e1374d11e --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/operations/ExtendDataOperation.java @@ -0,0 +1,275 @@ +package com.google.refine.freebase.operations; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.browsing.Engine; +import com.google.refine.browsing.FilteredRows; +import com.google.refine.browsing.RowVisitor; +import com.google.refine.freebase.FreebaseType; +import com.google.refine.freebase.util.FreebaseDataExtensionJob; +import com.google.refine.freebase.util.FreebaseDataExtensionJob.ColumnInfo; +import com.google.refine.freebase.util.FreebaseDataExtensionJob.DataExtension; +import com.google.refine.history.HistoryEntry; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Cell; +import com.google.refine.model.Column; +import com.google.refine.model.Project; +import com.google.refine.model.ReconCandidate; +import com.google.refine.model.Row; +import com.google.refine.model.changes.CellAtRow; +import com.google.refine.freebase.model.changes.DataExtensionChange; +import com.google.refine.operations.EngineDependentOperation; +import com.google.refine.operations.OperationRegistry; +import com.google.refine.process.LongRunningProcess; +import com.google.refine.process.Process; + +public class ExtendDataOperation extends EngineDependentOperation { + final protected String _baseColumnName; + final protected JSONObject _extension; + final protected int _columnInsertIndex; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + JSONObject engineConfig = obj.getJSONObject("engineConfig"); + + return new ExtendDataOperation( + engineConfig, + obj.getString("baseColumnName"), + obj.getJSONObject("extension"), + obj.getInt("columnInsertIndex") + ); + } + + public ExtendDataOperation( + JSONObject engineConfig, + String baseColumnName, + JSONObject extension, + int columnInsertIndex + ) { + super(engineConfig); + + _baseColumnName = baseColumnName; + _extension = extension; + _columnInsertIndex = columnInsertIndex; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.key("engineConfig"); writer.value(getEngineConfig()); + writer.key("columnInsertIndex"); writer.value(_columnInsertIndex); + writer.key("baseColumnName"); writer.value(_baseColumnName); + writer.key("extension"); writer.value(_extension); + writer.endObject(); + } + + protected String getBriefDescription(Project project) { + return "Extend data at index " + _columnInsertIndex + + " based on column " + _baseColumnName; + } + + protected String createDescription(Column column, List cellsAtRows) { + return "Extend data at index " + _columnInsertIndex + + " based on column " + column.getName() + + " by filling " + cellsAtRows.size(); + } + + public Process createProcess(Project project, Properties options) throws Exception { + return new ExtendDataProcess( + project, + getEngineConfig(), + getBriefDescription(null) + ); + } + + public class ExtendDataProcess extends LongRunningProcess implements Runnable { + final protected Project _project; + final protected JSONObject _engineConfig; + final protected long _historyEntryID; + protected int _cellIndex; + protected FreebaseDataExtensionJob _job; + + public ExtendDataProcess( + Project project, + JSONObject engineConfig, + String description + ) throws JSONException { + super(description); + _project = project; + _engineConfig = engineConfig; + _historyEntryID = HistoryEntry.allocateID(); + + _job = new FreebaseDataExtensionJob(_extension); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("id"); writer.value(hashCode()); + writer.key("description"); writer.value(_description); + writer.key("immediate"); writer.value(false); + writer.key("status"); writer.value(_thread == null ? "pending" : (_thread.isAlive() ? "running" : "done")); + writer.key("progress"); writer.value(_progress); + writer.endObject(); + } + + protected Runnable getRunnable() { + return this; + } + + protected void populateRowsWithMatches(List rowIndices) throws Exception { + Engine engine = new Engine(_project); + engine.initializeFromJSON(_engineConfig); + + Column column = _project.columnModel.getColumnByName(_baseColumnName); + if (column == null) { + throw new Exception("No column named " + _baseColumnName); + } + + _cellIndex = column.getCellIndex(); + + FilteredRows filteredRows = engine.getAllFilteredRows(); + filteredRows.accept(_project, new RowVisitor() { + List _rowIndices; + + public RowVisitor init(List rowIndices) { + _rowIndices = rowIndices; + return this; + } + + @Override + public void start(Project project) { + // nothing to do + } + + @Override + public void end(Project project) { + // nothing to do + } + + public boolean visit(Project project, int rowIndex, Row row) { + Cell cell = row.getCell(_cellIndex); + if (cell != null && cell.recon != null && cell.recon.match != null) { + _rowIndices.add(rowIndex); + } + + return false; + } + }.init(rowIndices)); + } + + protected int extendRows( + List rowIndices, + List dataExtensions, + int from, + int limit, + Map reconCandidateMap + ) { + Set ids = new HashSet(); + + int end; + for (end = from; end < limit && ids.size() < 10; end++) { + int index = rowIndices.get(end); + Row row = _project.rows.get(index); + Cell cell = row.getCell(_cellIndex); + + ids.add(cell.recon.match.id); + } + + Map map = null; + try { + map = _job.extend(ids, reconCandidateMap); + } catch (Exception e) { + map = new HashMap(); + } + + for (int i = from; i < end; i++) { + int index = rowIndices.get(i); + Row row = _project.rows.get(index); + Cell cell = row.getCell(_cellIndex); + String guid = cell.recon.match.id; + + if (map.containsKey(guid)) { + dataExtensions.add(map.get(guid)); + } else { + dataExtensions.add(null); + } + } + + return end; + } + + public void run() { + List rowIndices = new ArrayList(); + List dataExtensions = new ArrayList(); + + try { + populateRowsWithMatches(rowIndices); + } catch (Exception e2) { + // TODO : Not sure what to do here? + e2.printStackTrace(); + } + + int start = 0; + Map reconCandidateMap = new HashMap(); + + while (start < rowIndices.size()) { + int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap); + start = end; + + _progress = end * 100 / rowIndices.size(); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + if (_canceled) { + break; + } + } + } + + if (!_canceled) { + List columnNames = new ArrayList(); + for (ColumnInfo info : _job.columns) { + columnNames.add(StringUtils.join(info.names, " - ")); + } + + List columnTypes = new ArrayList(); + for (ColumnInfo info : _job.columns) { + columnTypes.add(info.expectedType); + } + + HistoryEntry historyEntry = new HistoryEntry( + _historyEntryID, + _project, + _description, + ExtendDataOperation.this, + new DataExtensionChange( + _baseColumnName, + _columnInsertIndex, + columnNames, + columnTypes, + rowIndices, + dataExtensions, + _historyEntryID) + ); + + _project.history.addEntry(historyEntry); + _project.processManager.onDoneProcess(this); + } + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java new file mode 100644 index 000000000..c10d29317 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/operations/ImportQADataOperation.java @@ -0,0 +1,106 @@ +package com.google.refine.freebase.operations; + +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.freebase.commands.UploadDataCommand; +import com.google.refine.history.HistoryEntry; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Cell; +import com.google.refine.model.Project; +import com.google.refine.model.Recon; +import com.google.refine.model.Row; +import com.google.refine.model.changes.MassReconChange; +import com.google.refine.operations.OperationRegistry; +import com.google.refine.util.ParsingUtilities; + +public class ImportQADataOperation extends AbstractOperation { + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + return new ImportQADataOperation(); + } + + public ImportQADataOperation() { + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription(null)); + writer.endObject(); + } + + @Override + protected String getBriefDescription(Project project) { + return "Import QA DAta"; + } + + @Override + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Integer jobID = (Integer) project.getMetadata().getPreferenceStore().get(UploadDataCommand.s_dataLoadJobIDPref); + if (jobID == null) { + throw new InternalError("Project is not associated with any data loading job."); + } + + Map reconIDToResult = new HashMap(); + + URL url = new URL("http://refinery.freebaseapps.com/get_answers/" + jobID); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setReadTimeout(30000); // 30 seconds + + LineNumberReader reader = new LineNumberReader(new InputStreamReader(conn.getInputStream())); + try { + String line; + while ((line = reader.readLine()) != null) { + JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(line); + long reconID = Long.parseLong(obj.getString("recon_id").substring(3)); + + reconIDToResult.put(reconID, obj.getString("result")); + } + } finally { + reader.close(); + } + + Map oldRecons = new HashMap(); + Map newRecons = new HashMap(); + + for (int r = 0; r < project.rows.size(); r++) { + Row row = project.rows.get(r); + + for (int c = 0; c < row.cells.size(); c++) { + Cell cell = row.cells.get(c); + if (cell != null && cell.recon != null) { + Recon oldRecon = cell.recon; + + if (reconIDToResult.containsKey(oldRecon.id)) { + Recon newRecon = oldRecon.dup(); + newRecon.setFeature(Recon.Feature_qaResult, reconIDToResult.get(oldRecon.id)); + + reconIDToResult.remove(oldRecon.id); + + oldRecons.put(oldRecon.id, oldRecon); + newRecons.put(oldRecon.id, newRecon); + } + } + } + } + + return new HistoryEntry( + historyEntryID, + project, + getBriefDescription(project), + this, + new MassReconChange(newRecons, oldRecons) + ); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java b/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java new file mode 100644 index 000000000..5b7a41f0d --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/operations/SaveProtographOperation.java @@ -0,0 +1,121 @@ +package com.google.refine.freebase.operations; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.freebase.protograph.Protograph; +import com.google.refine.history.Change; +import com.google.refine.history.HistoryEntry; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Project; +import com.google.refine.operations.OperationRegistry; +import com.google.refine.util.ParsingUtilities; +import com.google.refine.util.Pool; + +public class SaveProtographOperation extends AbstractOperation { + final protected Protograph _protograph; + + static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception { + return new SaveProtographOperation( + Protograph.reconstruct(obj.getJSONObject("protograph")) + ); + } + + public SaveProtographOperation(Protograph protograph) { + _protograph = protograph; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); writer.value(getBriefDescription()); + writer.key("protograph"); _protograph.write(writer, options); + writer.endObject(); + } + + protected String getBriefDescription() { + return "Save schema alignment skeleton"; + } + + @Override + protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception { + Change change = new ProtographChange(_protograph); + + return new HistoryEntry(historyEntryID, project, getBriefDescription(), SaveProtographOperation.this, change); + } + + static public class ProtographChange implements Change { + final protected Protograph _newProtograph; + protected Protograph _oldProtograph; + + public ProtographChange(Protograph protograph) { + _newProtograph = protograph; + } + + public void apply(Project project) { + synchronized (project) { + _oldProtograph = (Protograph) project.overlayModels.get("freebaseProtograph"); + + project.overlayModels.put("freebaseProtograph", _newProtograph); + } + } + + public void revert(Project project) { + synchronized (project) { + if (_oldProtograph == null) { + project.overlayModels.remove("freebaseProtograph"); + } else { + project.overlayModels.put("freebaseProtograph", _oldProtograph); + } + } + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n'); + writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n'); + writer.write("/ec/\n"); // end of change marker + } + + static public Change load(LineNumberReader reader, Pool pool) throws Exception { + Protograph oldProtograph = null; + Protograph newProtograph = null; + + String line; + while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("oldProtograph".equals(field) && value.length() > 0) { + oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); + } else if ("newProtograph".equals(field) && value.length() > 0) { + newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value)); + } + } + + ProtographChange change = new ProtographChange(newProtograph); + change._oldProtograph = oldProtograph; + + return change; + } + + static protected void writeProtograph(Protograph p, Writer writer) throws IOException { + if (p != null) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + p.write(jsonWriter, new Properties()); + } catch (JSONException e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java new file mode 100644 index 000000000..5a0ed007c --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/AnonymousNode.java @@ -0,0 +1,47 @@ +package com.google.refine.freebase.protograph; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseType; + +public class AnonymousNode implements Node, NodeWithLinks { + final public FreebaseType type; + final public List links = new LinkedList(); + + public AnonymousNode(FreebaseType type) { + this.type = type; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("anonymous"); + writer.key("type"); type.write(writer, options); + if (links != null) { + writer.key("links"); writer.array(); + for (Link link : links) { + link.write(writer, options); + } + writer.endArray(); + } + writer.endObject(); + } + + public void addLink(Link link) { + links.add(link); + } + + public Link getLink(int index) { + return links.get(index); + } + + public int getLinkCount() { + return links.size(); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/BooleanColumnCondition.java b/extensions/freebase/src/com/google/refine/freebase/protograph/BooleanColumnCondition.java new file mode 100644 index 000000000..bff519250 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/BooleanColumnCondition.java @@ -0,0 +1,42 @@ +package com.google.refine.freebase.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.refine.model.Column; +import com.google.refine.model.Project; +import com.google.refine.model.Row; + + +public class BooleanColumnCondition implements Condition { + final public String columnName; + + public BooleanColumnCondition(String columnName) { + this.columnName = columnName; + } + + @Override + public boolean test(Project project, int rowIndex, Row row) { + Column column = project.columnModel.getColumnByName(columnName); + if (column != null) { + Object o = row.getCellValue(column.getCellIndex()); + if (o != null) { + if (o instanceof Boolean) { + return ((Boolean) o).booleanValue(); + } else { + return Boolean.parseBoolean(o.toString()); + } + } + } + return false; + } + + @Override + public void write(JSONWriter writer, Properties options) throws JSONException { + writer.object(); + writer.key("columnName"); writer.value(columnName); + writer.endObject(); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/CellKeyNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/CellKeyNode.java new file mode 100644 index 000000000..338e01c1d --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/CellKeyNode.java @@ -0,0 +1,35 @@ +package com.google.refine.freebase.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseTopic; + +public class CellKeyNode extends CellNode { + final public FreebaseTopic namespace; + + public CellKeyNode( + FreebaseTopic namespace + ) { + this.namespace = namespace; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("cell-as-key"); + + writer.key("columnNames"); + writer.array(); + for (String name : columnNames) { + writer.value(name); + } + writer.endArray(); + + writer.key("namespace"); namespace.write(writer, options); + writer.endObject(); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/CellNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/CellNode.java new file mode 100644 index 000000000..5c64c376c --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/CellNode.java @@ -0,0 +1,8 @@ +package com.google.refine.freebase.protograph; + +import java.util.LinkedList; +import java.util.List; + +abstract public class CellNode implements Node { + final public List columnNames = new LinkedList(); +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java new file mode 100644 index 000000000..4a8923a5d --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/CellTopicNode.java @@ -0,0 +1,58 @@ +package com.google.refine.freebase.protograph; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseType; + +public class CellTopicNode extends CellNode implements NodeWithLinks { + final public FreebaseType type; + final public List links = new LinkedList(); + + public CellTopicNode( + FreebaseType type + ) { + this.type = type; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("cell-as-topic"); + writer.key("columnNames"); + writer.array(); + for (String name : columnNames) { + writer.value(name); + } + writer.endArray(); + if (type != null) { + writer.key("type"); type.write(writer, options); + } + if (links != null) { + writer.key("links"); writer.array(); + for (Link link : links) { + link.write(writer, options); + } + writer.endArray(); + } + + writer.endObject(); + } + + public void addLink(Link link) { + links.add(link); + } + + public Link getLink(int index) { + return links.get(index); + } + + public int getLinkCount() { + return links.size(); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/CellValueNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/CellValueNode.java new file mode 100644 index 000000000..1bf2ef43e --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/CellValueNode.java @@ -0,0 +1,36 @@ +package com.google.refine.freebase.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +public class CellValueNode extends CellNode { + final public String valueType; + final public String lang; + + public CellValueNode( + String valueType, + String lang + ) { + this.valueType = valueType; + this.lang = lang; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("cell-as-value"); + writer.key("columnNames"); + writer.array(); + for (String name : columnNames) { + writer.value(name); + } + writer.endArray(); + writer.key("valueType"); writer.value(valueType); + writer.key("lang"); writer.value(lang); + writer.endObject(); + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/Condition.java b/extensions/freebase/src/com/google/refine/freebase/protograph/Condition.java new file mode 100644 index 000000000..098f0fc69 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/Condition.java @@ -0,0 +1,9 @@ +package com.google.refine.freebase.protograph; + +import com.google.refine.Jsonizable; +import com.google.refine.model.Project; +import com.google.refine.model.Row; + +public interface Condition extends Jsonizable { + public boolean test(Project project, int rowIndex, Row row); +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/FreebaseTopicNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/FreebaseTopicNode.java new file mode 100644 index 000000000..940b81761 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/FreebaseTopicNode.java @@ -0,0 +1,48 @@ +package com.google.refine.freebase.protograph; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseTopic; + +public class FreebaseTopicNode implements Node, NodeWithLinks { + final public FreebaseTopic topic; + final public List links = new LinkedList(); + + public FreebaseTopicNode(FreebaseTopic topic) { + this.topic = topic; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("topic"); + writer.key("topic"); topic.write(writer, options); + if (links != null) { + writer.key("links"); writer.array(); + for (Link link : links) { + link.write(writer, options); + } + writer.endArray(); + } + + writer.endObject(); + } + + public void addLink(Link link) { + links.add(link); + } + + public Link getLink(int index) { + return links.get(index); + } + + public int getLinkCount() { + return links.size(); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/Link.java b/extensions/freebase/src/com/google/refine/freebase/protograph/Link.java new file mode 100644 index 000000000..7361e3087 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/Link.java @@ -0,0 +1,48 @@ +package com.google.refine.freebase.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.refine.Jsonizable; +import com.google.refine.freebase.FreebaseProperty; + +public class Link implements Jsonizable { + final public FreebaseProperty property; + final public Node target; + final public Condition condition; + final public boolean load; + + public Link(FreebaseProperty property, Node target, Condition condition, boolean load) { + this.property = property; + this.target = target; + this.condition = condition; + this.load = load; + } + + public FreebaseProperty getProperty() { + return property; + } + + public Node getTarget() { + return target; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("property"); property.write(writer, options); + if (target != null) { + writer.key("target"); + target.write(writer, options); + } + if (condition != null) { + writer.key("condition"); + condition.write(writer, options); + } + writer.endObject(); + } + +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/Node.java b/extensions/freebase/src/com/google/refine/freebase/protograph/Node.java new file mode 100644 index 000000000..406ae5247 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/Node.java @@ -0,0 +1,6 @@ +package com.google.refine.freebase.protograph; + +import com.google.refine.Jsonizable; + +public interface Node extends Jsonizable { +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/NodeWithLinks.java b/extensions/freebase/src/com/google/refine/freebase/protograph/NodeWithLinks.java new file mode 100644 index 000000000..debde9387 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/NodeWithLinks.java @@ -0,0 +1,9 @@ +package com.google.refine.freebase.protograph; + +public interface NodeWithLinks { + public void addLink(Link link); + + public int getLinkCount(); + + public Link getLink(int index); +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/Protograph.java b/extensions/freebase/src/com/google/refine/freebase/protograph/Protograph.java new file mode 100644 index 000000000..19aa226e6 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/Protograph.java @@ -0,0 +1,171 @@ +package com.google.refine.freebase.protograph; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseProperty; +import com.google.refine.freebase.FreebaseTopic; +import com.google.refine.freebase.FreebaseType; +import com.google.refine.model.OverlayModel; +import com.google.refine.model.Project; + +public class Protograph implements OverlayModel { + final protected List _rootNodes = new LinkedList(); + + public int getRootNodeCount() { + return _rootNodes.size(); + } + + public Node getRootNode(int index) { + return _rootNodes.get(index); + } + + @Override + public void onBeforeSave(Project project) { + } + + @Override + public void onAfterSave(Project project) { + } + + + @Override + public void dispose(Project project) { + } + + static public Protograph reconstruct(JSONObject o) throws JSONException { + Protograph g = new Protograph(); + + JSONArray rootNodes = o.getJSONArray("rootNodes"); + int count = rootNodes.length(); + + for (int i = 0; i < count; i++) { + JSONObject o2 = rootNodes.getJSONObject(i); + Node node = reconstructNode(o2); + if (node != null) { + g._rootNodes.add(node); + } + } + + return g; + } + + static protected Node reconstructNode(JSONObject o) throws JSONException { + Node node = null; + + String nodeType = o.getString("nodeType"); + if (nodeType.startsWith("cell-as-")) { + if ("cell-as-topic".equals(nodeType)) { + if (o.has("type")) { + node = new CellTopicNode( + reconstructType(o.getJSONObject("type")) + ); + } + } else if ("cell-as-value".equals(nodeType)) { + node = new CellValueNode( + o.getString("valueType"), + o.getString("lang") + ); + } else if ("cell-as-key".equals(nodeType)) { + node = new CellKeyNode( + reconstructTopic(o.getJSONObject("namespace")) + ); + } + + if (o.has("columnName") && !o.isNull("columnName")) { + ((CellNode) node).columnNames.add(o.getString("columnName")); + } + if (o.has("columnNames") && !o.isNull("columnNames")) { + JSONArray columnNames = o.getJSONArray("columnNames"); + int count = columnNames.length(); + + for (int c = 0; c < count; c++) { + ((CellNode) node).columnNames.add(columnNames.getString(c)); + } + } + } else if ("topic".equals(nodeType)) { + node = new FreebaseTopicNode(reconstructTopic(o.getJSONObject("topic"))); + } else if ("value".equals(nodeType)) { + node = new ValueNode( + o.get("value"), + o.getString("valueType"), + o.getString("lang") + ); + } else if ("anonymous".equals(nodeType)) { + node = new AnonymousNode(reconstructType(o.getJSONObject("type"))); + } + + if (node != null && node instanceof NodeWithLinks && o.has("links")) { + NodeWithLinks node2 = (NodeWithLinks) node; + + JSONArray links = o.getJSONArray("links"); + int linkCount = links.length(); + + for (int j = 0; j < linkCount; j++) { + JSONObject oLink = links.getJSONObject(j); + Condition condition = null; + + if (oLink.has("condition") && !oLink.isNull("condition")) { + JSONObject oCondition = oLink.getJSONObject("condition"); + if (oCondition.has("columnName") && !oCondition.isNull("columnName")) { + condition = new BooleanColumnCondition(oCondition.getString("columnName")); + } + } + + node2.addLink(new Link( + reconstructProperty(oLink.getJSONObject("property")), + oLink.has("target") && !oLink.isNull("target") ? + reconstructNode(oLink.getJSONObject("target")) : null, + condition, + oLink.has("load") && !oLink.isNull("load") ? + oLink.getBoolean("load") : true + )); + } + } + + return node; + } + + static protected FreebaseProperty reconstructProperty(JSONObject o) throws JSONException { + return new FreebaseProperty( + o.getString("id"), + o.getString("name") + ); + } + + static protected FreebaseType reconstructType(JSONObject o) throws JSONException { + return new FreebaseType( + o.getString("id"), + o.getString("name") + ); + } + + static protected FreebaseTopic reconstructTopic(JSONObject o) throws JSONException { + return new FreebaseTopic( + o.getString("id"), + o.getString("name") + ); + } + + public void write(JSONWriter writer, Properties options) throws JSONException { + writer.object(); + writer.key("rootNodes"); writer.array(); + + for (Node node : _rootNodes) { + node.write(writer, options); + } + + writer.endArray(); + writer.endObject(); + } + + static public Protograph load(Project project, JSONObject obj) throws Exception { + return reconstruct(obj); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java new file mode 100644 index 000000000..aa5296ec1 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/ValueNode.java @@ -0,0 +1,29 @@ +package com.google.refine.freebase.protograph; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +public class ValueNode implements Node { + final public Object value; + final public String valueType; + final public String lang; + + public ValueNode(Object value, String valueType, String lang) { + this.value = value; + this.valueType = valueType; + this.lang = lang; + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + + writer.object(); + writer.key("nodeType"); writer.value("value"); + writer.key("value"); writer.value(value); + writer.key("valueType"); writer.value(valueType); + writer.key("lang"); writer.value(lang); + writer.endObject(); + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java new file mode 100644 index 000000000..0d89e0ce3 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/MqlwriteLikeTransposedNodeFactory.java @@ -0,0 +1,346 @@ +package com.google.refine.freebase.protograph.transpose; + +import java.io.IOException; +import java.io.Writer; +import java.util.LinkedList; +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseProperty; +import com.google.refine.freebase.protograph.AnonymousNode; +import com.google.refine.freebase.protograph.CellKeyNode; +import com.google.refine.freebase.protograph.CellNode; +import com.google.refine.freebase.protograph.CellTopicNode; +import com.google.refine.freebase.protograph.CellValueNode; +import com.google.refine.freebase.protograph.FreebaseTopicNode; +import com.google.refine.freebase.protograph.Link; +import com.google.refine.freebase.protograph.ValueNode; +import com.google.refine.model.Cell; +import com.google.refine.model.Recon; +import com.google.refine.util.JSONUtilities; + +public class MqlwriteLikeTransposedNodeFactory implements TransposedNodeFactory { + protected Writer writer; + protected List rootObjects = new LinkedList(); + + private static final String TYPE = "type"; + private static final String ID = "id"; + private static final String NAME = "name"; + private static final String CREATE = "create"; + private static final String VALUE = "value"; + private static final String CONNECT = "connect"; + private static final String LANG = "lang"; + + public MqlwriteLikeTransposedNodeFactory(Writer writer) { + this.writer = writer; + } + + protected JSONArray getJSON() { + return new JSONArray(rootObjects); + } + + @Override + public void flush() throws IOException { + try { + JSONWriter jsonWriter = new JSONWriter(writer); + + jsonWriter.array(); + for (JSONObject obj : rootObjects) { + jsonWriter.value(obj); + } + jsonWriter.endArray(); + + } catch (JSONException e) { + e.printStackTrace(); + } + writer.flush(); + } + + abstract protected class JsonTransposedNode implements TransposedNode { + abstract public Object getJSON(); + } + + abstract protected class JsonObjectTransposedNode extends JsonTransposedNode { + abstract public JSONObject getJSONObject(); + + protected JSONObject obj; + + public Object getJSON() { + return getJSONObject(); + } + } + + protected class AnonymousTransposedNode extends JsonObjectTransposedNode { + JsonObjectTransposedNode parent; + FreebaseProperty property; + AnonymousNode node; + + protected AnonymousTransposedNode( + JsonObjectTransposedNode parent, + FreebaseProperty property, + AnonymousNode node + ) { + this.parent = parent; + this.property = property; + this.node = node; + } + + public JSONObject getJSONObject() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(TYPE, this.node.type.id); + obj.put(ID, (String) null); + obj.put(CREATE, "unconditional"); + } catch (JSONException e) { + e.printStackTrace(); + } + + linkTransposedNodeJSON(obj, parent, property); + } + + return obj; + } + } + + protected class CellTopicTransposedNode extends JsonObjectTransposedNode { + protected CellTopicNode node; + protected Cell cell; + + public CellTopicTransposedNode(CellTopicNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + @Override + public JSONObject getJSONObject() { + if (obj == null) { + obj = new JSONObject(); + try { + if (cell.recon != null && + cell.recon.judgment == Recon.Judgment.Matched && + cell.recon.match != null) { + obj.put(ID, cell.recon.match.id); + } else { + obj.put(ID, (String) null); + obj.put(NAME, cell.value.toString()); + obj.put(TYPE, node.type.id); + obj.put(CREATE, "unless_exists"); + } + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + + protected class CellValueTransposedNode extends JsonTransposedNode { + protected JSONObject obj; + protected CellValueNode node; + protected Cell cell; + + public CellValueTransposedNode(CellValueNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + public Object getJSON() { + if (obj == null) { + obj = new JSONObject(); + try { + JSONUtilities.putField(obj, VALUE, cell.value); + + obj.put(TYPE, node.valueType); + if ("/type/text".equals(node.valueType)) { + obj.put(LANG, node.lang); + } + + obj.put(CONNECT, "insert"); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + + protected class CellKeyTransposedNode extends JsonTransposedNode { + protected JSONObject obj; + protected CellKeyNode node; + protected Cell cell; + + public CellKeyTransposedNode(CellKeyNode node, Cell cell) { + this.node = node; + this.cell = cell; + } + + public Object getJSON() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(VALUE, cell.value.toString()); + + JSONObject nsObj = new JSONObject(); + nsObj.put(ID, node.namespace.id); + + obj.put("namespace", nsObj); + obj.put(CONNECT, "insert"); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + + protected class TopicTransposedNode extends JsonObjectTransposedNode { + protected FreebaseTopicNode node; + + public TopicTransposedNode(FreebaseTopicNode node) { + this.node = node; + } + + @Override + public JSONObject getJSONObject() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(ID, node.topic.id); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + + protected class ValueTransposedNode extends JsonTransposedNode { + protected JSONObject obj; + protected ValueNode node; + + public ValueTransposedNode(ValueNode node) { + this.node = node; + } + + public Object getJSON() { + if (obj == null) { + obj = new JSONObject(); + try { + obj.put(VALUE, node.value); + obj.put(TYPE, node.valueType); + if ("/type/text".equals(node.valueType)) { + obj.put(LANG, node.lang); + } + + obj.put(CONNECT, "insert"); + } catch (JSONException e) { + e.printStackTrace(); + } + } + return obj; + } + } + public TransposedNode transposeAnonymousNode( + TransposedNode parentNode, + Link link, + AnonymousNode node, int rowIndex) { + + return new AnonymousTransposedNode( + parentNode instanceof JsonObjectTransposedNode ? (JsonObjectTransposedNode) parentNode : null, + link != null ? link.property : null, + node + ); + } + + public TransposedNode transposeCellNode( + TransposedNode parentNode, + Link link, + CellNode node, + int rowIndex, + int cellIndex, + Cell cell) { + + JsonTransposedNode tnode = null; + if (node instanceof CellTopicNode) { + tnode = new CellTopicTransposedNode((CellTopicNode) node, cell); + } else if (node instanceof CellValueNode) { + tnode = new CellValueTransposedNode((CellValueNode) node, cell); + } else if (node instanceof CellKeyNode) { + tnode = new CellKeyTransposedNode((CellKeyNode) node, cell); + } + + if (tnode != null) { + processTransposedNode(tnode, parentNode, link != null ? link.property : null); + } + return tnode; + } + + public TransposedNode transposeTopicNode( + TransposedNode parentNode, + Link link, + FreebaseTopicNode node, int rowIndex) { + + JsonTransposedNode tnode = new TopicTransposedNode(node); + + processTransposedNode(tnode, parentNode, link != null ? link.property : null); + + return tnode; + } + + public TransposedNode transposeValueNode( + TransposedNode parentNode, + Link link, + ValueNode node, int rowIndex) { + + JsonTransposedNode tnode = new ValueTransposedNode(node); + + processTransposedNode(tnode, parentNode, link != null ? link.property : null); + + return tnode; + } + + protected void processTransposedNode( + JsonTransposedNode tnode, + TransposedNode parentNode, + FreebaseProperty property + ) { + + if (!(tnode instanceof AnonymousTransposedNode)) { + linkTransposedNodeJSON(tnode.getJSON(), parentNode, property); + } + } + + protected void linkTransposedNodeJSON( + Object obj, + TransposedNode parentNode, + FreebaseProperty property + ) { + + if (parentNode == null) { + if (obj instanceof JSONObject) { + rootObjects.add((JSONObject) obj); + } + } else if (parentNode instanceof JsonTransposedNode) { + JSONObject parentObj = ((JsonObjectTransposedNode) parentNode).getJSONObject(); + + try { + JSONArray a = null; + if (parentObj.has(property.id)) { + a = parentObj.getJSONArray(property.id); + } else { + a = new JSONArray(); + parentObj.put(property.id, a); + } + + a.put(a.length(), obj); + } catch (JSONException e) { + e.printStackTrace(); + } + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java new file mode 100644 index 000000000..043d865af --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNode.java @@ -0,0 +1,4 @@ +package com.google.refine.freebase.protograph.transpose; + +public interface TransposedNode { +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java new file mode 100644 index 000000000..f0813d25c --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TransposedNodeFactory.java @@ -0,0 +1,43 @@ +package com.google.refine.freebase.protograph.transpose; + +import java.io.IOException; + +import com.google.refine.freebase.protograph.AnonymousNode; +import com.google.refine.freebase.protograph.CellNode; +import com.google.refine.freebase.protograph.FreebaseTopicNode; +import com.google.refine.freebase.protograph.Link; +import com.google.refine.freebase.protograph.ValueNode; +import com.google.refine.model.Cell; + +public interface TransposedNodeFactory { + public TransposedNode transposeAnonymousNode( + TransposedNode parentNode, + Link link, + AnonymousNode node, int rowIndex + ); + + public TransposedNode transposeCellNode( + TransposedNode parentNode, + Link link, + CellNode node, + int rowIndex, + int cellIndex, + Cell cell + ); + + public TransposedNode transposeValueNode( + TransposedNode parentNode, + Link link, + ValueNode node, + int rowIndex + ); + + public TransposedNode transposeTopicNode( + TransposedNode parentNode, + Link link, + FreebaseTopicNode node, + int rowIndex + ); + + public void flush() throws IOException; +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java new file mode 100644 index 000000000..66ef0364b --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/Transposer.java @@ -0,0 +1,222 @@ +package com.google.refine.freebase.protograph.transpose; + +import java.util.LinkedList; +import java.util.List; + +import com.google.refine.browsing.FilteredRows; +import com.google.refine.browsing.RowVisitor; +import com.google.refine.expr.ExpressionUtils; +import com.google.refine.freebase.protograph.AnonymousNode; +import com.google.refine.freebase.protograph.CellNode; +import com.google.refine.freebase.protograph.CellTopicNode; +import com.google.refine.freebase.protograph.FreebaseTopicNode; +import com.google.refine.freebase.protograph.Link; +import com.google.refine.freebase.protograph.Node; +import com.google.refine.freebase.protograph.NodeWithLinks; +import com.google.refine.freebase.protograph.Protograph; +import com.google.refine.freebase.protograph.ValueNode; +import com.google.refine.model.Cell; +import com.google.refine.model.Column; +import com.google.refine.model.Project; +import com.google.refine.model.Row; +import com.google.refine.model.Recon.Judgment; + +public class Transposer { + static public void transpose( + Project project, + FilteredRows filteredRows, + Protograph protograph, + Node rootNode, + TransposedNodeFactory nodeFactory + ) { + transpose(project, filteredRows, protograph, rootNode, nodeFactory, 20); + } + + static public void transpose( + Project project, + FilteredRows filteredRows, + Protograph protograph, + Node rootNode, + TransposedNodeFactory nodeFactory, + int limit + ) { + Context rootContext = new Context(rootNode, null, null, limit); + + filteredRows.accept(project, new RowVisitor() { + Context rootContext; + Protograph protograph; + Node rootNode; + TransposedNodeFactory nodeFactory; + + @Override + public boolean visit(Project project, int rowIndex, Row row) { + if (rootContext.limit <= 0 || rootContext.count < rootContext.limit) { + descend(project, protograph, nodeFactory, rowIndex, row, rootNode, rootContext); + } + + if (rootContext.limit > 0 && rootContext.count > rootContext.limit) { + return true; + } + return false; + } + + @Override + public void start(Project project) { + // TODO Auto-generated method stub + + } + + @Override + public void end(Project project) { + // TODO Auto-generated method stub + + } + + public RowVisitor init( + Context rootContext, + Protograph protograph, + Node rootNode, + TransposedNodeFactory nodeFactory + ) { + this.rootContext = rootContext; + this.protograph = protograph; + this.rootNode = rootNode; + this.nodeFactory = nodeFactory; + + return this; + } + }.init(rootContext, protograph, rootNode, nodeFactory)); + } + + static protected void descend( + Project project, + Protograph protograph, + TransposedNodeFactory nodeFactory, + int rowIndex, + Row row, + Node node, + Context context + ) { + List tnodes = new LinkedList(); + + TransposedNode parentNode = context.parent == null ? null : context.parent.transposedNode; + Link link = context.parent == null ? null : context.link; + + if (node instanceof CellNode) { + CellNode node2 = (CellNode) node; + for (String columnName : node2.columnNames) { + Column column = project.columnModel.getColumnByName(columnName); + if (column != null) { + int cellIndex = column.getCellIndex(); + + Cell cell = row.getCell(cellIndex); + if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) { + if (node2 instanceof CellTopicNode && + (cell.recon == null || cell.recon.judgment == Judgment.None)) { + return; + } + + context.count++; + if (context.limit > 0 && context.count > context.limit) { + return; + } + + tnodes.add(nodeFactory.transposeCellNode( + parentNode, + link, + node2, + rowIndex, + cellIndex, + cell + )); + } + } + } + } else { + if (node instanceof AnonymousNode) { + tnodes.add(nodeFactory.transposeAnonymousNode( + parentNode, + link, + (AnonymousNode) node, + rowIndex + )); + } else if (node instanceof FreebaseTopicNode) { + tnodes.add(nodeFactory.transposeTopicNode( + parentNode, + link, + (FreebaseTopicNode) node, + rowIndex + )); + } else if (node instanceof ValueNode) { + tnodes.add(nodeFactory.transposeValueNode( + parentNode, + link, + (ValueNode) node, + rowIndex + )); + } + } + + if (node instanceof NodeWithLinks) { + NodeWithLinks node2 = (NodeWithLinks) node; + int linkCount = node2.getLinkCount(); + + for (int i = 0; i < linkCount; i++) { + Link link2 = node2.getLink(i); + if (link2.condition == null || link2.condition.test(project, rowIndex, row)) { + for (TransposedNode tnode : tnodes) { + context.transposedNode = tnode; + context.nullifySubContextNodes(); + + descend( + project, + protograph, + nodeFactory, + rowIndex, + row, + link2.getTarget(), + context.subContexts.get(i) + ); + } + } + } + } + } + + static class Context { + TransposedNode transposedNode; + List subContexts; + Context parent; + Link link; + int count; + int limit; + + Context(Node node, Context parent, Link link, int limit) { + this.parent = parent; + this.link = link; + this.limit = limit; + + if (node instanceof NodeWithLinks) { + NodeWithLinks node2 = (NodeWithLinks) node; + + int subContextCount = node2.getLinkCount(); + + subContexts = new LinkedList(); + for (int i = 0; i < subContextCount; i++) { + Link link2 = node2.getLink(i); + subContexts.add( + new Context(link2.getTarget(), this, link2, -1)); + } + } + } + + public void nullifySubContextNodes() { + if (subContexts != null) { + for (Context context : subContexts) { + context.transposedNode = null; + context.nullifySubContextNodes(); + } + } + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java new file mode 100644 index 000000000..748e15c13 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/protograph/transpose/TripleLoaderTransposedNodeFactory.java @@ -0,0 +1,725 @@ +package com.google.refine.freebase.protograph.transpose; + +import java.io.IOException; +import java.io.Writer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.google.refine.freebase.FreebaseProperty; +import com.google.refine.freebase.FreebaseTopic; +import com.google.refine.freebase.protograph.AnonymousNode; +import com.google.refine.freebase.protograph.CellKeyNode; +import com.google.refine.freebase.protograph.CellNode; +import com.google.refine.freebase.protograph.CellTopicNode; +import com.google.refine.freebase.protograph.CellValueNode; +import com.google.refine.freebase.protograph.FreebaseTopicNode; +import com.google.refine.freebase.protograph.Link; +import com.google.refine.freebase.protograph.ValueNode; +import com.google.refine.model.Cell; +import com.google.refine.model.Column; +import com.google.refine.model.Project; +import com.google.refine.model.Recon; +import com.google.refine.model.Recon.Judgment; +import com.google.refine.model.recon.ReconConfig; +import com.google.refine.model.recon.StandardReconConfig; + +public class TripleLoaderTransposedNodeFactory implements TransposedNodeFactory { + protected Project project; + + protected boolean start = true; + protected Writer writer; + protected WritingTransposedNode lastRootNode; + protected Map varPool = new HashMap(); + protected Map newTopicVars = new HashMap(); + protected Set serializedRecons = new HashSet(); + + protected long contextID = 0; + protected int contextRowIndex; + protected int contextRefCount = 0; + protected JSONObject contextTreeRoot; + + public TripleLoaderTransposedNodeFactory(Project project, Writer writer) { + this.project = project; + this.writer = writer; + } + + @Override + public void flush() throws IOException { + if (lastRootNode != null) { + lastRootNode.write(null, null, project, -1, -1, null); + lastRootNode = null; + + writeContextTreeNode(); + } + } + + protected void writeLine(String line) { + try { + if (start) { + start = false; + } else { + writer.write('\n'); + } + writer.write(line); + } catch (IOException e) { + // ignore + } + } + + protected void writeRecon( + StringBuffer sb, + Project project, + int rowIndex, + int cellIndex, + Cell cell + ) { + Recon recon = cell.recon; + + sb.append("\"rec"); sb.append(Long.toString(recon.id)); sb.append("\""); + contextRefCount++; + + if (!serializedRecons.contains(recon.id)) { + serializedRecons.add(recon.id); + + Column column = project.columnModel.getColumnByCellIndex(cellIndex); + + // qa:sample_group + { + StringBuffer sb2 = new StringBuffer(); + + sb2.append("{ \"s\" : \"rec"); + sb2.append(Long.toString(recon.id)); + sb2.append("\", \"p\" : \"qa:sample_group\", \"o\" : "); + sb2.append(JSONObject.quote(column.getName())); + sb2.append(", \"ignore\" : true }"); + + writeLine(sb2.toString()); + } + + // qa:recon_data + { + StringBuffer sb2 = new StringBuffer(); + + String s = cell.value instanceof String ? (String) cell.value : cell.value.toString(); + + sb2.append("{ \"s\" : \"rec"); + sb2.append(Long.toString(recon.id)); + sb2.append("\", \"p\" : \"qa:recon_data\", \"ignore\" : true, \"o\" : { "); + + sb2.append(" \"history_entry\" : "); sb2.append(Long.toString(recon.judgmentHistoryEntry)); + sb2.append(", \"text\" : "); sb2.append(JSONObject.quote(s)); + sb2.append(", \"column\" : "); sb2.append(JSONObject.quote(column.getName())); + sb2.append(", \"service\" : "); sb2.append(JSONObject.quote(recon.service)); + sb2.append(", \"action\" : "); sb2.append(JSONObject.quote(recon.judgmentAction)); + sb2.append(", \"batch\" : "); sb2.append(Integer.toString(recon.judgmentBatchSize)); + + if (recon.judgment == Judgment.Matched) { + sb2.append(", \"matchRank\" : "); sb2.append(Integer.toString(recon.matchRank)); + sb2.append(", \"id\" : "); sb2.append(JSONObject.quote(recon.match.id)); + } + + ReconConfig reconConfig = column.getReconConfig(); + if (reconConfig != null && reconConfig instanceof StandardReconConfig) { + StandardReconConfig standardReconConfig = (StandardReconConfig) reconConfig; + sb2.append(", \"type\" : "); sb2.append(JSONObject.quote(standardReconConfig.typeID)); + } + + sb2.append(" } }"); + + writeLine(sb2.toString()); + } + } + } + + protected void writeLine( + String subject, String predicate, Object object, + Project project, + int subjectRowIndex, int subjectCellIndex, Cell subjectCell, + int objectRowIndex, int objectCellIndex, Cell objectCell, + boolean ignore + ) { + if (subject != null && object != null) { + String s = object instanceof String ? + JSONObject.quote((String) object) : object.toString(); + + StringBuffer sb = new StringBuffer(); + sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"'); + sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"'); + sb.append(", \"o\" : "); sb.append(s); + if (subjectCell != null || objectCell != null) { + sb.append(", \"meta\" : { "); + + sb.append("\"recon\" : { "); + if (subjectCell != null) { + sb.append("\"s\" : "); + writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell); + } + if (objectCell != null) { + if (subjectCell != null) { + sb.append(", "); + } + sb.append("\"o\" : "); + writeRecon(sb, project, objectRowIndex, objectCellIndex, objectCell); + } + sb.append(" }"); + + sb.append(" }"); + } + if (ignore) { + sb.append(", \"ignore\" : true"); + } + sb.append(" }"); + + writeLine(sb.toString()); + } + } + + protected void writeLine( + String subject, String predicate, Object object, String lang, + Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell, + boolean ignore + ) { + if (subject != null && object != null) { + String s = object instanceof String ? + JSONObject.quote((String) object) : object.toString(); + + StringBuffer sb = new StringBuffer(); + sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"'); + sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"'); + sb.append(", \"o\" : "); sb.append(s); + sb.append(", \"lang\" : "); sb.append(lang); + + if (subjectCell != null) { + sb.append(", \"meta\" : { "); + sb.append("\"recon\" : { "); + sb.append("\"s\" : "); + writeRecon(sb, project, subjectRowIndex, subjectCellIndex, subjectCell); + sb.append(" }"); + sb.append(" }"); + } + if (ignore) { + sb.append(", \"ignore\" : true"); + } + sb.append(" }"); + + writeLine(sb.toString()); + } + } + + abstract protected class WritingTransposedNode implements TransposedNode { + JSONObject jsonContextNode; + boolean load; + + public Object write( + String subject, String predicate, Project project, + int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { + + return internalWrite( + subject, predicate, project, + subjectRowIndex, subjectCellIndex, subjectCell); + } + + abstract public Object internalWrite( + String subject, String predicate, Project project, + int subjectRowIndex, int subjectCellIndex, Cell subjectCell); + } + + abstract protected class TransposedNodeWithChildren extends WritingTransposedNode { + public List links = new LinkedList(); + public List rowIndices = new LinkedList(); + public List children = new LinkedList(); + + protected void writeChildren( + String subject, Project project, + int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { + + for (int i = 0; i < children.size(); i++) { + WritingTransposedNode child = children.get(i); + Link link = links.get(i); + String predicate = link.property.id; + + child.write(subject, predicate, project, + subjectRowIndex, subjectCellIndex, subjectCell); + } + } + } + + protected class AnonymousTransposedNode extends TransposedNodeWithChildren { + + //protected AnonymousTransposedNode(AnonymousNode node) { } + + public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { + if (children.size() == 0 || subject == null) { + return null; + } + + StringBuffer sb = new StringBuffer(); + sb.append("{ \"s\" : \""); sb.append(subject); sb.append('"'); + sb.append(", \"p\" : \""); sb.append(predicate); sb.append('"'); + sb.append(", \"o\" : { "); + + StringBuffer sbRecon = new StringBuffer(); + + boolean first = true; + boolean firstRecon = true; + + if (subjectCell.recon != null) { + sbRecon.append("\"s\" : "); + writeRecon(sbRecon, project, subjectRowIndex, subjectCellIndex, subjectCell); + + firstRecon = false; + } + + for (int i = 0; i < children.size(); i++) { + WritingTransposedNode child = children.get(i); + Link link = links.get(i); + + FreebaseProperty property = link.property; + + Object c = child.internalWrite(null, null, project, subjectRowIndex, subjectCellIndex, null); + if (c != null) { + if (first) { + first = false; + } else { + sb.append(", "); + } + sb.append("\"" + property.id + "\": "); + sb.append(c instanceof String ? JSONObject.quote((String) c) : c.toString()); + } + + if (child instanceof CellTopicTransposedNode) { + CellTopicTransposedNode child2 = (CellTopicTransposedNode) child; + Recon recon = child2.cell.recon; + + if (recon != null && + (recon.judgment == Judgment.Matched || recon.judgment == Judgment.New)) { + + if (firstRecon) { + firstRecon = false; + } else { + sbRecon.append(", "); + } + + sbRecon.append("\""); sbRecon.append(property.id); sbRecon.append("\" : "); + + writeRecon(sbRecon, project, + rowIndices.get(i), child2.cellIndex, child2.cell); + } + } + } + sb.append(" }, \"meta\" : { \"recon\" : { "); + sb.append(sbRecon.toString()); + sb.append(" } } }"); + + writeLine(sb.toString()); + + return null; + } + } + + protected class CellTopicTransposedNode extends TransposedNodeWithChildren { + protected CellTopicNode node; + protected int rowIndex; + protected int cellIndex; + protected Cell cell; + + public CellTopicTransposedNode(CellTopicNode node, int rowIndex, int cellIndex, Cell cell) { + this.node = node; + this.rowIndex = rowIndex; + this.cellIndex = cellIndex; + this.cell = cell; + } + + public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { + String id = null; + if (cell.recon != null && cell.recon.judgment != Recon.Judgment.None) { + int objectRowIndex = rowIndex; + int objectCellIndex = cellIndex; + Cell objectCell = cell; + + if (cell.recon.judgment == Recon.Judgment.Matched) { + id = cell.recon.match.id; + + } else if (cell.recon.judgment == Judgment.New) { + if (newTopicVars.containsKey(cell.recon.id)) { + id = newTopicVars.get(cell.recon.id); + } else { + Column column = project.columnModel.getColumnByCellIndex(cellIndex); + String columnName = column.getName(); + + long var = 0; + if (varPool.containsKey(columnName)) { + var = varPool.get(columnName); + } + varPool.put(columnName, var + 1); + + id = "$" + columnName.replaceAll("\\W+", "_") + "_" + var; + + String typeID = node.type.id; + + ReconConfig reconConfig = column.getReconConfig(); + if (reconConfig instanceof StandardReconConfig) { + typeID = ((StandardReconConfig) reconConfig).typeID; + } + + writeLine(id, "type", typeID, project, rowIndex, cellIndex, cell, -1, -1, (Cell) null, !load); + writeLine(id, "name", cell.value, project, -1, -1, (Cell) null, -1, -1, (Cell) null, !load); + + if (cell.recon != null) { + newTopicVars.put(cell.recon.id, id); + } + } + } else { + return null; + } + + if (subject != null) { + writeLine(subject, predicate, id, project, + subjectRowIndex, subjectCellIndex, subjectCell, + objectRowIndex, objectCellIndex, objectCell, !load); + } + + writeChildren(id, project, objectRowIndex, objectCellIndex, objectCell); + } + + return id; + } + } + + protected class CellValueTransposedNode extends WritingTransposedNode { + protected JSONObject obj; + protected CellValueNode node; + protected int rowIndex; + protected int cellIndex; + protected Cell cell; + + public CellValueTransposedNode(CellValueNode node, int rowIndex, int cellIndex, Cell cell) { + this.node = node; + this.rowIndex = rowIndex; + this.cellIndex = cellIndex; + this.cell = cell; + } + + public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { + if (subject != null) { + if ("/type/text".equals(node.lang)) { + writeLine(subject, predicate, cell.value, node.lang, project, + subjectRowIndex, subjectCellIndex, subjectCell, !load); + } else { + writeLine(subject, predicate, cell.value, project, + subjectRowIndex, subjectCellIndex, subjectCell, + -1, -1, null, !load); + } + } + + return cell.value; + } + } + + protected class CellKeyTransposedNode extends WritingTransposedNode { + protected CellKeyNode node; + protected int rowIndex; + protected int cellIndex; + protected Cell cell; + + public CellKeyTransposedNode(CellKeyNode node, int rowIndex, int cellIndex, Cell cell) { + this.node = node; + this.rowIndex = rowIndex; + this.cellIndex = cellIndex; + this.cell = cell; + } + + public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { + writeLine(subject, "key", node.namespace.id + "/" + cell.value, project, + subjectRowIndex, subjectCellIndex, subjectCell, + -1, -1, null, !load); + + return null; + } + } + + protected class TopicTransposedNode extends TransposedNodeWithChildren { + protected FreebaseTopicNode node; + + public TopicTransposedNode(FreebaseTopicNode node) { + this.node = node; + } + + public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { + writeLine(subject, predicate, node.topic.id, project, + subjectRowIndex, subjectCellIndex, subjectCell, + -1, -1, null, !load); + + writeChildren(node.topic.id, project, -1, -1, null); + + return node.topic.id; + } + } + + protected class ValueTransposedNode extends WritingTransposedNode { + protected ValueNode node; + + public ValueTransposedNode(ValueNode node) { + this.node = node; + } + + public Object internalWrite(String subject, String predicate, Project project, int subjectRowIndex, int subjectCellIndex, Cell subjectCell) { + if ("/type/text".equals(node.lang)) { + writeLine(subject, predicate, node.value, node.lang, project, + subjectRowIndex, subjectCellIndex, subjectCell, !load); + } else { + writeLine(subject, predicate, node.value, project, + subjectRowIndex, subjectCellIndex, subjectCell, + -1, -1, null, !load); + } + + return node.value; + } + } + + public TransposedNode transposeAnonymousNode( + TransposedNode parentNode, + Link link, + AnonymousNode node, int rowIndex) { + + WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode; + WritingTransposedNode tnode = new AnonymousTransposedNode(); + + tnode.load = + (parentNode2 == null || parentNode2.load) && + (link == null || link.load); + + processTransposedNode(tnode, parentNode, link, rowIndex); + + tnode.jsonContextNode = addJsonContext( + parentNode2 != null ? parentNode2.jsonContextNode : null, + link != null ? link.property.id : null, + null + ); + + return tnode; + } + + public TransposedNode transposeCellNode( + TransposedNode parentNode, + Link link, + CellNode node, + int rowIndex, + int cellIndex, + Cell cell) { + + WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode; + + WritingTransposedNode tnode = null; + if (node instanceof CellTopicNode) { + if (cell.recon != null && + (cell.recon.judgment == Judgment.Matched || + cell.recon.judgment == Judgment.New)) { + + tnode = new CellTopicTransposedNode( + (CellTopicNode) node, rowIndex, cellIndex, cell); + } + } else if (node instanceof CellValueNode) { + tnode = new CellValueTransposedNode((CellValueNode) node, rowIndex, cellIndex, cell); + } else if (node instanceof CellKeyNode) { + tnode = new CellKeyTransposedNode((CellKeyNode) node, rowIndex, cellIndex, cell); + } + + if (tnode != null) { + tnode.load = + (parentNode2 == null || parentNode2.load) && + (link == null || link.load); + + processTransposedNode(tnode, parentNode, link, rowIndex); + + tnode.jsonContextNode = addJsonContext( + parentNode2 != null ? parentNode2.jsonContextNode : null, + link != null ? link.property.id : null, + cell, + rowIndex + ); + } + return tnode; + } + + public TransposedNode transposeTopicNode( + TransposedNode parentNode, + Link link, + FreebaseTopicNode node, + int rowIndex) { + + WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode; + WritingTransposedNode tnode = new TopicTransposedNode(node); + + tnode.load = + (parentNode2 == null || parentNode2.load) && + (link == null || link.load); + + processTransposedNode(tnode, parentNode, link, rowIndex); + + tnode.jsonContextNode = addJsonContext( + parentNode2 != null ? parentNode2.jsonContextNode : null, + link != null ? link.property.id : null, + node.topic + ); + + return tnode; + } + + public TransposedNode transposeValueNode( + TransposedNode parentNode, + Link link, + ValueNode node, + int rowIndex) { + + WritingTransposedNode parentNode2 = (WritingTransposedNode) parentNode; + WritingTransposedNode tnode = new ValueTransposedNode(node); + + tnode.load = + (parentNode2 == null || parentNode2.load) && + (link == null || link.load); + + processTransposedNode(tnode, parentNode, link, rowIndex); + + tnode.jsonContextNode = addJsonContext( + parentNode2 != null ? parentNode2.jsonContextNode : null, + link != null ? link.property.id : null, + node.value + ); + + return tnode; + } + + protected void processTransposedNode( + WritingTransposedNode tnode, + TransposedNode parentNode, + Link link, + int rowIndex + ) { + if (parentNode != null) { + if (parentNode instanceof TransposedNodeWithChildren) { + TransposedNodeWithChildren parentNode2 = (TransposedNodeWithChildren) parentNode; + parentNode2.rowIndices.add(rowIndex); + parentNode2.children.add(tnode); + parentNode2.links.add(link); + } + } else { + addRootNode(tnode, rowIndex); + } + } + + protected JSONObject addJsonContext(JSONObject parent, String key, Object value) { + JSONObject o = new JSONObject(); + + try { + if (value instanceof FreebaseTopic) { + FreebaseTopic topic = (FreebaseTopic) value; + o.put("id", topic.id); + o.put("name", topic.name); + } else { + o.put("v", value); + } + } catch (JSONException e) { + // ignore + } + + connectJsonContext(parent, o, key); + return o; + } + + protected JSONObject addJsonContext(JSONObject parent, String key, Cell cell, int rowIndex) { + JSONObject o = new JSONObject(); + + connectJsonContext(parent, o, key); + + try { + if (cell != null) { + o.put("v", cell.value); + if (cell.recon != null) { + o.put("recon", "rec" + cell.recon.id); + + if (cell.recon.judgment == Judgment.Matched) { + o.put("id", cell.recon.match.id); + o.put("name", cell.recon.match.name); + } + + // qa:display_context + { + StringBuffer sb2 = new StringBuffer(); + + sb2.append("{ \"ignore\" : true, \"s\" : \"rec"); + sb2.append(Long.toString(cell.recon.id)); + sb2.append("\", \"p\" : \"qa:display_context\", \"o\" : \"ctx"); + sb2.append(Long.toString(contextID)); + sb2.append("\", \"meta\" : { \"row\" : "); + sb2.append(Integer.toString(rowIndex)); + sb2.append(" } }"); + + writeLine(sb2.toString()); + } + } + } + } catch (JSONException e) { + // ignore + } + + return o; + } + + protected void connectJsonContext(JSONObject parent, JSONObject o, String key) { + try { + if (parent == null) { + contextTreeRoot = o; + } else { + JSONArray a = null; + if (parent.has(key)) { + a = parent.getJSONArray(key); + } else { + a = new JSONArray(); + parent.put(key, a); + } + + a.put(o); + } + } catch (JSONException e) { + // ignore + } + } + + protected void addRootNode(WritingTransposedNode tnode, int rowIndex) { + if (lastRootNode != null) { + lastRootNode.write(null, null, project, -1, -1, null); + writeContextTreeNode(); + } + lastRootNode = tnode; + + contextTreeRoot = null; + contextRowIndex = rowIndex; + contextRefCount = 0; + contextID++; + } + + protected void writeContextTreeNode() { + if (contextTreeRoot != null && contextRefCount > 0) { + StringBuffer sb = new StringBuffer(); + + sb.append("{ \"ignore\" : true, \"s\" : \"ctx"); + sb.append(Long.toString(contextID)); + sb.append("\", \"p\" : \"qa:context_data\", \"o\" : { \"row\" : "); + sb.append(Integer.toString(contextRowIndex)); + sb.append(", \"data\" : "); + sb.append(contextTreeRoot.toString()); + sb.append(" } }"); + + writeLine(sb.toString()); + } + } +} diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java new file mode 100644 index 000000000..aa2459974 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseDataExtensionJob.java @@ -0,0 +1,417 @@ +/** + * + */ +package com.google.refine.freebase.util; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.io.StringWriter; +import java.io.Writer; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.freebase.FreebaseType; +import com.google.refine.util.JSONUtilities; +import com.google.refine.util.ParsingUtilities; +import com.google.refine.model.ReconCandidate; + +public class FreebaseDataExtensionJob { + static public class DataExtension { + final public Object[][] data; + + public DataExtension(Object[][] data) { + this.data = data; + } + } + + static public class ColumnInfo { + final public List names; + final public List path; + final public FreebaseType expectedType; + + protected ColumnInfo(List names, List path, FreebaseType expectedType) { + this.names = names; + this.path = path; + this.expectedType = expectedType; + } + } + + final public JSONObject extension; + final public int columnCount; + final public List columns = new ArrayList(); + + public FreebaseDataExtensionJob(JSONObject obj) throws JSONException { + this.extension = obj; + this.columnCount = (obj.has("properties") && !obj.isNull("properties")) ? + countColumns(obj.getJSONArray("properties"), columns, new ArrayList(), new ArrayList()) : 0; + } + + public Map extend( + Set ids, + Map reconCandidateMap + ) throws Exception { + StringWriter writer = new StringWriter(); + formulateQuery(ids, extension, writer); + + String query = writer.toString(); + InputStream is = doMqlRead(query); + try { + String s = ParsingUtilities.inputStreamToString(is); + JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); + + Map map = new HashMap(); + if (o.has("result")) { + JSONArray a = o.getJSONArray("result"); + int l = a.length(); + + for (int i = 0; i < l; i++) { + JSONObject o2 = a.getJSONObject(i); + String id = o2.getString("id"); + FreebaseDataExtensionJob.DataExtension ext = collectResult(o2, reconCandidateMap); + + if (ext != null) { + map.put(id, ext); + } + } + } + + return map; + } finally { + is.close(); + } + } + + protected FreebaseDataExtensionJob.DataExtension collectResult( + JSONObject obj, + Map reconCandidateMap + ) throws JSONException { + List rows = new ArrayList(); + + collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap); + + Object[][] data = new Object[rows.size()][columnCount]; + rows.toArray(data); + + return new DataExtension(data); + } + + protected void storeCell( + List rows, + int row, + int col, + Object value, + Map reconCandidateMap + ) { + while (row >= rows.size()) { + rows.add(new Object[columnCount]); + } + rows.get(row)[col] = value; + } + + protected void storeCell( + List rows, + int row, + int col, + JSONObject obj, + Map reconCandidateMap + ) throws JSONException { + String id = obj.getString("id"); + ReconCandidate rc; + if (reconCandidateMap.containsKey(id)) { + rc = reconCandidateMap.get(id); + } else { + rc = new ReconCandidate( + obj.getString("id"), + obj.getString("name"), + JSONUtilities.getStringArray(obj, "type"), + 100 + ); + + reconCandidateMap.put(id, rc); + } + + storeCell(rows, row, col, rc, reconCandidateMap); + } + + protected int[] collectResult( + List rows, + JSONObject extNode, + JSONObject resultNode, + int startRowIndex, + int startColumnIndex, + Map reconCandidateMap + ) throws JSONException { + String propertyID = extNode.getString("id"); + String expectedTypeID = extNode.getJSONObject("expected").getString("id"); + + JSONArray a = resultNode != null && resultNode.has(propertyID) && !resultNode.isNull(propertyID) ? + resultNode.getJSONArray(propertyID) : null; + + if (expectedTypeID.startsWith("/type/")) { + if (a != null) { + int l = a.length(); + for (int r = 0; r < l; r++) { + Object o = a.isNull(r) ? null : a.get(r); + if (o instanceof Serializable) { + storeCell(rows, startRowIndex++, startColumnIndex, o, reconCandidateMap); + } + } + } + + // note that we still take up a column even if we don't have any data + return new int[] { startRowIndex, startColumnIndex + 1 }; + } else { + boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties")); + boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included")); + + if (a != null && a.length() > 0) { + int maxColIndex = startColumnIndex; + + int l = a.length(); + for (int r = 0; r < l; r++) { + Object v = a.isNull(r) ? null : a.get(r); + JSONObject o = v != null && v instanceof JSONObject ? (JSONObject) v : null; + + int startColumnIndex2 = startColumnIndex; + int startRowIndex2 = startRowIndex; + + if (isOwnColumn) { + if (o != null) { + storeCell(rows, startRowIndex2++, startColumnIndex2++, o, reconCandidateMap); + } else { + storeCell(rows, startRowIndex2++, startColumnIndex2++, v, reconCandidateMap); + } + } + + if (hasSubProperties && o != null) { + int[] rowcol = collectResult( + rows, + extNode.getJSONArray("properties"), + o, + startRowIndex, + startColumnIndex2, + reconCandidateMap + ); + + startRowIndex2 = rowcol[0]; + startColumnIndex2 = rowcol[1]; + } + + startRowIndex = startRowIndex2; + maxColIndex = Math.max(maxColIndex, startColumnIndex2); + } + + return new int[] { startRowIndex, maxColIndex }; + } else { + return new int[] { + startRowIndex, + startColumnIndex + countColumns(extNode, null, new ArrayList(), new ArrayList()) + }; + } + } + } + + protected int[] collectResult( + List rows, + JSONArray subProperties, + JSONObject resultNode, + int startRowIndex, + int startColumnIndex, + Map reconCandidateMap + ) throws JSONException { + int maxStartRowIndex = startRowIndex; + + int k = subProperties.length(); + for (int c = 0; c < k; c++) { + int[] rowcol = collectResult( + rows, + subProperties.getJSONObject(c), + resultNode, + startRowIndex, + startColumnIndex, + reconCandidateMap + ); + + maxStartRowIndex = Math.max(maxStartRowIndex, rowcol[0]); + startColumnIndex = rowcol[1]; + } + + return new int[] { maxStartRowIndex, startColumnIndex }; + } + + + static protected InputStream doMqlRead(String query) throws IOException { + URL url = new URL("http://api.freebase.com/api/service/mqlread"); + + URLConnection connection = url.openConnection(); + connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + connection.setConnectTimeout(5000); + connection.setDoOutput(true); + + DataOutputStream dos = new DataOutputStream(connection.getOutputStream()); + try { + String body = "extended=1&query=" + ParsingUtilities.encode(query); + + dos.writeBytes(body); + } finally { + dos.flush(); + dos.close(); + } + + connection.connect(); + + return connection.getInputStream(); + } + + static protected void formulateQuery(Set ids, JSONObject node, Writer writer) throws JSONException { + JSONWriter jsonWriter = new JSONWriter(writer); + + jsonWriter.object(); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("id"); jsonWriter.value(null); + jsonWriter.key("id|="); + jsonWriter.array(); + for (String id : ids) { + if (id != null) { + jsonWriter.value(id); + } + } + jsonWriter.endArray(); + + formulateQueryNode(node.getJSONArray("properties"), jsonWriter); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); + } + + static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException { + String propertyID = node.getString("id"); + String expectedTypeID = node.getJSONObject("expected").getString("id"); + + writer.key(propertyID); + writer.array(); + { + if (!expectedTypeID.startsWith("/type/")) { // not literal + writer.object(); + writer.key("optional"); writer.value(true); + + boolean hasLimit = false; + if (node.has("constraints") && !node.isNull("constraints")) { + JSONObject constraints = node.getJSONObject("constraints"); + + String[] names = JSONObject.getNames(constraints); + for (String name : names) { + Object value = constraints.get(name); + if (name.equals("limit")) { + hasLimit = true; + } + + if (!name.contains(":") && + !name.equals("limit") && + !name.equals("optional") && + !name.equals("count") && + !name.equals("estimate-count") && + !name.equals("sort") && + !name.equals("return")) { + + if (name.startsWith("!")) { + name = "!c:" + name.substring(1); + } else { + name = "c:" + name; + } + } + writer.key(name); + writer.value(value); + } + } + if (!hasLimit) { + writer.key("limit"); writer.value(10); + } + + { + boolean hasSubProperties = (node.has("properties") && !node.isNull("properties")); + + if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) { + writer.key("name"); writer.value(null); + writer.key("id"); writer.value(null); + writer.key("type"); writer.array(); writer.endArray(); + } + + if (hasSubProperties) { + formulateQueryNode(node.getJSONArray("properties"), writer); + } + } + writer.endObject(); + } + } + writer.endArray(); + } + + static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException { + int l = propertiesA.length(); + + for (int i = 0; i < l; i++) { + formulateQueryNode(propertiesA.getJSONObject(i), writer); + } + } + + static protected int countColumns(JSONObject obj, List columns, List names, List path) throws JSONException { + String name = obj.getString("name"); + + List names2 = null; + List path2 = null; + if (columns != null) { + names2 = new ArrayList(names); + names2.add(name); + + path2 = new ArrayList(path); + path2.add(obj.getString("id")); + } + + if (obj.has("properties") && !obj.isNull("properties")) { + boolean included = (obj.has("included") && obj.getBoolean("included")); + if (included && columns != null) { + JSONObject expected = obj.getJSONObject("expected"); + + columns.add(new ColumnInfo(names2, path2, + new FreebaseType(expected.getString("id"), expected.getString("name")))); + } + + return (included ? 1 : 0) + + countColumns(obj.getJSONArray("properties"), columns, names2, path2); + } else { + if (columns != null) { + JSONObject expected = obj.getJSONObject("expected"); + + columns.add(new ColumnInfo(names2, path2, + new FreebaseType(expected.getString("id"), expected.getString("name")))); + } + return 1; + } + } + + static protected int countColumns(JSONArray a, List columns, List names, List path) throws JSONException { + int c = 0; + int l = a.length(); + for (int i = 0; i < l; i++) { + c += countColumns(a.getJSONObject(i), columns, names, path); + } + return c; + } +} \ No newline at end of file diff --git a/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java new file mode 100644 index 000000000..739c5e3c8 --- /dev/null +++ b/extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java @@ -0,0 +1,223 @@ +package com.google.refine.freebase.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; + +import oauth.signpost.OAuthConsumer; +import oauth.signpost.exception.OAuthCommunicationException; +import oauth.signpost.exception.OAuthExpectationFailedException; +import oauth.signpost.exception.OAuthMessageSignerException; + +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.params.CoreProtocolPNames; +import org.apache.http.util.EntityUtils; +import org.json.JSONException; +import org.json.JSONObject; + +import com.google.refine.ProjectManager; +import com.google.refine.RefineServlet; +import com.google.refine.oauth.Credentials; +import com.google.refine.oauth.OAuthUtilities; +import com.google.refine.oauth.Provider; + +public class FreebaseUtils { + + static final public String FREEBASE_HOST = "www.freebase.com"; + + static final private String FREEQ_URL = "http://data.labs.freebase.com/freeq/refine"; + + static final private String AGENT_ID = "/en/google_refine"; + + private static String getUserInfoURL(String host) { + return "http://" + host + "/api/service/user_info"; + } + + private static String getMQLWriteURL(String host) { + return "http://" + host + "/api/service/mqlwrite"; + } + + private static String getMQLReadURL(String host) { + return "http://" + host + "/api/service/mqlread"; + } + + private static String getUserAgent() { + return RefineServlet.FULLNAME; + } + + public static String getUserInfo(Credentials credentials, Provider provider) + throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException { + + OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); + + HttpGet httpRequest = new HttpGet(getUserInfoURL(provider.getHost())); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); + + // this is required by the Metaweb API to avoid XSS + httpRequest.setHeader("X-Requested-With", "1"); + + // sign the request with the oauth library + consumer.sign(httpRequest); + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); + } + + public static String getUserBadges(Provider provider, String user_id) + throws ClientProtocolException, IOException, JSONException { + + String query = "{" + + "'id' : '" + user_id + "'," + + "'!/type/usergroup/member' : [{" + + "'id' : null," + + "'key' : [{" + + "'namespace' : null" + + "}]" + + "}]" + + "}".replace("'", "\""); + + return mqlread(provider, query); + } + + public static String mqlread(Provider provider, String query) + throws ClientProtocolException, IOException, JSONException { + + JSONObject envelope = new JSONObject(); + envelope.put("query", new JSONObject(query)); + + List formparams = new ArrayList(); + formparams.add(new BasicNameValuePair("query", envelope.toString())); + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); + + HttpPost httpRequest = new HttpPost(getMQLReadURL(provider.getHost())); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); + httpRequest.setEntity(entity); + + // this is required by the Metaweb API to avoid XSS + httpRequest.setHeader("X-Requested-With", "1"); + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); + } + + public static String mqlwrite(Credentials credentials, Provider provider, String query) + throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, IOException, JSONException { + OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); + + JSONObject envelope = new JSONObject(); + envelope.put("query", new JSONObject(query)); + + List formparams = new ArrayList(); + formparams.add(new BasicNameValuePair("query", envelope.toString())); + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); + + HttpPost httpRequest = new HttpPost(getMQLWriteURL(provider.getHost())); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); + httpRequest.setEntity(entity); + + // this is required by the Metaweb API to avoid XSS + httpRequest.setHeader("X-Requested-With", "1"); + + // sign the request with the oauth library + consumer.sign(httpRequest); + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); + } + + public static String uploadTriples( + HttpServletRequest request, + String qa, + String source_name, + String source_id, + String mdo_id, + String triples + ) throws OAuthMessageSignerException, OAuthExpectationFailedException, OAuthCommunicationException, ClientProtocolException, JSONException, IOException { + + Provider provider = OAuthUtilities.getProvider(FREEBASE_HOST); + + Credentials credentials = Credentials.getCredentials(request, provider, Credentials.Type.ACCESS); + + JSONObject mdo_info = new JSONObject(); + mdo_info.put("name", source_name); + if (source_id != null) { + mdo_info.put("info_source",source_id); + } + + JSONObject user_info = new JSONObject(getUserInfo(credentials, provider)); + if (user_info.has("username")) { + + List formparams = new ArrayList(); + formparams.add(new BasicNameValuePair("user", user_info.getString("id"))); + formparams.add(new BasicNameValuePair("action_type", "LOAD_TRIPLE")); + formparams.add(new BasicNameValuePair("operator", user_info.getString("id"))); + formparams.add(new BasicNameValuePair("software_tool_used", AGENT_ID)); + formparams.add(new BasicNameValuePair("mdo_info", mdo_info.toString())); + formparams.add(new BasicNameValuePair("graphport", "sandbox")); + formparams.add(new BasicNameValuePair("payload", triples)); + formparams.add(new BasicNameValuePair("check_params", "false")); + if (mdo_id != null) { + formparams.add(new BasicNameValuePair("mdo_guid", mdo_id)); + } + if (Boolean.parseBoolean(qa)) { + formparams.add(new BasicNameValuePair("rabj", "true")); + } + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8"); + + HttpPost httpRequest = new HttpPost(getFreeQUrl()); + httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, getUserAgent()); + httpRequest.setEntity(entity); + + HttpPost surrogateRequest = new HttpPost(getUserInfoURL(FREEBASE_HOST)); + surrogateRequest.setEntity(entity); + + OAuthConsumer consumer = OAuthUtilities.getConsumer(credentials, provider); + + consumer.sign(surrogateRequest); + + Header[] h = surrogateRequest.getHeaders("Authorization"); + if (h.length > 0) { + httpRequest.setHeader("X-Freebase-Credentials", h[0].getValue()); + } else { + throw new RuntimeException("Couldn't find the oauth signature header in the surrogate request"); + } + + // execute the request + HttpClient httpClient = new DefaultHttpClient(); + HttpResponse httpResponse = httpClient.execute(httpRequest); + + // return the results + return EntityUtils.toString(httpResponse.getEntity()); + } else { + throw new RuntimeException("Invalid credentials"); + } + } + + static public String getFreeQUrl() { + String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq"); + return url != null ? url : FREEQ_URL; + } +}