diff --git a/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java b/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java index 00e23e8a2..92d242f29 100644 --- a/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java +++ b/src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java @@ -47,8 +47,7 @@ public class FreebaseDataExtensionJob { StringWriter writer = new StringWriter(); formulateQuery(guids, extension, writer); - URL url = new URL("http://api.freebase.com/api/service/mqlread"); - InputStream is = doPost(url, "query", writer.toString()); + InputStream is = doMqlRead(writer.toString()); try { String s = ParsingUtilities.inputStreamToString(is); JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s); @@ -143,7 +142,7 @@ public class FreebaseDataExtensionJob { boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties")); boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included")); - if (a != null) { + if (a != null && a.length() > 0) { int maxColIndex = startColumnIndex; int l = a.length(); @@ -166,16 +165,19 @@ public class FreebaseDataExtensionJob { startColumnIndex2 ); - startRowIndex = rowcol[0]; - maxColIndex = Math.max(maxColIndex, rowcol[1]); + startRowIndex2 = rowcol[0]; + startColumnIndex2 = rowcol[1]; } + + startRowIndex = startRowIndex2; + maxColIndex = Math.max(maxColIndex, startColumnIndex2); } return new int[] { startRowIndex, maxColIndex }; } else { return new int[] { startRowIndex, - countColumns(extNode, null) + startColumnIndex + countColumns(extNode, null) }; } } @@ -208,109 +210,112 @@ public class FreebaseDataExtensionJob { } -static protected InputStream doPost(URL url, String name, String load) throws IOException { - URLConnection connection = url.openConnection(); - connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); - connection.setConnectTimeout(5000); - connection.setDoOutput(true); - - DataOutputStream dos = new DataOutputStream(connection.getOutputStream()); - try { - String data = name + "=" + ParsingUtilities.encode(load); + static protected InputStream doMqlRead(String query) throws IOException { + URL url = new URL("http://api.freebase.com/api/service/mqlread"); + + URLConnection connection = url.openConnection(); + connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + connection.setConnectTimeout(5000); + connection.setDoOutput(true); - dos.writeBytes(data); - } finally { - dos.flush(); - dos.close(); + DataOutputStream dos = new DataOutputStream(connection.getOutputStream()); + try { + String body = "extended=1&query=" + ParsingUtilities.encode(query); + + dos.writeBytes(body); + } finally { + dos.flush(); + dos.close(); + } + + connection.connect(); + + return connection.getInputStream(); } - connection.connect(); - - return connection.getInputStream(); -} - -static protected void formulateQuery(Set guids, JSONObject node, Writer writer) throws JSONException { - JSONWriter jsonWriter = new JSONWriter(writer); - - jsonWriter.object(); - jsonWriter.key("query"); - jsonWriter.array(); + static protected void formulateQuery(Set guids, JSONObject node, Writer writer) throws JSONException { + JSONWriter jsonWriter = new JSONWriter(writer); + jsonWriter.object(); - - jsonWriter.key("guid"); jsonWriter.value(null); - jsonWriter.key("guid|="); - jsonWriter.array(); - for (String guid : guids) { - jsonWriter.value(guid); - } - jsonWriter.endArray(); - - formulateQueryNode(node.getJSONArray("properties"), jsonWriter); - - jsonWriter.endObject(); - jsonWriter.endArray(); - jsonWriter.endObject(); -} - -static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException { - String propertyID = node.getString("id"); - String expectedTypeID = node.getString("expected"); - - writer.key(propertyID); - writer.array(); - { - if (!expectedTypeID.startsWith("/type/")) { // not literal - writer.object(); - writer.key("limit"); writer.value(10); - { - boolean hasSubProperties = (node.has("properties") && !node.isNull("properties")); + jsonWriter.key("query"); + jsonWriter.array(); + jsonWriter.object(); + + jsonWriter.key("guid"); jsonWriter.value(null); + jsonWriter.key("guid|="); + jsonWriter.array(); + for (String guid : guids) { + jsonWriter.value(guid); + } + jsonWriter.endArray(); - if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) { - writer.key("name"); writer.value(null); - writer.key("id"); writer.value(null); - writer.key("guid"); writer.value(null); - writer.key("type"); writer.array(); writer.endArray(); - } - - if (hasSubProperties) { - formulateQueryNode(node.getJSONArray("properties"), writer); - } - } - writer.endObject(); - } + formulateQueryNode(node.getJSONArray("properties"), jsonWriter); + + jsonWriter.endObject(); + jsonWriter.endArray(); + jsonWriter.endObject(); } - writer.endArray(); -} - -static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException { - int l = propertiesA.length(); - for (int i = 0; i < l; i++) { - formulateQueryNode(propertiesA.getJSONObject(i), writer); - } -} - -static protected int countColumns(JSONObject obj, List columnNames) throws JSONException { - if (obj.has("properties") && !obj.isNull("properties")) { - boolean included = (obj.has("included") && obj.getBoolean("included")); - if (included && columnNames != null) { - columnNames.add(obj.getString("name")); + static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException { + String propertyID = node.getString("id"); + String expectedTypeID = node.getString("expected"); + + writer.key(propertyID); + writer.array(); + { + if (!expectedTypeID.startsWith("/type/")) { // not literal + writer.object(); + writer.key("limit"); writer.value(10); + writer.key("optional"); writer.value(true); + { + boolean hasSubProperties = (node.has("properties") && !node.isNull("properties")); + + if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) { + writer.key("name"); writer.value(null); + writer.key("id"); writer.value(null); + writer.key("guid"); writer.value(null); + writer.key("type"); writer.array(); writer.endArray(); + } + + if (hasSubProperties) { + formulateQueryNode(node.getJSONArray("properties"), writer); + } + } + writer.endObject(); + } } - return (included ? 1 : 0) + countColumns(obj.getJSONArray("properties"), columnNames); - } else { - if (columnNames != null) { - columnNames.add(obj.getString("name")); + writer.endArray(); + } + + static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException { + int l = propertiesA.length(); + + for (int i = 0; i < l; i++) { + formulateQueryNode(propertiesA.getJSONObject(i), writer); } - return 1; } -} - -static protected int countColumns(JSONArray a, List columnNames) throws JSONException { - int c = 0; - int l = a.length(); - for (int i = 0; i < l; i++) { - c += countColumns(a.getJSONObject(i), columnNames); + + static protected int countColumns(JSONObject obj, List columnNames) throws JSONException { + if (obj.has("properties") && !obj.isNull("properties")) { + boolean included = (obj.has("included") && obj.getBoolean("included")); + if (included && columnNames != null) { + columnNames.add(obj.getString("name")); + } + return (included ? 1 : 0) + countColumns(obj.getJSONArray("properties"), columnNames); + } else { + if (columnNames != null) { + columnNames.add(obj.getString("name")); + } + return 1; + } + } + + static protected int countColumns(JSONArray a, List columnNames) throws JSONException { + int c = 0; + int l = a.length(); + for (int i = 0; i < l; i++) { + c += countColumns(a.getJSONObject(i), columnNames); + } + return c; } - return c; -} } \ No newline at end of file diff --git a/src/main/webapp/scripts/dialogs/extend-data-preview-dialog.js b/src/main/webapp/scripts/dialogs/extend-data-preview-dialog.js index e602990f4..088007a77 100644 --- a/src/main/webapp/scripts/dialogs/extend-data-preview-dialog.js +++ b/src/main/webapp/scripts/dialogs/extend-data-preview-dialog.js @@ -5,7 +5,7 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) { this._extension = { properties: [] }; var self = this; - var frame = DialogSystem.createDialog(); + var frame = this._frame = DialogSystem.createDialog(); frame.width("900px").addClass("extend-data-preview-dialog"); var header = $('
').addClass("dialog-header").text("Add Columns from Freebase Based on Column " + column.name).appendTo(frame); @@ -25,7 +25,7 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) { 'Suggested Properties' + '' + '' + - '
' + + '
' + '' + '' ).appendTo(body); @@ -41,16 +41,100 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) { DialogSystem.dismissUntil(self._level - 1); }).appendTo(footer); - this._level = DialogSystem.showDialog(frame); + ExtendDataPreviewDialog.getAllProperties(column.reconConfig.type.id, function(properties) { + self._show(properties); + }); +}; + +ExtendDataPreviewDialog.getAllProperties = function(typeID, onDone) { + var query = { + id : typeID, + type : "/type/type", + "/freebase/type_hints/included_types" : [{ + optional: true, + properties : [{ + id : null, + name : null, + "/type/property/expected_type" : { + id : null, + "/freebase/type_hints/mediator" : [] + }, + sort : "name" + }] + }], + properties : [{ + id : null, + name : null, + "/type/property/expected_type" : { + id : null, + "/freebase/type_hints/mediator" : [] + }, + sort : "name" + }] + }; + + var allProperties = []; + var processProperty = function(property) { + var expectedType = property["/type/property/expected_type"]; + if (expectedType["/freebase/type_hints/mediator"].length > 0 && expectedType["/freebase/type_hints/mediator"][0]) { + + } else { + allProperties.push({ + id : property.id, + name : property.name, + expected : expectedType.id + }); + } + }; + var processProperties = function(properties) { + $.each(properties, function() { processProperty(this); }); + }; + + $.getJSON( + "http://api.freebase.com/api/service/mqlread?query=" + escape(JSON.stringify({ query : query })) + "&callback=?", + null, + function(o) { + if ("result" in o) { + processProperties(o.result.properties); + $.each(o.result["/freebase/type_hints/included_types"], function() { + processProperties(this.properties); + }) + + onDone(allProperties); + } else { + onDone([]); + } + }, + "jsonp" + ); +}; + +ExtendDataPreviewDialog.prototype._show = function(properties) { + this._level = DialogSystem.showDialog(this._frame); + + var self = this; + var container = this._elmts.suggestedPropertyContainer; + var renderSuggestedProperty = function(property) { + var div = $('
').addClass("suggested-property").appendTo(container); + $('').attr("href", "javascript:{}").text(property.name).appendTo(div).click(function() { + self._addProperty(property); + }); + }; + for (var i = 0; i < properties.length; i++) { + renderSuggestedProperty(properties[i]); + } var suggestConfig = { - type: '/type/property' + type: '/type/property', + schema: this._column.reconConfig.type.id }; - if ("reconConfig" in column) { - suggestConfig.schema = column.reconConfig.type.id; - } + this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) { - self._addProperty(data); + self._addProperty({ + id : data.id, + name: data.name, + expected: data["/type/property/expected_type"] + }); }); }; @@ -77,11 +161,7 @@ ExtendDataPreviewDialog.prototype._update = function() { }; ExtendDataPreviewDialog.prototype._addProperty = function(p) { - this._extension.properties.push({ - id : p.id, - name: p.name, - expected: p["/type/property/expected_type"] - }); + this._extension.properties.push(p); this._update(); }; @@ -93,6 +173,12 @@ ExtendDataPreviewDialog.prototype._renderPreview = function(data) { } var table = $('')[0]; + var trHead = table.insertRow(table.rows.length); + $(trHead.insertCell(trHead.cells.length)).text(this._column.name); + + for (var c = 0; c < data.columns.length; c++) { + $(trHead.insertCell(trHead.cells.length)).text(data.columns[c]); + } for (var r = 0; r < data.rows.length; r++) { var tr = table.insertRow(table.rows.length); diff --git a/src/main/webapp/scripts/views/data-table-column-header-ui.js b/src/main/webapp/scripts/views/data-table-column-header-ui.js index 941ec6119..462b05059 100644 --- a/src/main/webapp/scripts/views/data-table-column-header-ui.js +++ b/src/main/webapp/scripts/views/data-table-column-header-ui.js @@ -819,13 +819,17 @@ DataTableColumnHeaderUI.prototype._doAddColumn = function(initialExpression) { }; DataTableColumnHeaderUI.prototype._doAddColumnFromFreebase = function() { - var o = DataTableView.sampleVisibleRows(this._column); + if ("reconConfig" in this._column && "type" in this._column.reconConfig) { + var o = DataTableView.sampleVisibleRows(this._column); - new ExtendDataPreviewDialog( - this._column, - o.rowIndices, - function() {} - ); + new ExtendDataPreviewDialog( + this._column, + o.rowIndices, + function() {} + ); + } else { + alert("This column has not been reconciled yet."); + } }; DataTableColumnHeaderUI.prototype._doRemoveColumn = function() { diff --git a/src/main/webapp/styles/dialogs/extend-data-preview-dialog.css b/src/main/webapp/styles/dialogs/extend-data-preview-dialog.css index b0de89675..b8b1dcd83 100644 --- a/src/main/webapp/styles/dialogs/extend-data-preview-dialog.css +++ b/src/main/webapp/styles/dialogs/extend-data-preview-dialog.css @@ -1,10 +1,14 @@ .extend-data-preview-dialog .suggested-property-container { border: 1px solid #aaa; - padding: 10px; + padding: 5px; overflow: auto; height: 100%; } +.extend-data-preview-dialog .suggested-property { + padding: 5px; +} + .extend-data-preview-dialog input.property-suggest { display: block; padding: 2%; @@ -13,7 +17,15 @@ .extend-data-preview-dialog .preview-container { border: 1px solid #aaa; - padding: 10px; overflow: auto; height: 100%; -} \ No newline at end of file +} + +.extend-data-preview-dialog .preview-container table { + border-collapse: collapse; +} + +.extend-data-preview-dialog .preview-container td { + padding: 3px 5px; + border-bottom: 1px solid #ddd; +}