More work on the extend data preview dialog. Results now looks correct, but we still are not handling CVTs.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@295 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java

@@ -47,8 +47,7 @@ public class FreebaseDataExtensionJob {
         StringWriter writer = new StringWriter();
         formulateQuery(guids, extension, writer);
         
-        URL url = new URL("http://api.freebase.com/api/service/mqlread");
+        InputStream is = doMqlRead(writer.toString());
-        InputStream is = doPost(url, "query", writer.toString());
         try {
             String s = ParsingUtilities.inputStreamToString(is);
             JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
@@ -143,7 +142,7 @@ public class FreebaseDataExtensionJob {
             boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties")); 
             boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included"));
             
-            if (a != null) {
+            if (a != null && a.length() > 0) {
                 int maxColIndex = startColumnIndex;
                 
                 int l = a.length();
@@ -166,16 +165,19 @@ public class FreebaseDataExtensionJob {
                             startColumnIndex2
                         );
                         
-                        startRowIndex = rowcol[0];
+                        startRowIndex2 = rowcol[0];
-                        maxColIndex = Math.max(maxColIndex, rowcol[1]);
+                        startColumnIndex2 = rowcol[1];
                     }
+                    
+                    startRowIndex = startRowIndex2;
+                    maxColIndex = Math.max(maxColIndex, startColumnIndex2);
                 }
                 
                 return new int[] { startRowIndex, maxColIndex };
             } else {
                 return new int[] {
                     startRowIndex,
-                    countColumns(extNode, null)
+                    startColumnIndex + countColumns(extNode, null)
                 };
             }
         }
@@ -208,7 +210,9 @@ public class FreebaseDataExtensionJob {
     }
 
 
-static protected InputStream doPost(URL url, String name, String load) throws IOException {
+    static protected InputStream doMqlRead(String query) throws IOException {
+        URL url = new URL("http://api.freebase.com/api/service/mqlread");
+
         URLConnection connection = url.openConnection();
         connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
         connection.setConnectTimeout(5000);
@@ -216,9 +220,9 @@ static protected InputStream doPost(URL url, String name, String load) throws IO
         
         DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
         try {
-        String data = name + "=" + ParsingUtilities.encode(load);
+            String body = "extended=1&query=" + ParsingUtilities.encode(query);
             
-        dos.writeBytes(data);
+            dos.writeBytes(body);
         } finally {
             dos.flush();
             dos.close();
@@ -262,6 +266,7 @@ static protected void formulateQueryNode(JSONObject node, JSONWriter writer) thr
             if (!expectedTypeID.startsWith("/type/")) { // not literal
                 writer.object();
                 writer.key("limit"); writer.value(10);
+                writer.key("optional"); writer.value(true);
                 {
                     boolean hasSubProperties = (node.has("properties") && !node.isNull("properties")); 
                         

src/main/webapp/scripts/dialogs/extend-data-preview-dialog.js

@@ -5,7 +5,7 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
     this._extension = { properties: [] };
 
     var self = this;
-    var frame = DialogSystem.createDialog();
+    var frame = this._frame = DialogSystem.createDialog();
     frame.width("900px").addClass("extend-data-preview-dialog");
     
     var header = $('<div></div>').addClass("dialog-header").text("Add Columns from Freebase Based on Column " + column.name).appendTo(frame);
@@ -25,7 +25,7 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
                 '<td height="1">Suggested Properties</td>' +
             '</tr>' +
             '<tr>' +
-                '<td height="99%"><div class="suggested-property-container"></div></td>' +
+                '<td height="99%"><div class="suggested-property-container" bind="suggestedPropertyContainer"></div></td>' +
             '</tr>' +
         '</table></div>'
     ).appendTo(body);
@@ -41,16 +41,100 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
         DialogSystem.dismissUntil(self._level - 1);
     }).appendTo(footer);
     
-    this._level = DialogSystem.showDialog(frame);
+    ExtendDataPreviewDialog.getAllProperties(column.reconConfig.type.id, function(properties) {
+        self._show(properties);
+    });
+};
+
+ExtendDataPreviewDialog.getAllProperties = function(typeID, onDone) {
+    var query = {
+        id : typeID,
+        type : "/type/type",
+        "/freebase/type_hints/included_types" : [{
+            optional: true,
+            properties : [{
+                id : null,
+                name : null,
+                "/type/property/expected_type" : {
+                    id : null,
+                    "/freebase/type_hints/mediator" : []
+                },
+                sort : "name"
+            }]
+        }],
+        properties : [{
+            id : null,
+            name : null,
+            "/type/property/expected_type" : {
+                id : null,
+                "/freebase/type_hints/mediator" : []
+            },
+            sort : "name"
+        }]
+    };
+    
+    var allProperties = [];
+    var processProperty = function(property) {
+        var expectedType = property["/type/property/expected_type"];
+        if (expectedType["/freebase/type_hints/mediator"].length > 0 && expectedType["/freebase/type_hints/mediator"][0]) {
+            
+        } else {
+            allProperties.push({
+                id : property.id,
+                name : property.name,
+                expected : expectedType.id
+            });
+        }
+    };
+    var processProperties = function(properties) {
+        $.each(properties, function() { processProperty(this); });
+    };
+    
+    $.getJSON(
+        "http://api.freebase.com/api/service/mqlread?query=" + escape(JSON.stringify({ query : query })) + "&callback=?",
+        null,
+        function(o) {
+            if ("result" in o) {
+                processProperties(o.result.properties);
+                $.each(o.result["/freebase/type_hints/included_types"], function() {
+                    processProperties(this.properties);
+                })
+                
+                onDone(allProperties);
+            } else {
+                onDone([]);
+            }
+        },
+        "jsonp"
+    );    
+};
+
+ExtendDataPreviewDialog.prototype._show = function(properties) {
+    this._level = DialogSystem.showDialog(this._frame);
+    
+    var self = this;
+    var container = this._elmts.suggestedPropertyContainer;
+    var renderSuggestedProperty = function(property) {
+        var div = $('<div>').addClass("suggested-property").appendTo(container);
+        $('<a>').attr("href", "javascript:{}").text(property.name).appendTo(div).click(function() {
+            self._addProperty(property);
+        });
+    };
+    for (var i = 0; i < properties.length; i++) {
+        renderSuggestedProperty(properties[i]);
+    }
     
     var suggestConfig = {
-        type: '/type/property'
+        type: '/type/property',
+        schema: this._column.reconConfig.type.id
     };
-    if ("reconConfig" in column) {
+    
-        suggestConfig.schema = column.reconConfig.type.id;
-    }    
     this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
-        self._addProperty(data);
+        self._addProperty({
+            id : data.id,
+            name: data.name,
+            expected: data["/type/property/expected_type"]
+        });
     });
 };
 
@@ -77,11 +161,7 @@ ExtendDataPreviewDialog.prototype._update = function() {
 };
 
 ExtendDataPreviewDialog.prototype._addProperty = function(p) {
-    this._extension.properties.push({
+    this._extension.properties.push(p);
-        id : p.id,
-        name: p.name,
-        expected: p["/type/property/expected_type"]
-    });
     this._update();
 };
 
@@ -93,6 +173,12 @@ ExtendDataPreviewDialog.prototype._renderPreview = function(data) {
     }
     
     var table = $('<table>')[0];
+    var trHead = table.insertRow(table.rows.length);
+    $(trHead.insertCell(trHead.cells.length)).text(this._column.name);
+    
+    for (var c = 0; c < data.columns.length; c++) {
+        $(trHead.insertCell(trHead.cells.length)).text(data.columns[c]);
+    }
     
     for (var r = 0; r < data.rows.length; r++) {
         var tr = table.insertRow(table.rows.length);

src/main/webapp/scripts/views/data-table-column-header-ui.js

@@ -819,6 +819,7 @@ DataTableColumnHeaderUI.prototype._doAddColumn = function(initialExpression) {
 };
 
 DataTableColumnHeaderUI.prototype._doAddColumnFromFreebase = function() {
+    if ("reconConfig" in this._column && "type" in this._column.reconConfig) {
         var o = DataTableView.sampleVisibleRows(this._column);
     
         new ExtendDataPreviewDialog(
@@ -826,6 +827,9 @@ DataTableColumnHeaderUI.prototype._doAddColumnFromFreebase = function() {
             o.rowIndices, 
             function() {}
         );
+    } else {
+        alert("This column has not been reconciled yet.");
+    }
 };
 
 DataTableColumnHeaderUI.prototype._doRemoveColumn = function() {

src/main/webapp/styles/dialogs/extend-data-preview-dialog.css

@@ -1,10 +1,14 @@
 .extend-data-preview-dialog .suggested-property-container {
     border: 1px solid #aaa;
-    padding: 10px;
+    padding: 5px;
     overflow: auto;
     height: 100%;
 }
 
+.extend-data-preview-dialog .suggested-property {
+    padding: 5px;
+}
+
 .extend-data-preview-dialog input.property-suggest {
     display: block;
     padding: 2%;
@@ -13,7 +17,15 @@
 
 .extend-data-preview-dialog .preview-container {
     border: 1px solid #aaa;
-    padding: 10px;
     overflow: auto;
     height: 100%;
 }
+
+.extend-data-preview-dialog .preview-container table {
+    border-collapse: collapse;
+}
+
+.extend-data-preview-dialog .preview-container td {
+    padding: 3px 5px;
+    border-bottom: 1px solid #ddd;
+}