More work on the extend data preview dialog. Results now looks correct, but we still are not handling CVTs.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@295 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java

@@ -47,8 +47,7 @@ public class FreebaseDataExtensionJob {
         StringWriter writer = new StringWriter();
         formulateQuery(guids, extension, writer);
         
-        URL url = new URL("http://api.freebase.com/api/service/mqlread");
+        InputStream is = doMqlRead(writer.toString());
-        InputStream is = doPost(url, "query", writer.toString());
         try {
             String s = ParsingUtilities.inputStreamToString(is);
             JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
@@ -143,7 +142,7 @@ public class FreebaseDataExtensionJob {
             boolean hasSubProperties = (extNode.has("properties") && !extNode.isNull("properties")); 
             boolean isOwnColumn = !hasSubProperties || (extNode.has("included") && extNode.getBoolean("included"));
             
-            if (a != null) {
+            if (a != null && a.length() > 0) {
                 int maxColIndex = startColumnIndex;
                 
                 int l = a.length();
@@ -166,16 +165,19 @@ public class FreebaseDataExtensionJob {
                             startColumnIndex2
                         );
                         
-                        startRowIndex = rowcol[0];
+                        startRowIndex2 = rowcol[0];
-                        maxColIndex = Math.max(maxColIndex, rowcol[1]);
+                        startColumnIndex2 = rowcol[1];
                     }
+                    
+                    startRowIndex = startRowIndex2;
+                    maxColIndex = Math.max(maxColIndex, startColumnIndex2);
                 }
                 
                 return new int[] { startRowIndex, maxColIndex };
             } else {
                 return new int[] {
                     startRowIndex,
-                    countColumns(extNode, null)
+                    startColumnIndex + countColumns(extNode, null)
                 };
             }
         }
@@ -208,109 +210,112 @@ public class FreebaseDataExtensionJob {
     }
 
 
-static protected InputStream doPost(URL url, String name, String load) throws IOException {
+    static protected InputStream doMqlRead(String query) throws IOException {
-    URLConnection connection = url.openConnection();
+        URL url = new URL("http://api.freebase.com/api/service/mqlread");
-    connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+
-    connection.setConnectTimeout(5000);
+        URLConnection connection = url.openConnection();
-    connection.setDoOutput(true);
+        connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
-    
+        connection.setConnectTimeout(5000);
-    DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
+        connection.setDoOutput(true);
-    try {
-        String data = name + "=" + ParsingUtilities.encode(load);
         
-        dos.writeBytes(data);
+        DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
-    } finally {
+        try {
-        dos.flush();
+            String body = "extended=1&query=" + ParsingUtilities.encode(query);
-        dos.close();
+            
+            dos.writeBytes(body);
+        } finally {
+            dos.flush();
+            dos.close();
+        }
+        
+        connection.connect();
+        
+        return connection.getInputStream();
     }
     
-    connection.connect();
+    static protected void formulateQuery(Set<String> guids, JSONObject node, Writer writer) throws JSONException {
-    
+        JSONWriter jsonWriter = new JSONWriter(writer);
-    return connection.getInputStream();
+        
-}
-
-static protected void formulateQuery(Set<String> guids, JSONObject node, Writer writer) throws JSONException {
-    JSONWriter jsonWriter = new JSONWriter(writer);
-    
-    jsonWriter.object();
-    jsonWriter.key("query");
-        jsonWriter.array();
         jsonWriter.object();
-        
+        jsonWriter.key("query");
-            jsonWriter.key("guid"); jsonWriter.value(null);
+            jsonWriter.array();
-            jsonWriter.key("guid|=");
+            jsonWriter.object();
-                jsonWriter.array();
+            
-                for (String guid : guids) {
+                jsonWriter.key("guid"); jsonWriter.value(null);
-                    jsonWriter.value(guid);
+                jsonWriter.key("guid|=");
-                }
+                    jsonWriter.array();
-                jsonWriter.endArray();
+                    for (String guid : guids) {
-                
+                        jsonWriter.value(guid);
-            formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
+                    }
-        
+                    jsonWriter.endArray();
-        jsonWriter.endObject();
-        jsonWriter.endArray();
-    jsonWriter.endObject();
-}
-
-static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
-    String propertyID = node.getString("id");
-    String expectedTypeID = node.getString("expected");
-    
-    writer.key(propertyID);
-    writer.array();
-    {
-        if (!expectedTypeID.startsWith("/type/")) { // not literal
-            writer.object();
-            writer.key("limit"); writer.value(10);
-            {
-                boolean hasSubProperties = (node.has("properties") && !node.isNull("properties")); 
                     
-                if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
+                formulateQueryNode(node.getJSONArray("properties"), jsonWriter);
-                    writer.key("name"); writer.value(null);
+            
-                    writer.key("id"); writer.value(null);
+            jsonWriter.endObject();
-                    writer.key("guid"); writer.value(null);
+            jsonWriter.endArray();
-                    writer.key("type"); writer.array(); writer.endArray();
+        jsonWriter.endObject();
-                }
-                
-                if (hasSubProperties) {
-                    formulateQueryNode(node.getJSONArray("properties"), writer);
-                }
-            }
-            writer.endObject();
-        }
     }
-    writer.endArray();
-}
-
-static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
-    int l = propertiesA.length();
     
-    for (int i = 0; i < l; i++) {
+    static protected void formulateQueryNode(JSONObject node, JSONWriter writer) throws JSONException {
-        formulateQueryNode(propertiesA.getJSONObject(i), writer);
+        String propertyID = node.getString("id");
-    }
+        String expectedTypeID = node.getString("expected");
-}
+        
-
+        writer.key(propertyID);
-static protected int countColumns(JSONObject obj, List<String> columnNames) throws JSONException {
+        writer.array();
-    if (obj.has("properties") && !obj.isNull("properties")) {
+        {
-        boolean included = (obj.has("included") && obj.getBoolean("included"));
+            if (!expectedTypeID.startsWith("/type/")) { // not literal
-        if (included && columnNames != null) {
+                writer.object();
-            columnNames.add(obj.getString("name"));
+                writer.key("limit"); writer.value(10);
+                writer.key("optional"); writer.value(true);
+                {
+                    boolean hasSubProperties = (node.has("properties") && !node.isNull("properties")); 
+                        
+                    if (!hasSubProperties || (node.has("included") && node.getBoolean("included"))) {
+                        writer.key("name"); writer.value(null);
+                        writer.key("id"); writer.value(null);
+                        writer.key("guid"); writer.value(null);
+                        writer.key("type"); writer.array(); writer.endArray();
+                    }
+                    
+                    if (hasSubProperties) {
+                        formulateQueryNode(node.getJSONArray("properties"), writer);
+                    }
+                }
+                writer.endObject();
+            }
         }
-        return (included ? 1 : 0) + countColumns(obj.getJSONArray("properties"), columnNames);
+        writer.endArray();
-    } else {
+    }
-        if (columnNames != null) {
+    
-            columnNames.add(obj.getString("name"));
+    static protected void formulateQueryNode(JSONArray propertiesA, JSONWriter writer) throws JSONException {
+        int l = propertiesA.length();
+        
+        for (int i = 0; i < l; i++) {
+            formulateQueryNode(propertiesA.getJSONObject(i), writer);
         }
-        return 1;
     }
-}
+    
-
+    static protected int countColumns(JSONObject obj, List<String> columnNames) throws JSONException {
-static protected int countColumns(JSONArray a, List<String> columnNames) throws JSONException {
+        if (obj.has("properties") && !obj.isNull("properties")) {
-    int c = 0;
+            boolean included = (obj.has("included") && obj.getBoolean("included"));
-    int l = a.length();
+            if (included && columnNames != null) {
-    for (int i = 0; i < l; i++) {
+                columnNames.add(obj.getString("name"));
-        c += countColumns(a.getJSONObject(i), columnNames);
+            }
+            return (included ? 1 : 0) + countColumns(obj.getJSONArray("properties"), columnNames);
+        } else {
+            if (columnNames != null) {
+                columnNames.add(obj.getString("name"));
+            }
+            return 1;
+        }
+    }
+    
+    static protected int countColumns(JSONArray a, List<String> columnNames) throws JSONException {
+        int c = 0;
+        int l = a.length();
+        for (int i = 0; i < l; i++) {
+            c += countColumns(a.getJSONObject(i), columnNames);
+        }
+        return c;
     }
-    return c;
-}
 }

src/main/webapp/scripts/dialogs/extend-data-preview-dialog.js

@@ -5,7 +5,7 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
     this._extension = { properties: [] };
 
     var self = this;
-    var frame = DialogSystem.createDialog();
+    var frame = this._frame = DialogSystem.createDialog();
     frame.width("900px").addClass("extend-data-preview-dialog");
     
     var header = $('<div></div>').addClass("dialog-header").text("Add Columns from Freebase Based on Column " + column.name).appendTo(frame);
@@ -25,7 +25,7 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
                 '<td height="1">Suggested Properties</td>' +
             '</tr>' +
             '<tr>' +
-                '<td height="99%"><div class="suggested-property-container"></div></td>' +
+                '<td height="99%"><div class="suggested-property-container" bind="suggestedPropertyContainer"></div></td>' +
             '</tr>' +
         '</table></div>'
     ).appendTo(body);
@@ -41,16 +41,100 @@ function ExtendDataPreviewDialog(column, rowIndices, onDone) {
         DialogSystem.dismissUntil(self._level - 1);
     }).appendTo(footer);
     
-    this._level = DialogSystem.showDialog(frame);
+    ExtendDataPreviewDialog.getAllProperties(column.reconConfig.type.id, function(properties) {
+        self._show(properties);
+    });
+};
+
+ExtendDataPreviewDialog.getAllProperties = function(typeID, onDone) {
+    var query = {
+        id : typeID,
+        type : "/type/type",
+        "/freebase/type_hints/included_types" : [{
+            optional: true,
+            properties : [{
+                id : null,
+                name : null,
+                "/type/property/expected_type" : {
+                    id : null,
+                    "/freebase/type_hints/mediator" : []
+                },
+                sort : "name"
+            }]
+        }],
+        properties : [{
+            id : null,
+            name : null,
+            "/type/property/expected_type" : {
+                id : null,
+                "/freebase/type_hints/mediator" : []
+            },
+            sort : "name"
+        }]
+    };
+    
+    var allProperties = [];
+    var processProperty = function(property) {
+        var expectedType = property["/type/property/expected_type"];
+        if (expectedType["/freebase/type_hints/mediator"].length > 0 && expectedType["/freebase/type_hints/mediator"][0]) {
+            
+        } else {
+            allProperties.push({
+                id : property.id,
+                name : property.name,
+                expected : expectedType.id
+            });
+        }
+    };
+    var processProperties = function(properties) {
+        $.each(properties, function() { processProperty(this); });
+    };
+    
+    $.getJSON(
+        "http://api.freebase.com/api/service/mqlread?query=" + escape(JSON.stringify({ query : query })) + "&callback=?",
+        null,
+        function(o) {
+            if ("result" in o) {
+                processProperties(o.result.properties);
+                $.each(o.result["/freebase/type_hints/included_types"], function() {
+                    processProperties(this.properties);
+                })
+                
+                onDone(allProperties);
+            } else {
+                onDone([]);
+            }
+        },
+        "jsonp"
+    );    
+};
+
+ExtendDataPreviewDialog.prototype._show = function(properties) {
+    this._level = DialogSystem.showDialog(this._frame);
+    
+    var self = this;
+    var container = this._elmts.suggestedPropertyContainer;
+    var renderSuggestedProperty = function(property) {
+        var div = $('<div>').addClass("suggested-property").appendTo(container);
+        $('<a>').attr("href", "javascript:{}").text(property.name).appendTo(div).click(function() {
+            self._addProperty(property);
+        });
+    };
+    for (var i = 0; i < properties.length; i++) {
+        renderSuggestedProperty(properties[i]);
+    }
     
     var suggestConfig = {
-        type: '/type/property'
+        type: '/type/property',
+        schema: this._column.reconConfig.type.id
     };
-    if ("reconConfig" in column) {
+    
-        suggestConfig.schema = column.reconConfig.type.id;
-    }    
     this._elmts.addPropertyInput.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
-        self._addProperty(data);
+        self._addProperty({
+            id : data.id,
+            name: data.name,
+            expected: data["/type/property/expected_type"]
+        });
     });
 };
 
@@ -77,11 +161,7 @@ ExtendDataPreviewDialog.prototype._update = function() {
 };
 
 ExtendDataPreviewDialog.prototype._addProperty = function(p) {
-    this._extension.properties.push({
+    this._extension.properties.push(p);
-        id : p.id,
-        name: p.name,
-        expected: p["/type/property/expected_type"]
-    });
     this._update();
 };
 
@@ -93,6 +173,12 @@ ExtendDataPreviewDialog.prototype._renderPreview = function(data) {
     }
     
     var table = $('<table>')[0];
+    var trHead = table.insertRow(table.rows.length);
+    $(trHead.insertCell(trHead.cells.length)).text(this._column.name);
+    
+    for (var c = 0; c < data.columns.length; c++) {
+        $(trHead.insertCell(trHead.cells.length)).text(data.columns[c]);
+    }
     
     for (var r = 0; r < data.rows.length; r++) {
         var tr = table.insertRow(table.rows.length);

src/main/webapp/scripts/views/data-table-column-header-ui.js

@@ -819,13 +819,17 @@ DataTableColumnHeaderUI.prototype._doAddColumn = function(initialExpression) {
 };
 
 DataTableColumnHeaderUI.prototype._doAddColumnFromFreebase = function() {
-    var o = DataTableView.sampleVisibleRows(this._column);
+    if ("reconConfig" in this._column && "type" in this._column.reconConfig) {
+        var o = DataTableView.sampleVisibleRows(this._column);
     
-    new ExtendDataPreviewDialog(
+        new ExtendDataPreviewDialog(
-        this._column, 
+            this._column, 
-        o.rowIndices, 
+            o.rowIndices, 
-        function() {}
+            function() {}
-    );
+        );
+    } else {
+        alert("This column has not been reconciled yet.");
+    }
 };
 
 DataTableColumnHeaderUI.prototype._doRemoveColumn = function() {

src/main/webapp/styles/dialogs/extend-data-preview-dialog.css

@@ -1,10 +1,14 @@
 .extend-data-preview-dialog .suggested-property-container {
     border: 1px solid #aaa;
-    padding: 10px;
+    padding: 5px;
     overflow: auto;
     height: 100%;
 }
 
+.extend-data-preview-dialog .suggested-property {
+    padding: 5px;
+}
+
 .extend-data-preview-dialog input.property-suggest {
     display: block;
     padding: 2%;
@@ -13,7 +17,15 @@
 
 .extend-data-preview-dialog .preview-container {
     border: 1px solid #aaa;
-    padding: 10px;
     overflow: auto;
     height: 100%;
-}
+}
+
+.extend-data-preview-dialog .preview-container table {
+    border-collapse: collapse;
+}
+
+.extend-data-preview-dialog .preview-container td {
+    padding: 3px 5px;
+    border-bottom: 1px solid #ddd;
+}