More work on the recon UI. Standard services can now be added.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1038 7d457c2a-affb-35e4-300a-418c747d4874

main/src/com/metaweb/gridworks/commands/freebase/GuessTypesOfColumnCommand.java

@@ -20,6 +20,7 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.JSONWriter;
 
@@ -78,7 +79,7 @@ public class GuessTypesOfColumnCommand extends Command {
         }
     }
     
-    final static int s_sampleSize = 20;
+    final static int s_sampleSize = 10;
     
     /**
      * Run relevance searches for the first n cells in the given column and
@@ -111,10 +112,9 @@ public class GuessTypesOfColumnCommand extends Command {
             }
         }
         
+        StringWriter stringWriter = new StringWriter();
         try {
-            StringWriter stringWriter = new StringWriter();
             JSONWriter jsonWriter = new JSONWriter(stringWriter);
-            
             jsonWriter.object();
             for (int i = 0; i < samples.size(); i++) {
                 jsonWriter.key("q" + i);
@@ -126,8 +126,12 @@ public class GuessTypesOfColumnCommand extends Command {
                 jsonWriter.endObject();
             }
             jsonWriter.endObject();
-            
+        } catch (JSONException e) {
-            String queriesString = stringWriter.toString();
+            // ignore
+        }
+        
+        String queriesString = stringWriter.toString();
+        try {
             URL url = new URL(serviceUrl);
             URLConnection connection = url.openConnection();
             {
@@ -201,7 +205,7 @@ public class GuessTypesOfColumnCommand extends Command {
                 is.close();
             }
         } catch (Exception e) {
-            e.printStackTrace();
+            logger.error("Failed to guess cell types for load\n" + queriesString, e);
         }
         
         List<TypeGroup> types = new ArrayList<TypeGroup>(map.values());

main/src/com/metaweb/gridworks/model/recon/StandardReconConfig.java

@@ -330,6 +330,13 @@ public class StandardReconConfig extends ReconConfig {
                     score
                 );
                 
+                if (i == 0 && result.has("match") && result.getBoolean("match")) {
+                    recon.match = candidate;
+                    recon.matchRank = 0;
+                    recon.judgment = Judgment.Matched;
+                    recon.judgmentAction = "auto";
+                }
+                
                 recon.addCandidate(candidate);
                 count++;
             }
@@ -345,12 +352,6 @@ public class StandardReconConfig extends ReconConfig {
                 for (String typeID : candidate.types) {
                     if (this.typeID.equals(typeID)) {
                         recon.setFeature(Recon.Feature_typeMatch, true);
-                        if (autoMatch && candidate.score >= 100 && (count == 1 || candidate.score / recon.candidates.get(1).score >= 1.5)) {
-                            recon.match = candidate;
-                            recon.matchRank = 0;
-                            recon.judgment = Judgment.Matched;
-                            recon.judgmentAction = "auto";
-                        }
                         break;
                     }
                 }

main/webapp/modules/core/MOD-INF/controller.js

@@ -102,12 +102,14 @@ function init() {
             "styles/views/data-table-view.css",
             
             "styles/dialogs/expression-preview-dialog.css",
-            "styles/dialogs/recon-dialog.css",
             "styles/dialogs/clustering-dialog.css",
             "styles/dialogs/scatterplot-dialog.css",
             "styles/dialogs/freebase-loading-dialog.css",
             "styles/dialogs/extend-data-preview-dialog.css",
             
+            "styles/reconciliation/recon-dialog.css",
+            "styles/reconciliation/standard-service-panel.css",
+            
             "styles/protograph/schema-alignment-dialog.css"
         ]
     );

main/webapp/modules/core/scripts/reconciliation/add-namespaced-service-dialog.html

@@ -0,0 +1,13 @@
+<div class="dialog-frame" style="width: 300px;">
+  <div class="dialog-header" bind="dialogHeader">Add Namespaced Reconciliation Service</div>
+  <div class="dialog-body" bind="dialogBody"><div class="grid-layout layout-full layout-tighter"><table>
+    <tr><td>Namespace:</td></tr>
+    <tr><td><div class="input-container"><input bind="namespaceInput" /></td></tr>
+    <tr><td>Type of Entities (optional):</td></tr>
+    <tr><td><div class="input-container"><input bind="typeInput" /></td></tr>
+  </table></div></div>
+  <div class="dialog-footer" bind="dialogFooter">
+    <button bind="addButton">Add Service</button>
+    <button bind="cancelButton">Cancel</button>
+  </div>
+</div>

main/webapp/modules/core/scripts/reconciliation/add-standard-service-dialog.html

@@ -0,0 +1,11 @@
+<div class="dialog-frame" style="width: 500px;">
+  <div class="dialog-header" bind="dialogHeader">Add Standard Reconciliation Service</div>
+  <div class="dialog-body" bind="dialogBody">
+    <p>Enter the service's URL:</p>
+    <div class="input-container"><input value="http://" bind="input" /></div>
+  </div>
+  <div class="dialog-footer" bind="dialogFooter">
+    <button bind="addButton">Add Service</button>
+    <button bind="cancelButton">Cancel</button>
+  </div>
+</div>

main/webapp/modules/core/scripts/reconciliation/recon-dialog.html

@@ -1,19 +1,25 @@
 <div class="dialog-frame" style="width: 900px;">
   <div class="dialog-header" bind="dialogHeader"></div>
   <div class="dialog-body" bind="dialogBody">
-    <div class="grid-layout layout-normal layout-full"><table><tr>
+    <div class="grid-layout layout-normal layout-full grid-layout-for-ui"><table><tr>
       <td width="1%">
-        <div class="recon-dialog-service-header">Services and Extensions</div>
         <div class="recon-dialog-service-list" bind="serviceList"></div>
-        <div class="recon-dialog-service-controls">
-          <button bind="addStandardServiceButton">Add Standard Service...</button>
-        </div>
       </td>
-      <td><div class="recon-dialog-service-panel-container" bind="servicePanelContainer"></div></td>
+      <td><div class="recon-dialog-service-panel-container" bind="servicePanelContainer">
+        <div class="recon-dialog-service-panel-message" bind="servicePanelMessage">
+          Pick a Service or Extension on Left
+        </div>
+      </div></td>
     </tr></table></div>
   </div>
-  <div class="dialog-footer" bind="dialogFooter">
+  <div class="dialog-footer" bind="dialogFooter"><table width="100%"><tr>
-    <button bind="reconcileButton">Start Reconciling</button>
+    <td align="left">
-    <button bind="cancelButton">Cancel</button>
+      <button bind="addStandardServiceButton">Add Standard Service...</button>
+      <button bind="addNamespacedServiceButton">Add Namespaced Service...</button>
+    </td>
+    <td align="right">
+      <button bind="reconcileButton">Start Reconciling</button>
+      <button bind="cancelButton">Cancel</button>
+    </td>
   </div>
 </div>

main/webapp/modules/core/scripts/reconciliation/recon-dialog.js

@@ -11,7 +11,11 @@ ReconDialog.prototype._createDialog = function() {
     var dialog = $(DOM.loadHTML("core", "scripts/reconciliation/recon-dialog.html"));
 
     this._elmts = DOM.bind(dialog);
-    this._elmts.dialogHeader.text("Reconcile column " + this._column.name);
+    this._elmts.dialogHeader.text('Reconcile column "' + this._column.name + '"');
+    
+    this._elmts.addStandardServiceButton.click(function() { self._onAddStandardService(); });
+    this._elmts.addNamespacedServiceButton.click(function() { self._onAddNamespacedService(); });
+    
     this._elmts.reconcileButton.click(function() { self._onOK(); });
     this._elmts.cancelButton.click(function() { self._dismiss(); });
     
@@ -41,8 +45,21 @@ ReconDialog.prototype._dismiss = function() {
     DialogSystem.dismissUntil(this._level - 1);
 };
 
+ReconDialog.prototype._cleanDialog = function() {
+    for (var i = 0; i < this._serviceRecords.length; i++) {
+        var record = this._serviceRecords[i];
+        if (record.handler) {
+            record.handler.deactivate();
+        }
+        record.selector.remove();
+    }
+    this._serviceRecords = [];
+    this._selectedServiceRecordIndex = -1;
+};
+
 ReconDialog.prototype._populateDialog = function() {
     var self = this;
+    
     var services = ReconciliationManager.getAllServices();
     if (services.length > 0) {
         var renderService = function(service) {
@@ -66,8 +83,6 @@ ReconDialog.prototype._populateDialog = function() {
         for (var i = 0; i < services.length; i++) {
             renderService(services[i]);
         }
-    
-        this._selectService(this._serviceRecords[0]);
     }
 };
 
@@ -83,6 +98,8 @@ ReconDialog.prototype._selectService = function(record) {
                     }
                 }
                 
+                this._elmts.servicePanelMessage.hide();
+                
                 record.selector.addClass("selected");
                 if (record.handler) {
                     record.handler.activate();
@@ -99,3 +116,72 @@ ReconDialog.prototype._selectService = function(record) {
         }
     }
 };
+
+ReconDialog.prototype._refresh = function(newSelectIndex) {
+    this._cleanDialog();
+    this._populateDialog();
+    if (newSelectIndex >= 0) {
+        this._selectService(this._serviceRecords[newSelectIndex]);
+    }
+};
+
+ReconDialog.prototype._onAddStandardService = function() {
+    var self = this;
+    var dialog = $(DOM.loadHTML("core", "scripts/reconciliation/add-standard-service-dialog.html"));
+    var elmts = DOM.bind(dialog);
+    
+    var level = DialogSystem.showDialog(dialog);
+    var dismiss = function() {
+        DialogSystem.dismissUntil(level - 1);
+    };
+    
+    elmts.cancelButton.click(dismiss);
+    elmts.addButton.click(function() {
+        var url = $.trim(elmts.input[0].value);
+        if (url.length > 0) {
+            ReconciliationManager.registerStandardService(url, function(index) {
+                self._refresh(index);
+            });
+        }
+        dismiss();
+    });
+    elmts.input.focus().select();
+};
+
+ReconDialog.prototype._onAddNamespacedService = function() {
+    var self = this;
+    var dialog = $(DOM.loadHTML("core", "scripts/reconciliation/add-namespaced-service-dialog.html"));
+    var elmts = DOM.bind(dialog);
+    
+    var level = DialogSystem.showDialog(dialog);
+    var dismiss = function() {
+        DialogSystem.dismissUntil(level - 1);
+    };
+    
+    elmts.namespaceInput
+        .suggest({ type: '/type/namespace' })
+        .bind("fb-select", function(e, data) {
+            elmts.typeInput.focus();
+        });
+        
+    elmts.typeInput.suggestT({ type: '/type/type' });
+    
+    elmts.cancelButton.click(dismiss);
+    elmts.addButton.click(function() {
+        var namespaceData = elmts.namespaceInput.data("data.suggest");
+        var typeData = elmts.typeInput.data("data.suggest");
+        if (namespaceData) {
+            var url = "http://standard-reconcile.freebaseapps.com/namespace_reconcile?namespace=" + 
+                escape(namespaceData.id);
+            if (typeData) {
+                url += "&type=" + typeData.id;
+            }
+            
+            ReconciliationManager.registerStandardService(url, function(index) {
+                self._refresh(index);
+            });
+        }
+        dismiss();
+    });
+    elmts.namespaceInput.focus().data("suggest").textchange();
+};

main/webapp/modules/core/scripts/reconciliation/recon-manager.js

@@ -9,9 +9,11 @@ ReconciliationManager.getAllServices = function() {
 
 ReconciliationManager.registerService = function(service) {
     ReconciliationManager.customServices.push(service);
+    
+    return ReconciliationManager.customServices.length - 1;
 };
 
-ReconciliationManager.registerStandardService = function(url) {
+ReconciliationManager.registerStandardService = function(url, f) {
     $.ajax({
         async: false,
         url: url + (url.contains("?") ? "&" : "?") + "callback=?",
@@ -19,8 +21,15 @@ ReconciliationManager.registerStandardService = function(url) {
             data.url = url;
             data.ui = { "handler" : "ReconStandardServicePanel" };
             
+            index = ReconciliationManager.customServices.length + 
+                ReconciliationManager.standardServices.length;
+            
             ReconciliationManager.standardServices.push(data);
             ReconciliationManager.save();
+            
+            if (f) {
+                f(index);
+            }
         },
         dataType: "jsonp"
     });
@@ -57,7 +66,7 @@ ReconciliationManager.save = function(f) {
                 ReconciliationManager.standardServices = JSON.parse(data.value);
             } else {
                 ReconciliationManager.registerStandardService(
-                    "http://gridworks-helper.dfhuynh.user.dev.freebaseapps.com/reconcile");
+                    "http://standard-reconcile.dfhuynh.user.dev.freebaseapps.com/reconcile");
             }
         },
         dataType: "json"

main/webapp/modules/core/scripts/reconciliation/standard-service-panel.html

@@ -1,5 +1,10 @@
 <div class="recon-dialog-service-panel recon-dialog-standard-service-panel">
-  <div class="grid-layout layout-normal layout-full"><table>
+  <div class="grid-layout layout-normal layout-full grid-layout-for-ui"><table>
+    <tr>
+      <td colspan="2" style="text-align: right;">
+        &raquo; Access <a href="" target="_blank" bind="rawServiceLink">Service API</a>
+      </td>
+    </tr>
     <tr>
       <td>Reconcile each cell to an entity of one of these types:</td>
       <td>Also use relevant details from other columns:</td>

main/webapp/modules/core/scripts/reconciliation/standard-service-panel.js

@@ -50,6 +50,8 @@ ReconStandardServicePanel.prototype._constructUI = function() {
     this._panel = $(DOM.loadHTML("core", "scripts/reconciliation/standard-service-panel.html")).appendTo(this._container);
     this._elmts = DOM.bind(this._panel);
     
+    this._elmts.rawServiceLink.attr("href", this._service.url);
+    
     this._guessTypes(function() {
         self._populatePanel();
         self._wireEvents();
@@ -79,43 +81,57 @@ ReconStandardServicePanel.prototype._populatePanel = function() {
     /*
      *  Populate types
      */
-    var typeTableContainer = $('<div>')
+    if (this._types.length > 0) {
-        .addClass("grid-layout layout-tightest")
+        var typeTableContainer = $('<div>')
-        .appendTo(this._elmts.typeContainer);
+            .addClass("grid-layout layout-tightest")
+            .appendTo(this._elmts.typeContainer);
         
-    var typeTable = $('<table></table>').appendTo(typeTableContainer)[0];
+        var typeTable = $('<table></table>').appendTo(typeTableContainer)[0];
-    var createTypeChoice = function(type, check) {
-        var typeID = typeof type == "string" ? type : type.id;
-        var typeName = typeof type == "string" ? type : (type.name || type.id);
         
-        var tr = typeTable.insertRow(typeTable.rows.length);
+        var createTypeChoice = function(type, check) {
-        var td0 = tr.insertCell(0);
+            var typeID = typeof type == "string" ? type : type.id;
-        var td1 = tr.insertCell(1);
+            var typeName = typeof type == "string" ? type : (type.name || type.id);
         
-        td0.width = "1%";
+            var tr = typeTable.insertRow(typeTable.rows.length);
-        var radio = $('<input type="radio" name="type-choice">')
+            var td0 = tr.insertCell(0);
-            .attr("value", typeID)
+            var td1 = tr.insertCell(1);
-            .attr("typeName", typeName)
+        
-            .appendTo(td0)
+            td0.width = "1%";
-            .click(function() {
+            var radio = $('<input type="radio" name="type-choice">')
-                self._rewirePropertySuggests(this.value);
+                .attr("value", typeID)
-            });
+                .attr("typeName", typeName)
+                .appendTo(td0)
+                .click(function() {
+                    self._rewirePropertySuggests(this.value);
+                });
             
-        if (check) {
+            if (check) {
-            radio.attr("checked", "true");
+                radio.attr("checked", "true");
-        }
+            }
         
-        if (typeName == typeID) {
+            if (typeName == typeID) {
-            $(td1).html(typeName);
+                $(td1).html(typeName);
-        } else {
+            } else {
-            $(td1).html(
+                $(td1).html(
-                typeName + 
+                    typeName + 
-                '<br/>' +
+                    '<br/>' +
-                '<span class="type-id">' + typeID + '</span>');
+                    '<span class="type-id">' + typeID + '</span>');
+            }
+        };
+        for (var i = 0; i < this._types.length; i++) {
+            createTypeChoice(this._types[i], i === 0);
         }
-    };
+    } else {
-    for (var i = 0; i < this._types.length; i++) {
+        $('<div>')
-        createTypeChoice(this._types[i], i === 0);
+            .addClass("recon-dialog-standard-service-panel-message")
+            .text("Sorry, we can't suggest any type for your data. Please specify a type yourself below.")
+            .appendTo(this._elmts.typeContainer);
+            
+        this._panel
+            .find('input[name="type-choice"][value=""]')
+            .attr("checked", "true");
+            
+        this._panel.typeInput.focus();
     }
     
     /*
@@ -155,36 +171,54 @@ ReconStandardServicePanel.prototype._populatePanel = function() {
 };
 
 ReconStandardServicePanel.prototype._wireEvents = function() {
-    if (this._isInFreebaseIdentifierSpace()) {
+    var self = this;
-        var self = this;
+    var input = this._elmts.typeInput.unbind();
-        this._elmts.typeInput
+    
-            .suggestT({ type : '/type/type' })
+    if ("suggest" in this._service && "type" in this._service.suggest) {
-            .bind("fb-select", function(e, data) {
+        var suggestOptions = $.extend({}, this._service.suggest.type);
-                self._panel
+        input.suggestT(suggestOptions);
-                    .find('input[name="type-choice"][value=""]')
+    } else if (this._isInFreebaseSchemaSpace()) {
-                    .attr("checked", "true");
+        input.suggestT({ type : '/type/type' });
-                    
-                self._rewirePropertySuggests(data.id);
-            });
-        
-        this._rewirePropertySuggests(this._types[0].id);
     }
+    
+    input.bind("fb-select", function(e, data) {
+        self._panel
+            .find('input[name="type-choice"][value=""]')
+            .attr("checked", "true");
+            
+        self._rewirePropertySuggests(data.id);
+    });
+    
+    this._rewirePropertySuggests((this._types.length > 0) ? this._types[0] : null);
 };
 
 ReconStandardServicePanel.prototype._rewirePropertySuggests = function(type) {
-    if (this._isInFreebaseIdentifierSpace()) {
+    var inputs = this._panel
-        this._panel
+        .find('input[name="property"]')
-            .find('input[name="property"]')
+        .unbind();
-            .unbind().suggestP({
+        
-                type: '/type/property',
+    if ("property" in this._service && "property" in this._service.suggest) {
-                schema: (type) ? (typeof type == "string" ? type : type.id) : "/common/topic"
+        var suggestOptions = $.extend({}, this._service.suggest.property);
-            });
+        if (type) {
+            suggestOptions.schema = typeof type == "string" ? type : type.id;
+        }
+        inputs.suggestP(suggestOptions);
+    } else if (this._isInFreebaseSchemaSpace()) {
+        inputs.suggestP({
+            type: '/type/property',
+            schema: (type) ? (typeof type == "string" ? type : type.id) : "/common/topic"
+        });
     }
 };
 
 ReconStandardServicePanel.prototype._isInFreebaseIdentifierSpace = function() {
-    return "identifier-space" in this._service &&
+    return "identifierSpace" in this._service &&
-        this._service["identifier-space"].startsWith("http://rdf.freebase.com/");
+        this._service.identifierSpace == "http://rdf.freebase.com/ns/type.object.id";
+};
+
+ReconStandardServicePanel.prototype._isInFreebaseSchemaSpace = function() {
+    return "schemaSpace" in this._service &&
+        this._service.schemaSpace == "http://rdf.freebase.com/ns/type.object.id";
 };
 
 ReconStandardServicePanel.prototype.start = function() {

main/webapp/modules/core/styles/common.css

@@ -32,6 +32,9 @@ div.grid-layout > table > tbody > tr > th, div.grid-layout > table > tbody > tr
 div.grid-layout.grid-layout-for-text > table > tbody > tr > th, div.grid-layout.grid-layout-for-text > table > tbody > tr > td {
     vertical-align: middle;
 }
+div.grid-layout.grid-layout-for-ui > table > tbody > tr > th, div.grid-layout.grid-layout-for-ui > table > tbody > tr > td {
+    vertical-align: top;
+}
 div.grid-layout.layout-normal {
     margin: -10px;
 }

main/webapp/modules/core/styles/dialogs/recon-dialog.css

@@ -1,47 +0,0 @@
-.recon-dialog-service-header {
-    padding: 5px 10px;
-    font-weight: bold;
-    font-size: 120%;
-}
-.recon-dialog-service-list {
-    border: 1px solid #aaa;
-    padding: 1px;
-    overflow: auto;
-    width: 200px;
-    height: 400px;
-}
-
-.recon-dialog-service-controls {
-    padding: 5px 0px;
-}
-.recon-dialog-service-controls > button {
-    display: block;
-    width: 100%;
-}
-
-a.recon-dialog-service-selector {
-    display: block;
-    padding: 10px;
-    text-decoration: none;
-    color: black;
-}
-a.recon-dialog-service-selector:hover {
-    background: #eee;
-}
-a.recon-dialog-service-selector.selected {
-    background: #eee;
-    font-weight: bold;
-}
-
-.recon-dialog-standard-service-panel .type-id {
-    color: #888;
-}
-
-.recon-dialog-standard-service-panel .type-container,
-.recon-dialog-standard-service-panel .detail-container {
-    border: 1px solid #ccc;
-    padding: 10px;
-    max-height: 300px;
-    height: 300px !important;
-    overflow: auto;
-}

main/webapp/modules/core/styles/reconciliation/recon-dialog.css

@@ -0,0 +1,28 @@
+.recon-dialog-service-list {
+    border: 1px solid #aaa;
+    padding: 1px;
+    overflow: auto;
+    width: 200px;
+    height: 500px;
+}
+
+a.recon-dialog-service-selector {
+    display: block;
+    padding: 10px;
+    text-decoration: none;
+    color: black;
+}
+a.recon-dialog-service-selector:hover {
+    background: #eee;
+}
+a.recon-dialog-service-selector.selected {
+    background: #eee;
+    font-weight: bold;
+}
+
+.recon-dialog-service-panel-message {
+    text-align: center;
+    font-size: 200%;
+    padding: 50px 0;
+    color: #aaa;
+}

main/webapp/modules/core/styles/reconciliation/standard-service-panel.css

@@ -0,0 +1,19 @@
+.recon-dialog-standard-service-panel-message {
+    text-align: center;
+    font-size: 150%;
+    padding: 20px 0;
+    color: #aaa;
+}
+
+.recon-dialog-standard-service-panel .type-id {
+    color: #888;
+}
+
+.recon-dialog-standard-service-panel .type-container,
+.recon-dialog-standard-service-panel .detail-container {
+    border: 1px solid #ccc;
+    padding: 10px;
+    max-height: 300px;
+    height: 300px !important;
+    overflow: auto;
+}