Forgot to check in image files.

Added commands for judging similar cells.
Started to fix/unify terminologies for recon operations.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@116 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/GridworksServlet.java

@@ -37,6 +37,7 @@ import com.metaweb.gridworks.commands.recon.ApproveNewReconcileCommand;
 import com.metaweb.gridworks.commands.recon.ApproveReconcileCommand;
 import com.metaweb.gridworks.commands.recon.DiscardReconcileCommand;
 import com.metaweb.gridworks.commands.recon.JudgeOneCellCommand;
+import com.metaweb.gridworks.commands.recon.JudgeSimilarCellsCommand;
 import com.metaweb.gridworks.commands.recon.MatchSpecificTopicCommand;
 import com.metaweb.gridworks.commands.recon.ReconcileCommand;
 import com.metaweb.gridworks.commands.util.GetExpressionLanguageInfoCommand;
@@ -74,11 +75,12 @@ public class GridworksServlet extends HttpServlet {
         _commands.put("split-multi-value-cells", new SplitMultiValueCellsCommand());
 		
 		_commands.put("reconcile", new ReconcileCommand());
-		_commands.put("recon-approve-best-matches", new ApproveReconcileCommand());
+		_commands.put("recon-match-best-candidates", new ApproveReconcileCommand());
 		_commands.put("recon-mark-new-topics", new ApproveNewReconcileCommand());
 		_commands.put("recon-discard-judgments", new DiscardReconcileCommand());
 		_commands.put("recon-match-specific-topic-to-cells", new MatchSpecificTopicCommand());
 		_commands.put("recon-judge-one-cell", new JudgeOneCellCommand());
+        _commands.put("recon-judge-similar-cells", new JudgeSimilarCellsCommand());
 		
 		_commands.put("save-protograph", new SaveProtographCommand());
 		

src/main/java/com/metaweb/gridworks/commands/recon/JudgeOneCellCommand.java

@@ -10,6 +10,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.json.JSONWriter;
 
 import com.metaweb.gridworks.commands.Command;
+import com.metaweb.gridworks.expr.ExpressionUtils;
 import com.metaweb.gridworks.history.Change;
 import com.metaweb.gridworks.history.HistoryEntry;
 import com.metaweb.gridworks.model.Cell;
@@ -105,7 +106,7 @@ public class JudgeOneCellCommand extends Command {
 
 		protected HistoryEntry createHistoryEntry() throws Exception {
 			Cell cell = _project.rows.get(rowIndex).getCell(cellIndex);
-			if (cell == null || cell.value == null) {
+            if (cell == null || ExpressionUtils.isBlank(cell.value)) {
 				throw new Exception("Cell is blank");
 			}
 			

src/main/java/com/metaweb/gridworks/commands/recon/JudgeSimilarCellsCommand.java

@@ -0,0 +1,220 @@
+package com.metaweb.gridworks.commands.recon;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONWriter;
+
+import com.metaweb.gridworks.browsing.Engine;
+import com.metaweb.gridworks.browsing.FilteredRows;
+import com.metaweb.gridworks.browsing.RowVisitor;
+import com.metaweb.gridworks.commands.Command;
+import com.metaweb.gridworks.expr.ExpressionUtils;
+import com.metaweb.gridworks.history.HistoryEntry;
+import com.metaweb.gridworks.model.Cell;
+import com.metaweb.gridworks.model.Column;
+import com.metaweb.gridworks.model.Project;
+import com.metaweb.gridworks.model.Recon;
+import com.metaweb.gridworks.model.ReconCandidate;
+import com.metaweb.gridworks.model.Row;
+import com.metaweb.gridworks.model.changes.CellChange;
+import com.metaweb.gridworks.model.changes.MassCellChange;
+import com.metaweb.gridworks.process.QuickHistoryEntryProcess;
+
+public class JudgeSimilarCellsCommand extends Command {
+	@Override
+	public void doPost(HttpServletRequest request, HttpServletResponse response)
+			throws ServletException, IOException {
+		
+		try {
+			Project project = getProject(request);
+			Engine engine = getEngine(request, project);
+			
+			int rowIndex = Integer.parseInt(request.getParameter("row"));
+			int cellIndex = Integer.parseInt(request.getParameter("cell"));
+			String judgment = request.getParameter("judgment"); 
+
+			JudgeSimilarCellsProcess process = null;
+			
+			if (judgment != null) {
+				process = new JudgeSimilarCellsProcess(
+					project, 
+					engine,
+					"Judge one cell's recon result",
+					judgment,
+					rowIndex, 
+					cellIndex, 
+					request.getParameter("candidate")
+				);
+			} else {
+				ReconCandidate match = new ReconCandidate(
+					request.getParameter("topicID"),
+					request.getParameter("topicGUID"),
+					request.getParameter("topicName"),
+					request.getParameter("types").split(","),
+					100
+				);
+				
+				process = new JudgeSimilarCellsProcess(
+					project, 
+					engine,
+					"Judge one cell's recon result",
+					rowIndex, 
+					cellIndex, 
+					match
+				);
+			}
+			
+			boolean done = project.processManager.queueProcess(process);
+			if (done) {
+				JSONWriter writer = new JSONWriter(response.getWriter());
+				writer.object();
+				writer.key("code"); writer.value("ok");
+				writer.endObject();
+			} else {
+				respond(response, "{ \"code\" : \"pending\" }");
+			}
+		} catch (Exception e) {
+			respondException(response, e);
+		}
+	}
+	
+	protected class JudgeSimilarCellsProcess extends QuickHistoryEntryProcess {
+	    final Engine engine;
+		final int rowIndex;
+		final int cellIndex;
+		final String judgment;
+		final String candidateID;
+		final ReconCandidate match;
+		
+		JudgeSimilarCellsProcess(Project project, Engine engine, String briefDescription, String judgment, int rowIndex, int cellIndex, String candidateID) {
+			super(project, briefDescription);
+			this.engine = engine;
+			this.rowIndex = rowIndex;
+			this.cellIndex = cellIndex;
+			this.judgment = judgment;
+			this.candidateID = candidateID;
+			this.match = null;
+		}
+		
+		JudgeSimilarCellsProcess(Project project, Engine engine, String briefDescription, int rowIndex, int cellIndex, ReconCandidate match) {
+			super(project, briefDescription);
+			this.engine = engine;
+			this.rowIndex = rowIndex;
+			this.cellIndex = cellIndex;
+			this.judgment = null;
+			this.candidateID = null;
+			this.match = match;
+		}
+
+		protected HistoryEntry createHistoryEntry() throws Exception {
+			Cell cell = _project.rows.get(rowIndex).getCell(cellIndex);
+			if (cell == null || ExpressionUtils.isBlank(cell.value)) {
+				throw new Exception("Cell is blank");
+			}
+			
+			Column column = _project.columnModel.getColumnByCellIndex(cellIndex);
+			if (column == null) {
+				throw new Exception("No such column");
+			}
+			
+	        List<CellChange> cellChanges = new ArrayList<CellChange>(_project.rows.size());
+            String similarValue = cell.value.toString();
+	        
+	        RowVisitor rowVisitor = new RowVisitor() {
+	            List<CellChange> cellChanges;
+	            String similarValue;
+	            
+	            public RowVisitor init(List<CellChange> cellChanges, String similarValue) {
+	                this.cellChanges = cellChanges;
+	                this.similarValue = similarValue;
+	                return this;
+	            }
+	            
+	            public boolean visit(Project project, int rowIndex, Row row, boolean contextual) {
+	                Cell cell = row.getCell(cellIndex);
+	                if (cell != null && !ExpressionUtils.isBlank(cell.value) && similarValue.equals(cell.value)) {
+	                    Cell newCell = new Cell(
+                            cell.value, 
+                            cell.recon == null ? new Recon() : cell.recon.dup()
+                        );
+	                    
+	                    if (match != null) {
+	                        newCell.recon.judgment = Recon.Judgment.Matched;
+	                        newCell.recon.match = match;
+	                    } else {
+	                        if ("match".equals(judgment)) {
+	                            ReconCandidate match = null;
+	                            
+	                            if (cell.recon != null) {
+	                                for (ReconCandidate c : cell.recon.candidates) {
+	                                    if (candidateID.equals(c.topicID)) {
+	                                        match = c;
+	                                        break;
+	                                    }
+	                                }
+	                            }
+	                            if (match == null) {
+	                                return false;
+	                            }
+	                            
+	                            newCell.recon.judgment = Recon.Judgment.Matched;
+	                            newCell.recon.match = match;
+	                        } else if ("new".equals(judgment)) {
+	                            newCell.recon.judgment = Recon.Judgment.New;
+	                        } else if ("discard".equals(judgment)) {
+	                            newCell.recon.judgment = Recon.Judgment.None;
+	                            newCell.recon.match = null;
+	                        }
+	                    }
+	                    
+                        CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
+                        cellChanges.add(cellChange);
+	                }
+	                return false;
+	            }
+	        }.init(cellChanges, similarValue);
+	        
+	        FilteredRows filteredRows = engine.getAllFilteredRows(false);
+	        filteredRows.accept(_project, rowVisitor);
+	        
+	        String description = null;
+            if (match != null) {
+                description = "Match " + 
+                    cellChanges.size() + " cell(s) in column " + 
+                    column.getHeaderLabel() + " containing " +
+                    similarValue + " to topic " + 
+                    match.topicName + " (" +
+                    match.topicID + ")";
+            } else {
+                if ("match".equals(judgment)) {
+                    description = "Match " + 
+                        cellChanges.size() + " cell(s) in column " + 
+                        column.getHeaderLabel() + " containing " +
+                        similarValue + " to topic " + 
+                        candidateID;
+                } else if ("new".equals(judgment)) {
+                    description = "Mark to create new topic for " + 
+                        cellChanges.size() + " cell(s) in column " + 
+                        column.getHeaderLabel() + " containing " +
+                        similarValue;
+                } else if ("discard".equals(judgment)) {
+                    description = "Discard recon judgments for " + 
+                        cellChanges.size() + " cell(s) in column " + 
+                        column.getHeaderLabel() + " containing " +
+                        similarValue;
+                }
+            }
+	        
+	        MassCellChange massCellChange = new MassCellChange(
+                cellChanges, column.getHeaderLabel(), false);
+	        
+	        return new HistoryEntry(_project, description, null, massCellChange);
+		}
+	}
+}

src/main/java/com/metaweb/gridworks/model/AbstractOperation.java

@@ -15,6 +15,8 @@ import com.metaweb.gridworks.process.QuickHistoryEntryProcess;
  *  projects.
  */
 abstract public class AbstractOperation implements Serializable, Jsonizable {
+    private static final long serialVersionUID = 3916055862440019600L;
+
     public Process createProcess(Project project, Properties options) throws Exception {
 		return new QuickHistoryEntryProcess(project, getBriefDescription()) {
 			@Override

src/main/webapp/scripts/project/data-table-cell-ui.js

@@ -51,17 +51,27 @@ DataTableCellUI.prototype._render = function() {
         } else {
             $(divContent).html(cell.v);
             $('<span> </span>').appendTo(divContent);
-            $('<a href="javascript:{}">mark as new</a>')
-                .addClass("data-table-recon-action")
-                .appendTo(divContent).click(function(evt) {
-                    self._doMarkAsNew();
-                });
             
+            var ul = $('<div></div>').addClass("data-table-recon-candidates").appendTo(divContent);
             if (this._dataTableView._showRecon && "c" in r && r.c.length > 0) {
                 var candidates = r.c;
-                var ul = $('<ul></ul>').addClass("data-table-recon-candidates").appendTo(divContent);
                 var renderCandidate = function(candidate, index) {
-                    var li = $('<li></li>').appendTo(ul);
+                    var li = $('<div></div>').addClass("data-table-recon-candidate").appendTo(ul);
+                    
+                    $('<a href="javascript:{}">&nbsp;</a>')
+                        .addClass("data-table-recon-match-similar")
+                        .attr("title", "Match this topic to this cell and other cells with the same content")
+                        .appendTo(li).click(function(evt) {
+                            self._doMatchTopicToSimilarCells(candidate.id);
+                        });
+                        
+                    $('<a href="javascript:{}">&nbsp;</a>')
+                        .addClass("data-table-recon-match")
+                        .attr("title", "Match this topic to this cell")
+                        .appendTo(li).click(function(evt) {
+                            self._doMatchTopicToOneCell(candidate.id);
+                        });
+                        
                     $('<a></a>')
                         .addClass("data-table-recon-topic")
                         .attr("href", "http://www.freebase.com/view" + candidate.id)
@@ -75,11 +85,6 @@ DataTableCellUI.prototype._render = function() {
                         .appendTo(li);
                         
                     $('<span></span>').addClass("data-table-recon-score").text("(" + Math.round(candidate.score) + ")").appendTo(li);
-                    $('<a href="javascript:{}">match</a>')
-                        .addClass("data-table-recon-action")
-                        .appendTo(li).click(function(evt) {
-                            self._doSetAsMatch(candidate.id);
-                        });
                 };
                 
                 for (var i = 0; i < candidates.length; i++) {
@@ -87,6 +92,23 @@ DataTableCellUI.prototype._render = function() {
                 }
             }
             
+            var liNew = $('<div></div>').addClass("data-table-recon-candidate").appendTo(ul);
+            $('<a href="javascript:{}">&nbsp;</a>')
+                .addClass("data-table-recon-match-similar")
+                .attr("title", "Create a new topic for this cell and other cells with the same content")
+                .appendTo(liNew).click(function(evt) {
+                    self._doMatchNewTopicToSimilarCells();
+                });
+                
+            $('<a href="javascript:{}">&nbsp;</a>')
+                .addClass("data-table-recon-match")
+                .attr("title", "Create a new topic for this cell")
+                .appendTo(liNew).click(function(evt) {
+                    self._doMatchNewTopicToOneCell();
+                });
+                
+            $('<span>').text("(New topic)").appendTo(liNew);
+            
             $('<a href="javascript:{}"></a>')
                 .addClass("data-table-recon-search")
                 .click(function(evt) {
@@ -103,14 +125,22 @@ DataTableCellUI.prototype._doRematch = function() {
     this._doJudgment("discard");
 };
 
-DataTableCellUI.prototype._doMarkAsNew = function() {
+DataTableCellUI.prototype._doMatchNewTopicToOneCell = function() {
     this._doJudgment("new");
 };
 
-DataTableCellUI.prototype._doSetAsMatch = function(candidateID) {
+DataTableCellUI.prototype._doMatchNewTopicToSimilarCells = function() {
+    this._doJudgmentForSimilarCells("new");
+};
+
+DataTableCellUI.prototype._doMatchTopicToOneCell = function(candidateID) {
     this._doJudgment("match", { candidate : candidateID });
 };
 
+DataTableCellUI.prototype._doMatchTopicToSimilarCells = function(candidateID) {
+    this._doJudgmentForSimilarCells("match", { candidate : candidateID });
+};
+
 DataTableCellUI.prototype._doJudgment = function(judgment, params) {
     params = params || {};
     params.row = this._rowIndex;
@@ -119,10 +149,19 @@ DataTableCellUI.prototype._doJudgment = function(judgment, params) {
     this.doPostThenUpdate("recon-judge-one-cell", params);
 };
 
+DataTableCellUI.prototype._doJudgmentForSimilarCells = function(judgment, params) {
+    params = params || {};
+    params.row = this._rowIndex;
+    params.cell = this._cellIndex;
+    params.judgment = judgment;
+    
+    ui.dataTableView.doPostThenUpdate("recon-judge-similar-cells", params);
+};
+
 DataTableCellUI.prototype._searchForMatch = function() {
     var self = this;
     var frame = DialogSystem.createDialog();
-    frame.width("200px");
+    frame.width("400px");
     
     var header = $('<div></div>').addClass("dialog-header").text("Search for Match").appendTo(frame);
     var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
@@ -131,26 +170,41 @@ DataTableCellUI.prototype._searchForMatch = function() {
     $('<p></p>').text("Search Freebase for topic to match " + this._cell.v).appendTo(body);
     
     var input = $('<input />').attr("value", this._cell.v).appendTo($('<p></p>').appendTo(body));
+    var match = null;
     input.suggest({}).bind("fb-select", function(e, data) {
+        match = data;
+    });
+    
+    var pSimilar = $('<p></p>').appendTo(body);
+    var checkSimilar = $('<input type="checkbox" checked="true" />').appendTo(pSimilar);
+    $('<span>').text(" Match other cells with the same content as well").appendTo(pSimilar);
+    
+    $('<button></button>').text("Match").click(function() {
+        if (match != null) {
             var params = {
                 row: self._rowIndex,
                 cell: self._cellIndex,
-            topicID: data.id,
+                topicID: match.id,
-            topicGUID: data.guid,
+                topicGUID: match.guid,
-            topicName: data.name,
+                topicName: match.name,
-            types: $.map(data.type, function(elmt) { return elmt.id; }).join(",")
+                types: $.map(match.type, function(elmt) { return elmt.id; }).join(",")
             };
+            if (checkSimilar[0].checked) {
+                ui.dataTableView.doPostThenUpdate("recon-judge-similar-cells", params);
+            } else {
                 self.doPostThenUpdate("recon-judge-one-cell", params);
+            }
         
             DialogSystem.dismissUntil(level - 1);
-    });
+        }
+    }).appendTo(footer);
     
     $('<button></button>').text("Cancel").click(function() {
         DialogSystem.dismissUntil(level - 1);
     }).appendTo(footer);
     
     var level = DialogSystem.showDialog(frame);
-    input[0].focus();
+    input.focus().data("suggest").textchange();
 };
 
 DataTableCellUI.prototype.createUpdateFunction = function(onBefore) {

src/main/webapp/scripts/project/data-table-column-header-ui.js

@@ -195,24 +195,31 @@ DataTableColumnHeaderUI.prototype._createMenuForColumnHeader = function(elmt) {
                 },
                 {},
                 {
-                    label: "Approve Best Candidates",
+                    label: "Match Each Cell to Its Best Candidate",
-                    tooltip: "Approve best reconciliaton candidate per cell in this column for all current filtered rows",
+                    tooltip: "Match each cell to its best candidate in this column for all current filtered rows",
                     click: function() {
-                        self._doApproveBestCandidates();
+                        self._doReconMatchBestCandidates();
                     }
                 },
                 {
-                    label: "Approve As New Topics",
+                    label: "Create a New Topic for Each Cell",
-                    tooltip: "Set to create new topics for cells in this column for all current filtered rows",
+                    tooltip: "Mark to create one new topic for each cell in this column for all current filtered rows",
                     click: function() {
-                        self._doApproveNewTopics();
+                        self._doReconMarkNewTopics(false);
                     }
                 },
                 {
-                    label: "Discard Reconciliation Results",
+                    label: "Create One New Topic for All Cells",
+                    tooltip: "Mark to create one new, common topic for all cells in this column for all current filtered rows",
+                    click: function() {
+                        self._doReconMarkNewTopics(true);
+                    }
+                },
+                {
+                    label: "Discard Reconciliation Judgments",
                     tooltip: "Discard reconciliaton results in this column for all current filtered rows",
                     click: function() {
-                        self._doDiscardReconResults();
+                        self._doReconDiscardJudgments();
                     }
                 },
                 {},
@@ -438,21 +445,21 @@ DataTableColumnHeaderUI.prototype._doReconcile = function() {
 
 };
 
-DataTableColumnHeaderUI.prototype._doDiscardReconResults = function() {
+DataTableColumnHeaderUI.prototype._doReconDiscardJudgments = function() {
     this._dataTableView.doPostThenUpdate(
         "recon-discard-judgments",
         { columnName: this._column.headerLabel }
     );
 };
 
-DataTableColumnHeaderUI.prototype._doApproveBestCandidates = function() {
+DataTableColumnHeaderUI.prototype._doReconMatchBestCandidates = function() {
     this._dataTableView.doPostThenUpdate(
-        "recon-approve-best-matches",
+        "recon-match-best-candidates",
         { columnName: this._column.headerLabel }
     );
 };
 
-DataTableColumnHeaderUI.prototype._doApproveNewTopics = function() {
+DataTableColumnHeaderUI.prototype._doReconMarkNewTopics = function() {
     this._dataTableView.doPostThenUpdate(
         "recon-mark-new-topics",
         { columnName: this._column.headerLabel }
@@ -462,7 +469,7 @@ DataTableColumnHeaderUI.prototype._doApproveNewTopics = function() {
 DataTableColumnHeaderUI.prototype._doSearchToMatch = function() {
     var self = this;
     var frame = DialogSystem.createDialog();
-    frame.width("200px");
+    frame.width("400px");
     
     var header = $('<div></div>').addClass("dialog-header").text("Search for Match").appendTo(frame);
     var body = $('<div></div>').addClass("dialog-body").appendTo(frame);
@@ -491,7 +498,7 @@ DataTableColumnHeaderUI.prototype._doSearchToMatch = function() {
     }).appendTo(footer);
     
     var level = DialogSystem.showDialog(frame);
-    input[0].focus();
+    input.focus().data("suggest").textchange();
 };
 
 DataTableColumnHeaderUI.prototype._doAddColumn = function(initialExpression) {

src/main/webapp/scripts/project/recon-dialog.js

@@ -97,6 +97,6 @@ ReconDialog.prototype._createDialog = function() {
     
     var level = DialogSystem.showDialog(frame);
     
-    input[0].focus();
+    input.focus().data("suggest").textchange();
 };
 

src/main/webapp/styles/data-table-view.css

@@ -52,12 +52,15 @@ img.column-header-menu {
 }
 
 
-ul.data-table-recon-candidates {
+div.data-table-recon-candidates {
-    margin: 0.75em 1.5em;
+    margin: 0.5em 0;
-    padding: 0;
+    min-width: 15em;
-    list-style: circle;
     color: #88a;
 }
+div.data-table-recon-candidate {
+    padding-left: 35px;
+    position: relative;
+}
 
 a.data-table-recon-topic {
     text-decoration: none;
@@ -82,3 +85,35 @@ a.data-table-recon-action:hover, a.data-table-recon-search:hover {
     text-decoration: underline;
     color: #008;
 }
+
+a.data-table-recon-match {
+    display: block;
+    width: 16px;
+    height: 16px;
+    background-image: url('../images/checks-map.png');
+    background-repeat: no-repeat;
+    background-position: -17px 0px;
+    text-decoration: none;
+    position: absolute;
+    top: 0px;
+    left: 0px;
+}
+a.data-table-recon-match:hover {
+    background-position: 0px 0px;
+}
+
+a.data-table-recon-match-similar {
+    display: block;
+    width: 16px;
+    height: 16px;
+    background-image: url('../images/checks-map.png');
+    background-repeat: no-repeat;
+    background-position: -17px -17px;
+    text-decoration: none;
+    position: absolute;
+    top: 0px;
+    left: 16px;
+}
+a.data-table-recon-match-similar:hover {
+    background-position: 0px -17px;
+}