- make recon changes flush column precomputes

- fixed bug where recon features are not saved to file properly
- support selecting non-numeric, blank, and error choices in numeric range facets

git-svn-id: http://google-refine.googlecode.com/svn/trunk@265 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java

@@ -10,12 +10,16 @@ import com.metaweb.gridworks.model.Project;
 import com.metaweb.gridworks.model.Row;
 
 public class ExpressionNumericRowBinner implements RowVisitor {
-    final protected Evaluable         _evaluable;
+    final protected Evaluable       _evaluable;
     final protected int             _cellIndex;
-    final protected NumericBinIndex    _index;
+    final protected NumericBinIndex _index;
     
     final public int[] bins;
     
+    public int nonNumericCount;
+    public int blankCount;
+    public int errorCount;
+    
     public ExpressionNumericRowBinner(Evaluable evaluable, int cellIndex, NumericBinIndex index) {
         _evaluable = evaluable;
         _cellIndex = cellIndex;
@@ -30,26 +34,32 @@ public class ExpressionNumericRowBinner implements RowVisitor {
         ExpressionUtils.bind(bindings, row, rowIndex, cell);
         
         Object value = _evaluable.evaluate(bindings);
-        if (value != null) {
+        if (value != null && value.getClass().isArray()) {
-            if (value.getClass().isArray()) {
+            Object[] a = (Object[]) value;
-                Object[] a = (Object[]) value;
+            for (Object v : a) {
-                for (Object v : a) {
+                processValue(v);
-                    processValue(v);
-                }
-            } else {
-                processValue(value);
             }
+        } else {
+            processValue(value);
         }
         return false;
     }
     
     protected void processValue(Object value) {
-        if (value instanceof Number) {
+        if (ExpressionUtils.isError(value)) {
-            double d = ((Number) value).doubleValue();
+            errorCount++;
-            
+        } else if (ExpressionUtils.isNonBlankData(value)) {
-            int bin = (int) Math.round((d - _index.getMin()) / _index.getStep());
+            if (value instanceof Number) {
-            
+                double d = ((Number) value).doubleValue();
-            bins[bin]++;
+                
+                int bin = (int) Math.round((d - _index.getMin()) / _index.getStep());
+                
+                bins[bin]++;
+            } else {
+                nonNumericCount++;
+            }
+        } else {
+            blankCount++;
         }
     }
 }

src/main/java/com/metaweb/gridworks/browsing/facets/RangeFacet.java

@@ -13,24 +13,31 @@ import com.metaweb.gridworks.expr.Evaluable;
 import com.metaweb.gridworks.expr.MetaParser;
 import com.metaweb.gridworks.model.Column;
 import com.metaweb.gridworks.model.Project;
+import com.metaweb.gridworks.util.JSONUtilities;
 
 public class RangeFacet implements Facet {
     protected String     _name;
     protected String     _expression;
     protected String     _columnName;
     protected int        _cellIndex;
-    protected Evaluable _eval;
+    protected Evaluable  _eval;
     
     protected String    _mode;
     protected double    _min;
     protected double    _max;
     protected double    _step;
-    protected int[]        _baseBins;
+    protected int[]     _baseBins;
-    protected int[]        _bins;
+    protected int[]     _bins;
+    protected int       _nonNumericCount;
+    protected int       _blankCount;
+    protected int       _errorCount;
     
     protected double    _from;
     protected double    _to;
-    protected boolean    _selected;
+    protected boolean   _selected;
+    protected boolean   _selectNonNumeric;
+    protected boolean   _selectBlank;
+    protected boolean   _selectError;
     
     public RangeFacet() {
     }
@@ -71,6 +78,10 @@ public class RangeFacet implements Facet {
             }
         }
         
+        writer.key("nonNumericCount"); writer.value(_nonNumericCount);
+        writer.key("blankCount"); writer.value(_blankCount);
+        writer.key("errorCount"); writer.value(_errorCount);
+        
         writer.endObject();
     }
 
@@ -100,24 +111,32 @@ public class RangeFacet implements Facet {
                 _selected = true;
             }
         }
+        
+        _selectNonNumeric = JSONUtilities.getBoolean(o, "selectNonNumeric", true);
+        _selectBlank = JSONUtilities.getBoolean(o, "selectBlank", true);
+        _selectError = JSONUtilities.getBoolean(o, "selectError", true);
+        
+        if (!_selectNonNumeric || !_selectBlank || !_selectError) {
+            _selected = true;
+        }
     }
 
     public RowFilter getRowFilter() {
         if (_selected) {
             if ("min".equals(_mode)) {
-                return new ExpressionNumberComparisonRowFilter(_eval, _cellIndex) {
+                return new ExpressionNumberComparisonRowFilter(_eval, _cellIndex, _selectNonNumeric, _selectBlank, _selectError) {
                     protected boolean checkValue(double d) {
                         return d >= _from;
                     };
                 };
             } else if ("max".equals(_mode)) {
-                return new ExpressionNumberComparisonRowFilter(_eval, _cellIndex) {
+                return new ExpressionNumberComparisonRowFilter(_eval, _cellIndex, _selectNonNumeric, _selectBlank, _selectError) {
                     protected boolean checkValue(double d) {
                         return d <= _to;
                     };
                 };
             } else {
-                return new ExpressionNumberComparisonRowFilter(_eval, _cellIndex) {
+                return new ExpressionNumberComparisonRowFilter(_eval, _cellIndex, _selectNonNumeric, _selectBlank, _selectError) {
                     protected boolean checkValue(double d) {
                         return d >= _from && d <= _to;
                     };
@@ -157,5 +176,8 @@ public class RangeFacet implements Facet {
         filteredRows.accept(project, binner);
         
         _bins = binner.bins;
+        _nonNumericCount = binner.nonNumericCount;
+        _blankCount = binner.blankCount;
+        _errorCount = binner.errorCount;
     }
 }

src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java

@@ -9,12 +9,24 @@ import com.metaweb.gridworks.model.Project;
 import com.metaweb.gridworks.model.Row;
 
 abstract public class ExpressionNumberComparisonRowFilter implements RowFilter {
-    final protected Evaluable        _evaluable;
+    final protected Evaluable  _evaluable;
-    final protected int             _cellIndex;
+    final protected int        _cellIndex;
+    final protected boolean _selectNonNumeric;
+    final protected boolean _selectBlank;
+    final protected boolean _selectError;
     
-    public ExpressionNumberComparisonRowFilter(Evaluable evaluable, int cellIndex) {
+    public ExpressionNumberComparisonRowFilter(
+        Evaluable evaluable, 
+        int cellIndex,
+        boolean selectNonNumeric,
+        boolean selectBlank,
+        boolean selectError
+    ) {
         _evaluable = evaluable;
         _cellIndex = cellIndex;
+        _selectNonNumeric = selectNonNumeric;
+        _selectBlank = selectBlank;
+        _selectError = selectError;
     }
 
     public boolean filterRow(Project project, int rowIndex, Row row) {
@@ -24,22 +36,34 @@ abstract public class ExpressionNumberComparisonRowFilter implements RowFilter {
         ExpressionUtils.bind(bindings, row, rowIndex, cell);
         
         Object value = _evaluable.evaluate(bindings);
-        if (value != null) {
+        if (value != null && value.getClass().isArray()) {
-            if (value.getClass().isArray()) {
+            Object[] a = (Object[]) value;
-                Object[] a = (Object[]) value;
+            for (Object v : a) {
-                for (Object v : a) {
+                if (checkValue(v)) {
-                    if (v instanceof Number && checkValue(((Number) v).doubleValue())) {
-                        return true;
-                    }
-                }
-            } else {
-                if (value instanceof Number && checkValue(((Number) value).doubleValue())) {
                     return true;
                 }
             }
+        } else {
+            if (checkValue(value)) {
+                return true;
+            }
         }
         return false;
     }
+        
+    protected boolean checkValue(Object v) {
+        if (ExpressionUtils.isError(v)) {
+            return _selectError;
+        } else if (ExpressionUtils.isNonBlankData(v)) {
+            if (v instanceof Number) {
+                return checkValue(((Number) v).doubleValue());
+            } else {
+                return _selectNonNumeric;
+            }
+        } else {
+            return _selectBlank;
+        }
+    }
     
     abstract protected boolean checkValue(double d);
 }

src/main/java/com/metaweb/gridworks/model/Project.java

@@ -121,7 +121,7 @@ public class Project {
         
         writer.write("rowCount="); writer.write(Integer.toString(rows.size())); writer.write('\n');
         for (Row row : rows) {
-            row.save(writer); writer.write('\n');
+            row.save(writer, options); writer.write('\n');
         }
     }
     

src/main/java/com/metaweb/gridworks/model/Row.java

@@ -149,10 +149,10 @@ public class Row implements HasFields, Jsonizable {
         writer.endObject();
     }
     
-    public void save(Writer writer) {
+    public void save(Writer writer, Properties options) {
         JSONWriter jsonWriter = new JSONWriter(writer);
         try {
-            write(jsonWriter, new Properties());
+            write(jsonWriter, options);
         } catch (JSONException e) {
             e.printStackTrace();
         }

src/main/java/com/metaweb/gridworks/model/changes/MassRowChange.java

@@ -41,12 +41,12 @@ public class MassRowChange implements Change {
     public void save(Writer writer, Properties options) throws IOException {
         writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
         for (Row row : _newRows) {
-            row.save(writer);
+            row.save(writer, options);
             writer.write('\n');
         }
         writer.write("oldRowCount="); writer.write(Integer.toString(_oldRows.size())); writer.write('\n');
         for (Row row : _oldRows) {
-            row.save(writer);
+            row.save(writer, options);
             writer.write('\n');
         }
         writer.write("/ec/\n"); // end of change marker

src/main/java/com/metaweb/gridworks/model/changes/ReconChange.java

@@ -72,6 +72,8 @@ public class ReconChange extends MassCellChange {
             
             column.setReconConfig(_newReconConfig);
             column.setReconStats(_newReconStats);
+            
+            column.clearPrecomputes();
         }
     }
     
@@ -83,6 +85,8 @@ public class ReconChange extends MassCellChange {
             Column column = project.columnModel.getColumnByName(_commonColumnName);
             column.setReconConfig(_oldReconConfig);
             column.setReconStats(_oldReconStats);
+            
+            column.clearPrecomputes();
         }
     }
     

src/main/webapp/scripts/facets/range-facet.js

@@ -5,6 +5,13 @@ function RangeFacet(div, config, options) {
     
     this._from = ("from" in this._config) ? this._config.from : null;
     this._to = ("to" in this._config) ? this._config.to : null;
+    this._selectNonNumeric = true;
+    this._selectBlank = true;
+    this._selectError = true;
+    
+    this._nonNumericCount = 0;
+    this._blankCount = 0;
+    this._errorCount = 0;
     
     this._error = false;
     this._initializedUI = false;
@@ -23,6 +30,9 @@ RangeFacet.prototype._setDefaults = function() {
         this._to = this._config.max;
     }
     
+    this._selectNonNumeric = true;
+    this._selectBlank = true;
+    this._selectError = true;
 };
 
 RangeFacet.reconstruct = function(div, uiState) {
@@ -45,7 +55,10 @@ RangeFacet.prototype.getJSON = function() {
         name: this._config.name,
         mode: this._config.mode,
         expression: this._config.expression,
-        columnName: this._config.columnName
+        columnName: this._config.columnName,
+        selectNonNumeric: this._selectNonNumeric,
+        selectBlank: this._selectBlank,
+        selectError: this._selectError
     };
     
     if (this._config.mode == "min" || this._config.mode == "range") {
@@ -63,6 +76,10 @@ RangeFacet.prototype.getJSON = function() {
 };
 
 RangeFacet.prototype.hasSelection = function() {
+    if (!this._selectNonNumeric || !this._selectBlank || !this._selectError) {
+        return true;
+    }
+    
     switch (this._config.mode) {
     case "min":
         return this._from != null && (!this._initializedUI || this._from > this._config.min);
@@ -97,6 +114,10 @@ RangeFacet.prototype._initializeUI = function() {
             self._sliderDiv.slider("values", 0, self._from);
             self._sliderDiv.slider("values", 1, self._to);
         }
+        self._selectNonNumeric = true;
+        self._selectBlank = true;
+        self._selectError = true;
+        
         self._setRangeIndicators();
         self._updateRest();
     }).prependTo(headerDiv);
@@ -118,6 +139,7 @@ RangeFacet.prototype._initializeUI = function() {
     this._histogramDiv = $('<div></div>').addClass("facet-range-histogram").appendTo(bodyDiv);
     this._sliderDiv = $('<div></div>').addClass("facet-range-slider").appendTo(bodyDiv);
     this._statusDiv = $('<div></div>').addClass("facet-range-status").appendTo(bodyDiv);
+    this._otherChoicesDiv = $('<div></div>').addClass("facet-range-other-choices").appendTo(bodyDiv);
     
     var onSlide = function(event, ui) {
         switch (self._config.mode) {
@@ -164,6 +186,56 @@ RangeFacet.prototype._initializeUI = function() {
     
     this._sliderDiv.slider(sliderConfig);
     this._setRangeIndicators();
+    this._renderOtherChoices();
+};
+
+RangeFacet.prototype._renderOtherChoices = function() {
+    var self = this;
+    var container = this._otherChoicesDiv.empty();
+    
+    var table = $('<table>').attr("cellpadding", "0").attr("cellspacing", "1").css("white-space", "pre").appendTo(container)[0];
+    var tr0 = table.insertRow(0);
+    var tr1 = table.insertRow(1);
+    
+    var td00 = $(tr0.insertCell(0)).attr("width", "1%");
+    var nonNumericCheck = $('<input type="checkbox" />').appendTo(td00).change(function() {
+        self._selectNonNumeric = !self._selectNonNumeric;
+        self._updateRest();
+    });
+    if (this._selectNonNumeric) {
+        nonNumericCheck[0].checked = true;
+    }
+    
+    var td01 = $(tr0.insertCell(1)).attr("colspan", "3");
+    $('<span>').text("Non-numeric ").addClass("facet-choice-label").appendTo(td01);
+    $('<span>').text(this._nonNumericCount).addClass("facet-choice-count").appendTo(td01);
+    
+    var td10 = $(tr1.insertCell(0)).attr("width", "1%");
+    var blankCheck = $('<input type="checkbox" />').appendTo(td10).change(function() {
+        self._selectBlank = !self._selectBlank;
+        self._updateRest();
+    });
+    if (this._selectBlank) {
+        blankCheck[0].checked = true;
+    }
+    
+    var td11 = $(tr1.insertCell(1));
+    $('<span>').text("Blank ").addClass("facet-choice-label").appendTo(td11);
+    $('<span>').text(this._blankCount).addClass("facet-choice-count").appendTo(td11);
+    
+    var td12 = $(tr1.insertCell(2)).attr("width", "1%");
+    var errorCheck = $('<input type="checkbox" />').appendTo(td12).change(function() {
+        self._selectError = !self._selectError;
+        self._updateRest();
+    });
+    if (this._selectError) {
+        errorCheck[0].checked = true;
+    }
+    
+    var td13 = $(tr1.insertCell(3));
+    $('<span>').text("Error ").addClass("facet-choice-label").appendTo(td13);
+    $('<span>').text(this._errorCount).addClass("facet-choice-count").appendTo(td13);
+    
 };
 
 RangeFacet.prototype._setRangeIndicators = function() {
@@ -178,6 +250,7 @@ RangeFacet.prototype._setRangeIndicators = function() {
     default:
         text = this._from + " to " + this._to;
     }
+    
     this._statusDiv.text(text);
 };
 
@@ -204,6 +277,10 @@ RangeFacet.prototype.updateState = function(data) {
                 this._to = data.max;
             }
         }
+        
+        this._nonNumericCount = data.nonNumericCount;
+        this._blankCount = data.blankCount;
+        this._errorCount = data.errorCount;
     } else {
         this._error = true;
     }
@@ -255,6 +332,7 @@ RangeFacet.prototype.render = function() {
     }
     
     this._setRangeIndicators();
+    this._renderOtherChoices();
 };
 
 RangeFacet.prototype._reset = function() {

src/main/webapp/styles/project/browsing.css

@@ -125,7 +125,7 @@ img.facet-choice-link {
 .facet-range-slider {
 }
 .facet-range-status {
-    margin-top: 10px;
+    margin: 10px 0;
     text-align: center;
     color: #aaa;
 }