added a new command to get column metadata prior of creating the scatterplot half-matrix, this allows us to build a much more compact table and make the browser crawl a little less

git-svn-id: http://google-refine.googlecode.com/svn/trunk@481 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/GridworksServlet.java

@@ -38,6 +38,7 @@ import com.metaweb.gridworks.commands.info.GetAllProjectMetadataCommand;
 import com.metaweb.gridworks.commands.info.GetExpressionHistoryCommand;
 import com.metaweb.gridworks.commands.info.GetHistoryCommand;
 import com.metaweb.gridworks.commands.info.GetModelsCommand;
+import com.metaweb.gridworks.commands.info.GetColumnsInfoCommand;
 import com.metaweb.gridworks.commands.info.GetOperationsCommand;
 import com.metaweb.gridworks.commands.info.GetProcessesCommand;
 import com.metaweb.gridworks.commands.info.GetProjectMetadataCommand;
@@ -83,6 +84,7 @@ public class GridworksServlet extends HttpServlet {
         _commands.put("get-processes", new GetProcessesCommand());
         _commands.put("get-history", new GetHistoryCommand());
         _commands.put("get-operations", new GetOperationsCommand());
+        _commands.put("get-columns-info", new GetColumnsInfoCommand());
         _commands.put("get-scatterplot", new GetScatterplotCommand());
         
         _commands.put("undo-redo", new UndoRedoCommand());

src/main/java/com/metaweb/gridworks/browsing/charting/ScatterplotCharter.java

@@ -32,13 +32,13 @@ public class ScatterplotCharter {
 
     private static final int LIN = 0;
     private static final int LOG = 1;
-    private static final int RADIAL = 2;
+    private static final int POLAR = 2;
     
     private static int getAxisDim(String type) {
         if ("log".equals(type)) {
             return LOG;
-        } else if ("rad".equals(type) || "radial".equals(type)) {
+        } else if ("pol".equals(type) || "polar".equals(type)) {
-            return RADIAL;
+            return POLAR;
         } else {
             return LIN;
         }
@@ -108,17 +108,17 @@ public class ScatterplotCharter {
                 max_y = index_y.getMax();
             }
             
-            width = (o.has("w")) ? o.getInt("w") : 20;
+            width = (o.has("w")) ? o.getInt("w") : 50;
-            height = (o.has("h")) ? o.getInt("h") : 20;
+            height = (o.has("h")) ? o.getInt("h") : 50;
             
-            dot = (o.has("dot")) ? o.getDouble("dot") : 0.1d;
+            dot = (o.has("dot")) ? o.getDouble("dot") : 0.2d;
                         
             dim = (o.has("dim")) ? getAxisDim(o.getString("dim")) : LIN;
             
             delta_x = max_x - min_x;
             delta_y = max_y - min_y;
             
-            if (dim == RADIAL) {
+            if (dim == POLAR) {
                 rx = (o.has("rx")) ? o.getDouble("rx") : 0.0d;
                 ry = (o.has("ry")) ? o.getDouble("ry") : 0.0d;
             } else if (dim == LOG) {
@@ -181,7 +181,7 @@ public class ScatterplotCharter {
                     if (dim == LOG) {
                         x = Math.log10(xv - min_x) * w / log_delta_x - dot / 2;
                         y = Math.log10(yv - min_y) * h / log_delta_y - dot / 2;
-                    } else if (dim == RADIAL) {
+                    } else if (dim == POLAR) {
                         x = (xv - min_x) * w / delta_x - dot / 2;
                         y = (yv - min_y) * h / delta_y - dot / 2;
                     } else {

src/main/java/com/metaweb/gridworks/commands/info/GetAllProjectMetadataCommand.java

@@ -29,7 +29,6 @@ public class GetAllProjectMetadataCommand extends Command {
             Properties options = new Properties();
             
             writer.object();
-            
             writer.key("projects");
                 writer.object();
                 Map<Long, ProjectMetadata> m = ProjectManager.singleton.getAllProjectMetadata();
@@ -41,7 +40,6 @@ public class GetAllProjectMetadataCommand extends Command {
                     }
                 }
                 writer.endObject();
-                
             writer.endObject();
         } catch (JSONException e) {
             respondException(response, e);

src/main/java/com/metaweb/gridworks/commands/info/GetColumnsInfoCommand.java

@@ -0,0 +1,97 @@
+package com.metaweb.gridworks.commands.info;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.metaweb.gridworks.browsing.facets.NumericBinIndex;
+import com.metaweb.gridworks.commands.Command;
+import com.metaweb.gridworks.expr.Evaluable;
+import com.metaweb.gridworks.expr.MetaParser;
+import com.metaweb.gridworks.expr.ParsingException;
+import com.metaweb.gridworks.model.Column;
+import com.metaweb.gridworks.model.Project;
+
+public class GetColumnsInfoCommand extends Command {
+
+    public void doGet(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        
+        try {
+            //long start = System.currentTimeMillis();
+
+            response.setCharacterEncoding("UTF-8");
+            response.setHeader("Content-Type", "application/json");
+
+            Project project = getProject(request);
+            //Engine engine = getEngine(request, project);
+            
+            JSONWriter writer = new JSONWriter(response.getWriter());
+
+            writer.array();
+            for (Column column : project.columnModel.columns) {
+                writer.object();
+                    write(project, column, writer);
+                writer.endObject();
+            }
+            writer.endArray();
+            
+            //Gridworks.log("Obtained columns info in " + (System.currentTimeMillis() - start) + "ms");
+        } catch (Exception e) {
+            e.printStackTrace();
+            respondException(response, e);
+        }
+    }
+    
+    private NumericBinIndex getBinIndex(Project project, Column column) {
+        String expression = "value";
+        String key = "numeric-bin:" + expression;
+        Evaluable eval = null;
+        try {
+            eval = MetaParser.parse(expression);
+        } catch (ParsingException e) {
+            // this should never happen
+        }
+        NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
+        if (index == null) {
+            index = new NumericBinIndex(project, column.getName(), column.getCellIndex(), eval);
+            column.setPrecompute(key, index);
+        }
+        return index;
+    }
+    
+    private void write(Project project, Column column, JSONWriter writer) throws JSONException {
+        NumericBinIndex columnIndex = getBinIndex(project, column);
+        if (columnIndex != null) {
+            writer.key("name");
+            writer.value(column.getName());
+            boolean is_numeric = columnIndex.isNumeric();
+            writer.key("is_numeric");
+            writer.value(is_numeric);
+            writer.key("numeric_row_count");
+            writer.value(columnIndex.getNumericRowCount());
+            writer.key("non_numeric_row_count");
+            writer.value(columnIndex.getNonNumericRowCount());
+            writer.key("error_row_count");
+            writer.value(columnIndex.getErrorRowCount());
+            writer.key("blank_row_count");
+            writer.value(columnIndex.getBlankRowCount());
+            if (is_numeric) {
+                writer.key("min");
+                writer.value(columnIndex.getMin());
+                writer.key("max");
+                writer.value(columnIndex.getMax());
+                writer.key("step");
+                writer.value(columnIndex.getStep());
+            }
+        } else {
+            writer.key("error");
+            writer.value("error finding numeric information on the '" + column.getName() + "' column");
+        }
+    }
+}

src/main/java/com/metaweb/gridworks/commands/info/GetScatterplotCommand.java

@@ -23,6 +23,7 @@ public class GetScatterplotCommand extends Command {
         
         try {
             //long start = System.currentTimeMillis();
+            
             Project project = getProject(request);
             Engine engine = getEngine(request, project);
             JSONObject conf = getJsonParameter(request,"plotter");
@@ -38,7 +39,7 @@ public class GetScatterplotCommand extends Command {
                 sos.close();
             }
             
-            //Gridworks.log("drawn scatterplot in " + (System.currentTimeMillis() - start) + "ms");
+            //Gridworks.log("Drawn scatterplot in " + (System.currentTimeMillis() - start) + "ms");
         } catch (Exception e) {
             e.printStackTrace();
             respondException(response, e);

src/main/webapp/scripts/dialogs/scatterplot-dialog.js

@@ -1,8 +1,6 @@
 function ScatterplotDialog(column) {
     this._column = column;
     this._plot_method = "lin";
-    this._plot_size = Math.max(Math.floor("500" / theProject.columnModel.columns.length / 5) * 5,"20");
-    this._dot_size = 0.3;
     this._createDialog();
 }
 
@@ -30,8 +28,8 @@ ScatterplotDialog.prototype._createDialog = function() {
                         '<option selected="true">linear</option>' +
                         '<option>log-log</option>' +
                     '</select></span>' +
-                    '<span class="clustering-dialog-controls">Plot Size: <input bind="plotSize" type="test" size="2" value="' + this._plot_size + '"> px</span>' +
+                    '<span class="clustering-dialog-controls">Plot Size: <input bind="plotSize" type="test" size="2" value=""> px</span>' +
-                    '<span class="clustering-dialog-controls">Dot Size: <input bind="dotSize" type="test" size="2" value="' + this._dot_size + '"> px</span>' +
+                    '<span class="clustering-dialog-controls">Dot Size: <input bind="dotSize" type="test" size="2" value=""> px</span>' +
                 '</td>' +
             '</tr>' +
             '<tr>' +
@@ -84,78 +82,100 @@ ScatterplotDialog.prototype._createDialog = function() {
 
 ScatterplotDialog.prototype._renderMatrix = function() {
     var self = this;
-    var columns = theProject.columnModel.columns;
     
     var container = this._elmts.tableContainer.html(
-        '<div style="margin: 1em; font-size: 130%; color: #888;">Processing... <img src="/images/small-spinner.gif"></div>'
+        '<div style="margin: 1em; font-size: 130%; color: #888; background-color: white;">Processing... <img src="/images/small-spinner.gif"></div>'
     );
 
-    if (columns.length > 0) {
+    if (theProject.columnModel.columns.length > 0) {
-        var table = '<table class="scatterplot-matrix-table"><tbody>';
+        var params = {
-        
+            project: theProject.id
-        var createScatterplot = function(cx, cy) {
-            var title = cx + ' (x) vs. ' + cy + ' (y)';
-            var link = '<a href="javascript:{}" title="' + title + '" cx="' + cx + '" cy="' + cy + '">';
-            var plotter_params = { 
-                'cx' : cx, 
-                'cy' : cy,
-                'w' : self._plot_size * 3,
-                'h' : self._plot_size * 3,
-                'dot': self._dot_size,
-                'dim': self._plot_method
-            };
-            var params = {
-                project: theProject.id,
-                engine: JSON.stringify(ui.browsingEngine.getJSON()), 
-                plotter: JSON.stringify(plotter_params) 
-            };
-            var url = "/command/get-scatterplot?" + $.param(params);
-            return link + '<img src="' + url + '" width="' + self._plot_size + '" height="' + self._plot_size + '" /></a>';
         };
-
+        $.getJSON("/command/get-columns-info?" + $.param(params),function(data) {
-        for (var i = 0; i < columns.length; i++) {
+            if (data == null || typeof data.length == 'undefined') {
-            table += '<tr>';
+                container.html("Error calling 'get-columns-info'");
-            var div_class = "column_header";
+                return;
-            if (columns[i].name == this._column) div_class += " current_column";
-            table += '<td class="' + div_class + '" colspan="' + (i + 1) + '">' + columns[i].name + '</td>'
-            for (var j = i + 1; j < columns.length; j++) {
-                var cx = columns[i].name;
-                var cy = columns[j].name;
-                
-                var div_class = "scatterplot";
-                if (cx == this._column || cy == this._column) div_class += " current_column";
-                table += '<td><div class="' + div_class + '">' + createScatterplot(cx,cy) + '</div></td>';
             }
-            table += '</tr>';
-        }
                 
-        table += "</tbody></table>";
+            var columns = [];
+            for (var i = 0; i < data.length; i++) {
+                if (data[i].is_numeric) {
+                    columns.push(data[i]);
+                }
+            }
                 
-        var width = container.width();
+            if (typeof self._plot_size == 'undefined') {
-        container.empty().css("width", width + "px").append($(table));
+                self._plot_size = Math.max(Math.floor(500 / columns.length / 5) * 5,20);
+                self._dot_size = 0.4;
+                self._elmts.plotSize.val(self._plot_size);                
+                self._elmts.dotSize.val(self._dot_size);                
+            }
             
-        container.find("a").click(function() {
+            var table = '<table class="scatterplot-matrix-table"><tbody>';
-            var options = {
+            
-                "name" : $(this).attr("title"),
+            var createScatterplot = function(cx, cy) {
-                "x_columnName" : $(this).attr("cx"), 
+                var title = cx + ' (x) vs. ' + cy + ' (y)';
-                "y_columnName" : $(this).attr("cy"), 
+                var link = '<a href="javascript:{}" title="' + title + '" cx="' + cx + '" cy="' + cy + '">';
-                "x_expression" : "value",
+                var plotter_params = { 
-                "y_expression" : "value",
+                    'cx' : cx, 
-                "dot" : self._dot_size,
+                    'cy' : cy,
-                "dim" : self._plot_method
+                    'w' : self._plot_size * 3,
+                    'h' : self._plot_size * 3,
+                    'dot': self._dot_size,
+                    'dim': self._plot_method
+                };
+                var params = {
+                    project: theProject.id,
+                    engine: JSON.stringify(ui.browsingEngine.getJSON()), 
+                    plotter: JSON.stringify(plotter_params) 
+                };
+                var url = "/command/get-scatterplot?" + $.param(params);
+                return link + '<img src="' + url + '" width="' + self._plot_size + '" height="' + self._plot_size + '" /></a>';
             };
-            console.log(options);
-            ui.browsingEngine.addFacet("scatterplot", options);
-            //self._dismiss();
-        });
     
-        container.find(".scatterplot").hover(
+            for (var i = 0; i < columns.length; i++) {
-            function() {                
+                table += '<tr>';
-                $(this).find('img').addClass("hover");
+                var div_class = "column_header";
-            } , function() {
+                if (columns[i].name == self._column) div_class += " current_column";
-                $(this).find('img').removeClass("hover");
+                table += '<td class="' + div_class + '" colspan="' + (i + 1) + '">' + columns[i].name + '</td>'
+                for (var j = i + 1; j < columns.length; j++) {
+                    var cx = columns[i].name;
+                    var cy = columns[j].name;
+                    
+                    var div_class = "scatterplot";
+                    if (cx == self._column || cy == self._column) div_class += " current_column";
+                    table += '<td><div class="' + div_class + '">' + createScatterplot(cx,cy) + '</div></td>';
+                }
+                table += '</tr>';
             }
-        );
+    
+            table += "</tbody></table>";
+            
+            var width = container.width();
+            container.empty().css("width", width + "px").append($(table));
+            
+            container.find("a").click(function() {
+                var options = {
+                    "name" : $(this).attr("title"),
+                    "x_columnName" : $(this).attr("cx"), 
+                    "y_columnName" : $(this).attr("cy"), 
+                    "x_expression" : "value",
+                    "y_expression" : "value",
+                    "dot" : self._dot_size,
+                    "dim" : self._plot_method
+                };
+                ui.browsingEngine.addFacet("scatterplot", options);
+                //self._dismiss();
+            });
+        
+            container.find(".scatterplot").hover(
+                function() {                
+                    $(this).find('img').addClass("hover");
+                } , function() {
+                    $(this).find('img').removeClass("hover");
+                }
+            );
+        });
     } else {
         container.html(
             '<div style="margin: 2em;"><div style="font-size: 130%; color: #333;">There are no columns in this dataset</div></div>'

src/main/webapp/scripts/widgets/scatterplot-widget.js

@@ -97,12 +97,10 @@ ScatterplotWidget.prototype._render = function() {
                 
                 ctx.restore();
             }  
-            self._plotter.dot = "0.4";
             self._plotter.color = "000088";
             img2.src = self._get_image_url(self._plotter);
         }
     }  
-    self._plotter.dot = "0.4";
     self._plotter.color = "000000";
     img.src = self._get_image_url(self._plotter);
 };