Implemented inter-project joins.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@387 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/InterProjectModel.java

@@ -0,0 +1,132 @@
+package com.metaweb.gridworks;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.metaweb.gridworks.expr.ExpressionUtils;
+import com.metaweb.gridworks.expr.HasFieldsListImpl;
+import com.metaweb.gridworks.expr.WrappedRow;
+import com.metaweb.gridworks.model.Column;
+import com.metaweb.gridworks.model.Project;
+import com.metaweb.gridworks.model.Row;
+
+public class InterProjectModel {
+    static public class ProjectJoin {
+        final public long   fromProjectID;
+        final public String fromProjectColumnName;
+        final public long   toProjectID;
+        final public String toProjectColumnName;
+        
+        final public Map<Object, List<Integer>> valueToRowIndices = 
+            new HashMap<Object, List<Integer>>();
+        
+        ProjectJoin(
+            long   fromProjectID,
+            String fromProjectColumnName,
+            long   toProjectID,
+            String toProjectColumnName
+        ) {
+            this.fromProjectID = fromProjectID;
+            this.fromProjectColumnName = fromProjectColumnName;
+            this.toProjectID = toProjectID;
+            this.toProjectColumnName = toProjectColumnName;
+        }
+        
+        public HasFieldsListImpl getRows(Object value) {
+            if (ExpressionUtils.isNonBlankData(value) && valueToRowIndices.containsKey(value)) {
+                Project toProject = ProjectManager.singleton.getProject(toProjectID);
+                if (toProject != null) {
+                    HasFieldsListImpl rows = new HasFieldsListImpl();
+                    for (Integer r : valueToRowIndices.get(value)) {
+                        Row row = toProject.rows.get(r);
+                        rows.add(new WrappedRow(toProject, r, row));
+                    }
+                    
+                    return rows;
+                }
+            }
+            return null;
+        }
+    }
+    
+    protected Map<String, ProjectJoin> _joins = new HashMap<String, ProjectJoin>();
+    
+    public ProjectJoin getJoin(String fromProject, String fromColumn, String toProject, String toColumn) {
+        String key = fromProject + ";" + fromColumn + ";" + toProject + ";" + toColumn;
+        if (!_joins.containsKey(key)) {
+            ProjectJoin join = new ProjectJoin(
+                ProjectManager.singleton.getProjectID(fromProject), 
+                fromColumn, 
+                ProjectManager.singleton.getProjectID(toProject), 
+                toColumn
+            );
+            
+            computeJoin(join);
+            
+            _joins.put(key, join);
+        }
+        
+        return _joins.get(key);
+    }
+    
+    public void flushJoinsInvolvingProject(long projectID) {
+        for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
+            ProjectJoin join = entry.getValue();
+            if (join.fromProjectID == projectID || join.toProjectID == projectID) {
+                _joins.remove(entry.getKey());
+            }
+        }
+    }
+
+    public void flushJoinsInvolvingProjectColumn(long projectID, String columnName) {
+        for (Entry<String, ProjectJoin> entry : _joins.entrySet()) {
+            ProjectJoin join = entry.getValue();
+            if (join.fromProjectID == projectID && join.fromProjectColumnName.equals(columnName) || 
+                join.toProjectID == projectID && join.toProjectColumnName.equals(columnName)) {
+                _joins.remove(entry.getKey());
+            }
+        }
+    }
+
+    protected void computeJoin(ProjectJoin join) {
+        if (join.fromProjectID < 0 || join.toProjectID < 0) {
+            return;
+        }
+        
+        Project fromProject = ProjectManager.singleton.getProject(join.fromProjectID);
+        Project toProject = ProjectManager.singleton.getProject(join.toProjectID);
+        if (fromProject == null || toProject == null) {
+            return;
+        }
+        
+        Column fromColumn = fromProject.columnModel.getColumnByName(join.fromProjectColumnName);
+        Column toColumn = toProject.columnModel.getColumnByName(join.toProjectColumnName);
+        if (fromColumn == null || toColumn == null) {
+            return;
+        }
+        
+        for (Row fromRow : fromProject.rows) {
+            Object value = fromRow.getCellValue(fromColumn.getCellIndex());
+            if (ExpressionUtils.isNonBlankData(value)) {
+                if (!join.valueToRowIndices.containsKey(value)) {
+                    join.valueToRowIndices.put(value, new ArrayList<Integer>());
+                }
+            }
+        }
+        
+        int count = toProject.rows.size();
+        for (int r = 0; r < count; r++) {
+            Row toRow = toProject.rows.get(r);
+            
+            Object value = toRow.getCellValue(toColumn.getCellIndex());
+            if (ExpressionUtils.isNonBlankData(value)) {
+                if (join.valueToRowIndices.containsKey(value)) {
+                    join.valueToRowIndices.get(value).add(r);
+                }
+            }
+        }
+    }
+}

src/main/java/com/metaweb/gridworks/ProjectManager.java

@@ -12,6 +12,7 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import org.json.JSONArray;
 import org.json.JSONException;
@@ -41,6 +42,11 @@ public class ProjectManager {
      */
     transient protected Map<Long, Project> _projects;
     
+    /**
+     *  What caches the joins between projects.
+     */
+    transient protected InterProjectModel _interProjectModel = new InterProjectModel(); 
+    
     static public ProjectManager singleton;
     
     static public synchronized void initialize() {
@@ -137,6 +143,10 @@ public class ProjectManager {
         load();
     }
     
+    public InterProjectModel getInterProjectModel() {
+        return _interProjectModel;
+    }
+    
     public File getWorkspaceDir() {
         return _workspaceDir;
     }
@@ -208,6 +218,24 @@ public class ProjectManager {
         return _projectsMetadata.get(id);
     }
     
+    public ProjectMetadata getProjectMetadata(String name) {
+        for (ProjectMetadata pm : _projectsMetadata.values()) {
+            if (pm.getName().equals(name)) {
+                return pm;
+            }
+        }
+        return null;
+    }
+    
+    public long getProjectID(String name) {
+        for (Entry<Long, ProjectMetadata> entry : _projectsMetadata.entrySet()) {
+            if (entry.getValue().getName().equals(name)) {
+                return entry.getKey();
+            }
+        }
+        return -1;
+    }
+    
     public Map<Long, ProjectMetadata> getAllProjectMetadata() {
         return _projectsMetadata;
     }

src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNominalRowGrouper.java

@@ -22,6 +22,7 @@ public class ExpressionNominalRowGrouper implements RowVisitor {
      * Configuration
      */
     final protected Evaluable   _evaluable;
+    final protected String      _columnName;
     final protected int         _cellIndex;
     
     /*
@@ -31,8 +32,9 @@ public class ExpressionNominalRowGrouper implements RowVisitor {
     public int blankCount = 0;
     public int errorCount = 0;
     
-    public ExpressionNominalRowGrouper(Evaluable evaluable, int cellIndex) {
+    public ExpressionNominalRowGrouper(Evaluable evaluable, String columnName, int cellIndex) {
         _evaluable = evaluable;
+        _columnName = columnName;
         _cellIndex = cellIndex;
     }
     
@@ -40,7 +42,7 @@ public class ExpressionNominalRowGrouper implements RowVisitor {
         Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
 
         Properties bindings = ExpressionUtils.createBindings(project);
-        ExpressionUtils.bind(bindings, row, rowIndex, cell);
+        ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
         
         Object value = _evaluable.evaluate(bindings);
         if (value != null) {

src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java

@@ -19,6 +19,7 @@ public class ExpressionNumericRowBinner implements RowVisitor {
      * Configuration
      */
     final protected Evaluable       _evaluable;
+    final protected String          _columnName;
     final protected int             _cellIndex;
     final protected NumericBinIndex _index;     // base bins
     
@@ -31,8 +32,9 @@ public class ExpressionNumericRowBinner implements RowVisitor {
     public int blankCount;
     public int errorCount;
     
-    public ExpressionNumericRowBinner(Evaluable evaluable, int cellIndex, NumericBinIndex index) {
+    public ExpressionNumericRowBinner(Evaluable evaluable, String columnName, int cellIndex, NumericBinIndex index) {
         _evaluable = evaluable;
+        _columnName = columnName;
         _cellIndex = cellIndex;
         _index = index;
         bins = new int[_index.getBins().length];
@@ -42,7 +44,7 @@ public class ExpressionNumericRowBinner implements RowVisitor {
         Cell cell = row.getCell(_cellIndex);
 
         Properties bindings = ExpressionUtils.createBindings(project);
-        ExpressionUtils.bind(bindings, row, rowIndex, cell);
+        ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
         
         Object value = _evaluable.evaluate(bindings);
         if (value != null) {

src/main/java/com/metaweb/gridworks/browsing/facets/ListFacet.java

@@ -146,6 +146,7 @@ public class ListFacet implements Facet {
                 null :
                 new ExpressionEqualRowFilter(
                     _eval, 
+                    _columnName,
                     _cellIndex, 
                     createMatches(), 
                     _selectBlank, 
@@ -155,7 +156,7 @@ public class ListFacet implements Facet {
     public void computeChoices(Project project, FilteredRows filteredRows) {
         if (_eval != null && _errorMessage == null) {
             ExpressionNominalRowGrouper grouper = 
-                new ExpressionNominalRowGrouper(_eval, _cellIndex);
+                new ExpressionNominalRowGrouper(_eval, _columnName, _cellIndex);
             
             filteredRows.accept(project, grouper);
             

src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinIndex.java

@@ -27,7 +27,7 @@ public class NumericBinIndex {
     private double _step;
     private int[]  _bins;
     
-    public NumericBinIndex(Project project, int cellIndex, Evaluable eval) {
+    public NumericBinIndex(Project project, String columnName, int cellIndex, Evaluable eval) {
         Properties bindings = ExpressionUtils.createBindings(project);
         
         _min = Double.POSITIVE_INFINITY;
@@ -38,7 +38,7 @@ public class NumericBinIndex {
             Row row = project.rows.get(i);
             Cell cell = row.getCell(cellIndex);
 
-            ExpressionUtils.bind(bindings, row, i, cell);
+            ExpressionUtils.bind(bindings, row, i, columnName, cell);
             
             Object value = eval.evaluate(bindings);
             if (value != null) {

src/main/java/com/metaweb/gridworks/browsing/facets/RangeFacet.java

@@ -161,7 +161,7 @@ public class RangeFacet implements Facet {
         if (_eval != null && _errorMessage == null && _selected) {
             if ("min".equals(_mode)) {
                 return new ExpressionNumberComparisonRowFilter(
-                        _eval, _cellIndex, _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) {
+                        _eval, _columnName, _cellIndex, _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) {
                     
                     protected boolean checkValue(double d) {
                         return d >= _from;
@@ -169,7 +169,7 @@ public class RangeFacet implements Facet {
                 };
             } else if ("max".equals(_mode)) {
                 return new ExpressionNumberComparisonRowFilter(
-                        _eval, _cellIndex, _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) {
+                        _eval, _columnName, _cellIndex, _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) {
                     
                     protected boolean checkValue(double d) {
                         return d < _to;
@@ -177,7 +177,7 @@ public class RangeFacet implements Facet {
                 };
             } else {
                 return new ExpressionNumberComparisonRowFilter(
-                        _eval, _cellIndex, _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) {
+                        _eval, _columnName, _cellIndex, _selectNumeric, _selectNonNumeric, _selectBlank, _selectError) {
                     
                     protected boolean checkValue(double d) {
                         return d >= _from && d < _to;
@@ -196,7 +196,7 @@ public class RangeFacet implements Facet {
             String key = "numeric-bin:" + _expression;
             NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
             if (index == null) {
-                index = new NumericBinIndex(project, _cellIndex, _eval);
+                index = new NumericBinIndex(project, _columnName, _cellIndex, _eval);
                 column.setPrecompute(key, index);
             }
             
@@ -214,7 +214,7 @@ public class RangeFacet implements Facet {
             }
             
             ExpressionNumericRowBinner binner = 
-                new ExpressionNumericRowBinner(_eval, _cellIndex, index);
+                new ExpressionNumericRowBinner(_eval, _columnName, _cellIndex, index);
             
             filteredRows.accept(project, binner);
             

src/main/java/com/metaweb/gridworks/browsing/facets/TextSearchFacet.java

@@ -85,13 +85,13 @@ public class TextSearchFacet implements Facet {
         Evaluable eval = new VariableExpr("value");
         
         if ("regex".equals(_mode)) {
-            return new ExpressionStringComparisonRowFilter(eval, _cellIndex) {
+            return new ExpressionStringComparisonRowFilter(eval, _columnName, _cellIndex) {
                 protected boolean checkValue(String s) {
                     return _pattern.matcher(s).find();
                 };
             };
         } else {
-            return new ExpressionStringComparisonRowFilter(eval, _cellIndex) {
+            return new ExpressionStringComparisonRowFilter(eval, _columnName, _cellIndex) {
                 protected boolean checkValue(String s) {
                     return (_caseSensitive ? s : s.toLowerCase()).contains(_query);
                 };

src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionEqualRowFilter.java

@@ -16,6 +16,8 @@ import com.metaweb.gridworks.model.Row;
  */
 public class ExpressionEqualRowFilter implements RowFilter {
     final protected Evaluable       _evaluable; // the expression to evaluate
+    
+    final protected String          _columnName;
     final protected int             _cellIndex; // the expression is based on this column;
                                                 // -1 if based on no column in particular,
                                                 // for expression such as "row.starred".
@@ -26,12 +28,14 @@ public class ExpressionEqualRowFilter implements RowFilter {
     
     public ExpressionEqualRowFilter(
         Evaluable evaluable,
+        String columnName,
         int cellIndex, 
         Object[] matches, 
         boolean selectBlank, 
         boolean selectError
     ) {
         _evaluable = evaluable;
+        _columnName = columnName;
         _cellIndex = cellIndex;
         _matches = matches;
         _selectBlank = selectBlank;
@@ -42,7 +46,7 @@ public class ExpressionEqualRowFilter implements RowFilter {
         Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
         
         Properties bindings = ExpressionUtils.createBindings(project);
-        ExpressionUtils.bind(bindings, row, rowIndex, cell);
+        ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
         
         Object value = _evaluable.evaluate(bindings);
         if (value != null) {

src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionNumberComparisonRowFilter.java

@@ -17,6 +17,7 @@ import com.metaweb.gridworks.model.Row;
  */
 abstract public class ExpressionNumberComparisonRowFilter implements RowFilter {
     final protected Evaluable  _evaluable;
+    final protected String     _columnName;
     final protected int        _cellIndex;
     final protected boolean _selectNumeric;
     final protected boolean _selectNonNumeric;
@@ -25,6 +26,7 @@ abstract public class ExpressionNumberComparisonRowFilter implements RowFilter {
     
     public ExpressionNumberComparisonRowFilter(
         Evaluable evaluable,
+        String columnName,
         int cellIndex,
         boolean selectNumeric,
         boolean selectNonNumeric,
@@ -32,6 +34,7 @@ abstract public class ExpressionNumberComparisonRowFilter implements RowFilter {
         boolean selectError
     ) {
         _evaluable = evaluable;
+        _columnName = columnName;
         _cellIndex = cellIndex;
         _selectNumeric = selectNumeric;
         _selectNonNumeric = selectNonNumeric;
@@ -43,7 +46,7 @@ abstract public class ExpressionNumberComparisonRowFilter implements RowFilter {
         Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
 
         Properties bindings = ExpressionUtils.createBindings(project);
-        ExpressionUtils.bind(bindings, row, rowIndex, cell);
+        ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
         
         Object value = _evaluable.evaluate(bindings);
         if (value != null) {

src/main/java/com/metaweb/gridworks/browsing/filters/ExpressionStringComparisonRowFilter.java

@@ -14,10 +14,12 @@ import com.metaweb.gridworks.model.Row;
  */
 abstract public class ExpressionStringComparisonRowFilter implements RowFilter {
     final protected Evaluable _evaluable;
+    final protected String    _columnName;
     final protected int       _cellIndex;
     
-    public ExpressionStringComparisonRowFilter(Evaluable evaluable, int cellIndex) {
+    public ExpressionStringComparisonRowFilter(Evaluable evaluable, String columnName, int cellIndex) {
         _evaluable = evaluable;
+        _columnName = columnName;
         _cellIndex = cellIndex;
     }
 
@@ -25,7 +27,7 @@ abstract public class ExpressionStringComparisonRowFilter implements RowFilter {
         Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
         
         Properties bindings = ExpressionUtils.createBindings(project);
-        ExpressionUtils.bind(bindings, row, rowIndex, cell);
+        ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
         
         Object value = _evaluable.evaluate(bindings);
         if (value != null) {

src/main/java/com/metaweb/gridworks/commands/util/PreviewExpressionCommand.java

@@ -3,6 +3,7 @@ package com.metaweb.gridworks.commands.util;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.List;
 import java.util.Properties;
 
@@ -12,6 +13,7 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.json.JSONArray;
 import org.json.JSONException;
+import org.json.JSONObject;
 import org.json.JSONWriter;
 
 import com.metaweb.gridworks.commands.Command;
@@ -21,6 +23,8 @@ import com.metaweb.gridworks.expr.ExpressionUtils;
 import com.metaweb.gridworks.expr.HasFields;
 import com.metaweb.gridworks.expr.MetaParser;
 import com.metaweb.gridworks.expr.ParsingException;
+import com.metaweb.gridworks.expr.WrappedCell;
+import com.metaweb.gridworks.expr.WrappedRow;
 import com.metaweb.gridworks.model.Cell;
 import com.metaweb.gridworks.model.Project;
 import com.metaweb.gridworks.model.Row;
@@ -36,6 +40,7 @@ public class PreviewExpressionCommand extends Command {
             Project project = getProject(request);
             
             int cellIndex = Integer.parseInt(request.getParameter("cellIndex"));
+            String columnName = project.columnModel.getColumnByCellIndex(cellIndex).getName();
             
             String expression = request.getParameter("expression");
             String rowIndicesString = request.getParameter("rowIndices");
@@ -79,13 +84,13 @@ public class PreviewExpressionCommand extends Command {
                         Cell cell = row.getCell(cellIndex);
                             
                         try {
-                            ExpressionUtils.bind(bindings, row, rowIndex, cell);
+                            ExpressionUtils.bind(bindings, row, rowIndex, columnName, cell);
                             result = eval.evaluate(bindings);
                             
                             if (repeat) {
                                 for (int r = 0; r < repeatCount && ExpressionUtils.isStorable(result); r++) {
                                     Cell newCell = new Cell((Serializable) result, (cell != null) ? cell.recon : null);
-                                    ExpressionUtils.bind(bindings, row, rowIndex, newCell);
+                                    ExpressionUtils.bind(bindings, row, rowIndex, columnName, newCell);
                                     
                                     Object newResult = eval.evaluate(bindings);
                                     if (ExpressionUtils.isError(newResult)) {
@@ -102,20 +107,18 @@ public class PreviewExpressionCommand extends Command {
                         }
                     }
                     
-                    if (result != null && (result.getClass().isArray() || result instanceof List<?>)) {
+                    if (result == null) {
-                        writer.array();
+                        writer.value(null);
-                        if (result.getClass().isArray()) {
+                    } else if (ExpressionUtils.isError(result)) {
-                            for (Object v : (Object[]) result) {
+                        writer.object();
-                                writeValue(writer, v);
+                        writer.key("message"); writer.value(((EvalError) result).message);
-                            }
+                        writer.endObject();
                     } else {
-                            for (Object v : ExpressionUtils.toObjectList(result)) {
+                        StringBuffer sb = new StringBuffer();
-                                writeValue(writer, v);
+                        
-                            }
+                        writeValue(sb, result, false);
-                        }
+                        
-                        writer.endArray();
+                        writer.value(sb.toString());
-                    } else {
-                        writeValue(writer, result);
                     }
                 }
                 writer.endArray();
@@ -135,24 +138,57 @@ public class PreviewExpressionCommand extends Command {
         }
     }
     
-    static protected void writeValue(JSONWriter writer, Object v) throws JSONException {
+    static protected void writeValue(StringBuffer sb, Object v, boolean quote) throws JSONException {
         if (ExpressionUtils.isError(v)) {
-            writer.object();
+            sb.append("[error: " + ((EvalError) v).message + "]");
-            writer.key("message"); writer.value(((EvalError) v).message);
-            writer.endObject();
         } else {
-            if (v != null) {
+            if (v == null) {
-                if (v instanceof HasFields) {
+                sb.append("null");
-                    v = "[object " + v.getClass().getSimpleName() + "]";
+            } else {
+                if (v instanceof WrappedCell) {
+                    sb.append("[object Cell]");
+                } else if (v instanceof WrappedRow) {
+                    sb.append("[object Row]");
+                } else if (ExpressionUtils.isArray(v)) {
+                    Object[] a = (Object[]) v;
+                    sb.append("[ ");
+                    for (int i = 0; i < a.length; i++) {
+                        if (i > 0) {
+                            sb.append(", ");
+                        }
+                        writeValue(sb, a[i], true);
+                    }
+                    sb.append(" ]");
+                } else if (ExpressionUtils.isArrayOrList(v)) {
+                    List<Object> list = ExpressionUtils.toObjectList(v);
+                    sb.append("[ ");
+                    for (int i = 0; i < list.size(); i++) {
+                        if (i > 0) {
+                            sb.append(", ");
+                        }
+                        writeValue(sb, list.get(i), true);
+                    }
+                    sb.append(" ]");
+                } else if (v instanceof HasFields) {
+                    sb.append("[object " + v.getClass().getSimpleName() + "]");
                 } else if (v instanceof Calendar) {
                     Calendar c = (Calendar) v;
                     
-                    v = "[object " + 
+                    sb.append("[date " + 
-                        v.getClass().getSimpleName() + " " + 
+                        ParsingUtilities.dateToString(c.getTime()) +"]");
-                        ParsingUtilities.dateToString(c.getTime()) +"]";
+                } else if (v instanceof Date) {
+                    sb.append("[date " + 
+                            ParsingUtilities.dateToString((Date) v) +"]");
+                } else if (v instanceof String) {
+                    if (quote) {
+                        sb.append(JSONObject.quote((String) v));
+                    } else {
+                        sb.append((String) v);
+                    }
+                } else {
+                    sb.append(v.toString());
+                }
             }
-            }
-            writer.value(v);
         }
     }
 }

src/main/java/com/metaweb/gridworks/expr/CellTuple.java

@@ -0,0 +1,35 @@
+package com.metaweb.gridworks.expr;
+
+import java.util.Properties;
+
+import com.metaweb.gridworks.model.Cell;
+import com.metaweb.gridworks.model.Column;
+import com.metaweb.gridworks.model.Project;
+import com.metaweb.gridworks.model.Row;
+
+public class CellTuple implements HasFields {
+    final public Project project;
+    final public Row row;
+    
+    public CellTuple(Project project, Row row) {
+        this.project = project;
+        this.row = row;
+    }
+    
+    public Object getField(String name, Properties bindings) {
+        Column column = project.columnModel.getColumnByName(name);
+        if (column != null) {
+            int cellIndex = column.getCellIndex();
+            Cell cell = row.getCell(cellIndex);
+            
+            if (cell != null) {
+                return new WrappedCell(project, name, cell);
+            }
+        }
+        return null;
+    }
+
+    public boolean fieldAlsoHasFields(String name) {
+        return true;
+    }
+}

src/main/java/com/metaweb/gridworks/expr/ExpressionUtils.java

@@ -23,16 +23,22 @@ public class ExpressionUtils {
         return bindings;
     }
     
-    static public void bind(Properties bindings, Row row, int rowIndex, Cell cell) {
+    static public void bind(Properties bindings, Row row, int rowIndex, String columnName, Cell cell) {
-        bindings.put("row", row);
+        Project project = (Project) bindings.get("project");
+        
         bindings.put("rowIndex", rowIndex);
-        bindings.put("cells", row.getField("cells", bindings));
+        bindings.put("row", new WrappedRow(project, rowIndex, row));
+        bindings.put("cells", new CellTuple(project, row));
+        
+        if (columnName != null) {
+            bindings.put("columnName", columnName);
+        }
         
         if (cell == null) {
             bindings.remove("cell");
             bindings.remove("value");
         } else {
-            bindings.put("cell", cell);
+            bindings.put("cell", new WrappedCell(project, columnName, cell));
             if (cell.value == null) {
                 bindings.remove("value");
             } else {

src/main/java/com/metaweb/gridworks/expr/HasFields.java

@@ -8,4 +8,6 @@ import java.util.Properties;
  */
 public interface HasFields {
     public Object getField(String name, Properties bindings);
+    
+    public boolean fieldAlsoHasFields(String name);
 }

src/main/java/com/metaweb/gridworks/expr/HasFieldsList.java

@@ -0,0 +1,10 @@
+package com.metaweb.gridworks.expr;
+
+/**
+ * Interface for objects each of which is a list of HasFields objects of the
+ * same kind (e.g., list of cells). Its getField method thus returns either
+ * another HasFieldsList object or an array or java.util.List of objects. 
+ */
+public interface HasFieldsList extends HasFields {
+    public int length();
+}

src/main/java/com/metaweb/gridworks/expr/HasFieldsListImpl.java

@@ -0,0 +1,34 @@
+package com.metaweb.gridworks.expr;
+
+import java.util.ArrayList;
+import java.util.Properties;
+
+public class HasFieldsListImpl extends ArrayList<HasFields> implements HasFieldsList {
+    private static final long serialVersionUID = -8635194387420305802L;
+
+    public Object getField(String name, Properties bindings) {
+        int c = size();
+        if (c > 0 && get(0).fieldAlsoHasFields(name)) {
+            HasFieldsListImpl l = new HasFieldsListImpl();
+            for (int i = 0; i < size(); i++) {
+                l.add(i, (HasFields) this.get(i).getField(name, bindings));
+            }
+            return l;
+        } else {
+            Object[] r = new Object[this.size()];
+            for (int i = 0; i < r.length; i++) {
+                r[i] = this.get(i).getField(name, bindings);
+            }
+            return r;
+        }
+    }
+
+    public int length() {
+        return size();
+    }
+
+    public boolean fieldAlsoHasFields(String name) {
+        int c = size();
+        return (c > 0 && get(0).fieldAlsoHasFields(name));
+    }
+}

src/main/java/com/metaweb/gridworks/expr/WrappedCell.java

@@ -0,0 +1,26 @@
+package com.metaweb.gridworks.expr;
+
+import java.util.Properties;
+
+import com.metaweb.gridworks.model.Cell;
+import com.metaweb.gridworks.model.Project;
+
+public class WrappedCell implements HasFields {
+    final public Project project;
+    final public String columnName;
+    final public Cell cell;
+    
+    public WrappedCell(Project project, String columnName, Cell cell) {
+        this.project = project;
+        this.columnName = columnName;
+        this.cell = cell;
+    }
+    
+    public Object getField(String name, Properties bindings) {
+        return cell.getField(name, bindings);
+    }
+
+    public boolean fieldAlsoHasFields(String name) {
+        return cell.fieldAlsoHasFields(name);
+    }
+}

src/main/java/com/metaweb/gridworks/expr/WrappedRow.java

@@ -0,0 +1,104 @@
+package com.metaweb.gridworks.expr;
+
+import java.util.Properties;
+
+import com.metaweb.gridworks.model.Cell;
+import com.metaweb.gridworks.model.Column;
+import com.metaweb.gridworks.model.Project;
+import com.metaweb.gridworks.model.Row;
+
+public class WrappedRow implements HasFields {
+    final public Project project;
+    final public int rowIndex;
+    final public Row row;
+    
+    public WrappedRow(Project project, int rowIndex, Row row) {
+        this.project = project;
+        this.rowIndex = rowIndex;
+        this.row = row;
+    }
+    
+    public Object getField(String name, Properties bindings) {
+        if ("cells".equals(name)) {
+            return new CellTuple(project, row);
+        } else if ("index".equals(name)) {
+            return rowIndex;
+        } else if ("record".equals(name)) {
+            int rowIndex = (Integer) bindings.get("rowIndex");
+            int recordRowIndex = (row.contextRows != null && row.contextRows.size() > 0) ?
+                    row.contextRows.get(0) : rowIndex;
+            
+            return new Record(recordRowIndex, rowIndex);
+        } else if ("columnNames".equals(name)) {
+            Project project = (Project) bindings.get("project");
+            
+            return project.columnModel.getColumnNames();
+        } else {
+            return row.getField(name, bindings);
+        }
+    }
+
+    public boolean fieldAlsoHasFields(String name) {
+        return row.fieldAlsoHasFields(name);
+    }
+
+    protected class Record implements HasFields {
+        final int _recordRowIndex;
+        final int _currentRowIndex;
+        
+        protected Record(int recordRowIndex, int currentRowIndex) {
+            _recordRowIndex = recordRowIndex;
+            _currentRowIndex = currentRowIndex;
+        }
+
+        public Object getField(String name, Properties bindings) {
+            if ("cells".equals(name)) {
+                return new RecordCells(_recordRowIndex);
+            }
+            return null;
+        }
+
+        public boolean fieldAlsoHasFields(String name) {
+            return "cells".equals(name);
+        }
+    }
+    
+    protected class RecordCells implements HasFields {
+        final int _recordRowIndex;
+        
+        protected RecordCells(int recordRowIndex) {
+            _recordRowIndex = recordRowIndex;
+        }
+        
+        public Object getField(String name, Properties bindings) {
+            Column column = project.columnModel.getColumnByName(name);
+            if (column != null) {
+                Row recordRow = project.rows.get(_recordRowIndex);
+                int cellIndex = column.getCellIndex();
+                
+                HasFieldsListImpl cells = new HasFieldsListImpl();
+                
+                int recordIndex = recordRow.recordIndex;
+                int count = project.rows.size();
+                for (int r = _recordRowIndex; r < count; r++) {
+                    Row row = project.rows.get(r);
+                    if (row.recordIndex > recordIndex) {
+                        break;
+                    }
+                    
+                    Cell cell = row.getCell(cellIndex);
+                    if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
+                        cells.add(new WrappedCell(project, name, cell));
+                    }
+                }
+                
+                return cells;
+            }
+            return null;
+        }
+
+        public boolean fieldAlsoHasFields(String name) {
+            return true;
+        }
+    }
+}

src/main/java/com/metaweb/gridworks/expr/functions/Cross.java

@@ -0,0 +1,53 @@
+package com.metaweb.gridworks.expr.functions;
+
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.metaweb.gridworks.ProjectManager;
+import com.metaweb.gridworks.InterProjectModel.ProjectJoin;
+import com.metaweb.gridworks.expr.EvalError;
+import com.metaweb.gridworks.expr.WrappedCell;
+import com.metaweb.gridworks.gel.ControlFunctionRegistry;
+import com.metaweb.gridworks.gel.Function;
+import com.metaweb.gridworks.model.Project;
+
+public class Cross implements Function {
+
+    public Object call(Properties bindings, Object[] args) {
+        if (args.length == 3) {
+            // from project is implied
+            
+            Object wrappedCell = args[0]; // from cell
+            Object toProjectName = args[1];
+            Object toColumnName = args[2];
+            
+            if (wrappedCell != null && wrappedCell instanceof WrappedCell &&
+                toProjectName != null && toProjectName instanceof String &&
+                toColumnName != null && toColumnName instanceof String) {
+                
+                ProjectJoin join = ProjectManager.singleton.getInterProjectModel().getJoin(
+                    ProjectManager.singleton.getProjectMetadata(
+                            ((Project) bindings.get("project")).id).getName(),
+                    ((WrappedCell) wrappedCell).columnName,
+                    (String) toProjectName,
+                    (String) toColumnName
+                );
+                
+                return join.getRows(((WrappedCell) wrappedCell).cell.value);
+            }
+        }
+        return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a cell, a project name to join with, and a column name in that project");
+    }
+
+    public void write(JSONWriter writer, Properties options)
+        throws JSONException {
+    
+        writer.object();
+        writer.key("description"); writer.value("TODO");
+        writer.key("params"); writer.value("cell c, string projectName, string columnName");
+        writer.key("returns"); writer.value("array");
+        writer.endObject();
+    }
+}

src/main/java/com/metaweb/gridworks/gel/ControlFunctionRegistry.java

@@ -5,6 +5,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.Map.Entry;
 
+import com.metaweb.gridworks.expr.functions.Cross;
 import com.metaweb.gridworks.expr.functions.Get;
 import com.metaweb.gridworks.expr.functions.Length;
 import com.metaweb.gridworks.expr.functions.Slice;
@@ -170,6 +171,8 @@ public class ControlFunctionRegistry {
         registerFunction("or", new Or());
         registerFunction("not", new Not());
         
+        registerFunction("cross", new Cross());
+
         registerControl("if", new If());
         registerControl("with", new With());
         registerControl("forEach", new ForEach());

src/main/java/com/metaweb/gridworks/model/Cell.java

@@ -34,6 +34,10 @@ public class Cell implements HasFields, Jsonizable {
         return null;
     }
     
+    public boolean fieldAlsoHasFields(String name) {
+        return "recon".equals(name);
+    }
+
     public void write(JSONWriter writer, Properties options) throws JSONException {
         writer.object();
         if (ExpressionUtils.isError(value)) {

src/main/java/com/metaweb/gridworks/model/Recon.java

@@ -139,6 +139,10 @@ public class Recon implements HasFields, Jsonizable {
         return null;
     }
     
+    public boolean fieldAlsoHasFields(String name) {
+        return "match".equals(name) || "best".equals(name);
+    }
+    
     protected String judgmentToString() {
         return judgmentToString(judgment);
     }
@@ -148,6 +152,10 @@ public class Recon implements HasFields, Jsonizable {
             int index = s_featureMap.get(name);
             return index < features.length ? features[index] : null;
         }
+
+        public boolean fieldAlsoHasFields(String name) {
+            return false;
+        }
     }
 
     public void write(JSONWriter writer, Properties options)

src/main/java/com/metaweb/gridworks/model/ReconCandidate.java

@@ -40,6 +40,10 @@ public class ReconCandidate implements HasFields, Jsonizable {
         return null;
     }
     
+    public boolean fieldAlsoHasFields(String name) {
+        return false;
+    }
+
     public void write(JSONWriter writer, Properties options)
             throws JSONException {
         

src/main/java/com/metaweb/gridworks/model/Row.java

@@ -12,7 +12,7 @@ import org.json.JSONObject;
 import org.json.JSONWriter;
 
 import com.metaweb.gridworks.Jsonizable;
-import com.metaweb.gridworks.expr.ExpressionUtils;
+import com.metaweb.gridworks.expr.CellTuple;
 import com.metaweb.gridworks.expr.HasFields;
 import com.metaweb.gridworks.util.ParsingUtilities;
 
@@ -43,24 +43,14 @@ public class Row implements HasFields, Jsonizable {
             return flagged;
         } else if ("starred".equals(name)) {
             return starred;
-        } else if ("cells".equals(name)) {
-            return new Cells();
-        } else if ("index".equals(name)) {
-            return bindings.get("rowIndex");
-        } else if ("record".equals(name)) {
-        	int rowIndex = (Integer) bindings.get("rowIndex");
-        	int recordRowIndex = (contextRows != null && contextRows.size() > 0) ?
-        			contextRows.get(0) : rowIndex;
-        	
-            return new Record(recordRowIndex, rowIndex);
-        } else if ("columnNames".equals(name)) {
-        	Project project = (Project) bindings.get("project");
-        	
-        	return project.columnModel.getColumnNames();
         }
         return null;
     }
     
+    public boolean fieldAlsoHasFields(String name) {
+        return "cells".equals(name) || "record".equals(name);
+    }
+    
     public boolean isEmpty() {
         for (Cell cell : cells) {
             if (cell != null && cell.value != null && !isValueBlank(cell.value)) {
@@ -107,6 +97,10 @@ public class Row implements HasFields, Jsonizable {
         }
     }
     
+    public CellTuple getCellTuple(Project project) {
+        return new CellTuple(project, this);
+    }
+    
     public void write(JSONWriter writer, Properties options)
             throws JSONException {
         
@@ -182,82 +176,4 @@ public class Row implements HasFields, Jsonizable {
         return row;
     }
     
-    protected class Cells implements HasFields {
-        public Object getField(String name, Properties bindings) {
-            Project project = (Project) bindings.get("project");
-            Column column = project.columnModel.getColumnByName(name);
-            if (column != null) {
-                int cellIndex = column.getCellIndex();
-                return getCell(cellIndex);
-            }
-            return null;
-        }
-        
-    }
-    
-    protected static class Record implements HasFields {
-    	final int _recordRowIndex;
-    	final int _currentRowIndex;
-    	
-    	protected Record(int recordRowIndex, int currentRowIndex) {
-    		_recordRowIndex = recordRowIndex;
-    		_currentRowIndex = currentRowIndex;
-    	}
-
-		public Object getField(String name, Properties bindings) {
-	        if ("cells".equals(name)) {
-	            return new RecordCells(_recordRowIndex);
-	        }
-			return null;
-		}
-    }
-    
-    protected static class RecordCells implements HasFields {
-    	final int _recordRowIndex;
-    	
-    	protected RecordCells(int recordRowIndex) {
-    		_recordRowIndex = recordRowIndex;
-    	}
-    	
-		public Object getField(String name, Properties bindings) {
-            Project project = (Project) bindings.get("project");
-            Column column = project.columnModel.getColumnByName(name);
-            if (column != null) {
-            	Row recordRow = project.rows.get(_recordRowIndex);
-                int cellIndex = column.getCellIndex();
-                
-                CellTuple cells = new CellTuple();
-                
-            	int recordIndex = recordRow.recordIndex;
-            	int count = project.rows.size();
-            	for (int r = _recordRowIndex; r < count; r++) {
-            		Row row = project.rows.get(r);
-            		if (row.recordIndex > recordIndex) {
-            			break;
-            		}
-            		
-            		Cell cell = row.getCell(cellIndex);
-            		if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
-            			cells.add(cell);
-            		}
-            	}
-            	
-                return cells;
-            }
-            return null;
-		}
-    }
-    
-    protected static class CellTuple extends ArrayList<Cell> implements HasFields {
-
-        private static final long serialVersionUID = -651032866647686293L;
-
-		public Object getField(String name, Properties bindings) {
-			Object[] r = new Object[this.size()];
-			for (int i = 0; i < r.length; i++) {
-				r[i] = this.get(i).getField(name, bindings);
-			}
-			return r;
-		}
-    }
 }

src/main/java/com/metaweb/gridworks/operations/ColumnAdditionOperation.java

@@ -137,7 +137,7 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
             public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
                 Cell cell = row.getCell(cellIndex);
 
-                ExpressionUtils.bind(bindings, row, rowIndex, cell);
+                ExpressionUtils.bind(bindings, row, rowIndex, _baseColumnName, cell);
                 
                 Serializable v = ExpressionUtils.wrapStorable(eval.evaluate(bindings));
                 if (ExpressionUtils.isError(v)) {

src/main/java/com/metaweb/gridworks/operations/MassEditOperation.java

@@ -166,7 +166,7 @@ public class MassEditOperation extends EngineDependentMassCellOperation {
             public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
                 Cell cell = row.getCell(cellIndex);
 
-                ExpressionUtils.bind(bindings, row, rowIndex, cell);
+                ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
                 
                 Object v = eval.evaluate(bindings);
                 if (v != null) {

src/main/java/com/metaweb/gridworks/operations/TextTransformOperation.java

@@ -122,7 +122,7 @@ public class TextTransformOperation extends EngineDependentMassCellOperation {
                 Cell cell = row.getCell(cellIndex);
                 Object oldValue = cell != null ? cell.value : null;
 
-                ExpressionUtils.bind(bindings, row, rowIndex, cell);
+                ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
                 
                 Serializable newValue = ExpressionUtils.wrapStorable(eval.evaluate(bindings));
                 if (ExpressionUtils.isError(newValue)) {
@@ -138,7 +138,7 @@ public class TextTransformOperation extends EngineDependentMassCellOperation {
                     
                     if (_repeat) {
                         for (int i = 0; i < _repeatCount; i++) {
-                            ExpressionUtils.bind(bindings, row, rowIndex, newCell);
+                            ExpressionUtils.bind(bindings, row, rowIndex, _columnName, newCell);
                             
                             newValue = ExpressionUtils.wrapStorable(eval.evaluate(bindings));
                             if (ExpressionUtils.isError(newValue)) {

src/main/webapp/scripts/dialogs/expression-preview-dialog.js

@@ -322,17 +322,10 @@ ExpressionPreviewDialog.Widget.prototype._renderPreview = function(expression, d
     
     var renderValue = function(td, v) {
         if (v !== null && v !== undefined) {
-            if ($.isArray(v)) {
+            if ($.isPlainObject(v)) {
-                var a = [];
-                $.each(v, function() { a.push(JSON.stringify(this)); });
-                
-                td.text("[ " + a.join(", ") + " ]");
-            } else if ($.isPlainObject(v)) {
                 $('<span></span>').addClass("expression-preview-special-value").text("Error: " + v.message).appendTo(td);
-            } else if (typeof v === "string" && v.length == 0) {
-                $('<span>empty string</span>').addClass("expression-preview-special-value").appendTo(td);
             } else {
-                td.text(v.toString());
+                td.text(v);
             }
         } else {
             $('<span>null</span>').addClass("expression-preview-special-value").appendTo(td);