- switched to accessing versioned standard-reconcile app

- standardized preference keys to using dot separated format
- added support to override freeq url from workspace preferences
- added GEL controls: forEachIndex, forRange, filter
- enforced max-width on preview table columns in expression preview dialog
- added preservedAllTokens option to split GEL function

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1109 7d457c2a-affb-35e4-300a-418c747d4874

main/src/com/metaweb/gridworks/commands/expr/GetExpressionHistoryCommand.java

@@ -1,6 +1,7 @@
 package com.metaweb.gridworks.commands.expr;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -18,6 +19,10 @@ import com.metaweb.gridworks.preference.TopList;
 
 public class GetExpressionHistoryCommand extends Command {
     
+	static protected List<String> toExpressionList(Object o) {
+        return o == null ? new ArrayList<String>() : ((TopList) o).getList();
+	}
+
     @Override
     public void doGet(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
@@ -25,15 +30,10 @@ public class GetExpressionHistoryCommand extends Command {
         try {
             Project project = getProject(request);
             
-            List<String> localExpressions = 
+            List<String> localExpressions = toExpressionList(project.getMetadata().getPreferenceStore().get("scripting.expressions"));
-                ((TopList) project.getMetadata().getPreferenceStore().get("expressions"))
-                    .getList();
-            
             localExpressions = localExpressions.size() > 20 ? localExpressions.subList(0, 20) : localExpressions;
             
-            List<String> globalExpressions =
+            List<String> globalExpressions = toExpressionList(ProjectManager.singleton.getPreferenceStore().get("scripting.expressions"));
-                ((TopList) ProjectManager.singleton.getPreferenceStore().get("expressions"))
-                    .getList();
             
             Set<String> done = new HashSet<String>();
             

main/src/com/metaweb/gridworks/commands/expr/LogExpressionCommand.java

@@ -21,10 +21,10 @@ public class LogExpressionCommand extends Command {
             Project project = getProject(request);
             String expression = request.getParameter("expression");
             
-            ((TopList) project.getMetadata().getPreferenceStore().get("expressions"))
+            ((TopList) project.getMetadata().getPreferenceStore().get("scripting.expressions"))
                 .add(expression);
 
-            ((TopList) ProjectManager.singleton.getPreferenceStore().get("expressions"))
+            ((TopList) ProjectManager.singleton.getPreferenceStore().get("scripting.expressions"))
                 .add(expression);
             
             response.setCharacterEncoding("UTF-8");

main/src/com/metaweb/gridworks/expr/functions/strings/Split.java

@@ -14,28 +14,39 @@ import com.metaweb.gridworks.gel.Function;
 public class Split implements Function {
 
     public Object call(Properties bindings, Object[] args) {
-        if (args.length == 2) {
+        if (args.length >= 2 && args.length <= 3) {
+            boolean preserveAllTokens = false;
+            
             Object v = args[0];
             Object split = args[1];
+            if (args.length == 3) {
+                Object preserve = args[2];
+                if (preserve instanceof Boolean) {
+                    preserveAllTokens = ((Boolean) preserve);
+                }
+            }
+            
             if (v != null && split != null) {
                 String str = (v instanceof String ? (String) v : v.toString());
                 if (split instanceof String) {
-                    return StringUtils.splitByWholeSeparator(str, (String) split);
+                    return preserveAllTokens ?
+                        StringUtils.splitByWholeSeparatorPreserveAllTokens(str, (String) split) :
+                        StringUtils.splitByWholeSeparator(str, (String) split);
                 } else if (split instanceof Pattern) {
                     Pattern pattern = (Pattern) split;
                     return pattern.split(str);
                 }
             }
         }
-        return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings, or 1 string and 1 regex");
+        return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects 2 strings, or 1 string and 1 regex, followed by an optional boolean");
     }
     
     public void write(JSONWriter writer, Properties options)
         throws JSONException {
     
         writer.object();
-        writer.key("description"); writer.value("Returns the array of strings obtained by splitting s with separator sep");
+        writer.key("description"); writer.value("Returns the array of strings obtained by splitting s with separator sep. If preserveAllTokens is true, then empty segments are preserved.");
-        writer.key("params"); writer.value("string s, string or regex sep");
+        writer.key("params"); writer.value("string s, string or regex sep, optional boolean preserveAllTokens");
         writer.key("returns"); writer.value("array");
         writer.endObject();
     }

main/src/com/metaweb/gridworks/gel/ControlFunctionRegistry.java

@@ -64,8 +64,11 @@ import com.metaweb.gridworks.expr.functions.strings.Trim;
 import com.metaweb.gridworks.expr.functions.strings.Unescape;
 import com.metaweb.gridworks.expr.functions.strings.Unicode;
 import com.metaweb.gridworks.expr.functions.strings.UnicodeType;
+import com.metaweb.gridworks.gel.controls.Filter;
 import com.metaweb.gridworks.gel.controls.ForEach;
+import com.metaweb.gridworks.gel.controls.ForEachIndex;
 import com.metaweb.gridworks.gel.controls.ForNonBlank;
+import com.metaweb.gridworks.gel.controls.ForRange;
 import com.metaweb.gridworks.gel.controls.If;
 import com.metaweb.gridworks.gel.controls.IsBlank;
 import com.metaweb.gridworks.gel.controls.IsError;
@@ -188,6 +191,9 @@ public class ControlFunctionRegistry {
         registerControl("if", new If());
         registerControl("with", new With());
         registerControl("forEach", new ForEach());
+        registerControl("forEachIndex", new ForEachIndex());
+        registerControl("forRange", new ForRange());
+        registerControl("filter", new Filter());
         registerControl("forNonBlank", new ForNonBlank());
 
         registerControl("isNull", new IsNull());

main/src/com/metaweb/gridworks/gel/controls/Filter.java

@@ -0,0 +1,94 @@
+package com.metaweb.gridworks.gel.controls;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.metaweb.gridworks.expr.EvalError;
+import com.metaweb.gridworks.expr.Evaluable;
+import com.metaweb.gridworks.expr.ExpressionUtils;
+import com.metaweb.gridworks.gel.Control;
+import com.metaweb.gridworks.gel.ControlFunctionRegistry;
+import com.metaweb.gridworks.gel.ast.VariableExpr;
+
+public class Filter implements Control {
+    public String checkArguments(Evaluable[] args) {
+        if (args.length != 3) {
+            return ControlFunctionRegistry.getControlName(this) + " expects 3 arguments";
+        } else if (!(args[1] instanceof VariableExpr)) {
+            return ControlFunctionRegistry.getControlName(this) + 
+                " expects second argument to be a variable name";
+        }
+        return null;
+    }
+
+    public Object call(Properties bindings, Evaluable[] args) {
+        Object o = args[0].evaluate(bindings);
+        if (ExpressionUtils.isError(o)) {
+            return o;
+        } else if (!ExpressionUtils.isArrayOrCollection(o)) {
+            return new EvalError("First argument is not an array");
+        }
+        
+        String name = ((VariableExpr) args[1]).getName();
+        
+        Object oldValue = bindings.get(name);
+        try {
+            List<Object> results = null;
+            
+            if (o.getClass().isArray()) {
+                Object[] values = (Object[]) o;
+                
+                results = new ArrayList<Object>(values.length);
+                for (Object v : values) {
+                    bindings.put(name, v);
+                    
+                    Object r = args[2].evaluate(bindings);
+                    if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
+                    	results.add(v);
+                    }
+                }
+            } else {
+                Collection<Object> collection = ExpressionUtils.toObjectCollection(o);
+                
+                results = new ArrayList<Object>(collection.size());
+                
+                for (Object v : collection) {
+                    bindings.put(name, v);
+                    
+                    Object r = args[2].evaluate(bindings);
+                    if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
+                    	results.add(v);
+                    }
+                }
+            }
+            
+            return results.toArray(); 
+        } finally {
+            /*
+             *  Restore the old value bound to the variable, if any.
+             */
+            if (oldValue != null) {
+                bindings.put(name, oldValue);
+            } else {
+                bindings.remove(name);
+            }
+        }
+    }
+    
+    public void write(JSONWriter writer, Properties options)
+        throws JSONException {
+    
+        writer.object();
+        writer.key("description"); writer.value(
+            "Evaluates expression a to an array. Then for each array element, binds its value to variable name v, evaluates expression test which should return a boolean. If the boolean is true, pushes v onto the result array."
+        );
+        writer.key("params"); writer.value("expression a, variable v, expression test");
+        writer.key("returns"); writer.value("array");
+        writer.endObject();
+    }
+}

main/src/com/metaweb/gridworks/gel/controls/ForEachIndex.java

@@ -0,0 +1,108 @@
+package com.metaweb.gridworks.gel.controls;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.metaweb.gridworks.expr.EvalError;
+import com.metaweb.gridworks.expr.Evaluable;
+import com.metaweb.gridworks.expr.ExpressionUtils;
+import com.metaweb.gridworks.gel.Control;
+import com.metaweb.gridworks.gel.ControlFunctionRegistry;
+import com.metaweb.gridworks.gel.ast.VariableExpr;
+
+public class ForEachIndex implements Control {
+    public String checkArguments(Evaluable[] args) {
+        if (args.length != 4) {
+            return ControlFunctionRegistry.getControlName(this) + " expects 4 arguments";
+        } else if (!(args[1] instanceof VariableExpr)) {
+            return ControlFunctionRegistry.getControlName(this) + 
+                " expects second argument to be the index's variable name";
+        } else if (!(args[2] instanceof VariableExpr)) {
+            return ControlFunctionRegistry.getControlName(this) + 
+                " expects third argument to be the element's variable name";
+        }
+        return null;
+    }
+
+    public Object call(Properties bindings, Evaluable[] args) {
+        Object o = args[0].evaluate(bindings);
+        if (ExpressionUtils.isError(o)) {
+            return o;
+        } else if (!ExpressionUtils.isArrayOrCollection(o)) {
+            return new EvalError("First argument to forEach is not an array");
+        }
+        
+        String indexName = ((VariableExpr) args[1]).getName();
+        String elementName = ((VariableExpr) args[2]).getName();
+        
+        Object oldIndexValue = bindings.get(indexName);
+        Object oldElementValue = bindings.get(elementName);
+        try {
+            List<Object> results = null;
+            
+            if (o.getClass().isArray()) {
+                Object[] values = (Object[]) o;
+                
+                results = new ArrayList<Object>(values.length);
+                
+                for (int i = 0; i < values.length; i++) {
+                	Object v = values[i];
+                	
+                	bindings.put(indexName, i);
+                    bindings.put(elementName, v);
+                    
+                    Object r = args[3].evaluate(bindings);
+                    
+                    results.add(r);
+                }
+            } else {
+                List<Object> list = ExpressionUtils.toObjectList(o);
+                
+                results = new ArrayList<Object>(list.size());
+                
+                for (int i = 0; i < list.size(); i++) {
+                	Object v = list.get(i);
+                	
+                	bindings.put(indexName, i);
+                    bindings.put(elementName, v);
+                    
+                    Object r = args[3].evaluate(bindings);
+                    
+                    results.add(r);
+                }
+            }
+            
+            return results.toArray(); 
+        } finally {
+            /*
+             *  Restore the old values bound to the variables, if any.
+             */
+            if (oldIndexValue != null) {
+                bindings.put(indexName, oldIndexValue);
+            } else {
+                bindings.remove(indexName);
+            }
+            if (oldElementValue != null) {
+                bindings.put(elementName, oldElementValue);
+            } else {
+                bindings.remove(elementName);
+            }
+        }
+    }
+    
+    public void write(JSONWriter writer, Properties options)
+        throws JSONException {
+    
+        writer.object();
+        writer.key("description"); writer.value(
+            "Evaluates expression a to an array. Then for each array element, binds its index to variable i and its value to variable name v, evaluates expression e, and pushes the result onto the result array."
+        );
+        writer.key("params"); writer.value("expression a, variable i, variable v, expression e");
+        writer.key("returns"); writer.value("array");
+        writer.endObject();
+    }
+}

main/src/com/metaweb/gridworks/gel/controls/ForRange.java

@@ -0,0 +1,110 @@
+package com.metaweb.gridworks.gel.controls;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONWriter;
+
+import com.metaweb.gridworks.expr.EvalError;
+import com.metaweb.gridworks.expr.Evaluable;
+import com.metaweb.gridworks.expr.ExpressionUtils;
+import com.metaweb.gridworks.gel.Control;
+import com.metaweb.gridworks.gel.ControlFunctionRegistry;
+import com.metaweb.gridworks.gel.ast.VariableExpr;
+
+public class ForRange implements Control {
+    public String checkArguments(Evaluable[] args) {
+        if (args.length != 5) {
+            return ControlFunctionRegistry.getControlName(this) + " expects 5 arguments";
+        } else if (!(args[3] instanceof VariableExpr)) {
+            return ControlFunctionRegistry.getControlName(this) + 
+                " expects third argument to be the element's variable name";
+        }
+        return null;
+    }
+
+    public Object call(Properties bindings, Evaluable[] args) {
+        Object fromO = args[0].evaluate(bindings);
+        Object toO = args[1].evaluate(bindings);
+        Object stepO = args[2].evaluate(bindings);
+        
+        if (ExpressionUtils.isError(fromO)) {
+            return fromO;
+        } else if (ExpressionUtils.isError(toO)) {
+            return toO;
+        } else if (ExpressionUtils.isError(stepO)) {
+            return stepO;
+        } else if (!(fromO instanceof Number) || !(toO instanceof Number) || !(stepO instanceof Number)) {
+            return new EvalError("First, second, and third arguments of forRange must all be numbers");
+        }
+        
+        String indexName = ((VariableExpr) args[3]).getName();
+        Object oldIndexValue = bindings.get(indexName);
+
+        try {
+            List<Object> results = new ArrayList<Object>();
+            
+            if (isIntegral((Number) fromO) && isIntegral((Number) stepO)) {
+                long from = ((Number) fromO).longValue();
+                long step = ((Number) stepO).longValue();
+                double to = ((Number) toO).doubleValue();
+                
+                while (from < to) {
+                	bindings.put(indexName, from);
+                    
+                    Object r = args[4].evaluate(bindings);
+                    
+                    results.add(r);
+                    
+                    from += step;
+                }
+            } else {
+                double from = ((Number) fromO).longValue();
+                double step = ((Number) stepO).longValue();
+                double to = ((Number) toO).doubleValue();
+                
+                while (from < to) {
+                	bindings.put(indexName, from);
+                    
+                    Object r = args[4].evaluate(bindings);
+                    
+                    results.add(r);
+                    
+                    from += step;
+                }
+            }
+            return results.toArray(); 
+        } finally {
+            /*
+             *  Restore the old values bound to the variables, if any.
+             */
+            if (oldIndexValue != null) {
+                bindings.put(indexName, oldIndexValue);
+            } else {
+                bindings.remove(indexName);
+            }
+        }
+    }
+    
+    static private boolean isIntegral(Number o) {
+    	if (o instanceof Integer || o instanceof Long) {
+    		return true;
+    	} else {
+    		return (o.doubleValue() - o.longValue()) == 0;
+    	}
+    }
+    
+    public void write(JSONWriter writer, Properties options)
+        throws JSONException {
+    
+        writer.object();
+        writer.key("description"); writer.value(
+            "Iterates over the variable v starting at \"from\", incrementing by \"step\" each time while less than \"to\". At each iteration, evaluates expression e, and pushes the result onto the result array."
+        );
+        writer.key("params"); writer.value("number from, number to, number step, variable v, expression e");
+        writer.key("returns"); writer.value("array");
+        writer.endObject();
+    }
+}

main/src/com/metaweb/gridworks/util/FreebaseUtils.java

@@ -28,6 +28,7 @@ import org.json.JSONException;
 import org.json.JSONObject;
 
 import com.metaweb.gridworks.GridworksServlet;
+import com.metaweb.gridworks.ProjectManager;
 import com.metaweb.gridworks.oauth.Credentials;
 import com.metaweb.gridworks.oauth.OAuthUtilities;
 import com.metaweb.gridworks.oauth.Provider;
@@ -185,7 +186,7 @@ public class FreebaseUtils {
                 }
                 UrlEncodedFormEntity entity = new UrlEncodedFormEntity(formparams, "UTF-8");
     
-                HttpPost httpRequest = new HttpPost(FREEQ_URL);
+                HttpPost httpRequest = new HttpPost(getFreeQUrl());
                 httpRequest.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Gridworks " + GridworksServlet.getVersion());
                 httpRequest.setEntity(entity);
                 
@@ -241,4 +242,9 @@ public class FreebaseUtils {
         
         return allowed;
     }
+    
+    static public String getFreeQUrl() {
+    	String url = (String) ProjectManager.singleton.getPreferenceStore().get("freebase.freeq");
+    	return url != null ? url : FREEQ_URL;
+    }
 }

main/webapp/modules/core/scripts/dialogs/expression-preview-dialog.js

@@ -328,9 +328,9 @@ ExpressionPreviewDialog.Widget.prototype._renderPreview = function(expression, d
         
         $(tr.insertCell(0)).attr("width", "1%").html((this._rowIndices[i] + 1) + ".");
         
-        renderValue($(tr.insertCell(1)), this._values[i]);
+        renderValue($(tr.insertCell(1)).addClass("expression-preview-value"), this._values[i]);
         
-        var tdValue = $(tr.insertCell(2));
+        var tdValue = $(tr.insertCell(2)).addClass("expression-preview-value");
         if (this._results !== null) {
             var v = this._results[i];
             renderValue(tdValue, v);

main/webapp/modules/core/scripts/reconciliation/recon-dialog.js

@@ -181,7 +181,7 @@ ReconDialog.prototype._onAddNamespacedService = function() {
         var namespaceData = elmts.namespaceInput.data("data.suggest");
         var typeData = elmts.typeInput.data("data.suggest");
         if (namespaceData) {
-            var url = "http://standard-reconcile.freebaseapps.com/namespace_reconcile?namespace=" + 
+            var url = "http://1.standard-reconcile.freebaseapps.com/namespace_reconcile?namespace=" + 
                 escape(namespaceData.id);
             if (typeData) {
                 url += "&type=" + typeData.id;

main/webapp/modules/core/scripts/reconciliation/recon-manager.js

@@ -82,7 +82,7 @@ ReconciliationManager.save = function(f) {
         async: false,
         type: "POST",
         url: "/command/set-preference?" + $.param({ 
-            name: "standard-reconciliation-services" 
+            name: "reconciliation.standardServices" 
         }),
         data: { "value" : JSON.stringify(ReconciliationManager.standardServices) },
         success: function(data) {
@@ -101,7 +101,7 @@ ReconciliationManager.save = function(f) {
     $.ajax({
         async: false,
         url: "/command/get-preference?" + $.param({ 
-            name: "standard-reconciliation-services" 
+            name: "reconciliation.standardServices" 
         }),
         success: function(data) {
             if (data.value && data.value != "null") {
@@ -109,7 +109,7 @@ ReconciliationManager.save = function(f) {
                 ReconciliationManager._rebuildMap();
             } else {
                 ReconciliationManager.registerStandardService(
-                    "http://standard-reconcile.dfhuynh.user.dev.freebaseapps.com/reconcile");
+                    "http://1.standard-reconcile.freebaseapps.com/reconcile");
             }
         },
         dataType: "json"

main/webapp/modules/core/styles/dialogs/expression-preview-dialog.css

@@ -33,6 +33,11 @@ td.expression-preview-heading {
     font-weight: bold;
 }
 
+td.expression-preview-value {
+    max-width: 250px !important;
+    overflow-x: hidden;
+}
+
 .expression-preview-special-value {
     color: #aaa;
 }