Fixed minor bug introduced recently into the Export Project menu command.

Documented the commands.* packages.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@331 7d457c2a-affb-35e4-300a-418c747d4874

src/main/java/com/metaweb/gridworks/commands/Command.java

@@ -1,13 +1,9 @@
 package com.metaweb.gridworks.commands;
 
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.LineNumberReader;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
-import java.io.Reader;
 import java.io.StringWriter;
+import java.io.Writer;
 import java.util.Properties;
 
 import javax.servlet.ServletException;
@@ -15,31 +11,77 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.commons.lang.NotImplementedException;
-import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
-import org.json.JSONTokener;
 import org.json.JSONWriter;
 
 import com.metaweb.gridworks.Jsonizable;
 import com.metaweb.gridworks.ProjectManager;
 import com.metaweb.gridworks.browsing.Engine;
 import com.metaweb.gridworks.model.Project;
-import com.oreilly.servlet.multipart.FilePart;
+import com.metaweb.gridworks.util.ParsingUtilities;
-import com.oreilly.servlet.multipart.MultipartParser;
-import com.oreilly.servlet.multipart.ParamPart;
-import com.oreilly.servlet.multipart.Part;
 
+/**
+ * The super class of all calls that the client side can invoke, most of which
+ * are AJAX calls.
+ */
 public abstract class Command {
-    public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+    public void doPost(HttpServletRequest request, HttpServletResponse response) 
+        throws ServletException, IOException {
+        
         throw new NotImplementedException();
     };
     
-    public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+    public void doGet(HttpServletRequest request, HttpServletResponse response) 
+        throws ServletException, IOException {
+        
         throw new NotImplementedException();
     };
     
-    protected Project getProject(HttpServletRequest request) throws ServletException {
+    /**
+     * Utility function to get the browsing engine's configuration as a JSON object
+     * from the "engine" request parameter, most often in the POST body.
+     * 
+     * @param request
+     * @return
+     * @throws Exception
+     */
+    static protected JSONObject getEngineConfig(HttpServletRequest request) throws Exception {
+        String json = request.getParameter("engine");
+        if (json != null) {
+            return ParsingUtilities.evaluateJsonStringToObject(json);
+        }
+        return null;
+    }
+    
+    /**
+     * Utility function to reconstruct the browsing engine from the "engine" request parameter, 
+     * most often in the POST body.
+     * 
+     * @param request
+     * @param project
+     * @return
+     * @throws Exception
+     */
+    static protected Engine getEngine(HttpServletRequest request, Project project) throws Exception {
+        Engine engine = new Engine(project);
+        String json = request.getParameter("engine");
+        if (json != null) {
+            JSONObject o = ParsingUtilities.evaluateJsonStringToObject(json);
+            engine.initializeFromJSON(o);
+        }
+        return engine;
+    }
+    
+    /**
+     * Utility method for retrieving the Project object having the ID specified
+     * in the "project" URL parameter.
+     * 
+     * @param request
+     * @return
+     * @throws ServletException
+     */
+    static protected Project getProject(HttpServletRequest request) throws ServletException {
         try {
             Project p = ProjectManager.singleton.getProject(Long.parseLong(request.getParameter("project")));
             if (p != null) {
@@ -51,7 +93,7 @@ public abstract class Command {
         throw new ServletException("Missing or bad project URL parameter");
     }
     
-    protected int getIntegerParameter(HttpServletRequest request, String name, int def) throws ServletException {
+    static protected int getIntegerParameter(HttpServletRequest request, String name, int def) {
         try {
             return Integer.parseInt(request.getParameter(name));
         } catch (Exception e) {
@@ -60,24 +102,36 @@ public abstract class Command {
         return def;
     }
     
-    protected void respond(HttpServletResponse response, String content) throws IOException {
+    static protected JSONObject getJsonParameter(HttpServletRequest request, String name) {
-        response.setStatus(HttpServletResponse.SC_OK);
+        String value = request.getParameter(name);
-        
+        if (value != null) {
-        OutputStream os = response.getOutputStream();
+            try {
-        OutputStreamWriter osw = new OutputStreamWriter(os);
+                JSONObject o = ParsingUtilities.evaluateJsonStringToObject(value);
-        try {
+                
-            osw.write(content);
+                return o;
-        } finally {
+            } catch (JSONException e) {
-            osw.flush();
+            }
-            osw.close();
         }
+        return null;
     }
     
-    protected void respondJSON(HttpServletResponse response, Jsonizable o) throws IOException, JSONException {
+    static protected void respond(HttpServletResponse response, String content) 
+        throws IOException {
+        
+        response.setStatus(HttpServletResponse.SC_OK);
+        response.getWriter().write(content);
+    }
+    
+    static protected void respondJSON(HttpServletResponse response, Jsonizable o) 
+        throws IOException, JSONException {
+        
         respondJSON(response, o, new Properties());
     }
     
-    protected void respondJSON(HttpServletResponse response, Jsonizable o, Properties options) throws IOException, JSONException {
+    static protected void respondJSON(
+            HttpServletResponse response, Jsonizable o, Properties options) 
+            throws IOException, JSONException {
+        
         response.setCharacterEncoding("UTF-8");
         response.setHeader("Content-Type", "application/json");
         
@@ -86,7 +140,9 @@ public abstract class Command {
         o.write(writer, options);
     }
     
-    protected void respondException(HttpServletResponse response, Exception e) throws IOException {
+    static protected void respondException(HttpServletResponse response, Exception e) 
+        throws IOException {
+        
         e.printStackTrace();
         try {
             JSONObject o = new JSONObject();
@@ -108,84 +164,13 @@ public abstract class Command {
         }
     }
     
-    protected void redirect(HttpServletResponse response, String url) throws IOException {
+    static protected void redirect(HttpServletResponse response, String url) throws IOException {
         response.setStatus(HttpServletResponse.SC_OK);
         
-        String content = "<html><head><meta http-equiv=\"refresh\" content=\"1;url=" + url + "\"></head><body></body></html>";
+        Writer writer = response.getWriter();
-        response.getWriter().print(content);
+        writer.write("<html><head>");
+        writer.write("<meta http-equiv=\"refresh\" content=\"1;url=" + url + "\">");
+        writer.write("</head><body></body></html>");
     }
     
-    protected String readFileUpload(HttpServletRequest request, Properties properties) throws IOException {
-        StringBuffer sb = new StringBuffer();
-        try {
-            MultipartParser parser = new MultipartParser(request, 20 * 1024 * 1024);
-            Part part = null;
-            while ((part = parser.readNextPart()) != null) {
-                
-                if (part.isFile()) {
-                    Reader reader = new InputStreamReader(((FilePart) part).getInputStream());
-                    LineNumberReader lnr = new LineNumberReader(reader);
-                    try {
-                        String line = null;
-                        while ((line = lnr.readLine()) != null) {
-                            sb.append(line);
-                            sb.append('\n');
-                        }
-                    } finally {
-                        lnr.close();
-                    }
-                } else if (part.isParam()) {
-                    ParamPart paramPart = (ParamPart) part;
-                    properties.put(part.getName(), paramPart.getStringValue());
-                }
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        
-        return sb.toString();
-    }
-    
-    protected JSONObject getJsonParameter(HttpServletRequest request, String name) {
-        String value = request.getParameter(name);
-        if (value != null) {
-            try {
-                JSONObject o = jsonStringToObject(value);
-                
-                return o;
-            } catch (JSONException e) {
-            }
-        }
-        return null;
-    }
-    
-    protected JSONObject jsonStringToObject(String s) throws JSONException {
-        JSONTokener t = new JSONTokener(s);
-        JSONObject o = (JSONObject) t.nextValue();
-        return o;
-    }
-    
-    protected JSONArray jsonStringToArray(String s) throws JSONException {
-        JSONTokener t = new JSONTokener(s);
-        JSONArray a = (JSONArray) t.nextValue();
-        return a;
-    }
-    
-    protected JSONObject getEngineConfig(HttpServletRequest request) throws Exception {
-        String json = request.getParameter("engine");
-        if (json != null) {
-            return jsonStringToObject(json);
-        }
-        return null;
-    }
-    
-    protected Engine getEngine(HttpServletRequest request, Project project) throws Exception {
-        Engine engine = new Engine(project);
-        String json = request.getParameter("engine");
-        if (json != null) {
-            JSONObject o = jsonStringToObject(json);
-            engine.initializeFromJSON(o);
-        }
-        return engine;
-    }
 }

src/main/java/com/metaweb/gridworks/commands/EngineDependentCommand.java

@@ -13,6 +13,27 @@ import com.metaweb.gridworks.model.AbstractOperation;
 import com.metaweb.gridworks.model.Project;
 import com.metaweb.gridworks.process.Process;
 
+/**
+ * Convenient super class for commands that perform abstract operations on
+ * only the filtered rows based on the faceted browsing engine's configuration
+ * on the client side. 
+ * 
+ * The engine's configuration is passed over as a POST body parameter. It is 
+ * retrieved, de-serialized, and used to construct the abstract operation.
+ * The operation is then used to construct a process. The process is then
+ * queued for execution. If the process is not long running and there is no
+ * other queued process, then it gets executed right away, resulting in some
+ * change to the history. Otherwise, it is pending. The client side can
+ * decide how to update its UI depending on whether the process is done or
+ * still pending.
+ * 
+ * Note that there are interactions on the client side that change only 
+ * individual cells or individual rows (such as starring one row or editing 
+ * the text of one cell). These interactions do not depend on the faceted
+ * browsing engine's configuration, and so they don't invoke commands that
+ * subclass this class. See AnnotateOneRowCommand and EditOneCellCommand as
+ * examples. 
+ */
 abstract public class EngineDependentCommand extends Command {
     @Override
     public void doPost(HttpServletRequest request, HttpServletResponse response)

src/main/java/com/metaweb/gridworks/commands/edit/AnnotateOneRowCommand.java

@@ -30,7 +30,7 @@ public class AnnotateOneRowCommand extends Command {
             String starredString = request.getParameter("starred");
             if (starredString != null) {
                 boolean starred = "true".endsWith(starredString);
-                String description = starred ? "Star row " + rowIndex : "Unstar row " + rowIndex; 
+                String description = (starred ? "Star row " : "Unstar row ") + (rowIndex + 1); 
 
                 StarOneRowProcess process = new StarOneRowProcess(
                     project, 
@@ -76,7 +76,7 @@ public class AnnotateOneRowCommand extends Command {
         protected HistoryEntry createHistoryEntry() throws Exception {
             return new HistoryEntry(
                 _project, 
-                starred ? "Star row " + rowIndex : "Unstar row " + rowIndex, 
+                (starred ? "Star row " : "Unstar row ") + (rowIndex + 1), 
                 null, 
                 new RowStarChange(rowIndex, starred)
             );

src/main/java/com/metaweb/gridworks/commands/edit/ApplyOperationsCommand.java

@@ -16,6 +16,7 @@ import com.metaweb.gridworks.model.AbstractOperation;
 import com.metaweb.gridworks.model.Project;
 import com.metaweb.gridworks.operations.OperationRegistry;
 import com.metaweb.gridworks.process.Process;
+import com.metaweb.gridworks.util.ParsingUtilities;
 
 public class ApplyOperationsCommand extends Command {
     
@@ -26,7 +27,7 @@ public class ApplyOperationsCommand extends Command {
         Project project = getProject(request);
         String jsonString = request.getParameter("operations");
         try {
-            JSONArray a = jsonStringToArray(jsonString);
+            JSONArray a = ParsingUtilities.evaluateJsonStringToArray(jsonString);
             int count = a.length();
             for (int i = 0; i < count; i++) {
                 JSONObject obj = a.getJSONObject(i);

src/main/java/com/metaweb/gridworks/commands/edit/CreateProjectCommand.java

@@ -38,11 +38,25 @@ public class CreateProjectCommand extends Command {
             throws ServletException, IOException {
         
         try {
+            /*
+             * The uploaded file is in the POST body as a "file part". If
+             * we call request.getParameter() then the POST body will get
+             * read and we won't have a chance to parse the body ourselves.
+             * This is why we have to parse the URL for parameters ourselves.
+             * Don't call request.getParameter() before calling internalImport().
+             */
+            
             Properties options = ParsingUtilities.parseUrlParameters(request);
+            
             Project project = new Project();
             
             internalImport(request, project, options);
 
+            /*
+             * The import process above populates options with parameters
+             * in the POST body. That's why we're constructing the project
+             * metadata object after calling internalImport().
+             */
             ProjectMetadata pm = new ProjectMetadata();
             pm.setName(options.getProperty("project-name"));
             pm.setPassword(options.getProperty("project-password"));
@@ -64,7 +78,7 @@ public class CreateProjectCommand extends Command {
     
     protected void internalImport(
         HttpServletRequest    request,
-        Project                project,
+        Project               project,
         Properties            options
     ) throws Exception {
         MultipartParser parser = new MultipartParser(request, 1024 * 1024 * 1024);
@@ -73,91 +87,19 @@ public class CreateProjectCommand extends Command {
             Part part = null;
             String url = null;
             
-            int limit = -1;
-            int skip = 0;
-            
-            if (options.containsKey("limit")) {
-                String s = options.getProperty("limit");
-                try {
-                    limit = Integer.parseInt(s);
-                } catch (Exception e) {
-                }
-            }
-            if (options.containsKey("skip")) {
-                String s = options.getProperty("skip");
-                try {
-                    skip = Integer.parseInt(s);
-                } catch (Exception e) {
-                }
-            }
-            
             while ((part = parser.readNextPart()) != null) {
                 
                 if (part.isFile()) {
-                    FilePart filePart = (FilePart) part;
+                    internalImportFilePart((FilePart) part, project, options);
-                    BufferedInputStream inputStream = new BufferedInputStream(filePart.getInputStream());
                     
-                    Importer importer = guessImporter(options, null, filePart.getFileName());
-                    
-                    if (importer.takesReader()) {
-                        /*
-                         * NOTE(SM): The ICU4J char detection code requires the input stream to support mark/reset. 
-                         * Unfortunately, not all ServletInputStream implementations are marking, so we need do 
-                         * this memory-expensive wrapping to make it work. It's far from ideal but I don't have 
-                         * a more efficient solution.
-                         */
-                    	byte[] bytes = new byte[1024 * 4];
-                    	{
-                    		inputStream.mark(bytes.length);
-                    		inputStream.read(bytes);
-                    		inputStream.reset();
-                    	}
-                    	
-                        CharsetDetector detector = new CharsetDetector();
-                        detector.setDeclaredEncoding("utf8"); // the content on the web is encoded in UTF-8 so assume that
-                        
-                        Reader reader = null;
-                        CharsetMatch[] charsetMatches = detector.setText(bytes).detectAll();
-                        for (CharsetMatch charsetMatch : charsetMatches) {
-                        	try {
-                        		reader = new InputStreamReader(inputStream, charsetMatch.getName());
-                        		
-                                options.setProperty("encoding", charsetMatch.getName());
-                                options.setProperty("encoding_confidence", Integer.toString(charsetMatch.getConfidence()));
-                                
-                                Gridworks.log(
-                                	"Best encoding guess: " + 
-                                	charsetMatch.getName() + 
-                                	" [confidence: " + charsetMatch.getConfidence() + "]");
-                                
-                                break;
-                        	} catch (UnsupportedEncodingException e) {
-                        		// silent
-                        	}
-                        }
-                        
-                        if (reader == null) {
-                        	reader = new InputStreamReader(inputStream); // all else has failed
-                        }
-                        try {
-                            importer.read(reader, project, options, skip, limit);
-                        } finally {
-                            reader.close();
-                        }
-                    } else {
-                        try {
-                            importer.read(inputStream, project, options, skip, limit);
-                        } finally {
-                            inputStream.close();
-                        }
-                    }
                 } else if (part.isParam()) {
                     ParamPart paramPart = (ParamPart) part;
                     String paramName = paramPart.getName();
+                    
                     if (paramName.equals("raw-text")) {
                         StringReader reader = new StringReader(paramPart.getStringValue());
                         try {
-                            new TsvCsvImporter().read(reader, project, options, skip, limit);
+                            internalInvokeImporter(project, new TsvCsvImporter(), options, reader);
                         } finally {
                             reader.close();
                         }
@@ -170,18 +112,27 @@ public class CreateProjectCommand extends Command {
             }
             
             if (url != null && url.length() > 0) {
-                internalImportURL(request, project, options, url, skip, limit);
+                internalImportURL(request, project, options, url);
             }
         }
     }
     
+    protected void internalImportFilePart(
+        FilePart    filePart,
+        Project     project,
+        Properties  options
+    ) throws Exception {
+        
+        Importer importer = guessImporter(options, null, filePart.getFileName());
+        
+        internalInvokeImporter(project, importer, options, filePart.getInputStream(), null);
+    }
+    
     protected void internalImportURL(
         HttpServletRequest    request,
-        Project                project,
+        Project               project,
         Properties            options,
-        String                urlString,
+        String                urlString
-        int                 skip,
-        int                    limit
     ) throws Exception {
         URL url = new URL(urlString);
         URLConnection connection = null;
@@ -207,22 +158,126 @@ public class CreateProjectCommand extends Command {
                 connection.getContentType(),
                 url.getPath()
             );
-        
+            
-            if (importer.takesReader()) {
+            internalInvokeImporter(project, importer, options, inputStream, connection.getContentEncoding());
-                String encoding = connection.getContentEncoding();
-                
-                Reader reader = new InputStreamReader(
-                    inputStream, (encoding == null) ? "ISO-8859-1" : encoding);
-                            
-                importer.read(reader, project, options, skip, limit);
-            } else {
-                importer.read(inputStream, project, options, skip, limit);
-            }
         } finally {
             inputStream.close();
         }
     }
     
+    protected void internalInvokeImporter(
+        Project     project,
+        Importer    importer,
+        Properties  options,
+        InputStream rawInputStream,
+        String      encoding
+    ) throws Exception {
+        
+        int limit = -1;
+        int skip = 0;
+        
+        if (options.containsKey("limit")) {
+            String s = options.getProperty("limit");
+            try {
+                limit = Integer.parseInt(s);
+            } catch (Exception e) {
+            }
+        }
+        if (options.containsKey("skip")) {
+            String s = options.getProperty("skip");
+            try {
+                skip = Integer.parseInt(s);
+            } catch (Exception e) {
+            }
+        }
+        
+        BufferedInputStream inputStream = new BufferedInputStream(rawInputStream);
+        
+        if (importer.takesReader()) {
+            /*
+             * NOTE(SM): The ICU4J char detection code requires the input stream to support mark/reset. 
+             * Unfortunately, not all ServletInputStream implementations are marking, so we need do 
+             * this memory-expensive wrapping to make it work. It's far from ideal but I don't have 
+             * a more efficient solution.
+             */
+            byte[] bytes = new byte[1024 * 4];
+            {
+                inputStream.mark(bytes.length);
+                inputStream.read(bytes);
+                inputStream.reset();
+            }
+            
+            CharsetDetector detector = new CharsetDetector();
+            detector.setDeclaredEncoding("utf8"); // the content on the web is encoded in UTF-8 so assume that
+            
+            Reader reader = null;
+            CharsetMatch[] charsetMatches = detector.setText(bytes).detectAll();
+            for (CharsetMatch charsetMatch : charsetMatches) {
+                try {
+                    reader = new InputStreamReader(inputStream, charsetMatch.getName());
+                    
+                    options.setProperty("encoding", charsetMatch.getName());
+                    options.setProperty("encoding_confidence", Integer.toString(charsetMatch.getConfidence()));
+                    
+                    Gridworks.log(
+                        "Best encoding guess: " + 
+                        charsetMatch.getName() + 
+                        " [confidence: " + charsetMatch.getConfidence() + "]");
+                    
+                    break;
+                } catch (UnsupportedEncodingException e) {
+                    // silent
+                }
+            }
+            
+            if (reader == null) { // when all else fails
+                reader = encoding != null ?
+                        new InputStreamReader(inputStream, encoding) :
+                        new InputStreamReader(inputStream);
+            }
+            
+            try {
+                importer.read(reader, project, options, skip, limit);
+            } finally {
+                reader.close();
+            }
+        } else {
+            try {
+                importer.read(inputStream, project, options, skip, limit);
+            } finally {
+                inputStream.close();
+            }
+        }        
+    }
+    
+    protected void internalInvokeImporter(
+        Project     project,
+        Importer    importer,
+        Properties  options,
+        Reader      reader
+    ) throws Exception {
+        
+        int limit = -1;
+        int skip = 0;
+        
+        if (options.containsKey("limit")) {
+            String s = options.getProperty("limit");
+            try {
+                limit = Integer.parseInt(s);
+            } catch (Exception e) {
+            }
+        }
+        if (options.containsKey("skip")) {
+            String s = options.getProperty("skip");
+            try {
+                skip = Integer.parseInt(s);
+            } catch (Exception e) {
+            }
+        }
+        
+        importer.read(reader, project, options, skip, limit);
+    }
+    
     protected Importer guessImporter(
             Properties options, String contentType, String fileName) {
         

src/main/java/com/metaweb/gridworks/commands/edit/EditOneCellCommand.java

@@ -55,6 +55,10 @@ public class EditOneCellCommand extends Command {
             
             boolean done = project.processManager.queueProcess(process);
             if (done) {
+                /*
+                 * If the operation has been done, return the new cell's data
+                 * so the client side can update the cell's rendering right away.
+                 */
                 JSONWriter writer = new JSONWriter(response.getWriter());
                 writer.object();
                 writer.key("code"); writer.value("ok");

src/main/java/com/metaweb/gridworks/commands/edit/ExtendDataCommand.java

@@ -8,6 +8,7 @@ import com.metaweb.gridworks.commands.EngineDependentCommand;
 import com.metaweb.gridworks.model.AbstractOperation;
 import com.metaweb.gridworks.model.Project;
 import com.metaweb.gridworks.operations.ExtendDataOperation;
+import com.metaweb.gridworks.util.ParsingUtilities;
 
 public class ExtendDataCommand extends EngineDependentCommand {
     @Override
@@ -18,7 +19,7 @@ public class ExtendDataCommand extends EngineDependentCommand {
         int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex"));
         
         String jsonString = request.getParameter("extension");
-        JSONObject extension = jsonStringToObject(jsonString);
+        JSONObject extension = ParsingUtilities.evaluateJsonStringToObject(jsonString);
         
         return new ExtendDataOperation(
             engineConfig, 

src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java

@@ -57,7 +57,7 @@ public class ImportProjectCommand extends Command {
     ) throws Exception {
         MultipartParser parser = null;
         try {
-            parser = new MultipartParser(request, 20 * 1024 * 1024);
+            parser = new MultipartParser(request, 1024 * 1024 * 1024);
         } catch (Exception e) {
             // silent
         }

src/main/java/com/metaweb/gridworks/commands/edit/SaveProtographCommand.java

@@ -15,6 +15,7 @@ import com.metaweb.gridworks.model.Project;
 import com.metaweb.gridworks.operations.SaveProtographOperation;
 import com.metaweb.gridworks.process.Process;
 import com.metaweb.gridworks.protograph.Protograph;
+import com.metaweb.gridworks.util.ParsingUtilities;
 
 public class SaveProtographCommand extends Command {
     @Override
@@ -25,7 +26,7 @@ public class SaveProtographCommand extends Command {
             Project project = getProject(request);
             
             String jsonString = request.getParameter("protograph");
-            JSONObject json = jsonStringToObject(jsonString);
+            JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
             Protograph protograph = Protograph.reconstruct(json);
             
             AbstractOperation op = new SaveProtographOperation(protograph);

src/main/java/com/metaweb/gridworks/commands/info/ExportRowsCommand.java

@@ -2,6 +2,8 @@ package com.metaweb.gridworks.commands.info;
 
  import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Properties;
 
 import javax.servlet.ServletException;
@@ -18,7 +20,13 @@ import com.metaweb.gridworks.exporters.XlsExporter;
 import com.metaweb.gridworks.model.Project;
 
 public class ExportRowsCommand extends Command {
-
+    static protected Map<String, Exporter> s_formatToExporter = new HashMap<String, Exporter>();
+    static {
+        s_formatToExporter.put("tripleloader", new TripleloaderExporter());
+        s_formatToExporter.put("html", new HtmlTableExporter());
+        s_formatToExporter.put("xls", new XlsExporter());
+    }
+    
     public void doPost(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
         
@@ -27,15 +35,8 @@ public class ExportRowsCommand extends Command {
             Engine engine = getEngine(request, project);
             String format = request.getParameter("format");
             
-            
+            Exporter exporter = s_formatToExporter.get(format.toLowerCase());
-            Exporter exporter = null;
+            if (exporter == null){
-            if ("tripleloader".equalsIgnoreCase(format)) {
-                exporter = new TripleloaderExporter();
-            } else if ("html".equalsIgnoreCase(format)) {
-                exporter = new HtmlTableExporter();
-            } else if ("xls".equalsIgnoreCase(format)) {
-                exporter = new XlsExporter();
-            } else {
                 exporter = new TsvExporter();
             }
             

src/main/java/com/metaweb/gridworks/commands/info/GetAllProjectMetadataCommand.java

@@ -29,14 +29,15 @@ public class GetAllProjectMetadataCommand extends Command {
             
             writer.object();
             
-            writer.key("projects"); writer.object();
+            writer.key("projects");
-            
+                writer.object();
-            Map<Long, ProjectMetadata> m = ProjectManager.singleton.getAllProjectMetadata();
+                Map<Long, ProjectMetadata> m = ProjectManager.singleton.getAllProjectMetadata();
-            for (Long id : m.keySet()) {
+                for (Long id : m.keySet()) {
-                writer.key(id.toString());
+                    writer.key(id.toString());
-                m.get(id).write(writer, options);
+                    m.get(id).write(writer, options);
-            }
+                }
-            writer.endObject();
+                writer.endObject();
+                
             writer.endObject();
         } catch (JSONException e) {
             respondException(response, e);

src/main/java/com/metaweb/gridworks/commands/info/GetRowsCommand.java

@@ -38,8 +38,8 @@ public class GetRowsCommand extends Command {
             
             {
                 RowAccumulator acc = new RowAccumulator(start, limit) {
-                    JSONWriter    writer;
+                    JSONWriter  writer;
-                    Properties    options;
+                    Properties  options;
                     Properties  extra;
                     
                     public RowAccumulator init(JSONWriter writer, Properties options) {
@@ -53,8 +53,13 @@ public class GetRowsCommand extends Command {
                     }
                     
                     @Override
-                    public boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean includeDependent) {
+                    public boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean dependent) {
                         try {
+                            /*
+                             * Whatever that's in the "extra" field will be written out
+                             * by the row as well. This is how we can customize what the row
+                             * writes, in a limited way.
+                             */
                             if (contextual) {
                                 options.put("extra", extra);
                             } else {
@@ -99,19 +104,19 @@ public class GetRowsCommand extends Command {
             this.limit = limit;
         }
         
-        public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
+        public boolean visit(Project project, int rowIndex, Row row, boolean contextual, boolean dependent) {
             boolean r = false;
             
             if (total >= start && total < start + limit) {
-                r = internalVisit(rowIndex, row, includeContextual, includeDependent);
+                r = internalVisit(rowIndex, row, contextual, dependent);
             }
-            if (!includeContextual) {
+            if (!contextual) {
                 total++;
             }
             return r;
         }
         
-        protected boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean includeDependent) {
+        protected boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean dependent) {
             return false;
         }
     }

src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeOneCellCommand.java

@@ -61,6 +61,10 @@ public class ReconJudgeOneCellCommand extends Command {
             
             boolean done = project.processManager.queueProcess(process);
             if (done) {
+                /*
+                 * If the process is done, write back the cell's data so that the
+                 * client side can update its UI right away.
+                 */
                 JSONWriter writer = new JSONWriter(response.getWriter());
                 writer.object();
                 writer.key("code"); writer.value("ok");

src/main/java/com/metaweb/gridworks/commands/util/GuessTypesOfColumnCommand.java

@@ -78,6 +78,15 @@ public class GuessTypesOfColumnCommand extends Command {
     
     final static int s_sampleSize = 20;
     
+    /**
+     * Run relevance searches for the first n cells in the given column and
+     * count the types of the results. Return a sorted list of types, from most
+     * frequent to least. 
+     * 
+     * @param project
+     * @param column
+     * @return
+     */
     protected List<TypeGroup> guessTypes(Project project, Column column) {
         Map<String, TypeGroup> map = new HashMap<String, TypeGroup>();
         
@@ -117,7 +126,7 @@ public class GuessTypesOfColumnCommand extends Command {
             jsonWriter.endObject();
             
             StringBuffer sb = new StringBuffer();
-            sb.append("http://api.freebase.com/api/service/search?indent=1&queries=");
+            sb.append("http://api.freebase.com/api/service/search?queries=");
             sb.append(ParsingUtilities.encode(stringWriter.toString()));
             
             URL url = new URL(sb.toString());
@@ -146,7 +155,7 @@ public class GuessTypesOfColumnCommand extends Command {
                     
                     for (int j = 0; j < count; j++) {
                         JSONObject result = results.getJSONObject(j);
-                        double score = 1.0 / (1 + j); //result.getDouble("relevance:score");
+                        double score = 1.0 / (1 + j); // score by each result's rank
                         
                         JSONArray types = result.getJSONArray("type");
                         int typeCount = types.length();

src/main/java/com/metaweb/gridworks/commands/util/PreviewExpressionCommand.java

@@ -3,6 +3,7 @@ package com.metaweb.gridworks.commands.util;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Calendar;
+import java.util.List;
 import java.util.Properties;
 
 import javax.servlet.ServletException;
@@ -10,6 +11,7 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONWriter;
 
 import com.metaweb.gridworks.commands.Command;
@@ -55,7 +57,7 @@ public class PreviewExpressionCommand extends Command {
             response.setCharacterEncoding("UTF-8");
             response.setHeader("Content-Type", "application/json");
             
-            JSONArray rowIndices = jsonStringToArray(rowIndicesString);
+            JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString);
             int length = rowIndices.length();
             
             JSONWriter writer = new JSONWriter(response.getWriter());
@@ -100,23 +102,20 @@ public class PreviewExpressionCommand extends Command {
                         }
                     }
                     
-                    if (ExpressionUtils.isError(result)) {
+                    if (result != null && (result.getClass().isArray() || result instanceof List<?>)) {
-                        writer.object();
+                        writer.array();
-                        writer.key("message"); writer.value(((EvalError) result).message);
+                        if (result.getClass().isArray()) {
-                        writer.endObject();
+                            for (Object v : (Object[]) result) {
-                    } else {
+                                writeValue(writer, v);
-                        if (result != null) {
+                            }
-                            if (result instanceof HasFields) {
+                        } else {
-                                result = "[object " + result.getClass().getSimpleName() + "]";
+                            for (Object v : ExpressionUtils.toObjectList(result)) {
-                            } else if (result instanceof Calendar) {
+                                writeValue(writer, v);
-                                Calendar c = (Calendar) result;
-                                
-                                result = "[object " + 
-                                    result.getClass().getSimpleName() + " " + 
-                                    ParsingUtilities.dateToString(c.getTime()) +"]";
                             }
                         }
-                        writer.value(result);
+                        writer.endArray();
+                    } else {
+                        writeValue(writer, result);
                     }
                 }
                 writer.endArray();
@@ -135,4 +134,25 @@ public class PreviewExpressionCommand extends Command {
             respondException(response, e);
         }
     }
+    
+    static protected void writeValue(JSONWriter writer, Object v) throws JSONException {
+        if (ExpressionUtils.isError(v)) {
+            writer.object();
+            writer.key("message"); writer.value(((EvalError) v).message);
+            writer.endObject();
+        } else {
+            if (v != null) {
+                if (v instanceof HasFields) {
+                    v = "[object " + v.getClass().getSimpleName() + "]";
+                } else if (v instanceof Calendar) {
+                    Calendar c = (Calendar) v;
+                    
+                    v = "[object " + 
+                        v.getClass().getSimpleName() + " " + 
+                        ParsingUtilities.dateToString(c.getTime()) +"]";
+                }
+            }
+            writer.value(v);
+        }
+    }
 }

src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java

@@ -21,6 +21,7 @@ import com.metaweb.gridworks.model.Project;
 import com.metaweb.gridworks.model.ReconCandidate;
 import com.metaweb.gridworks.model.Row;
 import com.metaweb.gridworks.util.FreebaseDataExtensionJob;
+import com.metaweb.gridworks.util.ParsingUtilities;
 import com.metaweb.gridworks.util.FreebaseDataExtensionJob.ColumnInfo;
 import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension;
 
@@ -41,9 +42,9 @@ public class PreviewExtendDataCommand extends Command {
             }
             
             String jsonString = request.getParameter("extension");
-            JSONObject json = jsonStringToObject(jsonString);
+            JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
             
-            JSONArray rowIndices = jsonStringToArray(rowIndicesString);
+            JSONArray rowIndices = ParsingUtilities.evaluateJsonStringToArray(rowIndicesString);
             int length = rowIndices.length();
             int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex();
             

src/main/java/com/metaweb/gridworks/commands/util/PreviewProtographCommand.java

@@ -16,6 +16,7 @@ import com.metaweb.gridworks.protograph.Protograph;
 import com.metaweb.gridworks.protograph.transpose.MqlreadLikeTransposedNodeFactory;
 import com.metaweb.gridworks.protograph.transpose.Transposer;
 import com.metaweb.gridworks.protograph.transpose.TripleLoaderTransposedNodeFactory;
+import com.metaweb.gridworks.util.ParsingUtilities;
 
 public class PreviewProtographCommand extends Command {
     @Override
@@ -29,7 +30,7 @@ public class PreviewProtographCommand extends Command {
             response.setHeader("Content-Type", "application/json");
             
             String jsonString = request.getParameter("protograph");
-            JSONObject json = jsonStringToObject(jsonString);
+            JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
             Protograph protograph = Protograph.reconstruct(json);
             
             StringBuffer sb = new StringBuffer();

src/main/java/com/metaweb/gridworks/expr/functions/arrays/Join.java

@@ -7,6 +7,7 @@ import org.json.JSONException;
 import org.json.JSONWriter;
 
 import com.metaweb.gridworks.expr.EvalError;
+import com.metaweb.gridworks.expr.ExpressionUtils;
 import com.metaweb.gridworks.gel.ControlFunctionRegistry;
 import com.metaweb.gridworks.gel.Function;
 
@@ -33,7 +34,7 @@ public class Join implements Function {
 	                    }
 	                }
                 } else {
-	                for (Object o : (List<Object>) v) {
+	                for (Object o : ExpressionUtils.toObjectList(v)) {
 	                    if (o != null) {
 	                        if (sb.length() > 0) {
 	                            sb.append(separator);

src/main/java/com/metaweb/gridworks/util/JSONUtilities.java

@@ -93,5 +93,4 @@ public class JSONUtilities {
         }
         writer.endArray();
     }
-    
 }

src/main/webapp/scripts/project/menu-bar.js

@@ -187,7 +187,7 @@ MenuBar.prototype._doExportRows = function(format, ext) {
 };
 
 MenuBar.prototype._exportProject = function() {
-    var name = $(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
+    var name = $.trim(theProject.metadata.name.replace(/\W/g, ' ')).replace(/\s+/g, '-');
     var form = document.createElement("form");
     $(form)
         .css("display", "none")