Merge remote-tracking branch 'origin/master' into fusiontables-migration

Conflicts:
	extensions/freebase/module/langs/translation-default.json
	extensions/freebase/module/langs/translation-en.json
	extensions/freebase/module/langs/translation-it.json
	extensions/freebase/src/com/google/refine/freebase/commands/LoadLanguageCommand.java
	extensions/gdata/module/MOD-INF/controller.js
	extensions/gdata/module/langs/translation-default.json
	extensions/gdata/module/langs/translation-en.json
	extensions/gdata/module/langs/translation-it.json
	extensions/gdata/src/com/google/refine/extension/gdata/commands/LoadLanguageCommand.java
	main/src/com/google/refine/commands/lang/LoadLanguageCommand.java
	main/src/com/google/refine/commands/lang/SetLanguageCommand.java
	main/webapp/modules/core/langs/translation-default.json
	main/webapp/modules/core/langs/translation-en.json
	main/webapp/modules/core/langs/translation-it.json
	main/webapp/modules/core/scripts/index/lang-settings-ui.js

.gitignore

@@ -1,5 +1,8 @@
+*~
+\#*#
 *.DS_Store
 *.class
+.com.apple.timemachine.supported
 .import-temp/
 build/
 dist/

build.xml

@@ -14,7 +14,6 @@
     <property environment="env"/>
 
     <property name="version" value="trunk"/>
-    <property name="revision" value="rXXXX"/>
     <property name="full_version" value="0.0.0.0"/>
     <property name="build.dir" value="build"/>
     <property name="dist.dir" value="dist"/>
@@ -25,7 +24,7 @@
     <property name="appengine.version" value="1"/>
     <property name="appengine.sdk.dir" value="/opt/appengine"/>
         
-    <property name="fullname" value="openrefine-${version}-${revision}" />
+    <property name="fullname" value="openrefine-${version}" />
     
     <property name="main.dir" value="${basedir}/main" />
 
@@ -222,11 +221,10 @@
         
         <replace file="${built.webapp.dir}/WEB-INF/web.xml">
             <replacefilter token="$VERSION" value="${version}"/>
-            <replacefilter token="$REVISION" value="${revision}"/>
         </replace>
 
         <replace file="${built.webapp.dir}/WEB-INF/butterfly.properties">
-            <replacefilter token="../../extensions/" value="extensions"/>
+            <replacefilter token="../../extensions" value="extensions"/>
         </replace>
 
     </target>
@@ -254,14 +252,13 @@
 	    </classpath>
 	    <option value="-Xms256M"/>
 	    <option value="-Xmx1024M"/>
-	    <option value="-Drefine.version=${revision}"/>
+	    <option value="-Drefine.version=${version}"/>
 	    <option value="-Drefine.webapp=$APP_ROOT/Contents/Resource/${built.webapp.name}"/>
         </bundleapp>
 
 	<copy todir="${mac.dir}/OpenRefine.app/Contents/Resource">
           <fileset dir="${build.dir}" id="librarypathset" >
             <include name="${built.webapp.name}/**/**" />
-            <exclude name="**/*.class" />
           </fileset>
 	</copy>
 
@@ -277,6 +274,7 @@
     </target>
     
     <target name="windows" depends="jar, prepare_webapp">
+      <echo message="Full version ${full_version} and version ${version}"/>
         <mkdir dir="${windows.dir}"/>
         <taskdef 
             name="launch4j" 
@@ -294,7 +292,7 @@
                 <cp>server/lib/*.jar</cp>
             </classPath>
             <jre minVersion="1.6.0" jdkPreference="preferJre" initialHeapSize="256" maxHeapSize="1024">
-                <opt>-Djava.library.path=server/lib/native/windows -Drefine.version=${revision}</opt>
+                <opt>-Djava.library.path=server/lib/native/windows </opt>
             </jre>
             <versionInfo
                 fileVersion="${full_version}"
@@ -302,7 +300,7 @@
                 fileDescription="openrefine"
                 copyright="Copyright (c) 2013 OpenRefine contributors, 2010, Google, Inc."
                 productVersion="${full_version}"
-                txtProductVersion="${full_version}"
+                txtProductVersion="${version}"
                 productName="OpenRefine"
                 companyName="OpenRefine team"
                 internalName="openrefine"
@@ -341,7 +339,7 @@
         <copy file="${basedir}/LICENSE.txt" tofile="${windows.dir}/LICENSE.txt"/>
         
         <mkdir dir="${dist.dir}"/>
-        <zip destfile="${dist.dir}/openrefine-${version}-${revision}.zip" basedir="${windows.dir}/.." includes="${release.name}/**"/>
+        <zip destfile="${dist.dir}/openrefine-win-${version}.zip" basedir="${windows.dir}/.." includes="${release.name}/**"/>
     </target>
     
     <target name="linux" depends="jar, prepare_webapp">
@@ -372,7 +370,7 @@
         <copy file="${basedir}/refine" tofile="${linux.dir}/refine"/>
         
         <mkdir dir="${dist.dir}"/>
-        <tar longfile="gnu" compression="gzip" destfile="${dist.dir}/openrefine-${version}-${revision}.tar.gz">
+        <tar longfile="gnu" compression="gzip" destfile="${dist.dir}/openrefine-linux-${version}.tar.gz">
            <tarfileset dir="${linux.dir}/.." filemode="755">
                <include name="${release.name}/refine"/>
            </tarfileset>        

extensions/freebase/module/MOD-INF/controller.js

@@ -59,7 +59,6 @@ function init() {
   RS.registerCommand(module, "import-qa-data",        new Packages.com.google.refine.freebase.commands.ImportQADataCommand());
   RS.registerCommand(module, "mqlread",               new Packages.com.google.refine.freebase.commands.MQLReadCommand());
   RS.registerCommand(module, "mqlwrite",              new Packages.com.google.refine.freebase.commands.MQLWriteCommand());
-  RS.registerCommand(module, "load-language",		  new Packages.com.google.refine.freebase.commands.LoadLanguageCommand());
 
   var OR = Packages.com.google.refine.operations.OperationRegistry;
 

extensions/freebase/module/langs/translation-default.json

@@ -1,4 +1,4 @@
-{
+{
     "fb-schema-alignment": {
         "close-confirm": "There are unsaved changes. Close anyway?",
         "status-warning": "There are unsaved changes.",

extensions/freebase/module/langs/translation-en.json

@@ -1,4 +1,4 @@
-{
+{
     "fb-schema-alignment": {
         "close-confirm": "There are unsaved changes. Close anyway?",
         "status-warning": "There are unsaved changes.",

extensions/freebase/module/langs/translation-it.json

@@ -1,4 +1,4 @@
-{
+{
     "fb-schema-alignment": {
         "close-confirm": "Ci sono cambiamenti non salvati. Chiudere comunque?",
         "status-warning": "Ci sono cambiamenti non salvati.",

extensions/freebase/module/scripts/extension.js

@@ -39,11 +39,12 @@ var lang = navigator.language.split("-")[0]
 		|| navigator.userLanguage.split("-")[0];
 var dictionary = "";
 $.ajax({
-	url : "/command/freebase/load-language?",
+	url : "/command/core/load-language?",
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "freebase",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

extensions/freebase/src/com/google/refine/freebase/commands/LoadLanguageCommand.java

@@ -1,73 +0,0 @@
-
-package com.google.refine.freebase.commands;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.refine.ProjectManager;
-import com.google.refine.commands.Command;
-
-public class LoadLanguageCommand extends Command {
-
-    public LoadLanguageCommand() {
-        // TODO Auto-generated constructor stub
-    }
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        doPost(request, response);
-    }
-
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-
-        String rawDirectoryFile = request.getSession().getServletContext()
-                .getRealPath("extensions/freebase/module/langs/");
-        String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "");
-
-        BufferedReader reader = null;String param = null;
-        try {
-            param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
-        } catch (NullPointerException e) {
-        }
-        if (param == null) param = request.getParameter("lng");
-
-        String[] langs = param.split(" ");
-        try {
-            String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
-            reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-        } catch (FileNotFoundException e1) {
-            try {
-                String file = cleanedDirectory + File.separator + "translation-default.json";
-                reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-            } catch (FileNotFoundException e3) {
-                e3.printStackTrace();
-            }
-        }
-
-        String line = null;
-        String message = new String();
-        if (reader != null) {
-            while ((line = reader.readLine()) != null) {
-                // buffer.append(line);
-                message += line + System.getProperty("line.separator");
-            }
-        }
-
-        response.setCharacterEncoding("UTF-8");
-        response.setContentType("application/json");
-        response.getWriter().println(message);
-        response.getWriter().flush();
-        response.getWriter().close();
-    }
-
-}

extensions/freebase/src/com/google/refine/freebase/util/FreebaseUtils.java

@@ -430,6 +430,9 @@ public class FreebaseUtils {
 
     /**
      * This RPC call works for the Reconcile API, but MQLread is not supported over JSONRPC
+     * 
+     * NOTE: JSONRPC has been deprecated and replaced by HTTP Batch (which also
+     * doesn't support MQLread, so perhaps we should just remove this))
      */
     @SuppressWarnings("unused")
     static private JSONObject mqlreadRpc(String query) throws JSONException, UnsupportedEncodingException, IOException {

extensions/gdata/module/MOD-INF/controller.js

@@ -51,7 +51,6 @@ function init() {
   RS.registerCommand(module, "deauthorize", Packages.com.google.refine.extension.gdata.DeAuthorizeCommand());
   RS.registerCommand(module, "upload", Packages.com.google.refine.extension.gdata.UploadCommand());
   // TODO: Need a new OAUTH2 authorize command for FusionTables
-  RS.registerCommand(module, "load-language", Packages.com.google.refine.extension.gdata.commands.LoadLanguageCommand());
 
   // Register importer and exporter
   var IM = Packages.com.google.refine.importing.ImportingManager;

extensions/gdata/module/langs/translation-default.json

@@ -1,4 +1,4 @@
-{
+{
     "gdata-import": {
 		"preparing": "Preparing ...",
 		"creating": "Creating project ...",

extensions/gdata/module/langs/translation-en.json

@@ -1,4 +1,4 @@
-{
+{
     "gdata-import": {
 		"preparing": "Preparing ...",
 		"creating": "Creating project ...",

extensions/gdata/module/langs/translation-it.json

@@ -1,4 +1,4 @@
-{
+{
     "gdata-import": {
 		"preparing": "In preparazione ...",
 		"creating": "Creazione il progetto ...",

extensions/gdata/module/scripts/index/importing-controller.js

@@ -36,11 +36,12 @@ var lang = navigator.language.split("-")[0]
 		|| navigator.userLanguage.split("-")[0];
 var dictionary = "";
 $.ajax({
-	url : "/command/gdata/load-language?",
+	url : "/command/core/load-language?",
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "gdata",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

extensions/gdata/module/scripts/project/exporters.js

@@ -33,11 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 var dictionary = "";
 $.ajax({
-	url : "/command/gdata/load-language?",
+	url : "/command/core/load-language?",
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "gdata",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

extensions/gdata/src/com/google/refine/extension/gdata/commands/LoadLanguageCommand.java

@@ -1,74 +0,0 @@
-
-package com.google.refine.extension.gdata.commands;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.refine.ProjectManager;
-import com.google.refine.commands.Command;
-
-public class LoadLanguageCommand extends Command {
-
-    public LoadLanguageCommand() {
-        // TODO Auto-generated constructor stub
-    }
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        doPost(request, response);
-    }
-
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-
-        String rawDirectoryFile = request.getSession().getServletContext()
-                .getRealPath("extensions/gdata/module/langs/");
-        String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "");
-
-        BufferedReader reader = null;
-        String param = null;
-        try {
-            param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
-        } catch (NullPointerException e) {
-        }
-        if (param == null) param = request.getParameter("lng");
-
-        String[] langs = param.split(" ");
-        try {
-            String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
-            reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-        } catch (FileNotFoundException e1) {
-            try {
-                String file = cleanedDirectory + File.separator + "translation-default.json";
-                reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-            } catch (FileNotFoundException e3) {
-                e3.printStackTrace();
-            }
-        }
-
-        String line = null;
-        String message = new String();
-        if (reader != null) {
-            while ((line = reader.readLine()) != null) {
-                // buffer.append(line);
-                message += line + System.getProperty("line.separator");
-            }
-        }
-
-        response.setCharacterEncoding("UTF-8");
-        response.setContentType("application/json");
-        response.getWriter().println(message);
-        response.getWriter().flush();
-        response.getWriter().close();
-    }
-
-}

main/src/com/google/refine/ProjectManager.java

@@ -62,6 +62,13 @@ public abstract class ProjectManager {
     // last n expressions used across all projects
     static protected final int s_expressionHistoryMax = 100;
 
+    // If a project has been idle this long, flush it from memory
+    static protected final int PROJECT_FLUSH_DELAY = 1000 * 60 * 15; // 15 minutes
+    
+    // Don't spend more than this much time saving projects if doing a quick save
+    static protected final int QUICK_SAVE_MAX_TIME = 1000 * 30; // 30 secs
+
+
     protected Map<Long, ProjectMetadata> _projectsMetadata;
     protected PreferenceStore            _preferenceStore;
 
@@ -119,7 +126,6 @@ public abstract class ProjectManager {
             _projectsMetadata.put(project.id, projectMetadata);
         }
     }
- //----------Load from data store to memory----------------
 
     /**
      * Load project metadata from data storage
@@ -135,7 +141,6 @@ public abstract class ProjectManager {
      */
     protected abstract Project loadProject(long id);
 
-    //------------Import and Export from Refine archive-----------------
     /**
      * Import project from a Refine archive
      * @param projectID
@@ -154,7 +159,6 @@ public abstract class ProjectManager {
     public abstract void exportProject(long projectId, TarOutputStream tos) throws IOException;
 
 
- //------------Save to record store------------
     /**
      * Saves a project and its metadata to the data store
      * @param id
@@ -194,8 +198,9 @@ public abstract class ProjectManager {
     /**
      * Save project to the data store
      * @param project
+     * @throws IOException 
      */
-    protected abstract void saveProject(Project project);
+    protected abstract void saveProject(Project project) throws IOException;
 
     /**
      * Save workspace and all projects to data store
@@ -227,9 +232,6 @@ public abstract class ProjectManager {
         }
     }
 
-    static protected final int s_projectFlushDelay = 1000 * 60 * 60; // 1 hour
-    static protected final int s_quickSaveTimeout = 1000 * 30; // 30 secs
-
     /**
      * Saves all projects to the data store
      * @param allModified
@@ -256,7 +258,7 @@ public abstract class ProjectManager {
                         records.add(new SaveRecord(project, msecsOverdue));
 
                     } else if (!project.getProcessManager().hasPending()
-                              && startTimeOfSave.getTime() - project.getLastSave().getTime() > s_projectFlushDelay) {
+                              && startTimeOfSave.getTime() - project.getLastSave().getTime() > PROJECT_FLUSH_DELAY) {
                         
                         /*
                          *  It's been a while since the project was last saved and it hasn't been
@@ -289,19 +291,36 @@ public abstract class ProjectManager {
 
             for (int i = 0;
                  i < records.size() &&
-                    (allModified || (new Date().getTime() - startTimeOfSave.getTime() < s_quickSaveTimeout));
+                    (allModified || (new Date().getTime() - startTimeOfSave.getTime() < QUICK_SAVE_MAX_TIME));
                  i++) {
 
                 try {
                     saveProject(records.get(i).project);
                 } catch (Exception e) {
                     e.printStackTrace();
+                    // In case we're running low on memory, free as much as we can
+                    disposeUnmodifiedProjects();
+                }
+            }
+        }
+    }
+
+    /**
+     * Flush all unmodified projects from memory.
+     */
+    protected void disposeUnmodifiedProjects() {
+        synchronized (this) {
+            for (long id : _projectsMetadata.keySet()) {
+                ProjectMetadata metadata = getProjectMetadata(id);
+                Project project = _projects.get(id);
+                if (project != null && !project.getProcessManager().hasPending() 
+                        && metadata.getModified().getTime() < project.getLastSave().getTime()) {
+                        _projects.remove(id).dispose();
                 }
             }
         }
     }
 
-    //--------------Get from memory--------------
     /**
      * Gets the InterProjectModel from memory
      */
@@ -405,7 +424,6 @@ public abstract class ProjectManager {
      */
     public abstract HistoryEntryManager getHistoryEntryManager();
 
-    //-------------remove project-----------
 
     /**
      * Remove the project from the data store
@@ -434,7 +452,6 @@ public abstract class ProjectManager {
         }
     }
 
-    //--------------Miscellaneous-----------
 
     /**
      * Sets the flag for long running operations.  This will prevent

main/src/com/google/refine/ProjectMetadata.java

@@ -54,6 +54,7 @@ import com.google.refine.util.ParsingUtilities;
 public class ProjectMetadata implements Jsonizable {
     private final Date     _created;
     private Date           _modified;
+    private Date written = null;
     private String         _name;
     private String         _password;
 
@@ -71,9 +72,14 @@ public class ProjectMetadata implements Jsonizable {
     }
 
     public ProjectMetadata() {
-        _created = new Date();
+        this(new Date());
         _modified = _created;
-        preparePreferenceStore(_preferenceStore);
+    }
+
+    public ProjectMetadata(Date created, Date modified, String name) {
+        this(created);
+        _modified = modified;
+        _name = name;
     }
     
     @Override
@@ -103,16 +109,36 @@ public class ProjectMetadata implements Jsonizable {
         }
         
         writer.endObject();
+        
+        if ("save".equals(options.getProperty("mode"))) {
+            written = new Date();
+        }
     }
 
-    public void write(JSONWriter jsonWriter) throws Exception {
+    public boolean isDirty() {
+        return written == null || _modified.after(written);
+    }
+    
+    public void write(JSONWriter jsonWriter) throws JSONException  {
+        write(jsonWriter, false);
+    }
+    
+    /**
+     * @param jsonWriter writer to save metadatea to
+     * @param onlyIfDirty true to not write unchanged metadata
+     * @throws JSONException
+     */
+    public void write(JSONWriter jsonWriter, boolean onlyIfDirty) throws JSONException  {
+        if (!onlyIfDirty || isDirty()) {
             Properties options = new Properties();
             options.setProperty("mode", "save");
 
             write(jsonWriter, options);
         }
+    }
     
     static public ProjectMetadata loadFromJSON(JSONObject obj) {
+        // TODO: Is this correct?  It's using modified date for creation date
         ProjectMetadata pm = new ProjectMetadata(JSONUtilities.getDate(obj, "modified", new Date()));
 
         pm._modified = JSONUtilities.getDate(obj, "modified", new Date());
@@ -157,6 +183,8 @@ public class ProjectMetadata implements Jsonizable {
             }
         }
 
+        pm.written = new Date(); // Mark it as not needing writing until modified
+        
         return pm;
     }
     

main/src/com/google/refine/RefineServlet.java

@@ -370,7 +370,10 @@ public class RefineServlet extends Butterfly {
     }
     
     static public void setUserAgent(HttpURLConnection httpConnection) {
-        httpConnection.addRequestProperty("User-Agent", "OpenRefine/" + FULL_VERSION);
+        httpConnection.addRequestProperty("User-Agent", getUserAgent());
     }
 
+    static public String getUserAgent() {
+        return "OpenRefine/" + FULL_VERSION;
+    }
 }

main/src/com/google/refine/commands/lang/LoadLanguageCommand.java

@@ -7,18 +7,28 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Arrays;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
 import com.google.refine.ProjectManager;
 import com.google.refine.commands.Command;
+import com.google.refine.preference.PreferenceStore;
+
+import edu.mit.simile.butterfly.ButterflyModule;
+
 
 public class LoadLanguageCommand extends Command {
 
     public LoadLanguageCommand() {
-        // TODO Auto-generated constructor stub
+    	super();
     }
 
     @Override
@@ -30,44 +40,47 @@ public class LoadLanguageCommand extends Command {
     public void doPost(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
 
-        String rawDirectoryFile = request.getSession().getServletContext().getRealPath("webapp/modules/langs/");
+        String modname = request.getParameter("module");
-        String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "main"
+        if (modname == null) {
-                + File.separator);
+            modname = "core";
+        }
+        ButterflyModule module = this.servlet.getModule(modname);
 
-        BufferedReader reader = null;
+        String[] langs = request.getParameterValues("lang");
-        String param = null;
+        if (langs == null || "".equals(langs[0])) {
+            PreferenceStore ps = ProjectManager.singleton.getPreferenceStore();
+            if (ps != null) {
+                langs = new String[] {(String) ps.get("userLang")};
+            }
+        }
+        langs = Arrays.copyOf(langs, langs.length+1);
+        langs[langs.length-1] = "default";
+
+        JSONObject json = null;
+        boolean loaded = false;
+        for (String lang : langs) {
+            File langFile = new File(module.getPath(), "langs" + File.separator + "translation-" + lang + ".json");
             try {
-            param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
+                Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(langFile), "UTF-8"));
-        } catch (NullPointerException e) {
+                json = new JSONObject(new JSONTokener(reader));
-        }
-        if (param == null) param = request.getParameter("lng");
-
-        String[] langs = param.split(" ");
-        try {
-            String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
-            reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-        } catch (FileNotFoundException e1) {
-            try {
-                String file = cleanedDirectory + File.separator + "translation-default.json";
-                reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-            } catch (FileNotFoundException e3) {
-                e3.printStackTrace();
-            }
-        }
-
-        String line = null;
-        String message = new String();
-        if (reader != null) {
-            while ((line = reader.readLine()) != null) {
-                // buffer.append(line);
-                message += line + System.getProperty("line.separator");
-            }
-        }
-
                 response.setCharacterEncoding("UTF-8");
                 response.setContentType("application/json");
-        response.getWriter().println(message);
+                json.write(response.getWriter());
                 response.getWriter().flush();
                 response.getWriter().close();
+                loaded = true;
+                break;
+            } catch (FileNotFoundException e1) {
+                json = null;
+                continue;
+            } catch (JSONException e) {
+                json = null;
+                logger.error("JSON error reading/writing language file", e);
+                continue;
+            }
+        }
+        if (!loaded) {
+        	logger.error("Failed to load any language files");
+        }
     }
 }

main/src/com/google/refine/commands/lang/SetLanguageCommand.java

@@ -1,33 +0,0 @@
-
-package com.google.refine.commands.lang;
-
-import java.io.IOException;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.refine.ProjectManager;
-import com.google.refine.commands.Command;
-import com.google.refine.preference.PreferenceStore;
-
-public class SetLanguageCommand extends Command {
-
-    public SetLanguageCommand() {
-        // TODO Auto-generated constructor stub
-    }
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        doPost(request, response);
-    }
-
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        String lang = request.getParameter("lng");
-        PreferenceStore pref = ProjectManager.singleton.getPreferenceStore();
-        
-        pref.put("userLang", lang);
-    }
-
-}

main/src/com/google/refine/expr/functions/ToDate.java

@@ -33,12 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.expr.functions;
 
+import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.GregorianCalendar;
+import java.util.Locale;
 import java.util.Properties;
 
+import org.apache.commons.lang.StringUtils;
 import org.json.JSONException;
 import org.json.JSONWriter;
 
@@ -99,29 +102,56 @@ public class ToDate implements Function {
         }
 
         // "o, format1, format2 (optional), ..."
+        Locale locale = Locale.getDefault();
         if (args.length>=2) {
             for (int i=1;i<args.length;i++) {
                 if (!(args[i] instanceof String)) {
                     // skip formats that aren't strings
                     continue;
                 }
-                String format  = (String) args[i];
+                String format  = StringUtils.trim((String) args[i]);
-                SimpleDateFormat formatter;
+                DateFormat formatter;
+                // Attempt to parse first string as a language tag
+                if (i == 1) {
+                    // Locale possibleLocale = Locale.forLanguageTag(format); // Java 1.7+ only
+                    Locale possibleLocale;
+                    int c = format.indexOf('_');
+                    if (c > 0) {
+                        possibleLocale = new Locale(format.substring(0, c),format.substring(c+1));
+                    } else {
+                        possibleLocale = new Locale(format);
+                    }
+                    boolean valid = false;
+                    for (Locale l : DateFormat.getAvailableLocales()) {
+                        if (l.equals(possibleLocale)) {
+                            locale = possibleLocale;
+                            valid = true;
+                            break;
+                        }
+                    }
+                    if (valid) { // If we got a valid locale
+                        if (args.length == 2) { // No format strings to try, process using default
+                          formatter = DateFormat.getDateInstance(DateFormat.DEFAULT, locale);
+                          formatter.setLenient(true);
+                          GregorianCalendar date = parse(o1, formatter);
+                          if (date != null) {
+                              return date;
+                          } else {
+                              return new EvalError("Unable to parse as date");
+                          }
+                        }
+                        continue; // Don't try to process locale string as a format string if it was valid
+                    }
+                }
                 try {
-                    formatter = new SimpleDateFormat(format);
+                    formatter = new SimpleDateFormat(format,locale);
                 } catch (IllegalArgumentException e) {
                     return new EvalError("Unknown date format");
                 }
-                Date date = null;
+                formatter.setLenient(true);
-                try {
+                GregorianCalendar date = parse(o1, formatter);
-                    date = formatter.parse(o1);
-                } catch (java.text.ParseException e) {
-                    continue;
-                }
                 if (date != null) {
-                    GregorianCalendar c = new GregorianCalendar();
+                    return date;
-                    c.setTime(date);
-                    return c;
                 }
             }
             return new EvalError("Unable to parse as date");
@@ -131,6 +161,18 @@ public class ToDate implements Function {
     }
 
 
+    private GregorianCalendar parse(String o1, DateFormat formatter) {
+        try {
+            Date date = formatter.parse(o1);
+            GregorianCalendar c = new GregorianCalendar();
+            c.setTime(date);
+            return c;
+        } catch (java.text.ParseException e) {
+            return null;
+        }
+    }
+
+
     @Override
     public void write(JSONWriter writer, Properties options)
     throws JSONException {

main/src/com/google/refine/grel/controls/Filter.java

@@ -81,7 +81,11 @@ public class Filter implements Control {
                 
                 results = new ArrayList<Object>(values.length);
                 for (Object v : values) {
+                    if (v != null) {
                         bindings.put(name, v);
+                    } else {
+                        bindings.remove(name);
+                    }
                     
                     Object r = args[2].evaluate(bindings);
                     if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
@@ -97,7 +101,11 @@ public class Filter implements Control {
                     try {
                         Object v = a.get(i);
                         
+                        if (v != null) {
                             bindings.put(name, v);
+                        } else {
+                            bindings.remove(name);
+                        }
                         
                         Object r = args[2].evaluate(bindings);
                         if (r instanceof Boolean && ((Boolean) r).booleanValue()) {
@@ -113,7 +121,11 @@ public class Filter implements Control {
                 results = new ArrayList<Object>(collection.size());
                 
                 for (Object v : collection) {
+                    if (v != null) {
                         bindings.put(name, v);
+                    } else {
+                        bindings.remove(name);
+                    }
                     
                     Object r = args[2].evaluate(bindings);
                     if (r instanceof Boolean && ((Boolean) r).booleanValue()) {

main/src/com/google/refine/grel/controls/ForEach.java

@@ -81,7 +81,11 @@ public class ForEach implements Control {
                 
                 results = new ArrayList<Object>(values.length);
                 for (Object v : values) {
+                    if (v != null) {
                         bindings.put(name, v);
+                    } else {
+                        bindings.remove(name);
+                    }
                     
                     Object r = args[2].evaluate(bindings);
                     
@@ -96,7 +100,11 @@ public class ForEach implements Control {
                     try {
                         Object v = a.get(i);
                         
+                        if (v != null) {
                             bindings.put(name, v);
+                        } else {
+                            bindings.remove(name);
+                        }
                         
                         Object r = args[2].evaluate(bindings);
                         
@@ -111,7 +119,11 @@ public class ForEach implements Control {
                 results = new ArrayList<Object>(collection.size());
                 
                 for (Object v : collection) {
+                    if (v != null) {
                         bindings.put(name, v);
+                    } else {
+                        bindings.remove(name);
+                    }
                     
                     Object r = args[2].evaluate(bindings);
                     

main/src/com/google/refine/importers/ExcelImporter.java

@@ -43,6 +43,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.poi.POIXMLException;
 import org.apache.poi.common.usermodel.Hyperlink;
 import org.apache.poi.hssf.usermodel.HSSFDateUtil;
 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
@@ -115,6 +116,10 @@ public class ExcelImporter extends TabularImportingParserBase {
             }                
         } catch (IOException e) {
             logger.error("Error generating parser UI initialization data for Excel file", e);
+        } catch (IllegalArgumentException e) {
+            logger.error("Error generating parser UI initialization data for Excel file (only Excel 97 & later supported)", e);
+        } catch (POIXMLException e) {
+            logger.error("Error generating parser UI initialization data for Excel file - invalid XML", e);
         }
         
         return options;
@@ -153,6 +158,20 @@ public class ExcelImporter extends TabularImportingParserBase {
                e
            ));
             return;
+        } catch (IllegalArgumentException e) {
+            exceptions.add(new ImportException(
+                    "Attempted to parse as an Excel file but failed. " +
+                    "Only Excel 97 and later formats are supported.",
+                    e
+                ));
+                return;
+        } catch (POIXMLException e) {
+            exceptions.add(new ImportException(
+                    "Attempted to parse as an Excel file but failed. " +
+                    "Invalid XML.",
+                    e
+                ));
+                return;
         }
         
         int[] sheets = JSONUtilities.getIntArray(options, "sheets");

main/src/com/google/refine/importers/FixedWidthImporter.java

@@ -46,7 +46,7 @@ public class FixedWidthImporter extends TabularImportingParserBase {
 
             JSONUtilities.safePut(options, "headerLines", 0);
             JSONUtilities.safePut(options, "columnWidths", columnWidths);
-            JSONUtilities.safePut(options, "guessCellValueTypes", true);
+            JSONUtilities.safePut(options, "guessCellValueTypes", false);
         }
         return options;
     }

main/src/com/google/refine/importers/LineBasedImporter.java

@@ -29,7 +29,7 @@ public class LineBasedImporter extends TabularImportingParserBase {
         
         JSONUtilities.safePut(options, "linesPerRow", 1);
         JSONUtilities.safePut(options, "headerLines", 0);
-        JSONUtilities.safePut(options, "guessCellValueTypes", true);
+        JSONUtilities.safePut(options, "guessCellValueTypes", false);
         
         return options;
     }

main/src/com/google/refine/importers/RdfTripleImporter.java

@@ -86,6 +86,7 @@ public class RdfTripleImporter extends ImportingParserBase {
             JSONObject options, List<Exception> exceptions) {
         
         Graph graph;
+        try {
             switch (mode) {
             case NT:
                 graph = rdfReader.parseNTriples(input);
@@ -99,6 +100,10 @@ public class RdfTripleImporter extends ImportingParserBase {
             default:
                 throw new IllegalArgumentException("Unknown parsing mode");    
             }
+        } catch (Exception e) {
+            exceptions.add(e);
+            return;
+        }
         
         ClosableIterable<Triple> triples = graph.find(ANY_SUBJECT_NODE, ANY_PREDICATE_NODE, ANY_OBJECT_NODE);
         try {

main/src/com/google/refine/importers/SeparatorBasedImporter.java

@@ -73,7 +73,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
         String separator = guessSeparator(job, fileRecords);
         JSONUtilities.safePut(options, "separator", separator != null ? separator : "\\t");
         
-        JSONUtilities.safePut(options, "guessCellValueTypes", true);
+        JSONUtilities.safePut(options, "guessCellValueTypes", false);
         JSONUtilities.safePut(options, "processQuotes", true);
 
         return options;
@@ -99,9 +99,9 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
         boolean strictQuotes = JSONUtilities.getBoolean(options, "strictQuotes", false);
         
         final CSVParser parser = new CSVParser(
-            sep.toCharArray()[0],//HACK changing string to char - won't work for multi-char separators.
+            sep,
             CSVParser.DEFAULT_QUOTE_CHARACTER,
-            (char) 127, // we don't want escape processing try DEL as a rare character until we can turn it off
+            (char) 0, // we don't want escape processing
             strictQuotes,
             CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE,
             !processQuotes);
@@ -168,6 +168,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
         return guessSeparator(file, encoding, false); // quotes off for backward compatibility
     }
 
+    // TODO: Move this to the CSV project?
     static public Separator guessSeparator(File file, String encoding, boolean handleQuotes) {
         try {
             InputStream is = new FileInputStream(file);
@@ -190,7 +191,9 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
                     if (s.length() == 0) {
                         continue;
                     }
+                    if (!inQuote) {
                         lineCount++;
+                    }
                     
                     for (int i = 0; i < s.length(); i++) {
                         char c = s.charAt(i);
@@ -212,12 +215,14 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
                         }
                     }
                     
+                    if (!inQuote) {
                         for (Separator separator : separators) {
                             separator.totalCount += separator.currentLineCount;
                             separator.totalOfSquaredCount += separator.currentLineCount * separator.currentLineCount;
                             separator.currentLineCount = 0;
                         }
                     }
+                }
                 
                 if (separators.size() > 0) {
                     for (Separator separator : separators) {
@@ -231,14 +236,16 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
                     Collections.sort(separators, new Comparator<Separator>() {
                         @Override
                         public int compare(Separator sep0, Separator sep1) {
-                            return Double.compare(sep0.stddev, sep1.stddev);
+                            return Double.compare(sep0.stddev / sep0.averagePerLine, 
+                            sep1.stddev / sep1.averagePerLine);
                         }
                     });
-                    for (Separator separator : separators) {
+                    
+                    Separator separator = separators.get(0);
                     if (separator.stddev / separator.averagePerLine < 0.1) {
                         return separator;
                     }
-                    }
+                   
                 }
             } finally {
                 lineNumberReader.close();

main/src/com/google/refine/importing/ImportingJob.java

@@ -46,6 +46,7 @@ import org.json.JSONObject;
 import org.json.JSONWriter;
 
 import com.google.refine.Jsonizable;
+import com.google.refine.ProjectManager;
 import com.google.refine.ProjectMetadata;
 import com.google.refine.model.Project;
 import com.google.refine.util.JSONUtilities;
@@ -97,9 +98,9 @@ public class ImportingJob implements Jsonizable {
         }
     }
     
-    public void setProjectID(long id2) {
+    public void setProjectID(long projectID) {
         synchronized (config) {
-            JSONUtilities.safePut(config, "projectID", project.id);
+            JSONUtilities.safePut(config, "projectID", projectID);
         }
     }
 
@@ -167,6 +168,11 @@ public class ImportingJob implements Jsonizable {
         if (project != null) {
             project.dispose();
         }
+        
+        // Make sure all projects have been saved in case we run out of memory 
+        // or have some other catastrophe on import
+        ProjectManager.singleton.save(true);
+        
         project = new Project();
         metadata = new ProjectMetadata();
     }

main/src/com/google/refine/importing/ImportingManager.java

@@ -35,6 +35,7 @@ package com.google.refine.importing;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -44,7 +45,6 @@ import java.util.Properties;
 import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
-import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.commons.io.FileUtils;
 import org.json.JSONException;
@@ -83,7 +83,7 @@ public class ImportingManager {
     
     static private RefineServlet servlet;
     static private File importDir;
-    final static private Map<Long, ImportingJob> jobs = new ConcurrentHashMap<Long, ImportingJob>();
+    final static private Map<Long, ImportingJob> jobs = Collections.synchronizedMap(new HashMap<Long, ImportingJob>());
     
     // Mapping from format to label, e.g., "text" to "Text files", "text/xml" to "XML files"
     final static public Map<String, Format> formatToRecord = new HashMap<String, Format>();
@@ -289,7 +289,11 @@ public class ImportingManager {
     
     static private void cleanUpStaleJobs() {
         long now = System.currentTimeMillis();
-        for (Long id : jobs.keySet()) {
+        HashSet<Long> keys;
+        synchronized(jobs) {
+            keys = new HashSet<Long>(jobs.keySet());
+        }
+        for (Long id : keys) {
             ImportingJob job = jobs.get(id);
             if (job != null && !job.updating && now - job.lastTouched > s_stalePeriod) {
                 job.dispose();

main/src/com/google/refine/importing/ImportingUtilities.java

@@ -42,7 +42,6 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.UnsupportedEncodingException;
-import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
@@ -65,6 +64,14 @@ import org.apache.commons.fileupload.ProgressListener;
 import org.apache.commons.fileupload.disk.DiskFileItemFactory;
 import org.apache.commons.fileupload.servlet.ServletFileUpload;
 import org.apache.commons.fileupload.util.Streams;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DecompressingHttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.util.EntityUtils;
 import org.apache.tools.bzip2.CBZip2InputStream;
 import org.apache.tools.tar.TarEntry;
 import org.apache.tools.tar.TarInputStream;
@@ -210,16 +217,15 @@ public class ImportingUtilities {
             }
         });
 
-        @SuppressWarnings("rawtypes")
+        @SuppressWarnings("unchecked")
-        List tempFiles = upload.parseRequest(request);
+        List<FileItem> tempFiles = (List<FileItem>)upload.parseRequest(request);
         
         progress.setProgress("Uploading data ...", -1);
-        parts: for (Object obj : tempFiles) {
+        parts: for (FileItem fileItem : tempFiles) {
             if (progress.isCanceled()) {
                 break;
             }
             
-            FileItem fileItem = (FileItem) obj;
             InputStream stream = fileItem.getInputStream();
             
             String name = fileItem.getFieldName().toLowerCase();
@@ -244,10 +250,10 @@ public class ImportingUtilities {
                         calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
                     
                     JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
+                    JSONUtilities.append(fileRecords, fileRecord);
                     
                     clipboardCount++;
                     
-                    JSONUtilities.append(fileRecords, fileRecord);
                 } else if (name.equals("download")) {
                     String urlString = Streams.asString(stream);
                     URL url = new URL(urlString);
@@ -272,56 +278,77 @@ public class ImportingUtilities {
                         }
                     }
 
-                    URLConnection urlConnection = url.openConnection();
+                    if ("http".equals(url.getProtocol()) || "https".equals(url.getProtocol())) {
-                    urlConnection.setConnectTimeout(5000);
+                        DefaultHttpClient client = new DefaultHttpClient();
-                    if (urlConnection instanceof HttpURLConnection) {
+                        DecompressingHttpClient httpclient = 
-                        HttpURLConnection httpConnection = (HttpURLConnection) urlConnection;
+                                new DecompressingHttpClient(client);
-                        RefineServlet.setUserAgent(httpConnection);
+                        HttpGet httpGet = new HttpGet(url.toURI());
+                        httpGet.setHeader("User-Agent", RefineServlet.getUserAgent());
+                        if ("https".equals(url.getProtocol())) {
+                            // HTTPS only - no sending password in the clear over HTTP
+                            String userinfo = url.getUserInfo();
+                            if (userinfo != null) {
+                                int s = userinfo.indexOf(':');
+                                if (s > 0) {
+                                    String user = userinfo.substring(0, s);
+                                    String pw = userinfo.substring(s + 1, userinfo.length());
+                                    client.getCredentialsProvider().setCredentials(
+                                            new AuthScope(url.getHost(), 443),
+                                            new UsernamePasswordCredentials(user, pw));
+                                }
+                            }
                         }
-                    // TODO: Set Accept-Encoding on connection so we don't get stuff we can't handle?
-                    urlConnection.connect();
  
-                    InputStream stream2 = urlConnection.getInputStream();
+                        HttpResponse response = httpclient.execute(httpGet);
+                        
                         try {
-                        String localname = url.getPath();
+                            response.getStatusLine();
-                        if (localname.isEmpty() || localname.endsWith("/")) {
+                            HttpEntity entity = response.getEntity();
-                            localname = localname + "temp";
+                            if (entity == null) {
+                                throw new Exception("No content found in " + url.toString());
                             }
-                        File file = allocateFile(rawDataDir, localname);
+                            InputStream stream2 = entity.getContent();
-                        
+                            String encoding = null;
-                        int contentLength = urlConnection.getContentLength();
+                            if (entity.getContentEncoding() != null) {
-                        if (contentLength > 0) {
+                                encoding = entity.getContentEncoding().getValue();
-                            update.totalExpectedSize += contentLength;
                             }
-                        
+                            JSONUtilities.safePut(fileRecord, "declaredEncoding", encoding);
-                        JSONUtilities.safePut(fileRecord, "declaredEncoding", urlConnection.getContentEncoding());
+                            String contentType = null;
-                        JSONUtilities.safePut(fileRecord, "declaredMimeType", urlConnection.getContentType());
+                            if (entity.getContentType().getValue() != null) {
-                        JSONUtilities.safePut(fileRecord, "fileName", file.getName());
+                                contentType = entity.getContentType().getValue();
-                        JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
-
-                        progress.setProgress("Downloading " + urlString,
-                            calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
-                        
-                        long actualLength = saveStreamToFile(stream2, file, update);
-                        JSONUtilities.safePut(fileRecord, "size", actualLength);
-                        if (actualLength == 0) {
-                            throw new Exception("No content found in " + urlString);
-                        } else if (contentLength >= 0) {
-                            update.totalExpectedSize += (actualLength - contentLength);
-                        } else {
-                            update.totalExpectedSize += actualLength;
                             }
-                        progress.setProgress("Saving " + urlString + " locally",
+                            JSONUtilities.safePut(fileRecord, "declaredMimeType", contentType);
-                            calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
+                            if (saveStream(stream2, url, rawDataDir, progress, update, 
-                        
+                                    fileRecord, fileRecords,
-                        if (postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress)) {
+                                    entity.getContentLength())) {
+                                archiveCount++;
+                            }
+                            downloadCount++;
+                            EntityUtils.consume(entity);
+                        } finally {
+                            httpGet.releaseConnection();
+                        }
+                    } else {
+                        // Fallback handling for non HTTP connections (only FTP?)
+                        URLConnection urlConnection = url.openConnection();
+                        urlConnection.setConnectTimeout(5000);
+                        urlConnection.connect();
+                        InputStream stream2 = urlConnection.getInputStream();
+                        JSONUtilities.safePut(fileRecord, "declaredEncoding", 
+                                urlConnection.getContentEncoding());
+                        JSONUtilities.safePut(fileRecord, "declaredMimeType", 
+                                urlConnection.getContentType());
+                        try {
+                            if (saveStream(stream2, url, rawDataDir, progress, 
+                                    update, fileRecord, fileRecords,
+                                    urlConnection.getContentLength())) {
                                 archiveCount++;
                             }
-
                             downloadCount++;
                         } finally {
                             stream2.close();
                         }
+                    }
                 } else {
                     String value = Streams.asString(stream);
                     parameters.put(name, value);
@@ -361,8 +388,8 @@ public class ImportingUtilities {
         }
         
         // Delete all temp files.
-        for (Object obj : tempFiles) {
+        for (FileItem fileItem : tempFiles) {
-            ((FileItem)obj).delete();
+            fileItem.delete();
         }
         
         JSONUtilities.safePut(retrievalRecord, "uploadCount", uploadCount);
@@ -371,6 +398,37 @@ public class ImportingUtilities {
         JSONUtilities.safePut(retrievalRecord, "archiveCount", archiveCount);
     }
 
+    private static boolean saveStream(InputStream stream, URL url, File rawDataDir, final Progress progress,
+            final SavingUpdate update, JSONObject fileRecord, JSONArray fileRecords, long length)
+            throws IOException, Exception {
+        String localname = url.getPath();
+        if (localname.isEmpty() || localname.endsWith("/")) {
+            localname = localname + "temp";
+        }
+        File file = allocateFile(rawDataDir, localname);
+
+        JSONUtilities.safePut(fileRecord, "fileName", file.getName());
+        JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
+
+        update.totalExpectedSize += length;
+
+        progress.setProgress("Downloading " + url.toString(),
+            calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
+        
+        long actualLength = saveStreamToFile(stream, file, update);
+        JSONUtilities.safePut(fileRecord, "size", actualLength);
+        if (actualLength == 0) {
+            throw new Exception("No content found in " + url.toString());
+        } else if (length >= 0) {
+            update.totalExpectedSize += (actualLength - length);
+        } else {
+            update.totalExpectedSize += actualLength;
+        }
+        progress.setProgress("Saving " + url.toString() + " locally",
+            calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
+        return postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress);
+    }
+    
     static public String getRelativePath(File file, File dir) {
         String location = file.getAbsolutePath().substring(dir.getAbsolutePath().length());
         return (location.startsWith(File.separator)) ? location.substring(1) : location;
@@ -627,17 +685,13 @@ public class ImportingUtilities {
     static public InputStream tryOpenAsCompressedFile(File file, String mimeType, String contentEncoding) {
         String fileName = file.getName();
         try {
-            /*
-             * TODO: Do we need to support MIME types as well as content encodings?
-             * application/x-bzip2
-             * application/x-gzip
-             * multipart/x-gzip
-             */
             if (fileName.endsWith(".gz") 
                     || "gzip".equals(contentEncoding) 
-                    || "x-gzip".equals(contentEncoding)) {                
+                    || "x-gzip".equals(contentEncoding)
+                    || "application/x-gzip".equals(mimeType)) {                
                 return new GZIPInputStream(new FileInputStream(file));
-            } else if (fileName.endsWith(".bz2")) {
+            } else if (fileName.endsWith(".bz2")
+                    ||"application/x-bzip2".equals(mimeType)) {
                 InputStream is = new FileInputStream(file);
                 is.mark(4);
                 if (!(is.read() == 'B' && is.read() == 'Z')) {
@@ -694,6 +748,15 @@ public class ImportingUtilities {
         return encoding;
     }
 
+    /**
+     * Figure out the best (most common) format for the set of files, select
+     * all files which match that format, and return the format found.
+     * 
+     * @param job ImportingJob object
+     * @param retrievalRecord JSON object containing "files" key with all our files
+     * @param fileSelectionIndexes JSON array of selected file indices matching best format
+     * @return best (highest frequency) format
+     */
     static public String autoSelectFiles(ImportingJob job, JSONObject retrievalRecord, JSONArray fileSelectionIndexes) {
         final Map<String, Integer> formatToCount = new HashMap<String, Integer>();
         List<String> formats = new ArrayList<String>();

main/src/com/google/refine/io/FileProjectManager.java

@@ -62,7 +62,7 @@ import com.google.refine.model.Project;
 import com.google.refine.preference.TopList;
 
 public class FileProjectManager extends ProjectManager {
-    final static protected String s_projectDirNameSuffix = ".project";
+    final static protected String PROJECT_DIR_SUFFIX = ".project";
 
     protected File                       _workspaceDir;
 
@@ -72,6 +72,8 @@ public class FileProjectManager extends ProjectManager {
         if (singleton == null) {
             logger.info("Using workspace directory: {}", dir.getAbsolutePath());
             singleton = new FileProjectManager(dir);
+            // This needs our singleton set, thus the unconventional control flow
+            ((FileProjectManager) singleton).recover();
         }
 
     }
@@ -85,7 +87,6 @@ public class FileProjectManager extends ProjectManager {
         }
 
         load();
-        recover();
     }
 
     public File getWorkspaceDir() {
@@ -93,7 +94,7 @@ public class FileProjectManager extends ProjectManager {
     }
 
     static public File getProjectDir(File workspaceDir, long projectID) {
-        File dir = new File(workspaceDir, projectID + s_projectDirNameSuffix);
+        File dir = new File(workspaceDir, projectID + PROJECT_DIR_SUFFIX);
         if (!dir.exists()) {
             dir.mkdir();
         }
@@ -114,6 +115,9 @@ public class FileProjectManager extends ProjectManager {
     public boolean loadProjectMetadata(long projectID) {
         synchronized (this) {
             ProjectMetadata metadata = ProjectMetadataUtilities.load(getProjectDir(projectID));
+            if (metadata == null) {
+                metadata = ProjectMetadataUtilities.recover(getProjectDir(projectID), projectID);
+            }
             if (metadata != null) {
                 _projectsMetadata.put(projectID, metadata);
                 return true;
@@ -217,7 +221,7 @@ public class FileProjectManager extends ProjectManager {
     }
 
     @Override
-    protected void saveProject(Project project){
+    protected void saveProject(Project project) throws IOException{
         ProjectUtilities.save(project);
     }
 
@@ -227,7 +231,6 @@ public class FileProjectManager extends ProjectManager {
     }
 
 
-
     /**
      * Save the workspace's data out to file in a safe way: save to a temporary file first
      * and rename it to the real file.
@@ -237,7 +240,12 @@ public class FileProjectManager extends ProjectManager {
         synchronized (this) {
             File tempFile = new File(_workspaceDir, "workspace.temp.json");
             try {
-                saveToFile(tempFile);
+                if (!saveToFile(tempFile)) {
+                    // If the save wasn't really needed, just keep what we had
+                    tempFile.delete();
+                    logger.info("Skipping unnecessary workspace save");
+                    return;
+                }
             } catch (Exception e) {
                 e.printStackTrace();
 
@@ -248,21 +256,23 @@ public class FileProjectManager extends ProjectManager {
             File file = new File(_workspaceDir, "workspace.json");
             File oldFile = new File(_workspaceDir, "workspace.old.json");
 
+            if (oldFile.exists()) {
+                oldFile.delete();
+            }
+            
             if (file.exists()) {
                 file.renameTo(oldFile);
             }
 
             tempFile.renameTo(file);
-            if (oldFile.exists()) {
-                oldFile.delete();
-            }
 
             logger.info("Saved workspace");
         }
     }
 
-    protected void saveToFile(File file) throws IOException, JSONException {
+    protected boolean saveToFile(File file) throws IOException, JSONException {
         FileWriter writer = new FileWriter(file);
+        boolean saveWasNeeded = false;
         try {
             JSONWriter jsonWriter = new JSONWriter(writer);
             jsonWriter.object();
@@ -272,11 +282,9 @@ public class FileProjectManager extends ProjectManager {
                 ProjectMetadata metadata = _projectsMetadata.get(id);
                 if (metadata != null) {
                     jsonWriter.value(id);
-
+                    if (metadata.isDirty()) {
-                    try {
                         ProjectMetadataUtilities.save(metadata, getProjectDir(id));
-                    } catch (Exception e) {
+                        saveWasNeeded = true;
-                        e.printStackTrace();
                     }
                 }
             }
@@ -284,12 +292,14 @@ public class FileProjectManager extends ProjectManager {
             writer.write('\n');
 
             jsonWriter.key("preferences");
+            saveWasNeeded |= _preferenceStore.isDirty();
             _preferenceStore.write(jsonWriter, new Properties());
 
             jsonWriter.endObject();
         } finally {
             writer.close();
         }
+        return saveWasNeeded;
     }
 
 
@@ -386,11 +396,12 @@ public class FileProjectManager extends ProjectManager {
     }
 
     protected void recover() {
+        boolean recovered = false;
         for (File file : _workspaceDir.listFiles()) {
             if (file.isDirectory() && !file.isHidden()) {
-                String name = file.getName();
+                String dirName = file.getName();
-                if (file.getName().endsWith(s_projectDirNameSuffix)) {
+                if (file.getName().endsWith(PROJECT_DIR_SUFFIX)) {
-                    String idString = name.substring(0, name.length() - s_projectDirNameSuffix.length());
+                    String idString = dirName.substring(0, dirName.length() - PROJECT_DIR_SUFFIX.length());
                     long id = -1;
                     try {
                         id = Long.parseLong(idString);
@@ -400,19 +411,22 @@ public class FileProjectManager extends ProjectManager {
 
                     if (id > 0 && !_projectsMetadata.containsKey(id)) {
                         if (loadProjectMetadata(id)) {
-                            logger.info(
+                            logger.info("Recovered project named " 
-                                    "Recovered project named " + 
+                                    + getProjectMetadata(id).getName()
-                                            getProjectMetadata(id).getName() +
+                                    + " in directory " + dirName);
-                                            " in directory " + name);
+                            recovered = true;
                         } else {
-                            logger.warn("Failed to recover project in directory " + name);
+                            logger.warn("Failed to recover project in directory " + dirName);
 
-                            file.renameTo(new File(file.getParentFile(), name + ".corrupted"));
+                            file.renameTo(new File(file.getParentFile(), dirName + ".corrupted"));
                         }
                     }
                 }
             }
         }
+        if (recovered) {
+            saveWorkspace();
+        }
     }
 
     @Override

main/src/com/google/refine/io/ProjectMetadataUtilities.java

@@ -36,9 +36,14 @@ package com.google.refine.io;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.FileReader;
+import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.util.Date;
+import java.util.List;
 
+import org.apache.commons.lang.StringUtils;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.JSONTokener;
 import org.json.JSONWriter;
@@ -46,36 +51,31 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.refine.ProjectMetadata;
+import com.google.refine.model.Project;
 
 
 public class ProjectMetadataUtilities {
     final static Logger logger = LoggerFactory.getLogger("project_metadata_utilities");
 
-    public static void save(ProjectMetadata projectMeta, File projectDir) throws Exception {
+    public static void save(ProjectMetadata projectMeta, File projectDir) throws JSONException, IOException  {
         File tempFile = new File(projectDir, "metadata.temp.json");
-        try {
         saveToFile(projectMeta, tempFile);
-        } catch (Exception e) {
-            e.printStackTrace();
-
-            logger.warn("Failed to save project metadata");
-            return;
-        }
 
         File file = new File(projectDir, "metadata.json");
         File oldFile = new File(projectDir, "metadata.old.json");
 
+        if (oldFile.exists()) {
+            oldFile.delete();
+        }
+        
         if (file.exists()) {
             file.renameTo(oldFile);
         }
 
         tempFile.renameTo(file);
-        if (oldFile.exists()) {
-            oldFile.delete();
-        }
     }
 
-    protected static void saveToFile(ProjectMetadata projectMeta, File metadataFile) throws Exception {
+    protected static void saveToFile(ProjectMetadata projectMeta, File metadataFile) throws JSONException, IOException   {
         Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile));
         try {
             JSONWriter jsonWriter = new JSONWriter(writer);
@@ -104,6 +104,45 @@ public class ProjectMetadataUtilities {
         return null;
     }
     
+    /**
+     * Reconstruct the project metadata on a best efforts basis.  The name is
+     * gone, so build something descriptive from the column names.  Recover the
+     * creation and modification times based on whatever files are available.
+     * 
+     * @param projectDir the project directory
+     * @param id the proejct id
+     * @return
+     */
+    static public ProjectMetadata recover(File projectDir, long id) {
+        ProjectMetadata pm = null;
+        Project p = ProjectUtilities.load(projectDir, id);
+        if (p != null) {
+            List<String> columnNames = p.columnModel.getColumnNames();
+            String tempName = "<recovered project> - " + columnNames.size() 
+                    + " cols X " + p.rows.size() + " rows - "
+                    + StringUtils.join(columnNames,'|');
+            p.dispose();
+            long ctime = System.currentTimeMillis();
+            long mtime = 0;
+
+            File dataFile = new File(projectDir, "data.zip");
+            ctime = mtime = dataFile.lastModified();
+
+            File historyDir = new File(projectDir,"history");
+            File[] files = historyDir.listFiles();
+            if (files != null) {
+                for (File f : files) {
+                    long time = f.lastModified();
+                    ctime = Math.min(ctime, time);
+                    mtime = Math.max(mtime, time);
+                }
+            }
+            pm = new ProjectMetadata(new Date(ctime),new Date(mtime), tempName);
+            logger.error("Partially recovered missing metadata project in directory " + projectDir + " - " + tempName);
+        }
+        return pm;
+    }
+
     static protected ProjectMetadata loadFromFile(File metadataFile) throws Exception {
         FileReader reader = new FileReader(metadataFile);
         try {

main/src/com/google/refine/io/ProjectUtilities.java

@@ -35,6 +35,7 @@ package com.google.refine.io;
 
 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 import java.util.zip.ZipOutputStream;
@@ -50,7 +51,7 @@ import com.google.refine.util.Pool;
 public class ProjectUtilities {
     final static Logger logger = LoggerFactory.getLogger("project_utilities");
 
-    synchronized public static void save(Project project) {
+    synchronized public static void save(Project project) throws IOException {
         synchronized (project) {
             long id = project.id;
             File dir = ((FileProjectManager)ProjectManager.singleton).getProjectDir(id);
@@ -58,11 +59,15 @@ public class ProjectUtilities {
             File tempFile = new File(dir, "data.temp.zip");
             try {
                 saveToFile(project, tempFile);
-            } catch (Exception e) {
+            } catch (IOException e) {
                 e.printStackTrace();
-
                 logger.warn("Failed to save project {}", id);
-                return;
+                try {
+                    tempFile.delete();
+                } catch (Exception e2) {
+                    // just ignore - file probably was never created.
+                }
+                throw e;
             }
 
             File file = new File(dir, "data.zip");
@@ -83,7 +88,7 @@ public class ProjectUtilities {
         }
     }
 
-    protected static void saveToFile(Project project, File file) throws Exception {
+    protected static void saveToFile(Project project, File file) throws IOException  {
         ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file));
         try {
             Pool pool = new Pool();

main/src/com/google/refine/model/ColumnModel.java

@@ -283,12 +283,17 @@ public class ColumnModel implements Jsonizable {
         _nameToColumn = new HashMap<String, Column>();
         _cellIndexToColumn = new HashMap<Integer, Column>();
         _columnNames = new ArrayList<String>();
-        
+        int maxCellIndex = -1;
         for (Column column : columns) {
             _nameToColumn.put(column.getName(), column);
-            _cellIndexToColumn.put(column.getCellIndex(), column);
+            int cidx = column.getCellIndex();
+            if (cidx > maxCellIndex) {
+                maxCellIndex = cidx;
+            }
+            _cellIndexToColumn.put(cidx, column);
             _columnNames.add(column.getName());
         }
+        _maxCellIndex = maxCellIndex;
     }
     
     /**

main/src/com/google/refine/model/Project.java

@@ -98,6 +98,9 @@ public class Project {
         this.history = new History(this);
     }
     
+    /**
+     * Free/dispose of project data from memory.
+     */
     public void dispose() {
         for (OverlayModel overlayModel : overlayModels.values()) {
             try {
@@ -107,6 +110,7 @@ public class Project {
             }
         }
         ProjectManager.singleton.getInterProjectModel().flushJoinsInvolvingProject(this.id);
+        // The rest of the project should get garbage collected when we return.
     }
 
     public Date getLastSave(){
@@ -190,6 +194,7 @@ public class Project {
     ) throws Exception {
         long start = System.currentTimeMillis();
         
+        // version of Refine which wrote the file
         /* String version = */ reader.readLine();
         
         Project project = new Project(id);

main/src/com/google/refine/model/recon/StandardReconConfig.java

@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.model.recon;
 
 import java.io.DataOutputStream;
-import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringWriter;
 import java.net.HttpURLConnection;
@@ -316,11 +315,11 @@ public class StandardReconConfig extends ReconConfig {
             }
             
             if (connection.getResponseCode() >= 400) {
-                // TODO: Retry with backoff on 500 errors?
                 InputStream is = connection.getErrorStream();
-                throw new IOException("Failed  - code:" 
+                logger.error("Failed  - code:" 
                         + Integer.toString(connection.getResponseCode()) 
-                + " message: " + is == null ? "" : ParsingUtilities.inputStreamToString(is));
+                        + " message: " + is == null ? ""
+                                : ParsingUtilities.inputStreamToString(is));
             } else {
                 InputStream is = connection.getInputStream();
                 try {

main/src/com/google/refine/util/Pool.java

@@ -55,6 +55,10 @@ import com.google.refine.RefineServlet;
 import com.google.refine.model.Recon;
 import com.google.refine.model.ReconCandidate;
 
+/**
+ * A serializable pool of ReconCandidates indexed by ID.
+ *
+ */
 public class Pool implements Jsonizable {
     final protected Map<String, Recon> recons = new HashMap<String, Recon>();
     

main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java

@@ -131,6 +131,13 @@ public class ToFromConversionTests extends RefineTest {
       Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parse("2012-03-01"));
       // Multiple format strings should get tried sequentially until one succeeds or all are exhausted
       Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parse("2012-03-01"));
+      // First string can be a locale identifier instead of a format string
+      Assert.assertEquals(invoke("toDate", "2013-06-01","zh"), CalendarParser.parse("2013-06-01")); 
+      Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01")); 
+      Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_HK","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01")); 
+      Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_TW","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01")); 
+      Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_CN","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01")); 
+
         // Date
         // Calendar
         // String

main/tests/server/src/com/google/refine/tests/importers/ExcelImporterTests.java

@@ -183,7 +183,7 @@ public class ExcelImporterTests extends ImporterTest {
 
         File file = null;
         try {
-            file = File.createTempFile("oepnrefine-importer-test", xml ? ".xlsx" : ".xls");
+            file = File.createTempFile("openrefine-importer-test", xml ? ".xlsx" : ".xls");
             file.deleteOnExit();
             OutputStream outputStream = new FileOutputStream(file);
             wb.write(outputStream);

main/webapp/modules/core/MOD-INF/controller.js

@@ -99,7 +99,6 @@ function registerCommands() {
   RS.registerCommand(module, "key-value-columnize", new Packages.com.google.refine.commands.cell.KeyValueColumnizeCommand());
 
   RS.registerCommand(module, "load-language", Packages.com.google.refine.commands.lang.LoadLanguageCommand());
-  RS.registerCommand(module, "set-language", Packages.com.google.refine.commands.lang.SetLanguageCommand());
   
   RS.registerCommand(module, "add-column", new Packages.com.google.refine.commands.column.AddColumnCommand());
   RS.registerCommand(module, "add-column-by-fetching-urls", new Packages.com.google.refine.commands.column.AddColumnByFetchingURLsCommand());
@@ -385,7 +384,7 @@ function init() {
       "externals/jquery-1.7.2.min.js",
       "externals/jquery.cookie.js",
       "externals/jquery.eventstack-0.3.js",
-      "externals/suggest/suggest-4_2.min.js",
+      "externals/suggest/suggest-4_3.js",
       "externals/jquery-ui/jquery-ui-1.8.20.custom.min.js",
       "externals/imgareaselect/jquery.imgareaselect.js",
       "externals/date.js",
@@ -451,7 +450,7 @@ function init() {
     "project/styles",
     module,
     [
-      "externals/suggest/css/suggest-4_2.min.css",
+      "externals/suggest/css/suggest-4_3.min.css",
       "externals/jquery-ui/css/ui-lightness/jquery-ui-1.8.20.custom.css",
       "externals/imgareaselect/css/imgareaselect-default.css",
 
@@ -491,7 +490,7 @@ function init() {
     [
       "externals/jquery-1.7.2.min.js",
       "externals/jquery.cookie.js",
-      "externals/suggest/suggest-4_2.min.js",
+      "externals/suggest/suggest-4_3.js",
       "externals/jquery-ui/jquery-ui-1.8.20.custom.min.js",
       "externals/imgareaselect/jquery.imgareaselect.js",
       "externals/date.js",
@@ -503,7 +502,7 @@ function init() {
     "preferences/styles",
     module,
     [
-      "externals/suggest/css/suggest-4_2.min.css",
+      "externals/suggest/css/suggest-4_3.min.css",
       "externals/jquery-ui/css/ui-lightness/jquery-ui-1.8.20.custom.css",
       "styles/jquery-ui-overrides.less",
       "styles/common.less",

main/webapp/modules/core/externals/suggest/css/suggest-4_3.css

@@ -539,7 +539,7 @@ Freebase Suggest Attribution
 
 .fbs-attribution {
     padding-right: 72px; /* accommodate attribution background image */
-    background-image: url(template/img/fbs-attribution.png);
+    background-image: url(//www.gstatic.com/freebase/img/freebase-cc-by-61x23.png);
     background-repeat: no-repeat;
     background-position: center right;
     min-height: 15px;

main/webapp/modules/core/externals/suggest/suggest-4_2.min.js

@@ -1,82 +0,0 @@
-
-/*
- * Copyright 2012, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * Additional Licenses for Third Party components can be found here:
- * http://wiki.freebase.com/wiki/Freebase_Site_License
- *
- */
-(function(c,r){if(!("console"in window)){var p=window.console={};p.log=p.warn=p.error=p.debug=function(){}}c(function(){var a=c("<div>");c(document.body).append(a);var b=setTimeout(function(){if(c.cleanData){var a=c.cleanData;c.cleanData=function(b){for(var f=0,g;null!=(g=b[f]);f++)c(g).triggerHandler("remove");a(b)}}else{var b=c.fn.remove;c.fn.remove=function(a,e){return this.each(function(){e||(!a||c.filter(a,[this]).length)&&c("*",this).add([this]).each(function(){c(this).triggerHandler("remove")});
-return b.call(c(this),a,e)})}}},1);a.bind("remove",function(){clearTimeout(b)});a.remove()});var q={key:1,filter:1,spell:1,exact:1,lang:1,scoring:1,prefixed:1,stemmed:1,format:1,mql_output:1,output:1,type:1};c.suggest=function(a,b){c.fn[a]=function(b){this.length||console.warn("Suggest: invoked on empty element set");return this.each(function(){this.nodeName&&("INPUT"===this.nodeName.toUpperCase()?this.type&&"TEXT"!==this.type.toUpperCase()&&console.warn("Suggest: unsupported INPUT type: "+this.type):console.warn("Suggest: unsupported DOM element: "+
-this.nodeName));var g=c.data(this,a);g&&g._destroy();c.data(this,a,new c.suggest[a](this,b))._init()})};c.suggest[a]=function(b,g){var d=this,f=this.options=c.extend(!0,{},c.suggest.defaults,c.suggest[a].defaults,g),i=f.css_prefix=f.css_prefix||"",h=f.css;this.name=a;c.each(h,function(a){h[a]=i+h[a]});f.ac_param={};c.each(q,function(a){var b=f[a];null===b||""===b||(f.ac_param[a]=b)});f.flyout_lang=null;if(f.ac_param.lang){var j=f.ac_param.lang;c.isArray(j)&&j.length&&(j=j.join(","));j&&(f.flyout_lang=
-j)}this._status={START:"",LOADING:"",SELECT:"",ERROR:""};f.status&&(f.status instanceof Array&&3<=f.status.length)&&(this._status.START=f.status[0]||"",this._status.LOADING=f.status[1]||"",this._status.SELECT=f.status[2]||"",4===f.status.length&&(this._status.ERROR=f.status[3]||""));var j=this.status=c('<div style="display:none;">').addClass(h.status),l=this.list=c("<ul>").addClass(h.list),k=this.pane=c('<div style="display:none;" class="fbs-reset">').addClass(h.pane);k.append(j).append(l);f.parent?
-c(f.parent).append(k):(k.css("position","absolute"),f.zIndex&&k.css("z-index",f.zIndex),c(document.body).append(k));k.bind("mousedown",function(a){d.input.data("dont_hide",true);a.stopPropagation()}).bind("mouseup",function(a){d.input.data("dont_hide")&&d.input.focus();d.input.removeData("dont_hide");a.stopPropagation()}).bind("click",function(a){a.stopPropagation();if(a=d.get_selected()){d.onselect(a,true);d.hide_all()}});l.hover(function(a){d.hoverover_list(a)},function(a){d.hoverout_list(a)});
-this.input=c(b).attr("autocomplete","off").unbind(".suggest").bind("remove.suggest",function(){d._destroy()}).bind("keydown.suggest",function(a){d.keydown(a)}).bind("keypress.suggest",function(a){d.keypress(a)}).bind("keyup.suggest",function(a){d.keyup(a)}).bind("blur.suggest",function(a){d.blur(a)}).bind("textchange.suggest",function(){d.textchange()}).bind("focus.suggest",function(a){d.focus(a)}).bind(c.browser.msie?"paste.suggest":"input.suggest",function(){clearTimeout(d.paste_timeout);d.paste_timeout=
-setTimeout(function(){d.textchange()},0)});this.onresize=function(){d.invalidate_position();if(k.is(":visible")){d.position();if(f.flyout&&d.flyoutpane&&d.flyoutpane.is(":visible")){var a=d.get_selected();a&&d.flyout_position(a)}}};c(window).bind("resize.suggest",this.onresize).bind("scroll.suggest",this.onresize)};c.suggest[a].prototype=c.extend({},c.suggest.prototype,b)};c.suggest.prototype={_init:function(){},_destroy:function(){this.pane.remove();this.list.remove();this.input.unbind(".suggest");
-c(window).unbind("resize.suggest",this.onresize).unbind("scroll.suggest",this.onresize);this.input.removeData("data.suggest")},invalidate_position:function(){self._position=null},status_start:function(){this.hide_all();this.status.siblings().hide();this._status.START&&(this.status.text(this._status.START).show(),this.pane.is(":visible")||(this.position(),this.pane_show()));this._status.LOADING&&this.status.removeClass("loading")},status_loading:function(){this.status.siblings().show();this._status.LOADING?
-(this.status.addClass("loading").text(this._status.LOADING).show(),this.pane.is(":visible")||(this.position(),this.pane_show())):this.status.hide()},status_select:function(){this.status.siblings().show();this._status.SELECT?this.status.text(this._status.SELECT).show():this.status.hide();this._status.LOADING&&this.status.removeClass("loading")},status_error:function(){this.status.siblings().show();this._status.ERROR?this.status.text(this._status.ERROR).show():this.status.hide();this._status.LOADING&&
-this.status.removeClass("loading")},focus:function(a){""===this.input.val()?this.status_start():this.focus_hook(a)},focus_hook:function(){!this.input.data("data.suggest")&&(!this.pane.is(":visible")&&c("."+this.options.css.item,this.list).length)&&(this.position(),this.pane_show())},keydown:function(a){var b=a.keyCode;if(9===b)this.tab(a);else if(38===b||40===b)a.shiftKey||a.preventDefault()},keypress:function(a){var b=a.keyCode;38===b||40===b?a.shiftKey||a.preventDefault():13===b&&this.enter(a)},
-keyup:function(a){var b=a.keyCode;if(38===b)a.preventDefault(),this.up(a);else if(40===b)a.preventDefault(),this.down(a);else if(a.ctrlKey&&77===b)c(".fbs-more-link",this.pane).click();else if(c.suggest.is_char(a)){clearTimeout(this.keypress.timeout);var e=this;this.keypress.timeout=setTimeout(function(){e.textchange()},0)}else 27===b&&this.escape(a);return!0},blur:function(){this.input.data("dont_hide")||(this.input.data("data.suggest"),this.hide_all())},tab:function(a){if(!a.shiftKey&&!a.metaKey&&
-!a.ctrlKey){var a=this.options,a=this.pane.is(":visible")&&c("."+a.css.item,this.list).length,b=this.get_selected();a&&b&&(this.onselect(b),this.hide_all())}},enter:function(a){var b=this.options;if(this.pane.is(":visible")){if(a.shiftKey){this.shift_enter(a);a.preventDefault();return}if(c("."+b.css.item,this.list).length){var e=this.get_selected();if(e){this.onselect(e);this.hide_all();a.preventDefault();return}if(!b.soft&&(this.input.data("data.suggest"),c("."+this.options.css.item+":visible",this.list).length)){this.updown(!1);
-a.preventDefault();return}}}b.soft?this.soft_enter():a.preventDefault()},soft_enter:function(){},shift_enter:function(){},escape:function(){this.hide_all()},up:function(a){this.updown(!0,a.ctrlKey||a.shiftKey)},down:function(a){this.updown(!1,null,a.ctrlKey||a.shiftKey)},updown:function(a,b,e){var g=this.options.css,d=this.list;if(this.pane.is(":visible")){var f=c("."+g.item+":visible",d);if(f.length){var d=c(f[0]),f=c(f[f.length-1]),i=this.get_selected()||[];clearTimeout(this.ignore_mouseover.timeout);
-this._ignore_mouseover=!1;a?b?this._goto(d):i.length?i[0]==d[0]?(d.removeClass(g.selected),this.input.val(this.input.data("original.suggest")),this.hoverout_list()):(a=i.prevAll("."+g.item+":visible:first"),this._goto(a)):this._goto(f):e?this._goto(f):i.length?i[0]==f[0]?(f.removeClass(g.selected),this.input.val(this.input.data("original.suggest")),this.hoverout_list()):(a=i.nextAll("."+g.item+":visible:first"),this._goto(a)):this._goto(d)}}else a||this.textchange()},_goto:function(a){a.trigger("mouseover.suggest");
-var b=a.data("data.suggest");this.input.val(b?b.name:this.input.data("original.suggest"));this.scroll_to(a)},scroll_to:function(a){var b=this.list,c=b.scrollTop(),g=c+b.innerHeight(),d=a.outerHeight(),a=a.prevAll().length*d,d=a+d;a<c?(this.ignore_mouseover(),b.scrollTop(a)):d>g&&(this.ignore_mouseover(),b.scrollTop(c+d-g))},textchange:function(){this.input.removeData("data.suggest");this.input.trigger("fb-textchange",this);var a=this.input.val();""===a?this.status_start():(this.status_loading(),this.request(a))},
-request:function(){},response:function(a){if(a&&("cost"in a&&this.trackEvent(this.name,"response","cost",a.cost),this.check_response(a))){var b=[];c.isArray(a)?b=a:"result"in a&&(b=a.result);var e=c.map(arguments,function(a){return a});this.response_hook.apply(this,e);var g=null,d=this,f=this.options;c.each(b,function(b,c){if(!c.id&&c.mid)c.id=c.mid;var e=d.create_item(c,a).bind("mouseover.suggest",function(a){d.mouseover_item(a)});e.data("data.suggest",c);d.list.append(e);b===0&&(g=e)});this.input.data("original.suggest",
-this.input.val());if(0===c("."+f.css.item,this.list).length&&f.nomatch){b=c('<li class="fbs-nomatch">');if("string"===typeof f.nomatch)b.text(f.nomatch);else if(f.nomatch.title&&b.append(c('<em class="fbs-nomatch-text">').text(f.nomatch.title)),f.nomatch.heading&&b.append(c("<h3>").text(f.nomatch.heading)),(f=f.nomatch.tips)&&f.length){var i=c('<ul class="fbs-search-tips">');c.each(f,function(a,b){i.append(c("<li>").text(b))});b.append(i)}b.bind("click.suggest",function(a){a.stopPropagation()});this.list.append(b)}e.push(g);
-this.show_hook.apply(this,e);this.position();this.pane_show()}},pane_show:function(){var a=!1;c("> li",this.list).length&&(a=!0);a||this.pane.children(":not(."+this.options.css.list+")").each(function(){if("none"!=c(this).css("display"))return a=!0,!1});if(a)if(this.options.animate){var b=this;this.pane.slideDown("fast",function(){b.input.trigger("fb-pane-show",b)})}else this.pane.show(),this.input.trigger("fb-pane-show",this);else this.pane.hide(),this.input.trigger("fb-pane-hide",this)},create_item:function(a){var b=
-this.options.css,e=c("<li>").addClass(b.item),a=c("<label>").text(a.name);e.append(c("<div>").addClass(b.item_name).append(a));return e},mouseover_item:function(a){if(!this._ignore_mouseover){a=a.target;"li"!==a.nodeName.toLowerCase()&&(a=c(a).parents("li:first"));var b=c(a),e=this.options.css;c("."+e.item,this.list).each(function(){this!==b[0]&&c(this).removeClass(e.selected)});b.hasClass(e.selected)||(b.addClass(e.selected),this.mouseover_item_hook(b))}},mouseover_item_hook:function(){},hoverover_list:function(){},
-hoverout_list:function(){},check_response:function(){return!0},response_hook:function(){this.list.empty()},show_hook:function(){this.status_select()},position:function(){var a=this.pane,b=this.options;if(!b.parent){if(!self._position){var e=this.input,g=e.offset(),d=e.outerWidth(!0),f=e.outerHeight(!0);g.top+=f;var i=a.outerWidth(),h=a.outerHeight(),j=g.top+h/2,l=c(window).scrollLeft(),e=c(window).scrollTop(),k=c(window).width(),n=c(window).height()+e,m=!0;"left"==b.align?m=!0:"right"==b.align?m=
-!1:g.left>l+k/2&&(m=!1);m||(m=g.left-(i-d),m>l&&(g.left=m));j>n&&(b=g.top-f-h,b>e&&(g.top=b));this._position=g}a.css({top:this._position.top,left:this._position.left})}},ignore_mouseover:function(){this._ignore_mouseover=!0;var a=this;this.ignore_mouseover.timeout=setTimeout(function(){a.ignore_mouseover_reset()},1E3)},ignore_mouseover_reset:function(){this._ignore_mouseover=!1},get_selected:function(){var a=null,b=this.options.css.selected;c("li",this.list).each(function(){var e=c(this);if(e.hasClass(b)&&
-e.is(":visible"))return a=e,!1});return a},onselect:function(a){var b=a.data("data.suggest");b&&(this.input.val(b.name).data("data.suggest",b).trigger("fb-select",b),this.trackEvent(this.name,"fb-select","index",a.prevAll().length))},trackEvent:function(a,b,c,g){this.input.trigger("fb-track-event",{category:a,action:b,label:c,value:g})},hide_all:function(){this.pane.hide();this.input.trigger("fb-pane-hide",this)}};c.extend(c.suggest,{defaults:{status:["Start typing to get suggestions...","Searching...",
-"Select an item from the list:","Sorry, something went wrong. Please try again later"],soft:!1,nomatch:"no matches",css:{pane:"fbs-pane",list:"fbs-list",item:"fbs-item",item_name:"fbs-item-name",selected:"fbs-selected",status:"fbs-status"},css_prefix:null,parent:null,animate:!1,zIndex:null},strongify:function(a,b){var e,g=a.toLowerCase().indexOf(b.toLowerCase());if(0<=g){var d=b.length;e=document.createTextNode(a.substring(0,g));var f=c("<strong>").text(a.substring(g,g+d)),g=document.createTextNode(a.substring(g+
-d));e=c("<div>").append(e).append(f).append(g)}else e=c("<div>").text(a);return e},keyCode:{CAPS_LOCK:20,CONTROL:17,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,INSERT:45,LEFT:37,NUMPAD_ENTER:108,PAGE_DOWN:34,PAGE_UP:33,RIGHT:39,SHIFT:16,SPACE:32,TAB:9,UP:38,OPTION:18,APPLE:224},is_char:function(a){if("keypress"===a.type){if((a.metaKey||a.ctrlKey)&&118===a.charCode)return!0;if("isChar"in a)return a.isChar}else{var b=c.suggest.keyCode.not_char;b||(b={},c.each(c.suggest.keyCode,function(a,c){b[""+c]=1}),
-c.suggest.keyCode.not_char=b);return!(""+a.keyCode in b)}},parse_input:function(a){for(var b=/(\S+)\:(?:\"([^\"]+)\"|(\S+))/g,e=a,g=[],d={},f=b.exec(a);f;)f[1]in q?d[f[1]]=c.isEmptyObject(f[2])?f[3]:f[2]:g.push(f[0]),e=e.replace(f[0],""),f=b.exec(a);e=c.trim(e.replace(/\s+/g," "));return[e,g,d]},mqlkey_fast:/^[_A-Za-z0-9][A-Za-z0-9_-]*$/,mqlkey_slow:/^(?:[A-Za-z0-9]|\$[A-F0-9]{4})(?:[A-Za-z0-9_-]|\$[A-F0-9]{4})*$/,check_mql_key:function(a){return c.suggest.mqlkey_fast.test(a)||c.suggest.mqlkey_slow.test(a)?
-!0:!1},check_mql_id:function(a){if(0===a.indexOf("/")){a=a.split("/");a.shift();if(!(1==a.length&&""===a[0]))for(var b=0,e=a.length;b<e;b++)if(!c.suggest.check_mql_key(a[b]))return!1;return!0}return!1},is_system_type:function(a){return null==a?!1:0===a.indexOf("/type/")}});var s=c.suggest.prototype._destroy,t=c.suggest.prototype.show_hook;c.suggest("suggest",{_init:function(){var a=this,b=this.options;null==b.flyout_service_url&&(b.flyout_service_url=b.service_url);this.flyout_url=b.flyout_service_url;
-b.flyout_service_path&&(this.flyout_url+=b.flyout_service_path);this.flyout_url=this.flyout_url.replace(/\$\{key\}/g,b.key);null==b.flyout_image_service_url&&(b.flyout_image_service_url=b.service_url);this.flyout_image_url=b.flyout_image_service_url;b.flyout_image_service_path&&(this.flyout_image_url+=b.flyout_image_service_path);this.flyout_image_url=this.flyout_image_url.replace(/\$\{key\}/g,b.key);c.suggest.cache||(c.suggest.cache={});if(b.flyout&&(this.flyoutpane=c('<div style="display:none;" class="fbs-reset">').addClass(b.css.flyoutpane),
-b.flyout_parent?c(b.flyout_parent).append(this.flyoutpane):(this.flyoutpane.css("position","absolute"),b.zIndex&&this.flyoutpane.css("z-index",b.zIndex),c(document.body).append(this.flyoutpane)),this.flyoutpane.hover(function(b){a.hoverover_list(b)},function(b){a.hoverout_list(b)}).bind("mousedown.suggest",function(b){b.stopPropagation();a.pane.click()}),c.suggest.flyout||(c.suggest.flyout={}),!c.suggest.flyout.cache))c.suggest.flyout.cache={}},_destroy:function(){s.call(this);this.flyoutpane&&this.flyoutpane.remove();
-this.input.removeData("request.count.suggest");this.input.removeData("flyout.request.count.suggest")},shift_enter:function(){this.options.suggest_new&&(this.suggest_new(),this.hide_all())},hide_all:function(){this.pane.hide();this.flyoutpane&&this.flyoutpane.hide();this.input.trigger("fb-pane-hide",this);this.input.trigger("fb-flyoutpane-hide",this)},request:function(a,b){var e=this,g=this.options,d=a,f=g.ac_param.filter||[],i=null;"string"===c.type(f)&&(f=[f]);f=f.slice();if(g.advanced){var h=c.suggest.parse_input(d),
-d=h[0];h[1].length&&f.push("(all "+h[1].join(" ")+")");i=h[2];c.suggest.check_mql_id(d)&&(f.push('(any alias{start}:"'+d+'" mid:"'+d+'")'),i.prefixed=!0,d="")}h={};h[g.query_param_name]=d;b&&(h.cursor=b);c.extend(h,g.ac_param,i);f.length&&(h.filter=f);var j=g.service_url+g.service_path+"?"+c.param(h,!0);if(d=c.suggest.cache[j])this.response(d,b?b:-1,!0);else{clearTimeout(this.request.timeout);var l={url:g.service_url+g.service_path,data:h,traditional:!0,beforeSend:function(){var a=e.input.data("request.count.suggest")||
-0;a||e.trackEvent(e.name,"start_session");a=a+1;e.trackEvent(e.name,"request","count",a);e.input.data("request.count.suggest",a)},success:function(d){c.suggest.cache[j]=d;d.prefix=a;e.response(d,b?b:-1)},error:function(a){e.status_error();e.trackEvent(e.name,"request","error",{url:this.url,response:a?a.responseText:""});e.input.trigger("fb-error",Array.prototype.slice.call(arguments))},complete:function(a){a&&e.trackEvent(e.name,"request","tid",a.getResponseHeader("X-Metaweb-TID"))},dataType:"jsonp",
-cache:!0};this.request.timeout=setTimeout(function(){c.ajax(l)},g.xhr_delay)}},create_item:function(a,b){var e=this.options.css,g=c("<li>").addClass(e.item),d=c("<label>").append(c.suggest.strongify(a.name||a.id,b.prefix)),f=c("<div>").addClass(e.item_name).append(d),i=a.notable;a.under&&c(":first",d).append(c("<small>").text(" ("+a.under+")"));(null!=i&&c.suggest.is_system_type(i.id)||null!=this.options.scoring&&"SCHEMA"===this.options.scoring.toUpperCase())&&c(":first",d).append(c("<small>").text(" ("+
-a.id+")"));g.append(f);e=c("<div>").addClass(e.item_type);i&&i.name?e.text(i.name):this.options.show_id&&a.id&&e.text(a.id);f.prepend(e);return g},mouseover_item_hook:function(a){a=a.data("data.suggest");this.options.flyout&&a&&this.flyout_request(a)},check_response:function(a){return a.prefix===this.input.val()},response_hook:function(a,b){this.flyoutpane&&this.flyoutpane.hide();0<b?c(".fbs-more",this.pane).remove():this.list.empty()},show_hook:function(a,b,e){t.apply(this,[a]);var g=this.options,
-d=this,f=this.pane,i=this.list,h=a.result,j=c(".fbs-more",f),l=c(".fbs-suggestnew",f);c(".fbs-status",f);var k=a.correction;if(k&&k.length){var n=c('<a class="fbs-spell-link" href="#">').append(k[0]).bind("click.suggest",function(a){a.preventDefault();a.stopPropagation();d.input.val(k[0]).trigger("textchange")});d.status.empty().append("Search instead for ").append(n).show()}h&&h.length&&"cursor"in a?(j.length||(h=c('<a class="fbs-more-link" href="#" title="(Ctrl+m)">view more</a>'),j=c('<div class="fbs-more">').append(h),
-h.bind("click.suggest",function(a){a.preventDefault();a.stopPropagation();a=c(this).parent(".fbs-more");d.more(a.data("cursor.suggest"))}),i.after(j)),j.data("cursor.suggest",a.cursor),j.show()):j.remove();g.suggest_new?(l.length||(a=c('<button class="fbs-suggestnew-button">'),a.text(g.suggest_new),l=c('<div class="fbs-suggestnew">').append('<div class="fbs-suggestnew-description">Your item not in the list?</div>').append(a).append('<span class="fbs-suggestnew-shortcut">(Shift+Enter)</span>').bind("click.suggest",
-function(a){a.stopPropagation();d.suggest_new(a)}),f.append(l)),l.show()):l.remove();e&&(e.length&&0<b)&&(b=e.prevAll().length*e.outerHeight(),i.scrollTop(),i.animate({scrollTop:b},"slow",function(){e.trigger("mouseover.suggest")}))},suggest_new:function(){var a=this.input.val();""!==a&&(this.input.data("data.suggest",a).trigger("fb-select-new",a),this.trackEvent(this.name,"fb-select-new","index","new"),this.hide_all())},more:function(a){if(a){var b=this.input.data("original.suggest");null!==b&&this.input.val(b);
-this.request(this.input.val(),a);this.trackEvent(this.name,"more","cursor",a)}return!1},flyout_request:function(a){var b=this,e=this.options,g=this.flyoutpane.data("data.suggest");if(g&&a.id===g.id)this.flyoutpane.is(":visible")||(a=this.get_selected(),this.flyout_position(a),this.flyoutpane.show(),this.input.trigger("fb-flyoutpane-show",this));else if((g=c.suggest.flyout.cache[a.id])&&g.id&&g.html)this.flyout_response(g);else{var d=a.id,f={url:this.flyout_url.replace(/\$\{id\}/g,a.id),traditional:!0,
-beforeSend:function(){var a=b.input.data("flyout.request.count.suggest")||0,a=a+1;b.trackEvent(b.name,"flyout.request","count",a);b.input.data("flyout.request.count.suggest",a)},success:function(a){a["req:id"]=d;a.result&&a.result.length&&(a.html=c.suggest.suggest.create_flyout(a.result[0],b.flyout_image_url));c.suggest.flyout.cache[d]=a;b.flyout_response(a)},error:function(a){b.trackEvent(b.name,"flyout","error",{url:this.url,response:a?a.responseText:""})},complete:function(a){a&&b.trackEvent(b.name,
-"flyout","tid",a.getResponseHeader("X-Metaweb-TID"))},dataType:"jsonp",cache:!0};e.flyout_lang&&(f.data={lang:e.flyout_lang});clearTimeout(this.flyout_request.timeout);this.flyout_request.timeout=setTimeout(function(){c.ajax(f)},e.xhr_delay);this.input.trigger("fb-request-flyout",f)}},flyout_response:function(a){var b=this.pane,c=this.get_selected()||[];if(b.is(":visible")&&c.length&&(b=c.data("data.suggest"))&&a["req:id"]===b.id&&a.html)this.flyoutpane.html(a.html),this.flyout_position(c),this.flyoutpane.show().data("data.suggest",
-b),this.input.trigger("fb-flyoutpane-show",this)},flyout_position:function(a){if(!this.options.flyout_parent){var b=this.pane,e=this.flyoutpane,g=this.options.css,d=r,f=parseInt(e.css("top"),10),i=parseInt(e.css("left"),10),h=b.offset(),j=b.outerWidth(),l=e.outerHeight(),k=e.outerWidth();if("bottom"===this.options.flyout)d=h,j=this.input.offset(),d.top=h.top<j.top?d.top-l:d.top+b.outerHeight(),e.addClass(g.flyoutpane+"-bottom");else{d=a.offset();a=a.outerHeight();d.left+=j;var n=d.left+k,b=c(document.body).scrollLeft(),
-m=c(window).width()+b;d.top=d.top+a-l;d.top<h.top&&(d.top=h.top);n>m&&(h=d.left-(j+k),h>b&&(d.left=h));e.removeClass(g.flyoutpane+"-bottom")}d.top===f&&d.left===i||e.css({top:d.top,left:d.left})}},hoverout_list:function(){this.flyoutpane&&!this.get_selected()&&this.flyoutpane.hide()}});c.extend(c.suggest.suggest,{defaults:{filter:null,spell:"always",exact:!1,scoring:null,lang:null,key:null,prefixed:!0,stemmed:null,format:null,advanced:!0,show_id:!0,query_param_name:"query",service_url:"https://www.googleapis.com/freebase/v1",
-service_path:"/search",align:null,flyout:!0,flyout_service_url:null,flyout_service_path:"/search?filter=(all mid:${id})&output=(notable:/client/summary description type)&key=${key}",flyout_image_service_url:null,flyout_image_service_path:"/image${id}?maxwidth=75&key=${key}&errorid=/freebase/no_image_png",flyout_parent:null,suggest_new:null,nomatch:{title:"No suggested matches",heading:"Tips on getting better suggestions:",tips:["Enter more or fewer characters","Add words related to your original search",
-"Try alternate spellings","Check your spelling"]},css:{item_type:"fbs-item-type",flyoutpane:"fbs-flyout-pane"},xhr_delay:200},get_value:function(a,b,e){if(null==a||null==b||0==b.length)return null;c.isArray(b)||(b=[b]);var g=null;c.each(b,function(b,c){g=a[c];if(null==g)return!1;a=g;return!0});if(e){if(null==g)return[];c.isArray(g)||(g=[g]);var d=[];c.each(g,function(a,b){if("object"===c.type(b))if(null!=b.name)b=b.name;else if(b.id||b.mid)b=b.id||b.mid;else if(null!=b.value){var e=[];c.each(b,function(a,
-b){"value"!==a&&e.push(b)});b=b.value;e.length&&(b+=" ("+e.join(", ")+")")}c.isArray(b)&&b.length&&(b=b[0].value);null!=b&&d.push(b)});return d}return null==g?null:g},is_commons_id:function(a){return/^\/base\//.test(a)||/^\/user\//.test(a)?!1:!0},create_flyout:function(a,b){var e=c.suggest.suggest.get_value,g=c.suggest.is_system_type,d=a.name,f=null,i=null,h=[],j=[],l={};if((f=e(a,"notable"))&&f.name)j.push(f.name),l[f.name]=!0;f&&g(f.id)?f=a.id:(f=a.mid,i=b.replace(/\$\{id\}/g,f));var g="freebase",
-k=e(a,["output","description","wikipedia"],!0);k&&k.length?g="wikipedia":k=e(a,["output","description","freebase"],!0);var k=k&&k.length?k[0]:null,n=e(a,["output","notable:/client/summary"]);if(n){var m=e(n,"/common/topic/notable_paths");m&&m.length&&c.each(m,function(a,b){var c=e(n,b,true);if(c&&c.length){var d=b.split("/").pop();h.push([d,c.join(", ")])}})}(m=e(a,["output","type","/type/object/type"],!0))&&m.length&&c.each(m,function(a,b){if(!l[b]){j.push(b);l[b]=true}});var o=c('<div class="fbs-flyout-content">');
-d&&o.append(c('<h1 id="fbs-flyout-title">').text(d));o.append(c('<h3 class="fbs-topic-properties fbs-flyout-id">').text(f));c.each(h,function(a,b){o.append(c('<h3 class="fbs-topic-properties">').append(c("<strong>").text(b[0]+": ")).append(document.createTextNode(b[1])))});k&&o.append(c('<p class="fbs-topic-article">').append(c('<em class="fbs-citation">').text("["+g+"] ")).append(document.createTextNode(k)));i&&(o.children().addClass("fbs-flyout-image-true"),o.prepend(c('<img id="fbs-topic-image" class="fbs-flyout-image-true" src="'+
-i+'">')));d=c('<span class="fbs-flyout-types">').text(j.slice(0,10).join(", "));d=c('<div class="fbs-attribution">').append(d);return c("<div>").append(o).append(d).html()}});document.createElement("input")})(jQuery);

main/webapp/modules/core/externals/suggest/suggest-4_3.js

@@ -1657,9 +1657,11 @@
       flyout_service_url: null,
 
       // flyout_service_url + flyout_service_path =
-      // url to search with output=(notable:/client/summary description type).
+      // url to search with
+      // output=(notable:/client/summary (description citation) type).
       flyout_service_path: "/search?filter=(all mid:${id})&" +
-          "output=(notable:/client/summary description type)&key=${key}",
+          "output=(notable:/client/summary " +
+          "(description citation provenance) type)&key=${key}",
 
       // default is service_url if NULL
       flyout_image_service_url: null,
@@ -1699,6 +1701,21 @@
       xhr_delay: 200
     },
 
+    /**
+     * Get a value from an object multiple levels deep.
+     */
+     get_value_by_keys: function(obj, var_args) {
+       var keys = $.isArray(var_args) ? var_args :
+           Array.prototype.slice.call(arguments, 1);
+       for (var i = 0; i < keys.length; i++) {
+         obj = obj[keys[i]];
+         if (obj == null) {
+           break;
+         }
+       }
+       return obj;
+     },
+
     /**
      * Utility method to get values of an object specified by one or more
      * (nested) keys. For example:
@@ -1721,15 +1738,7 @@
       if (!$.isArray(path)) {
         path = [path];
       }
-      var v = null;
+      var v =  $.suggest.suggest.get_value_by_keys(obj, path);
-      $.each(path, function(i, p){
-        v = obj[p];
-        if (v == null) {
-          return false;
-        }
-        obj = v;
-        return true;
-      });
       if (resolve_search_values) {
         if (v == null) {
           return [];
@@ -1782,17 +1791,20 @@
 
     /**
      * Create the flyout html content given the search result
-     * containing output=(notable:/client/summary description type).
+     * containing output=(notable:/client/summary \
+     * (description citation provenance) type).
      * The resulting html will be cached for optimization.
      *
      * @param data:Object - The search result with
-     *     output=(notable:/client/summary description type).
+     *   output=(notable:/client/summary \
+     *   (description citation provenance) type)
      * @param flyout_image_url:String - The url template for the image url.
      *   The substring, "${id}", will be replaced by data.id. It is assumed all
      *   parameters to the flyout image service (api key, dimensions, etc.) is
      *   already encoded into the url template.
      */
     create_flyout: function(data, flyout_image_url) {
+      var get_value_by_keys = $.suggest.suggest.get_value_by_keys;
       var get_value = $.suggest.suggest.get_value;
       var is_system_type = $.suggest.is_system_type;
       var is_commons_id = $.suggest.suggest.is_commons_id;
@@ -1815,21 +1827,53 @@
         id = data['mid'];
         image = flyout_image_url.replace(/\$\{id\}/g, id);
       }
-      var description_src = 'freebase';
+
-      var description = get_value(
+      var desc_text = null;
-          data, ['output', 'description', 'wikipedia'], true);
+      var desc_source = null;
-      if (description && description.length) {
+      var desc_provider = null;
-        description_src = 'wikipedia';
+      var desc_statement = null;
+      var descs = get_value_by_keys(
+          data, 'output', 'description', '/common/topic/description') || [];
+      if (descs.length) {
+        var best = descs[0];
+        $.each(descs, function(i, desc) {
+          if (get_value_by_keys(desc, 'citation', 0, 'mid') == '/m/0d07ph') {
+            // Prefer 'Wikipedia" descriptions (/m/0d07ph).
+            best = desc;
+            return false;
           }
-      else {
+          return true;
-        description = get_value(
+        });
-            data, ['output', 'description', 'freebase'], true);
+        if ($.isArray(best.value) && best.value.length) {
+          desc_text = best.value[0].value;
+        } else {
+          desc_text = best.value;
         }
-      if (description && description.length) {
+        if (get_value_by_keys(best, 'provenance', 0, 'restrictions', 0) ==
-        description = description[0];
+            'REQUIRES_CITATION') {
+          desc_source = get_value_by_keys(best, 'provenance', 0, 'source', 0);
+          desc_provider =
+              get_value_by_keys(best, 'citation', 'provider', 0, 'name');
+          if (desc_provider && $.isArray(desc_provider) &&
+              desc_provider.length) {
+            desc_provider = desc_provider[0].value;
           }
-      else {
+          desc_statement = get_value_by_keys(best, 'citation', 'statement', 0);
-        description = null;
+          if (desc_statement && desc_statement.value) {
+            desc_statement = desc_statement.value;
+          }
+        }
+      } else {
+        // Handle "old" output description format.
+        $.each(['wikipedia', 'freebase'], function(i, key) {
+          descs = get_value(data, ['output', 'description', key], true);
+          if (descs && descs.length) {
+            desc_text = descs[0];
+            desc_provider = key;
+            return false;
+          }
+          return true;
+        });
       }
       var summary = get_value(data, ['output', 'notable:/client/summary']);
       if (summary) {
@@ -1838,6 +1882,7 @@
           $.each(notable_paths, function(i, path) {
             var values = get_value(summary, path, true);
             if (values && values.length) {
+              values = values.slice(0, 3);
               var prop_text = path.split('/').pop();
               notable_props.push([prop_text, values.join(', ')]);
             }
@@ -1861,17 +1906,28 @@
       content
           .append($('<h3 class="fbs-topic-properties fbs-flyout-id">')
           .text(id));
+      notable_props = notable_props.slice(0, 3);
       $.each(notable_props, function(i, prop) {
           content.append($('<h3 class="fbs-topic-properties">')
               .append($('<strong>').text(prop[0] + ': '))
               .append(document.createTextNode(prop[1])));
           });
-      if (description) {
+      if (desc_text) {
-        content.append(
+        var article = $('<p class="fbs-topic-article">');
-            $('<p class="fbs-topic-article">')
+        if (desc_provider) {
-                .append($('<em class="fbs-citation">')
+          if (desc_source) {
-                    .text('[' + description_src + '] '))
+            article.append($('<a class="fbs-citation">')
-                .append(document.createTextNode(description)));
+                .attr('href', desc_source)
+                .attr('title', desc_statement || desc_provider)
+                .text('[' + desc_provider + ']'));
+          } else {
+            article.append($('<em class="fbs-citation">')
+                .attr('title', desc_statement || desc_provider)
+                .text('[' + desc_provider + '] '));
+          }
+        }
+        article.append(document.createTextNode(' ' + desc_text));
+        content.append(article);
       }
       if (image) {
         content.children().addClass('fbs-flyout-image-true');
@@ -1880,7 +1936,7 @@
               image + '">'));
       }
       var flyout_types = $('<span class="fbs-flyout-types">')
-        .text(notable_types.slice(0, 10).join(', '));
+        .text(notable_types.slice(0, 3).join(', '));
       var footer = $('<div class="fbs-attribution">').append(flyout_types);
 
       return $('<div>')

main/webapp/modules/core/externals/suggest/suggest-4_3.min.js

@@ -0,0 +1,83 @@
+
+/*
+ * Copyright 2012, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Additional Licenses for Third Party components can be found here:
+ * http://wiki.freebase.com/wiki/Freebase_Site_License
+ *
+ */
+(function(b,v){if(!("console"in window)){var p=window.console={};p.log=p.warn=p.error=p.debug=function(){}}b(function(){var a=b("<div>");b(document.body).append(a);var c=setTimeout(function(){if(b.cleanData){var a=b.cleanData;b.cleanData=function(c){for(var d=0,g;null!=(g=c[d]);d++)b(g).triggerHandler("remove");a(c)}}else{var c=b.fn.remove;b.fn.remove=function(a,d){return this.each(function(){d||(!a||b.filter(a,[this]).length)&&b("*",this).add([this]).each(function(){b(this).triggerHandler("remove")});
+return c.call(b(this),a,d)})}}},1);a.bind("remove",function(){clearTimeout(c)});a.remove()});var u={key:1,filter:1,spell:1,exact:1,lang:1,scoring:1,prefixed:1,stemmed:1,format:1,mql_output:1,output:1};b.suggest=function(a,c){b.fn[a]=function(c){this.length||console.warn("Suggest: invoked on empty element set");return this.each(function(){this.nodeName&&("INPUT"===this.nodeName.toUpperCase()?this.type&&"TEXT"!==this.type.toUpperCase()&&console.warn("Suggest: unsupported INPUT type: "+this.type):console.warn("Suggest: unsupported DOM element: "+
+this.nodeName));var g=b.data(this,a);g&&g._destroy();b.data(this,a,new b.suggest[a](this,c))._init()})};b.suggest[a]=function(c,g){var f=this,d=this.options=b.extend(!0,{},b.suggest.defaults,b.suggest[a].defaults,g),i=d.css_prefix=d.css_prefix||"",h=d.css;this.name=a;b.each(h,function(a){h[a]=i+h[a]});d.ac_param={};b.each(u,function(a){var c=d[a];null===c||""===c||(d.ac_param[a]=c)});d.flyout_lang=null;if(d.ac_param.lang){var j=d.ac_param.lang;b.isArray(j)&&j.length&&(j=j.join(","));j&&(d.flyout_lang=
+j)}this._status={START:"",LOADING:"",SELECT:"",ERROR:""};d.status&&(d.status instanceof Array&&3<=d.status.length)&&(this._status.START=d.status[0]||"",this._status.LOADING=d.status[1]||"",this._status.SELECT=d.status[2]||"",4===d.status.length&&(this._status.ERROR=d.status[3]||""));var j=this.status=b('<div style="display:none;">').addClass(h.status),m=this.list=b("<ul>").addClass(h.list),l=this.pane=b('<div style="display:none;" class="fbs-reset">').addClass(h.pane);l.append(j).append(m);d.parent?
+b(d.parent).append(l):(l.css("position","absolute"),d.zIndex&&l.css("z-index",d.zIndex),b(document.body).append(l));l.bind("mousedown",function(a){f.input.data("dont_hide",true);a.stopPropagation()}).bind("mouseup",function(a){f.input.data("dont_hide")&&f.input.focus();f.input.removeData("dont_hide");a.stopPropagation()}).bind("click",function(a){a.stopPropagation();if(a=f.get_selected()){f.onselect(a,true);f.hide_all()}});m.hover(function(a){f.hoverover_list(a)},function(a){f.hoverout_list(a)});
+this.input=b(c).attr("autocomplete","off").unbind(".suggest").bind("remove.suggest",function(){f._destroy()}).bind("keydown.suggest",function(a){f.keydown(a)}).bind("keypress.suggest",function(a){f.keypress(a)}).bind("keyup.suggest",function(a){f.keyup(a)}).bind("blur.suggest",function(a){f.blur(a)}).bind("textchange.suggest",function(){f.textchange()}).bind("focus.suggest",function(a){f.focus(a)}).bind(b.browser.msie?"paste.suggest":"input.suggest",function(){clearTimeout(f.paste_timeout);f.paste_timeout=
+setTimeout(function(){f.textchange()},0)});this.onresize=function(){f.invalidate_position();if(l.is(":visible")){f.position();if(d.flyout&&f.flyoutpane&&f.flyoutpane.is(":visible")){var a=f.get_selected();a&&f.flyout_position(a)}}};b(window).bind("resize.suggest",this.onresize).bind("scroll.suggest",this.onresize)};b.suggest[a].prototype=b.extend({},b.suggest.prototype,c)};b.suggest.prototype={_init:function(){},_destroy:function(){this.pane.remove();this.list.remove();this.input.unbind(".suggest");
+b(window).unbind("resize.suggest",this.onresize).unbind("scroll.suggest",this.onresize);this.input.removeData("data.suggest")},invalidate_position:function(){self._position=null},status_start:function(){this.hide_all();this.status.siblings().hide();this._status.START&&(this.status.text(this._status.START).show(),this.pane.is(":visible")||(this.position(),this.pane_show()));this._status.LOADING&&this.status.removeClass("loading")},status_loading:function(){this.status.siblings().show();this._status.LOADING?
+(this.status.addClass("loading").text(this._status.LOADING).show(),this.pane.is(":visible")||(this.position(),this.pane_show())):this.status.hide()},status_select:function(){this.status.siblings().show();this._status.SELECT?this.status.text(this._status.SELECT).show():this.status.hide();this._status.LOADING&&this.status.removeClass("loading")},status_error:function(){this.status.siblings().show();this._status.ERROR?this.status.text(this._status.ERROR).show():this.status.hide();this._status.LOADING&&
+this.status.removeClass("loading")},focus:function(a){""===this.input.val()?this.status_start():this.focus_hook(a)},focus_hook:function(){!this.input.data("data.suggest")&&(!this.pane.is(":visible")&&b("."+this.options.css.item,this.list).length)&&(this.position(),this.pane_show())},keydown:function(a){var c=a.keyCode;if(9===c)this.tab(a);else if(38===c||40===c)a.shiftKey||a.preventDefault()},keypress:function(a){var c=a.keyCode;38===c||40===c?a.shiftKey||a.preventDefault():13===c&&this.enter(a)},
+keyup:function(a){var c=a.keyCode;if(38===c)a.preventDefault(),this.up(a);else if(40===c)a.preventDefault(),this.down(a);else if(a.ctrlKey&&77===c)b(".fbs-more-link",this.pane).click();else if(b.suggest.is_char(a)){clearTimeout(this.keypress.timeout);var e=this;this.keypress.timeout=setTimeout(function(){e.textchange()},0)}else 27===c&&this.escape(a);return!0},blur:function(){this.input.data("dont_hide")||(this.input.data("data.suggest"),this.hide_all())},tab:function(a){if(!a.shiftKey&&!a.metaKey&&
+!a.ctrlKey){var a=this.options,a=this.pane.is(":visible")&&b("."+a.css.item,this.list).length,c=this.get_selected();a&&c&&(this.onselect(c),this.hide_all())}},enter:function(a){var c=this.options;if(this.pane.is(":visible")){if(a.shiftKey){this.shift_enter(a);a.preventDefault();return}if(b("."+c.css.item,this.list).length){var e=this.get_selected();if(e){this.onselect(e);this.hide_all();a.preventDefault();return}if(!c.soft&&(this.input.data("data.suggest"),b("."+this.options.css.item+":visible",this.list).length)){this.updown(!1);
+a.preventDefault();return}}}c.soft?this.soft_enter():a.preventDefault()},soft_enter:function(){},shift_enter:function(){},escape:function(){this.hide_all()},up:function(a){this.updown(!0,a.ctrlKey||a.shiftKey)},down:function(a){this.updown(!1,null,a.ctrlKey||a.shiftKey)},updown:function(a,c,e){var g=this.options.css,f=this.list;if(this.pane.is(":visible")){var d=b("."+g.item+":visible",f);if(d.length){var f=b(d[0]),d=b(d[d.length-1]),i=this.get_selected()||[];clearTimeout(this.ignore_mouseover.timeout);
+this._ignore_mouseover=!1;a?c?this._goto(f):i.length?i[0]==f[0]?(f.removeClass(g.selected),this.input.val(this.input.data("original.suggest")),this.hoverout_list()):(a=i.prevAll("."+g.item+":visible:first"),this._goto(a)):this._goto(d):e?this._goto(d):i.length?i[0]==d[0]?(d.removeClass(g.selected),this.input.val(this.input.data("original.suggest")),this.hoverout_list()):(a=i.nextAll("."+g.item+":visible:first"),this._goto(a)):this._goto(f)}}else a||this.textchange()},_goto:function(a){a.trigger("mouseover.suggest");
+var c=a.data("data.suggest");this.input.val(c?c.name:this.input.data("original.suggest"));this.scroll_to(a)},scroll_to:function(a){var c=this.list,b=c.scrollTop(),g=b+c.innerHeight(),f=a.outerHeight(),a=a.prevAll().length*f,f=a+f;a<b?(this.ignore_mouseover(),c.scrollTop(a)):f>g&&(this.ignore_mouseover(),c.scrollTop(b+f-g))},textchange:function(){this.input.removeData("data.suggest");this.input.trigger("fb-textchange",this);var a=this.input.val();""===a?this.status_start():(this.status_loading(),this.request(a))},
+request:function(){},response:function(a){if(a&&("cost"in a&&this.trackEvent(this.name,"response","cost",a.cost),this.check_response(a))){var c=[];b.isArray(a)?c=a:"result"in a&&(c=a.result);var e=b.map(arguments,function(a){return a});this.response_hook.apply(this,e);var g=null,f=this,d=this.options;b.each(c,function(c,b){if(!b.id&&b.mid)b.id=b.mid;var d=f.create_item(b,a).bind("mouseover.suggest",function(a){f.mouseover_item(a)});d.data("data.suggest",b);f.list.append(d);c===0&&(g=d)});this.input.data("original.suggest",
+this.input.val());if(0===b("."+d.css.item,this.list).length&&d.nomatch){c=b('<li class="fbs-nomatch">');if("string"===typeof d.nomatch)c.text(d.nomatch);else if(d.nomatch.title&&c.append(b('<em class="fbs-nomatch-text">').text(d.nomatch.title)),d.nomatch.heading&&c.append(b("<h3>").text(d.nomatch.heading)),(d=d.nomatch.tips)&&d.length){var i=b('<ul class="fbs-search-tips">');b.each(d,function(a,c){i.append(b("<li>").text(c))});c.append(i)}c.bind("click.suggest",function(a){a.stopPropagation()});this.list.append(c)}e.push(g);
+this.show_hook.apply(this,e);this.position();this.pane_show()}},pane_show:function(){var a=!1;b("> li",this.list).length&&(a=!0);a||this.pane.children(":not(."+this.options.css.list+")").each(function(){if("none"!=b(this).css("display"))return a=!0,!1});if(a)if(this.options.animate){var c=this;this.pane.slideDown("fast",function(){c.input.trigger("fb-pane-show",c)})}else this.pane.show(),this.input.trigger("fb-pane-show",this);else this.pane.hide(),this.input.trigger("fb-pane-hide",this)},create_item:function(a){var c=
+this.options.css,e=b("<li>").addClass(c.item),a=b("<label>").text(a.name);e.append(b("<div>").addClass(c.item_name).append(a));return e},mouseover_item:function(a){if(!this._ignore_mouseover){a=a.target;"li"!==a.nodeName.toLowerCase()&&(a=b(a).parents("li:first"));var c=b(a),e=this.options.css;b("."+e.item,this.list).each(function(){this!==c[0]&&b(this).removeClass(e.selected)});c.hasClass(e.selected)||(c.addClass(e.selected),this.mouseover_item_hook(c))}},mouseover_item_hook:function(){},hoverover_list:function(){},
+hoverout_list:function(){},check_response:function(){return!0},response_hook:function(){this.list.empty()},show_hook:function(){this.status_select()},position:function(){var a=this.pane,c=this.options;if(!c.parent){if(!self._position){var e=this.input,g=e.offset(),f=e.outerWidth(!0),d=e.outerHeight(!0);g.top+=d;var i=a.outerWidth(),h=a.outerHeight(),j=g.top+h/2,m=b(window).scrollLeft(),e=b(window).scrollTop(),l=b(window).width(),n=b(window).height()+e,k=!0;"left"==c.align?k=!0:"right"==c.align?k=
+!1:g.left>m+l/2&&(k=!1);k||(k=g.left-(i-f),k>m&&(g.left=k));j>n&&(c=g.top-d-h,c>e&&(g.top=c));this._position=g}a.css({top:this._position.top,left:this._position.left})}},ignore_mouseover:function(){this._ignore_mouseover=!0;var a=this;this.ignore_mouseover.timeout=setTimeout(function(){a.ignore_mouseover_reset()},1E3)},ignore_mouseover_reset:function(){this._ignore_mouseover=!1},get_selected:function(){var a=null,c=this.options.css.selected;b("li",this.list).each(function(){var e=b(this);if(e.hasClass(c)&&
+e.is(":visible"))return a=e,!1});return a},onselect:function(a){var c=a.data("data.suggest");c&&(this.input.val(c.name).data("data.suggest",c).trigger("fb-select",c),this.trackEvent(this.name,"fb-select","index",a.prevAll().length))},trackEvent:function(a,c,b,g){this.input.trigger("fb-track-event",{category:a,action:c,label:b,value:g})},hide_all:function(){this.pane.hide();this.input.trigger("fb-pane-hide",this)}};b.extend(b.suggest,{defaults:{status:["Start typing to get suggestions...","Searching...",
+"Select an item from the list:","Sorry, something went wrong. Please try again later"],soft:!1,nomatch:"no matches",css:{pane:"fbs-pane",list:"fbs-list",item:"fbs-item",item_name:"fbs-item-name",selected:"fbs-selected",status:"fbs-status"},css_prefix:null,parent:null,animate:!1,zIndex:null},strongify:function(a,c){var e,g=a.toLowerCase().indexOf(c.toLowerCase());if(0<=g){var f=c.length;e=document.createTextNode(a.substring(0,g));var d=b("<strong>").text(a.substring(g,g+f)),g=document.createTextNode(a.substring(g+
+f));e=b("<div>").append(e).append(d).append(g)}else e=b("<div>").text(a);return e},keyCode:{CAPS_LOCK:20,CONTROL:17,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,INSERT:45,LEFT:37,NUMPAD_ENTER:108,PAGE_DOWN:34,PAGE_UP:33,RIGHT:39,SHIFT:16,SPACE:32,TAB:9,UP:38,OPTION:18,APPLE:224},is_char:function(a){if("keypress"===a.type){if((a.metaKey||a.ctrlKey)&&118===a.charCode)return!0;if("isChar"in a)return a.isChar}else{var c=b.suggest.keyCode.not_char;c||(c={},b.each(b.suggest.keyCode,function(a,b){c[""+b]=1}),
+b.suggest.keyCode.not_char=c);return!(""+a.keyCode in c)}},parse_input:function(a){for(var c=/(\S+)\:(?:\"([^\"]+)\"|(\S+))/g,e=a,g=[],f={},d=c.exec(a);d;)d[1]in u?f[d[1]]=b.isEmptyObject(d[2])?d[3]:d[2]:g.push(d[0]),e=e.replace(d[0],""),d=c.exec(a);e=b.trim(e.replace(/\s+/g," "));return[e,g,f]},mqlkey_fast:/^[_A-Za-z0-9][A-Za-z0-9_-]*$/,mqlkey_slow:/^(?:[A-Za-z0-9]|\$[A-F0-9]{4})(?:[A-Za-z0-9_-]|\$[A-F0-9]{4})*$/,check_mql_key:function(a){return b.suggest.mqlkey_fast.test(a)||b.suggest.mqlkey_slow.test(a)?
+!0:!1},check_mql_id:function(a){if(0===a.indexOf("/")){a=a.split("/");a.shift();if(!(1==a.length&&""===a[0]))for(var c=0,e=a.length;c<e;c++)if(!b.suggest.check_mql_key(a[c]))return!1;return!0}return!1},is_system_type:function(a){return null==a?!1:0===a.indexOf("/type/")}});var w=b.suggest.prototype._destroy,x=b.suggest.prototype.show_hook;b.suggest("suggest",{_init:function(){var a=this,c=this.options;null==c.flyout_service_url&&(c.flyout_service_url=c.service_url);this.flyout_url=c.flyout_service_url;
+c.flyout_service_path&&(this.flyout_url+=c.flyout_service_path);this.flyout_url=this.flyout_url.replace(/\$\{key\}/g,c.key);null==c.flyout_image_service_url&&(c.flyout_image_service_url=c.service_url);this.flyout_image_url=c.flyout_image_service_url;c.flyout_image_service_path&&(this.flyout_image_url+=c.flyout_image_service_path);this.flyout_image_url=this.flyout_image_url.replace(/\$\{key\}/g,c.key);b.suggest.cache||(b.suggest.cache={});if(c.flyout&&(this.flyoutpane=b('<div style="display:none;" class="fbs-reset">').addClass(c.css.flyoutpane),
+c.flyout_parent?b(c.flyout_parent).append(this.flyoutpane):(this.flyoutpane.css("position","absolute"),c.zIndex&&this.flyoutpane.css("z-index",c.zIndex),b(document.body).append(this.flyoutpane)),this.flyoutpane.hover(function(b){a.hoverover_list(b)},function(b){a.hoverout_list(b)}).bind("mousedown.suggest",function(b){b.stopPropagation();a.pane.click()}),b.suggest.flyout||(b.suggest.flyout={}),!b.suggest.flyout.cache))b.suggest.flyout.cache={}},_destroy:function(){w.call(this);this.flyoutpane&&this.flyoutpane.remove();
+this.input.removeData("request.count.suggest");this.input.removeData("flyout.request.count.suggest")},shift_enter:function(){this.options.suggest_new&&(this.suggest_new(),this.hide_all())},hide_all:function(){this.pane.hide();this.flyoutpane&&this.flyoutpane.hide();this.input.trigger("fb-pane-hide",this);this.input.trigger("fb-flyoutpane-hide",this)},request:function(a,c){var e=this,g=this.options,f=a,d=g.ac_param.filter||[],i=null;"string"===b.type(d)&&(d=[d]);d=d.slice();if(g.advanced){var h=b.suggest.parse_input(f),
+f=h[0];h[1].length&&d.push("(all "+h[1].join(" ")+")");i=h[2];b.suggest.check_mql_id(f)&&(d.push('(any alias{start}:"'+f+'" mid:"'+f+'")'),i.prefixed=!0,f="")}h={};h[g.query_param_name]=f;c&&(h.cursor=c);b.extend(h,g.ac_param,i);d.length&&(h.filter=d);var j=g.service_url+g.service_path+"?"+b.param(h,!0);if(f=b.suggest.cache[j])this.response(f,c?c:-1,!0);else{clearTimeout(this.request.timeout);var m={url:g.service_url+g.service_path,data:h,traditional:!0,beforeSend:function(){var a=e.input.data("request.count.suggest")||
+0;a||e.trackEvent(e.name,"start_session");a=a+1;e.trackEvent(e.name,"request","count",a);e.input.data("request.count.suggest",a)},success:function(d){b.suggest.cache[j]=d;d.prefix=a;e.response(d,c?c:-1)},error:function(a){e.status_error();e.trackEvent(e.name,"request","error",{url:this.url,response:a?a.responseText:""});e.input.trigger("fb-error",Array.prototype.slice.call(arguments))},complete:function(a){a&&e.trackEvent(e.name,"request","tid",a.getResponseHeader("X-Metaweb-TID"))},dataType:"jsonp",
+cache:!0};this.request.timeout=setTimeout(function(){b.ajax(m)},g.xhr_delay)}},create_item:function(a,c){var e=this.options.css,g=b("<li>").addClass(e.item),f=b("<label>").append(b.suggest.strongify(a.name||a.id,c.prefix)),d=b("<div>").addClass(e.item_name).append(f),i=a.notable;a.under&&b(":first",f).append(b("<small>").text(" ("+a.under+")"));(null!=i&&b.suggest.is_system_type(i.id)||null!=this.options.scoring&&"SCHEMA"===this.options.scoring.toUpperCase())&&b(":first",f).append(b("<small>").text(" ("+
+a.id+")"));g.append(d);e=b("<div>").addClass(e.item_type);i&&i.name?e.text(i.name):this.options.show_id&&a.id&&e.text(a.id);d.prepend(e);return g},mouseover_item_hook:function(a){a=a.data("data.suggest");this.options.flyout&&a&&this.flyout_request(a)},check_response:function(a){return a.prefix===this.input.val()},response_hook:function(a,c){this.flyoutpane&&this.flyoutpane.hide();0<c?b(".fbs-more",this.pane).remove():this.list.empty()},show_hook:function(a,c,e){x.apply(this,[a]);var g=this.options,
+f=this,d=this.pane,i=this.list,h=a.result,j=b(".fbs-more",d),m=b(".fbs-suggestnew",d);b(".fbs-status",d);var l=a.correction;if(l&&l.length){var n=b('<a class="fbs-spell-link" href="#">').append(l[0]).bind("click.suggest",function(a){a.preventDefault();a.stopPropagation();f.input.val(l[0]).trigger("textchange")});f.status.empty().append("Search instead for ").append(n).show()}h&&h.length&&"cursor"in a?(j.length||(h=b('<a class="fbs-more-link" href="#" title="(Ctrl+m)">view more</a>'),j=b('<div class="fbs-more">').append(h),
+h.bind("click.suggest",function(a){a.preventDefault();a.stopPropagation();a=b(this).parent(".fbs-more");f.more(a.data("cursor.suggest"))}),i.after(j)),j.data("cursor.suggest",a.cursor),j.show()):j.remove();g.suggest_new?(m.length||(a=b('<button class="fbs-suggestnew-button">'),a.text(g.suggest_new),m=b('<div class="fbs-suggestnew">').append('<div class="fbs-suggestnew-description">Your item not in the list?</div>').append(a).append('<span class="fbs-suggestnew-shortcut">(Shift+Enter)</span>').bind("click.suggest",
+function(a){a.stopPropagation();f.suggest_new(a)}),d.append(m)),m.show()):m.remove();e&&(e.length&&0<c)&&(c=e.prevAll().length*e.outerHeight(),i.scrollTop(),i.animate({scrollTop:c},"slow",function(){e.trigger("mouseover.suggest")}))},suggest_new:function(){var a=this.input.val();""!==a&&(this.input.data("data.suggest",a).trigger("fb-select-new",a),this.trackEvent(this.name,"fb-select-new","index","new"),this.hide_all())},more:function(a){if(a){var b=this.input.data("original.suggest");null!==b&&this.input.val(b);
+this.request(this.input.val(),a);this.trackEvent(this.name,"more","cursor",a)}return!1},flyout_request:function(a){var c=this,e=this.options,g=this.flyoutpane.data("data.suggest");if(g&&a.id===g.id)this.flyoutpane.is(":visible")||(a=this.get_selected(),this.flyout_position(a),this.flyoutpane.show(),this.input.trigger("fb-flyoutpane-show",this));else if((g=b.suggest.flyout.cache[a.id])&&g.id&&g.html)this.flyout_response(g);else{var f=a.id,d={url:this.flyout_url.replace(/\$\{id\}/g,a.id),traditional:!0,
+beforeSend:function(){var a=c.input.data("flyout.request.count.suggest")||0,a=a+1;c.trackEvent(c.name,"flyout.request","count",a);c.input.data("flyout.request.count.suggest",a)},success:function(a){a["req:id"]=f;a.result&&a.result.length&&(a.html=b.suggest.suggest.create_flyout(a.result[0],c.flyout_image_url));b.suggest.flyout.cache[f]=a;c.flyout_response(a)},error:function(a){c.trackEvent(c.name,"flyout","error",{url:this.url,response:a?a.responseText:""})},complete:function(a){a&&c.trackEvent(c.name,
+"flyout","tid",a.getResponseHeader("X-Metaweb-TID"))},dataType:"jsonp",cache:!0};e.flyout_lang&&(d.data={lang:e.flyout_lang});clearTimeout(this.flyout_request.timeout);this.flyout_request.timeout=setTimeout(function(){b.ajax(d)},e.xhr_delay);this.input.trigger("fb-request-flyout",d)}},flyout_response:function(a){var b=this.pane,e=this.get_selected()||[];if(b.is(":visible")&&e.length&&(b=e.data("data.suggest"))&&a["req:id"]===b.id&&a.html)this.flyoutpane.html(a.html),this.flyout_position(e),this.flyoutpane.show().data("data.suggest",
+b),this.input.trigger("fb-flyoutpane-show",this)},flyout_position:function(a){if(!this.options.flyout_parent){var c=this.pane,e=this.flyoutpane,g=this.options.css,f=v,d=parseInt(e.css("top"),10),i=parseInt(e.css("left"),10),h=c.offset(),j=c.outerWidth(),m=e.outerHeight(),l=e.outerWidth();if("bottom"===this.options.flyout)f=h,j=this.input.offset(),f.top=h.top<j.top?f.top-m:f.top+c.outerHeight(),e.addClass(g.flyoutpane+"-bottom");else{f=a.offset();a=a.outerHeight();f.left+=j;var n=f.left+l,c=b(document.body).scrollLeft(),
+k=b(window).width()+c;f.top=f.top+a-m;f.top<h.top&&(f.top=h.top);n>k&&(h=f.left-(j+l),h>c&&(f.left=h));e.removeClass(g.flyoutpane+"-bottom")}f.top===d&&f.left===i||e.css({top:f.top,left:f.left})}},hoverout_list:function(){this.flyoutpane&&!this.get_selected()&&this.flyoutpane.hide()}});b.extend(b.suggest.suggest,{defaults:{filter:null,spell:"always",exact:!1,scoring:null,lang:null,key:null,prefixed:!0,stemmed:null,format:null,advanced:!0,show_id:!0,query_param_name:"query",service_url:"https://www.googleapis.com/freebase/v1",
+service_path:"/search",align:null,flyout:!0,flyout_service_url:null,flyout_service_path:"/search?filter=(all mid:${id})&output=(notable:/client/summary (description citation provenance) type)&key=${key}",flyout_image_service_url:null,flyout_image_service_path:"/image${id}?maxwidth=75&key=${key}&errorid=/freebase/no_image_png",flyout_parent:null,suggest_new:null,nomatch:{title:"No suggested matches",heading:"Tips on getting better suggestions:",tips:["Enter more or fewer characters","Add words related to your original search",
+"Try alternate spellings","Check your spelling"]},css:{item_type:"fbs-item-type",flyoutpane:"fbs-flyout-pane"},xhr_delay:200},get_value_by_keys:function(a,c){for(var e=b.isArray(c)?c:Array.prototype.slice.call(arguments,1),g=0;g<e.length&&!(a=a[e[g]],null==a);g++);return a},get_value:function(a,c,e){if(null==a||null==c||0==c.length)return null;b.isArray(c)||(c=[c]);a=b.suggest.suggest.get_value_by_keys(a,c);if(e){if(null==a)return[];b.isArray(a)||(a=[a]);var g=[];b.each(a,function(a,c){if("object"===
+b.type(c))if(null!=c.name)c=c.name;else if(c.id||c.mid)c=c.id||c.mid;else if(null!=c.value){var e=[];b.each(c,function(a,b){"value"!==a&&e.push(b)});c=c.value;e.length&&(c+=" ("+e.join(", ")+")")}b.isArray(c)&&c.length&&(c=c[0].value);null!=c&&g.push(c)});return g}return null==a?null:a},is_commons_id:function(a){return/^\/base\//.test(a)||/^\/user\//.test(a)?!1:!0},create_flyout:function(a,c){var e=b.suggest.suggest.get_value_by_keys,g=b.suggest.suggest.get_value,f=b.suggest.is_system_type,d=a.name,
+i=null,h=null,j=[],m=[],l={};if((i=g(a,"notable"))&&i.name)m.push(i.name),l[i.name]=!0;i&&f(i.id)?i=a.id:(i=a.mid,h=c.replace(/\$\{id\}/g,i));var n=null,k=f=null,r=null,s=e(a,"output","description","/common/topic/description")||[];if(s.length){var o=s[0];b.each(s,function(a,b){if(e(b,"citation",0,"mid")=="/m/0d07ph"){o=b;return false}return true});n=b.isArray(o.value)&&o.value.length?o.value[0].value:o.value;if("REQUIRES_CITATION"==e(o,"provenance",0,"restrictions",0)){f=e(o,"provenance",0,"source",
+0);if((k=e(o,"citation","provider",0,"name"))&&b.isArray(k)&&k.length)k=k[0].value;if((r=e(o,"citation","statement",0))&&r.value)r=r.value}}else b.each(["wikipedia","freebase"],function(b,c){if((s=g(a,["output","description",c],true))&&s.length){n=s[0];k=c;return false}return true});var p=g(a,["output","notable:/client/summary"]);if(p){var t=g(p,"/common/topic/notable_paths");t&&t.length&&b.each(t,function(a,b){var c=g(p,b,true);if(c&&c.length){var c=c.slice(0,3),d=b.split("/").pop();j.push([d,c.join(", ")])}})}(t=
+g(a,["output","type","/type/object/type"],!0))&&t.length&&b.each(t,function(a,b){if(!l[b]){m.push(b);l[b]=true}});var q=b('<div class="fbs-flyout-content">');d&&q.append(b('<h1 id="fbs-flyout-title">').text(d));q.append(b('<h3 class="fbs-topic-properties fbs-flyout-id">').text(i));j=j.slice(0,3);b.each(j,function(a,c){q.append(b('<h3 class="fbs-topic-properties">').append(b("<strong>").text(c[0]+": ")).append(document.createTextNode(c[1])))});n&&(d=b('<p class="fbs-topic-article">'),k&&(f?d.append(b('<a class="fbs-citation">').attr("href",
+f).attr("title",r||k).text("["+k+"]")):d.append(b('<em class="fbs-citation">').attr("title",r||k).text("["+k+"] "))),d.append(document.createTextNode(" "+n)),q.append(d));h&&(q.children().addClass("fbs-flyout-image-true"),q.prepend(b('<img id="fbs-topic-image" class="fbs-flyout-image-true" src="'+h+'">')));h=b('<span class="fbs-flyout-types">').text(m.slice(0,3).join(", "));h=b('<div class="fbs-attribution">').append(h);return b("<div>").append(q).append(h).html()}});document.createElement("input")})(jQuery);

main/webapp/modules/core/langs/translation-default.json

@@ -1,4 +1,4 @@
-{
+{
     "core-index": {
 		"slogan": "A power tool for working with messy data",
 		"help": "Help",
@@ -226,7 +226,6 @@
 		"long-format": "Long locale format",
 		"full-format": "Full locale format",
 		"custom": "Custom",
-		"help": "Help",
 		"local-time": "Use local time zone",
 		"omit-time": "Omit time",
 		"out-col-header": "Output column headers",
@@ -264,7 +263,6 @@
 		"facet-by-count": "Facet by choice counts",
 		"edit-based-col": "Edit Facet's Expression based on Column",
 		"edit-facet-exp": "Edit Facet's Expression",
-		"current-exp": "Current Expression",
 		"set-max-choices": "Set the maximum number of choices shown in each text facet (too many will slow down the application)",
 		"case-sensitive": "case sensitive",
 		"regular-exp": "regular expression",
@@ -410,7 +408,7 @@
 		"sort": "Sort",
 		"collapse-expand": "Collapse/expand columns to make viewing the data more convenient",
 		"collapse-this": "Collapse this column",
-		"collapse-all": "Collapse all other columns",
+		"collapse-other": "Collapse all other columns",
 		"collapse-left": "Collapse all columns to left",
 		"collapse-right": "Collapse all columns to right",
 		"reconcile": "Reconcile",
@@ -560,7 +558,6 @@
 		"view": "View",
 		"collapse-all": "Collapse all columns",
 		"expand-all": "Expand all columns",
-		"remove-sort": "Remove sort",
 		"reorder-perma": "Reorder rows permanently",
 		"by": "By",
 		"custom-text-trans": "Custom text transform on column",

main/webapp/modules/core/langs/translation-en.json

@@ -1,4 +1,4 @@
-{
+{
     "core-index": {
 		"slogan": "A power tool for working with messy data",
 		"help": "Help",
@@ -226,7 +226,6 @@
 		"long-format": "Long locale format",
 		"full-format": "Full locale format",
 		"custom": "Custom",
-		"help": "Help",
 		"local-time": "Use local time zone",
 		"omit-time": "Omit time",
 		"out-col-header": "Output column headers",
@@ -264,7 +263,6 @@
 		"facet-by-count": "Facet by choice counts",
 		"edit-based-col": "Edit Facet's Expression based on Column",
 		"edit-facet-exp": "Edit Facet's Expression",
-		"current-exp": "Current Expression",
 		"set-max-choices": "Set the maximum number of choices shown in each text facet (too many will slow down the application)",
 		"case-sensitive": "case sensitive",
 		"regular-exp": "regular expression",
@@ -410,7 +408,7 @@
 		"sort": "Sort",
 		"collapse-expand": "Collapse/expand columns to make viewing the data more convenient",
 		"collapse-this": "Collapse this column",
-		"collapse-all": "Collapse all other columns",
+		"collapse-other": "Collapse all other columns",
 		"collapse-left": "Collapse all columns to left",
 		"collapse-right": "Collapse all columns to right",
 		"reconcile": "Reconcile",
@@ -560,7 +558,6 @@
 		"view": "View",
 		"collapse-all": "Collapse all columns",
 		"expand-all": "Expand all columns",
-		"remove-sort": "Remove sort",
 		"reorder-perma": "Reorder rows permanently",
 		"by": "By",
 		"custom-text-trans": "Custom text transform on column",

main/webapp/modules/core/langs/translation-it.json

@@ -1,4 +1,4 @@
-{
+{
     "core-index": {
 		"slogan": "Uno strumento potente, per lavorare con dati sporchi",
 		"help": "Aiuto",
@@ -226,7 +226,6 @@
 		"long-format": "Formato lungo",
 		"full-format": "Formato completo",
 		"custom": "Personalizzato",
-		"help": "Aiuto",
 		"local-time": "Usa fuso orario locale",
 		"omit-time": "Ometti orario",
 		"out-col-header": "Esporta le intestazioni delle colonne",
@@ -264,7 +263,6 @@
 		"facet-by-count": "Faccetta per quantità alternative",
 		"edit-based-col": "Modifica l'espressione della faccetta basandoti sulla colonna",
 		"edit-facet-exp": "Modifica l'espressione della faccetta",
-		"current-exp": "Espressione corrente",
 		"set-max-choices": "Imposta il massimo numero di alternative da mostrare in ogni faccetta di testo (se troppe l'applicazione subirà rallentamenti)",
 		"case-sensitive": "case sensitive",
 		"regular-exp": "espressione regolare",
@@ -410,7 +408,7 @@
 		"sort": "Ordina",
 		"collapse-expand": "Collassa/espandi colonne per rendere più comoda la visualizzazione",
 		"collapse-this": "Collassa questa colonna",
-		"collapse-all": "Collassa tutte le altre colonne",
+		"collapse-other": "Collassa tutte le altre colonne",
 		"collapse-left": "Collassa tutte le colonne a sinistra",
 		"collapse-right": "Collassa tutte le colonne a destra",
 		"reconcile": "Riconcilia",
@@ -560,7 +558,6 @@
 		"view": "Vista",
 		"collapse-all": "Collassa tutte le colonne",
 		"expand-all": "Espandi tutte le colonne",
-		"remove-sort": "Rimuovi ordinamento",
 		"reorder-perma": "Riordina righe permanentemente",
 		"by": "Per",
 		"custom-text-trans": "Trasformazione di testo personalizzata sulla colonna",

main/webapp/modules/core/scripts/index.js

@@ -45,7 +45,8 @@ $.ajax({
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "core",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

main/webapp/modules/core/scripts/index/create-project-ui.js

@@ -54,7 +54,7 @@ Refine.CreateProjectUI = function(elmt) {
   $('#or-create-from').text($.i18n._('core-index-create')["from"]);
   
   $('#create-project-progress-cancel-button').text($.i18n._('core-buttons')["cancel"]);
-  $('#create-project-error-ok-button').text($.i18n._('core-buttons')["ok"]);
+  $('#create-project-error-ok-button').html($.i18n._('core-buttons')["ok"]);
   
   $.post(
     "command/core/get-importing-configuration",

main/webapp/modules/core/scripts/index/lang-settings-ui.js

@@ -11,11 +11,12 @@ Refine.SetLanguageUI = function(elmt) {
 	
 	this._elmts.set_lan_btn.bind('click', function(e) {		
 		$.ajax({
-			url : "/command/core/set-language?",
+			url : "/command/core/set-preference?",
 			type : "POST",
 			async : false,
 			data : {
-				lng : $("#langDD option:selected").val()
+			  name : "userLang",
+				value : $("#langDD option:selected").val()
 			},
 			success : function(data) {
 				alert($.i18n._('core-index-lang')["page-reload"]);

main/webapp/modules/core/scripts/index/parser-interfaces/json-parser-ui.js

@@ -119,10 +119,10 @@ Refine.JsonParserUI.prototype._initialize = function() {
     this._optionContainerElmts.limitInput[0].value = this._config.limit.toString();
   }
   if (this._config.trimStrings) {
-    this._optionContainerElmts.trimStringsCheckbox.attr("checked", "checked");
+    this._optionContainerElmts.trimStringsCheckbox.attr("checked", "unchecked");
   }
   if (this._config.guessCellValueTypes) {
-    this._optionContainerElmts.guessCellValueTypesCheckbox.attr("checked", "checked");
+    this._optionContainerElmts.guessCellValueTypesCheckbox.attr("checked", "unchecked");
   }
   if (this._config.storeEmptyStrings) {
     this._optionContainerElmts.storeEmptyStringsCheckbox.attr("checked", "checked");

main/webapp/modules/core/scripts/index/parser-interfaces/xml-parser-ui.js

@@ -107,19 +107,19 @@ Refine.XmlParserUI.prototype._initialize = function() {
   $('#or-import-rows').text($.i18n._('core-index-parser')["rows-data"]);
   $('#or-import-load').text($.i18n._('core-index-parser')["load-at-most"]);
   $('#or-import-preserve').text($.i18n._('core-index-parser')["preserve-empty"]);
-  $('#or-import-trim').text($.i18n._('core-index-parser')["trim"]);
+  $('#or-import-trim').html($.i18n._('core-index-parser')["trim"]);
-  $('#or-import-parseCell').text($.i18n._('core-index-parser')["parse-cell"]);
+  $('#or-import-parseCell').html($.i18n._('core-index-parser')["parse-cell"]);
-  $('#or-import-store').text($.i18n._('core-index-parser')["store-source"]);
+  $('#or-import-store').html($.i18n._('core-index-parser')["store-source"]);
   
   if (this._config.limit > 0) {
     this._optionContainerElmts.limitCheckbox.attr("checked", "checked");
     this._optionContainerElmts.limitInput[0].value = this._config.limit.toString();
   }
   if (this._config.trimStrings) {
-    this._optionContainerElmts.trimStringsCheckbox.attr("checked", "checked");
+    this._optionContainerElmts.trimStringsCheckbox.attr("checked", "unchecked");
   }
   if (this._config.guessCellValueTypes) {
-    this._optionContainerElmts.guessCellValueTypesCheckbox.attr("checked", "checked");
+    this._optionContainerElmts.guessCellValueTypesCheckbox.attr("checked", "unchecked");
   }
   if (this._config.storeEmptyStrings) {
     this._optionContainerElmts.storeEmptyStringsCheckbox.attr("checked", "checked");

main/webapp/modules/core/scripts/project.js

@@ -42,7 +42,8 @@ $.ajax({
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "core",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

main/webapp/modules/core/scripts/project/process-panel.js

@@ -149,9 +149,10 @@ ProcessPanel.prototype._render = function(newData) {
     for (var i = 0; i < processes.length; i++) {
       var process = processes[i];
       if (process.status != "pending") {
-        Refine.setTitle(process.progress + "% "+elmts.or_proj_undo.html($.i18n._('core-project')["complete"]));
+        // TODO: We should be using formatting, not string concatenation here
+        Refine.setTitle(process.progress + "% "+$.i18n._('core-project')["complete"]);
         this._elmts.progressDescription.text(process.description);
-        this._elmts.progressSpan.text(process.progress  + '% '+elmts.or_proj_undo.html($.i18n._('core-project')["complete"]));
+        this._elmts.progressSpan.text(process.progress  + '% '+$.i18n._('core-project')["complete"]);
       }
       if ("onDone" in process) {
         newProcessMap[process.id] = process;
@@ -170,7 +171,7 @@ ProcessPanel.prototype._render = function(newData) {
       .click(function() {
         self._cancelAll();
         $(this).text($.i18n._('core-project')["canceling"]).unbind();
-      })
+      });
     
     this._div.fadeIn(200);
   }

main/webapp/modules/core/scripts/views/data-table/cell-ui.js

@@ -105,7 +105,7 @@ DataTableCellUI.prototype._render = function() {
       .appendTo(divContent);
 
       if (service && (service.view) && (service.view.url)) {
-        a.attr("href", service.view.url.replace("{{id}}", match.id));
+        a.attr("href", encodeURI(service.view.url.replace("{{id}}", match.id)));
       } else if (ReconciliationManager.isFreebaseIdOrMid(r.identifierSpace)) {
         a.attr("href", "http://www.freebase.com/view" + match.id);
       }
@@ -148,18 +148,18 @@ DataTableCellUI.prototype._render = function() {
             .text(candidate.name)
             .appendTo(li);
 
-            // TODO: replace view URL with local code?
             if ((service) && (service.view) && (service.view.url)) {
-              a.attr("href", service.view.url.replace("{{id}}", candidate.id));
+              a.attr("href", encodeURI(service.view.url.replace("{{id}}", candidate.id)));
             } else if (ReconciliationManager.isFreebaseIdOrMid(r.identifierSpace)) {
               a.attr("href", "http://www.freebase.com/view" + candidate.id);
             }
 
             var preview = null;
-            if ((service) && (service.preview)) {
+            if ((service) && (service.preview) 
+                && service.preview.url.indexOf("http://www.freebase.com/widget/topic") < 0) {
               preview = service.preview;
             } else if (ReconciliationManager.isFreebaseIdOrMid(r.identifierSpace)) {
-              preview = DataTableCellUI.topicBlockPreview;
+              preview = DataTableCellUI.internalPreview;
             }
             if (preview) {
               a.click(function(evt) {
@@ -436,30 +436,37 @@ DataTableCellUI.prototype._postProcessSeveralCells = function(command, params, b
   );
 };
 
-// FIXME: Topic Blocks are gone
+DataTableCellUI.internalPreview = {
-DataTableCellUI.topicBlockPreview = {
+  srchurl: 'https://www.googleapis.com/freebase/v1/search?filter=(all mid:${id})&output=(notable:/client/summary description type)&key='+CustomSuggest.FREEBASE_API_KEY+"&callback=?",
-  url: 'http://www.freebase.com/widget/topic{{id}}?mode=content&blocks=[{"block":"full_info"},{"block":"article_props"}]',
+  imgurl : 'https://www.googleapis.com/freebase/v1/image${id}?maxwidth=75&errorid=/freebase/no_image_png&key='+CustomSuggest.FREEBASE_API_KEY,
   width: 430,
   height: 300
 };
 
-// TODO: Inject code to format using Suggest here?
 DataTableCellUI.prototype._previewCandidateTopic = function(candidate, elmt, preview) {
   var self = this;
   var id = candidate.id;
-  var url = preview.url.replace("{{id}}", id);
-
   var fakeMenu = MenuSystem.createMenu();
   fakeMenu
   .width(414)
   .addClass('data-table-topic-popup')
   .html(DOM.loadHTML("core", "scripts/views/data-table/cell-recon-preview-popup-header.html"));
 
+  if (preview && preview.url) { // Service has a preview URL associated with it
+    var url = encodeURI(preview.srch.replace("{{id}}", id));
     var iframe = $('<iframe></iframe>')
     .width(preview.width)
     .height(preview.height)
     .attr("src", url)
     .appendTo(fakeMenu);
+  } else { // Otherwise use our internal preview
+    var url = encodeURI(DataTableCellUI.internalPreview.srchurl.replace("\${id}", id));
+    $.ajax(url,{dataType:"jsonp"}).done(function(searchResponse) {
+      var data = searchResponse.result[0];
+      var html = $.suggest.suggest.create_flyout(data, preview.imgurl);
+      fakeMenu.append(html);
+    });
+  }
 
   MenuSystem.showMenu(fakeMenu, function(){});
   MenuSystem.positionMenuLeftRight(fakeMenu, $(elmt));

main/webapp/modules/core/scripts/views/data-table/column-header-ui.js

@@ -151,7 +151,7 @@ DataTableColumnHeaderUI.prototype._createMenuForColumnHeader = function(elmt) {
           }
         },
         {
-          label: $.i18n._('core-views')["collapse-all"],
+          label: $.i18n._('core-views')["collapse-other"],
           click: function() {
             var collapsedColumnNames = {};
             for (var i = 0; i < theProject.columnModel.columns.length; i++) {

main/webapp/modules/core/scripts/views/data-table/menu-reconcile.js

@@ -240,7 +240,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
             ui.browsingEngine.addFacet(
                 "list", 
                 {
-                  "name" : column.name + ": "+$.i18n._('core-views')["best-cand-type.match"],
+                  "name" : column.name + ": "+$.i18n._('core-views')["best-cand-type-match"],
                   "columnName" : column.name, 
                   "expression" : 'forNonBlank(cell.recon.features.typeMatch, v, v, if(isNonBlank(value), "(unreconciled)", "(blank)"))'
                 },

refine

@@ -186,7 +186,7 @@ get_version() {
         fail "Must specify a version number"
     fi
 
-    NUM_VERSION=`echo $VERSION | sed -E 's/[a-zA-Z]+/./g'`
+    NUM_VERSION=`echo $VERSION | sed -E 's/-.*//g'`
     
     if [ "${NUM_VERSION}" = "" ] ; then
         fail "${VERSION} is not a valid version number"
@@ -208,12 +208,14 @@ get_version() {
 get_revision() {
     if [ -d ".svn" ] ; then
         INFO=`svn info`
+        REVISION=`echo $INFO | sed s/^$VERSION-//`
     elif [ -d ".git" ] ; then
         INFO=`git describe`
+        REVISION=`echo $INFO`
+	REVISION=${REVISION:4}
     else
         error "cannot obtain revision, exiting!"
     fi
-    REVISION=`echo $INFO | sed s/^$VERSION-//`
 }
     
 download() {
@@ -470,7 +472,7 @@ ant() {
     
     #export ANT_OPTS="-Xmx1024M"
     
-    "$ANT" -f build.xml $ANT_PARAMS -Dversion="$VERSION" -Dfull_version="$FULL_VERSION" -Drevision="$REVISION" $1 || error "Error while running ant task '$1'"
+    "$ANT" -f build.xml $ANT_PARAMS -Dversion="$VERSION" -Dfull_version="$FULL_VERSION"  $1 || error "Error while running ant task '$1'"
 }
 
 # ----------------------------------------------------------------------------------------------
@@ -482,15 +484,16 @@ dist() {
     
     echo "All distributions were built and are located at $REFINE_DIST_DIR"
     echo 
-    echo "Upload them to the distibution site, then prepend the GoogleRefineReleases array at"
+    echo "Upload them to the distibution site, then prepend the releases array at"
     echo
-    echo "   http://code.google.com/p/google-refine/source/browse/support/releases2.js"
+    echo "   https://github.com/OpenRefine/OpenRefine/tree/gh-pages/javascript/releases.js"
     echo
     echo "with"
     echo 
     echo "    {"
     echo "        \"description\": \"OpenRefine ${VERSION}\","
     echo "        \"version\": \"${VERSION}\","
+    # TODO: We need to modify version checking for the future
     echo "        \"revision\": \"${REVISION}\""
     echo "    },"
     echo
@@ -535,10 +538,15 @@ mac_dist() {
         rm "$REFINE_BUILD_DIR/temp_refine.dmg"
     fi
 
+# Sign the bundle with a self-signed cert so OS X doesn't frustrate users by making app invisible
+    codesign -s "OpenRefine Code Signing" "$REFINE_BUILD_DIR/mac/OpenRefine.app"
+    spctl --assess --type execute --verbose=4 "$REFINE_BUILD_DIR/mac/OpenRefine.app"
+
     TITLE="OpenRefine $VERSION"
     echo "Building MacOSX DMG for $TITLE"
     hdiutil create -srcfolder "$REFINE_BUILD_DIR/mac" -volname "$TITLE" -fs HFS+ -fsargs "-c c=64,a=16,e=16" -format UDRW -size ${SIZE}m "$REFINE_BUILD_DIR/temp_refine.dmg" || error "can't create empty DMG"
-    DEVICE=`hdiutil attach -readwrite -noverify -noautoopen "$REFINE_BUILD_DIR/temp_refine.dmg" | egrep '^/dev/' | sed 1q | awk '{print $1}'`
+    DEVICE=`hdiutil attach -readwrite -noverify -noautoopen "$REFINE_BUILD_DIR/temp_refine.dmg" | egrep '^/dev/' | sed -e "s/^\/dev\///g" -e 1q  | awk '{print $1}'`
+    echo $DEVICE
     hdiutil attach "$REFINE_BUILD_DIR/temp_refine.dmg" || error "Can't attach temp DMG"
     
     echo '
@@ -567,16 +575,17 @@ mac_dist() {
     
     sync
     sync
+    sleep 3
     hdiutil detach $DEVICE
     
-    if [ -f "$REFINE_DIST_DIR/openrefine-$VERSION-$REVISION.dmg" ] ; then
+    if [ -f "$REFINE_DIST_DIR/openrefine-mac-$VERSION.dmg" ] ; then
-        rm "$REFINE_DIST_DIR/openrefine-$VERSION-$REVISION.dmg"
+        rm "$REFINE_DIST_DIR/openrefine-mac-$VERSION.dmg"
     fi
     
-    hdiutil convert "$REFINE_BUILD_DIR/temp_refine.dmg" -format UDZO -imagekey zlib-level=9 -o "$REFINE_DIST_DIR/openrefine-$VERSION-$REVISION.dmg" || error "Error compressing DMG"
+    hdiutil convert "$REFINE_BUILD_DIR/temp_refine.dmg" -format UDZO -imagekey zlib-level=9 -o "$REFINE_DIST_DIR/openrefine-mac-$VERSION.dmg" || error "Error compressing DMG"
-    hdiutil internet-enable -yes "$REFINE_DIST_DIR/openrefine-$VERSION-$REVISION.dmg" || error "Error internet-enabling DMG"
+    hdiutil internet-enable -yes "$REFINE_DIST_DIR/openrefine-mac-$VERSION.dmg" || error "Error internet-enabling DMG"
     
-    #rm -f "$REFINE_BUILD_DIR/temp_refine.dmg"
+    rm -f "$REFINE_BUILD_DIR/temp_refine.dmg"
 }
 
 test() {

refine.ini

@@ -5,10 +5,10 @@ no_proxy="localhost,127.0.0.1"
 #REFINE_PORT=3334
 #REFINE_HOST=127.0.0.1
 #REFINE_WEBAPP=main\webapp
-REFINE_MEMORY=3000M
+REFINE_MEMORY=1400M
 
 # Some sample configurations. These have no defaults.
 #ANT_HOME=C:\grefine\tools\apache-ant-1.8.1
 #JAVA_HOME=C:\Program Files\Java\jdk1.6.0_25
 #JAVA_OPTIONS=-XX:+UseParallelGC -verbose:gc -Drefine.headless=true
-JAVA_OPTIONS=-Drefine.headless=true
+#JAVA_OPTIONS=-Drefine.data_dir=C:\Users\user\AppData\Roaming\OpenRefine

server/src/com/google/refine/Refine.java

@@ -40,6 +40,7 @@ import java.io.File;
 import java.io.FileFilter;
 import java.io.IOException;
 import java.lang.reflect.Method;
+import java.net.BindException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -115,6 +116,7 @@ public class Refine {
         boolean headless = Configurations.getBoolean("refine.headless",false);
         if (headless) {
             System.setProperty("java.awt.headless", "true");
+            logger.info("Running in headless mode");
         } else {
             try {
                 RefineClient client = new RefineClient();
@@ -195,7 +197,12 @@ class RefineServer extends Server {
         }
         
         // start the server
+        try {
             this.start();
+        } catch (BindException e) {
+            logger.error("Failed to start server - is there another copy running already on this port/address?");
+            throw e;
+        }
         
         configure(context);
     }
@@ -302,6 +309,7 @@ class RefineServer extends Server {
         }
         
         File dataDir = null;
+        File grefineDir = null;
         File gridworksDir = null;
         
         String os = System.getProperty("os.name").toLowerCase();
@@ -312,10 +320,13 @@ class RefineServer extends Server {
                 // so we're using a library that uses JNI to ask directly the win32 APIs, 
                 // it's not elegant but it's the safest bet.
                 
+                dataDir = new File(fixWindowsUnicodePath(JDataPathSystem.getLocalSystem()
+                        .getLocalDataPath("OpenRefine").getPath()));
+
                 DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Google");
 
                 // new: ./Google/Refine old: ./Gridworks
-                dataDir = new File(new File(fixWindowsUnicodePath(localDataPath.getPath())), "Refine");
+                grefineDir = new File(new File(fixWindowsUnicodePath(localDataPath.getPath())), "Refine");
                 gridworksDir = new File(fixWindowsUnicodePath(JDataPathSystem.getLocalSystem()
                         .getLocalDataPath("Gridworks").getPath()));
             } catch (Error e) {
@@ -344,17 +355,20 @@ class RefineServer extends Server {
                     parentDir = new File(".");
                 }
                 
-                dataDir = new File(new File(parentDir, "Google"), "Refine");
+                dataDir = new File(parentDir, "OpenRefine");
+                grefineDir = new File(new File(parentDir, "Google"), "Refine");
                 gridworksDir = new File(parentDir, "Gridworks");
             }
         } else if (os.contains("mac os x")) {
             // on macosx, use "~/Library/Application Support"
             String home = System.getProperty("user.home");
             
-            // TODO: Update needed (again)
+            String data_home = (home != null) ? home + "/Library/Application Support/OpenRefine" : ".openrefine";
-            String data_home = (home != null) ? home + "/Library/Application Support/Google/Refine" : ".google-refine";
             dataDir = new File(data_home);
             
+            String grefine_home = (home != null) ? home + "/Library/Application Support/Google/Refine" : ".google-refine";
+            grefineDir = new File(grefine_home);
+            
             String gridworks_home = (home != null) ? home + "/Library/Application Support/Gridworks" : ".gridworks"; 
             gridworksDir = new File(gridworks_home);
         } else { // most likely a UNIX flavor
@@ -369,16 +383,27 @@ class RefineServer extends Server {
                 data_home = home + "/.local/share";
             }
             
-            dataDir = new File(data_home + "/google/refine");
+            dataDir = new File(data_home + "/openrefine");
+            grefineDir = new File(data_home + "/google/refine");
             gridworksDir = new File(data_home + "/gridworks");
         }
         
-        // If refine data dir doesn't exist, try to find and move gridworks data dir over
+        // If refine data dir doesn't exist, try to find and move Google Refine or Gridworks data dir over
-        if (!dataDir.exists() && gridworksDir.exists()) {
+        if (!dataDir.exists()) {
-            if (!dataDir.getParentFile().mkdirs()) {
+            if (grefineDir.exists()) {
-                logger.error("FAILED to create parent directory for workspace rename target " 
+                if (gridworksDir.exists()) {
-                        + dataDir.getParent());
+                    logger.warn("Found both Gridworks: " + gridworksDir
+                            + " & Googld Refine dirs " + grefineDir) ;
+                }
+                if (grefineDir.renameTo(dataDir)) {
+                    logger.info("Renamed Google Refine directory " + grefineDir 
+                            + " to " + dataDir);
                 } else {
+                    logger.error("FAILED to rename Google Refine directory " 
+                            + grefineDir 
+                            + " to " + dataDir);
+                } 
+            } else if (gridworksDir.exists()) {
                 if (gridworksDir.renameTo(dataDir)) {
                     logger.info("Renamed Gridworks directory " + gridworksDir 
                             + " to " + dataDir);
@@ -434,6 +459,8 @@ class RefineClient extends JFrame implements ActionListener {
     
     private static final long serialVersionUID = 7886547342175227132L;
 
+    final static Logger logger = LoggerFactory.getLogger("refine-client");
+
     public static boolean MACOSX = (System.getProperty("os.name").toLowerCase().startsWith("mac os x"));
     
     private URI uri;
@@ -484,6 +511,9 @@ class RefineClient extends JFrame implements ActionListener {
     } 
     
     private void openBrowser() {
+        if (!Desktop.isDesktopSupported()) {
+            logger.warn("Java Desktop class not supported on this platform.  Please open %s in your browser",uri.toString());
+        }
         try {
             Desktop.getDesktop().browse(uri);
         } catch (IOException e) {

unsign

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+# Deactivates any embedded code signatures in a Mach-O binary.
+
+module MachO
+  class Unsign
+    def self.unsign(filename)
+      File.open(filename, "r+") do |f|
+        Unsign.new(f).headers
+      end
+    end
+
+    attr_accessor :headers
+
+    protected
+
+    FatHeader = Struct.new(:cpu_type, :cpu_subtype, :offset, :size, :align, :mach)
+    MachHeader = Struct.new(:cpu_type, :cpu_subtype, :filetype, :ncmds, :sizeofcmds, :flags, :reserved, :cmds)
+    LoadCommand = Struct.new(:cmd, :cmdsize)
+
+    def initialize(f)
+      @f = f
+      @headers = process
+    end
+
+    def debug(message)
+      puts message if ENV["DEBUG"]
+    end
+
+    def word_type
+      @big_endian ? 'N' : 'V'
+    end
+
+    def patch_code_signature(lc)
+      # just change LC_CODE_SIGNATURE to a high value that will be ignored by the loader
+      debug "PATCHING LC_CODE_SIGNATURE"
+      @f.seek(-8, IO::SEEK_CUR)
+      @f.write([0xff, lc.cmdsize].pack("#{word_type}2"))
+      lc
+    end
+
+    def process_mach
+      len = @x86_64 ? 7 : 6
+      header = MachHeader.new(*@f.read(len*4).unpack("#{word_type}#{len}"))
+      debug "MACH HEADER: #{header.inspect}"
+      header.cmds = (1..(header.ncmds)).collect do
+        lc = LoadCommand.new(*@f.read(8).unpack("#{word_type}2"))
+        debug "LOAD COMMAND: #{lc.inspect}"
+
+        lc = case lc.cmd
+        when 0x1d then patch_code_signature(lc)
+        else lc
+        end
+
+        @f.seek(lc.cmdsize - 8, IO::SEEK_CUR)
+
+        lc
+      end
+      header
+    end
+
+    def process_fat
+      num_arches, = @f.read(4).unpack("N")
+      arches = (1..num_arches).collect do
+        FatHeader.new(*@f.read(20).unpack("N5"))
+      end
+      debug "FAT HEADER: #{arches.inspect}"
+      arches.each do |arch|
+        @f.seek(arch.offset)
+        arch.mach = process
+      end
+      arches
+    end
+
+    def process
+      magic, = @f.read(4).unpack("N")
+      debug "MAGIC: 0x%08x" % magic
+      case magic
+      when 0xcafebabe then @big_endian, @x86_64 = false, false; process_fat
+      when 0xfeedface then @big_endian, @x86_64 = true,  false; process_mach
+      when 0xcffaedfe then @big_endian, @x86_64 = false, true;  process_mach
+      when 0xcefaedfe then @big_endian, @x86_64 = false, false; process_mach
+      else raise "unknown magic: 0x%08x" % magic
+      end
+    end
+  end
+end
+
+# command line driver
+if __FILE__ == $0
+  if ARGV.empty?
+    $stderr.puts "usage:  #{$0} filename ..."
+    exit 1
+  end
+
+  ARGV.each do |filename|
+    puts "removing signatures from: #{filename}"
+    MachO::Unsign::unsign(filename)
+  end
+end