Clean up localization - fixes #760, modifies pull request #755

- make all file loading relative to module base
- move core language files into appropriate place
- eliminate all SetLanguage commands and use SetPreference instead
- eliminate all LoadLanguage commands except for core's
- fix duplicate keys in JSON language files
- remove BOM from JSON language files

OPEN - task 760: Translations not being loaded from built kit 
http://github.com/OpenRefine/OpenRefine/issues/issue/760

extensions/freebase/module/MOD-INF/controller.js

@@ -59,7 +59,6 @@ function init() {
   RS.registerCommand(module, "import-qa-data",        new Packages.com.google.refine.freebase.commands.ImportQADataCommand());
   RS.registerCommand(module, "mqlread",               new Packages.com.google.refine.freebase.commands.MQLReadCommand());
   RS.registerCommand(module, "mqlwrite",              new Packages.com.google.refine.freebase.commands.MQLWriteCommand());
-  RS.registerCommand(module, "load-language",		  new Packages.com.google.refine.freebase.commands.LoadLanguageCommand());
 
   var OR = Packages.com.google.refine.operations.OperationRegistry;
 

extensions/freebase/module/langs/translation-default.json

@@ -1,4 +1,4 @@
-{
+{
     "fb-schema-alignment": {
         "close-confirm": "There are unsaved changes. Close anyway?",
         "status-warning": "There are unsaved changes.",
@@ -112,4 +112,4 @@
 		"load-sandbox": "Load to Sandbox",
 		"ok": "Ok"
     }
-}
+}

extensions/freebase/module/langs/translation-en.json

@@ -1,4 +1,4 @@
-{
+{
     "fb-schema-alignment": {
         "close-confirm": "There are unsaved changes. Close anyway?",
         "status-warning": "There are unsaved changes.",
@@ -122,4 +122,4 @@
 		"load-sandbox": "Load to Sandbox",
 		"ok": "Ok"
     }
-}
+}

extensions/freebase/module/langs/translation-it.json

@@ -1,4 +1,4 @@
-{
+{
     "fb-schema-alignment": {
         "close-confirm": "Ci sono cambiamenti non salvati. Chiudere comunque?",
         "status-warning": "Ci sono cambiamenti non salvati.",
@@ -122,4 +122,4 @@
 		"load-sandbox": "Carica nella Sandbox",
 		"ok": "Ok"
     }
-}
+}

extensions/freebase/module/scripts/extension.js

@@ -39,11 +39,12 @@ var lang = navigator.language.split("-")[0]
 		|| navigator.userLanguage.split("-")[0];
 var dictionary = "";
 $.ajax({
-	url : "/command/freebase/load-language?",
+	url : "/command/core/load-language?",
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "freebase",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

extensions/freebase/src/com/google/refine/freebase/commands/LoadLanguageCommand.java

@@ -1,73 +0,0 @@
-
-package com.google.refine.freebase.commands;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.refine.ProjectManager;
-import com.google.refine.commands.Command;
-
-public class LoadLanguageCommand extends Command {
-
-    public LoadLanguageCommand() {
-        // TODO Auto-generated constructor stub
-    }
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        doPost(request, response);
-    }
-
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-
-        String rawDirectoryFile = request.getSession().getServletContext()
-                .getRealPath("extensions/freebase/module/langs/");
-        String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "");
-
-        BufferedReader reader = null;String param = null;
-        try {
-            param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
-        } catch (NullPointerException e) {
-        }
-        if (param == null) param = request.getParameter("lng");
-
-        String[] langs = param.split(" ");
-        try {
-            String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
-            reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-        } catch (FileNotFoundException e1) {
-            try {
-                String file = cleanedDirectory + File.separator + "translation-default.json";
-                reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-            } catch (FileNotFoundException e3) {
-                e3.printStackTrace();
-            }
-        }
-
-        String line = null;
-        String message = new String();
-        if (reader != null) {
-            while ((line = reader.readLine()) != null) {
-                // buffer.append(line);
-                message += line + System.getProperty("line.separator");
-            }
-        }
-
-        response.setCharacterEncoding("UTF-8");
-        response.setContentType("application/json");
-        response.getWriter().println(message);
-        response.getWriter().flush();
-        response.getWriter().close();
-    }
-
-}

extensions/gdata/module/MOD-INF/controller.js

@@ -43,7 +43,6 @@ function init() {
   var RS = Packages.com.google.refine.RefineServlet;
   RS.registerCommand(module, "deauthorize", Packages.com.google.refine.extension.gdata.DeAuthorizeCommand());
   RS.registerCommand(module, "upload", Packages.com.google.refine.extension.gdata.UploadCommand());
-  RS.registerCommand(module, "load-language", Packages.com.google.refine.extension.gdata.commands.LoadLanguageCommand());
 
   // Register importer and exporter
   var IM = Packages.com.google.refine.importing.ImportingManager;

extensions/gdata/module/langs/translation-default.json

@@ -1,4 +1,4 @@
-{
+{
     "gdata-import": {
 		"preparing": "Preparing ...",
 		"creating": "Creating project ...",
@@ -53,4 +53,4 @@
 		"authorize-label": "OpenRefine - Authorization",
 		"authorized-label": "Authorization process completed. Close this window and return to OpenRefine."
 	}
-}
+}

extensions/gdata/module/langs/translation-en.json

@@ -1,4 +1,4 @@
-{
+{
     "gdata-import": {
 		"preparing": "Preparing ...",
 		"creating": "Creating project ...",
@@ -53,4 +53,4 @@
 		"authorize-label": "OpenRefine - Authorization",
 		"authorized-label": "Authorization process completed. Close this window and return to OpenRefine."
 	}
-}
+}

extensions/gdata/module/langs/translation-it.json

@@ -1,4 +1,4 @@
-{
+{
     "gdata-import": {
 		"preparing": "In preparazione ...",
 		"creating": "Creazione il progetto ...",
@@ -53,4 +53,4 @@
 		"authorize-label": "OpenRefine - Autorizzazione",
 		"authorized-label": "Processo di autorizzazione completato. Chiudi questa finestra e torna ad OpenRefine."
 	}
-}
+}

extensions/gdata/module/scripts/index/importing-controller.js

@@ -36,11 +36,12 @@ var lang = navigator.language.split("-")[0]
 		|| navigator.userLanguage.split("-")[0];
 var dictionary = "";
 $.ajax({
-	url : "/command/gdata/load-language?",
+	url : "/command/core/load-language?",
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "gdata",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

extensions/gdata/module/scripts/project/exporters.js

@@ -33,11 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 var dictionary = "";
 $.ajax({
-	url : "/command/gdata/load-language?",
+	url : "/command/core/load-language?",
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "gdata",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

extensions/gdata/src/com/google/refine/extension/gdata/commands/LoadLanguageCommand.java

@@ -1,74 +0,0 @@
-
-package com.google.refine.extension.gdata.commands;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.refine.ProjectManager;
-import com.google.refine.commands.Command;
-
-public class LoadLanguageCommand extends Command {
-
-    public LoadLanguageCommand() {
-        // TODO Auto-generated constructor stub
-    }
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        doPost(request, response);
-    }
-
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-
-        String rawDirectoryFile = request.getSession().getServletContext()
-                .getRealPath("extensions/gdata/module/langs/");
-        String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "");
-
-        BufferedReader reader = null;
-        String param = null;
-        try {
-            param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
-        } catch (NullPointerException e) {
-        }
-        if (param == null) param = request.getParameter("lng");
-
-        String[] langs = param.split(" ");
-        try {
-            String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
-            reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-        } catch (FileNotFoundException e1) {
-            try {
-                String file = cleanedDirectory + File.separator + "translation-default.json";
-                reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
-            } catch (FileNotFoundException e3) {
-                e3.printStackTrace();
-            }
-        }
-
-        String line = null;
-        String message = new String();
-        if (reader != null) {
-            while ((line = reader.readLine()) != null) {
-                // buffer.append(line);
-                message += line + System.getProperty("line.separator");
-            }
-        }
-
-        response.setCharacterEncoding("UTF-8");
-        response.setContentType("application/json");
-        response.getWriter().println(message);
-        response.getWriter().flush();
-        response.getWriter().close();
-    }
-
-}

main/src/com/google/refine/commands/lang/LoadLanguageCommand.java

@@ -7,18 +7,28 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Arrays;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
 import com.google.refine.ProjectManager;
 import com.google.refine.commands.Command;
+import com.google.refine.preference.PreferenceStore;
+
+import edu.mit.simile.butterfly.ButterflyModule;
+
 
 public class LoadLanguageCommand extends Command {
 
     public LoadLanguageCommand() {
-        // TODO Auto-generated constructor stub
+    	super();
     }
 
     @Override
@@ -30,44 +40,47 @@ public class LoadLanguageCommand extends Command {
     public void doPost(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
 
-        String rawDirectoryFile = request.getSession().getServletContext().getRealPath("webapp/modules/langs/");
+        String modname = request.getParameter("module");
-        String cleanedDirectory = rawDirectoryFile.replace("main" + File.separator + "webapp" + File.separator, "main"
+        if (modname == null) {
-                + File.separator);
+            modname = "core";
-
-        BufferedReader reader = null;
-        String param = null;
-        try {
-            param = (String) ProjectManager.singleton.getPreferenceStore().get("userLang");
-        } catch (NullPointerException e) {
         }
-        if (param == null) param = request.getParameter("lng");
+        ButterflyModule module = this.servlet.getModule(modname);
 
-        String[] langs = param.split(" ");
+        String[] langs = request.getParameterValues("lang");
-        try {
+        if (langs == null || "".equals(langs[0])) {
-            String file = cleanedDirectory + File.separator + "translation-" + langs[0] + ".json";
+            PreferenceStore ps = ProjectManager.singleton.getPreferenceStore();
-            reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
+            if (ps != null) {
-        } catch (FileNotFoundException e1) {
+                langs = new String[] {(String) ps.get("userLang")};
+            }
+        }
+        langs = Arrays.copyOf(langs, langs.length+1);
+        langs[langs.length-1] = "default";
+
+        JSONObject json = null;
+        boolean loaded = false;
+        for (String lang : langs) {
+            File langFile = new File(module.getPath(), "langs" + File.separator + "translation-" + lang + ".json");
             try {
-                String file = cleanedDirectory + File.separator + "translation-default.json";
+                Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(langFile), "UTF-8"));
-                reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
+                json = new JSONObject(new JSONTokener(reader));
-            } catch (FileNotFoundException e3) {
+                response.setCharacterEncoding("UTF-8");
-                e3.printStackTrace();
+                response.setContentType("application/json");
+                json.write(response.getWriter());
+                response.getWriter().flush();
+                response.getWriter().close();
+                loaded = true;
+                break;
+            } catch (FileNotFoundException e1) {
+                json = null;
+                continue;
+            } catch (JSONException e) {
+                json = null;
+                logger.error("JSON error reading/writing language file", e);
+                continue;
             }
         }
-
+        if (!loaded) {
-        String line = null;
+        	logger.error("Failed to load any language files");
-        String message = new String();
-        if (reader != null) {
-            while ((line = reader.readLine()) != null) {
-                // buffer.append(line);
-                message += line + System.getProperty("line.separator");
-            }
         }
-
-        response.setCharacterEncoding("UTF-8");
-        response.setContentType("application/json");
-        response.getWriter().println(message);
-        response.getWriter().flush();
-        response.getWriter().close();
     }
 }

main/src/com/google/refine/commands/lang/SetLanguageCommand.java

@@ -1,33 +0,0 @@
-
-package com.google.refine.commands.lang;
-
-import java.io.IOException;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.refine.ProjectManager;
-import com.google.refine.commands.Command;
-import com.google.refine.preference.PreferenceStore;
-
-public class SetLanguageCommand extends Command {
-
-    public SetLanguageCommand() {
-        // TODO Auto-generated constructor stub
-    }
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        doPost(request, response);
-    }
-
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
-            throws ServletException, IOException {
-        String lang = request.getParameter("lng");
-        PreferenceStore pref = ProjectManager.singleton.getPreferenceStore();
-        
-        pref.put("userLang", lang);
-    }
-
-}

main/webapp/modules/core/MOD-INF/controller.js

@@ -99,7 +99,6 @@ function registerCommands() {
   RS.registerCommand(module, "key-value-columnize", new Packages.com.google.refine.commands.cell.KeyValueColumnizeCommand());
 
   RS.registerCommand(module, "load-language", Packages.com.google.refine.commands.lang.LoadLanguageCommand());
-  RS.registerCommand(module, "set-language", Packages.com.google.refine.commands.lang.SetLanguageCommand());
   
   RS.registerCommand(module, "add-column", new Packages.com.google.refine.commands.column.AddColumnCommand());
   RS.registerCommand(module, "add-column-by-fetching-urls", new Packages.com.google.refine.commands.column.AddColumnByFetchingURLsCommand());

main/webapp/modules/core/langs/translation-default.json

@@ -1,4 +1,4 @@
-{
+{
     "core-index": {
 		"slogan": "A power tool for working with messy data",
 		"help": "Help",
@@ -226,7 +226,6 @@
 		"long-format": "Long locale format",
 		"full-format": "Full locale format",
 		"custom": "Custom",
-		"help": "Help",
 		"local-time": "Use local time zone",
 		"omit-time": "Omit time",
 		"out-col-header": "Output column headers",
@@ -264,7 +263,6 @@
 		"facet-by-count": "Facet by choice counts",
 		"edit-based-col": "Edit Facet's Expression based on Column",
 		"edit-facet-exp": "Edit Facet's Expression",
-		"current-exp": "Current Expression",
 		"set-max-choices": "Set the maximum number of choices shown in each text facet (too many will slow down the application)",
 		"case-sensitive": "case sensitive",
 		"regular-exp": "regular expression",
@@ -410,7 +408,7 @@
 		"sort": "Sort",
 		"collapse-expand": "Collapse/expand columns to make viewing the data more convenient",
 		"collapse-this": "Collapse this column",
-		"collapse-all": "Collapse all other columns",
+		"collapse-other": "Collapse all other columns",
 		"collapse-left": "Collapse all columns to left",
 		"collapse-right": "Collapse all columns to right",
 		"reconcile": "Reconcile",
@@ -560,7 +558,6 @@
 		"view": "View",
 		"collapse-all": "Collapse all columns",
 		"expand-all": "Expand all columns",
-		"remove-sort": "Remove sort",
 		"reorder-perma": "Reorder rows permanently",
 		"by": "By",
 		"custom-text-trans": "Custom text transform on column",
@@ -666,4 +663,4 @@
 		"transpose": "Transpose",
 		"apply-to-all": "Apply to All Identical Cells"
 	}
-}
+}

main/webapp/modules/core/langs/translation-en.json

@@ -1,4 +1,4 @@
-{
+{
     "core-index": {
 		"slogan": "A power tool for working with messy data",
 		"help": "Help",
@@ -226,7 +226,6 @@
 		"long-format": "Long locale format",
 		"full-format": "Full locale format",
 		"custom": "Custom",
-		"help": "Help",
 		"local-time": "Use local time zone",
 		"omit-time": "Omit time",
 		"out-col-header": "Output column headers",
@@ -264,7 +263,6 @@
 		"facet-by-count": "Facet by choice counts",
 		"edit-based-col": "Edit Facet's Expression based on Column",
 		"edit-facet-exp": "Edit Facet's Expression",
-		"current-exp": "Current Expression",
 		"set-max-choices": "Set the maximum number of choices shown in each text facet (too many will slow down the application)",
 		"case-sensitive": "case sensitive",
 		"regular-exp": "regular expression",
@@ -410,7 +408,7 @@
 		"sort": "Sort",
 		"collapse-expand": "Collapse/expand columns to make viewing the data more convenient",
 		"collapse-this": "Collapse this column",
-		"collapse-all": "Collapse all other columns",
+		"collapse-other": "Collapse all other columns",
 		"collapse-left": "Collapse all columns to left",
 		"collapse-right": "Collapse all columns to right",
 		"reconcile": "Reconcile",
@@ -560,7 +558,6 @@
 		"view": "View",
 		"collapse-all": "Collapse all columns",
 		"expand-all": "Expand all columns",
-		"remove-sort": "Remove sort",
 		"reorder-perma": "Reorder rows permanently",
 		"by": "By",
 		"custom-text-trans": "Custom text transform on column",
@@ -666,4 +663,4 @@
 		"transpose": "Transpose",
 		"apply-to-all": "Apply to All Identical Cells"
 	}
-}
+}

main/webapp/modules/core/langs/translation-it.json

@@ -1,4 +1,4 @@
-{
+{
     "core-index": {
 		"slogan": "Uno strumento potente, per lavorare con dati sporchi",
 		"help": "Aiuto",
@@ -226,7 +226,6 @@
 		"long-format": "Formato lungo",
 		"full-format": "Formato completo",
 		"custom": "Personalizzato",
-		"help": "Aiuto",
 		"local-time": "Usa fuso orario locale",
 		"omit-time": "Ometti orario",
 		"out-col-header": "Esporta le intestazioni delle colonne",
@@ -264,7 +263,6 @@
 		"facet-by-count": "Faccetta per quantità alternative",
 		"edit-based-col": "Modifica l'espressione della faccetta basandoti sulla colonna",
 		"edit-facet-exp": "Modifica l'espressione della faccetta",
-		"current-exp": "Espressione corrente",
 		"set-max-choices": "Imposta il massimo numero di alternative da mostrare in ogni faccetta di testo (se troppe l'applicazione subirà rallentamenti)",
 		"case-sensitive": "case sensitive",
 		"regular-exp": "espressione regolare",
@@ -410,7 +408,7 @@
 		"sort": "Ordina",
 		"collapse-expand": "Collassa/espandi colonne per rendere più comoda la visualizzazione",
 		"collapse-this": "Collassa questa colonna",
-		"collapse-all": "Collassa tutte le altre colonne",
+		"collapse-other": "Collassa tutte le altre colonne",
 		"collapse-left": "Collassa tutte le colonne a sinistra",
 		"collapse-right": "Collassa tutte le colonne a destra",
 		"reconcile": "Riconcilia",
@@ -560,7 +558,6 @@
 		"view": "Vista",
 		"collapse-all": "Collassa tutte le colonne",
 		"expand-all": "Espandi tutte le colonne",
-		"remove-sort": "Rimuovi ordinamento",
 		"reorder-perma": "Riordina righe permanentemente",
 		"by": "Per",
 		"custom-text-trans": "Trasformazione di testo personalizzata sulla colonna",
@@ -666,4 +663,4 @@
 		"transpose": "Transponi",
 		"apply-to-all": "Applica a tutte le celle identiche"
 	}
-}
+}

main/webapp/modules/core/scripts/index.js

@@ -45,7 +45,8 @@ $.ajax({
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "core",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

main/webapp/modules/core/scripts/index/lang-settings-ui.js

@@ -11,11 +11,12 @@ Refine.SetLanguageUI = function(elmt) {
 	
 	this._elmts.set_lan_btn.bind('click', function(e) {		
 		$.ajax({
-			url : "/command/core/set-language?",
+			url : "/command/core/set-preference?",
 			type : "POST",
 			async : false,
 			data : {
-				lng : $("#langDD option:selected").val()
+			  name : "userLang",
+				value : $("#langDD option:selected").val()
 			},
 			success : function(data) {
 				alert($.i18n._('core-index-lang')["page-reload"]);

main/webapp/modules/core/scripts/project.js

@@ -42,7 +42,8 @@ $.ajax({
 	type : "POST",
 	async : false,
 	data : {
-		lng : lang
+	  module : "core",
+//		lang : lang
 	},
 	success : function(data) {
 		dictionary = data;

main/webapp/modules/core/scripts/views/data-table/column-header-ui.js

@@ -151,7 +151,7 @@ DataTableColumnHeaderUI.prototype._createMenuForColumnHeader = function(elmt) {
           }
         },
         {
-          label: $.i18n._('core-views')["collapse-all"],
+          label: $.i18n._('core-views')["collapse-other"],
           click: function() {
             var collapsedColumnNames = {};
             for (var i = 0; i < theProject.columnModel.columns.length; i++) {