Merge pull request #2210 from OpenRefine/issue-2209-language-fallback

Add server-side language fallback.

main/src/com/google/refine/commands/lang/LoadLanguageCommand.java

@@ -35,11 +35,14 @@ import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.UnsupportedEncodingException;
 import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map.Entry;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
 import com.google.refine.ProjectManager;
@@ -86,32 +89,34 @@ public class LoadLanguageCommand extends Command {
         }
         
         // Default language is English
-        langs = Arrays.copyOf(langs, langs.length+1);
+        if (langs.length == 0 || langs[langs.length-1] != "en" ) {
-        langs[langs.length-1] = "en";
+            langs = Arrays.copyOf(langs, langs.length+1);
+            langs[langs.length-1] = "en";
+        }
 
-        ObjectNode json = null;
+        ObjectNode translations = null;
-        boolean loaded = false;
+        for (int i = langs.length-1; i >= 0; i--) {
-        for (String lang : langs) {
+            if (langs[i] == null) continue;
-            if (lang == null) continue;
+            ObjectNode json = loadLanguage(this.servlet, modname, langs[i]);
-            json = loadLanguage(this.servlet, modname, lang);
             if (json != null) {
-                response.setCharacterEncoding("UTF-8");
+                if (translations == null) {
-                response.setContentType("application/json");
+                    translations = json;
-                try {
+                } else {
-                    ObjectNode node = ParsingUtilities.mapper.createObjectNode();
+                    translations = mergeLanguages(json, translations);
-                    node.put("dictionary", json);
-                    node.put("lang", new TextNode(lang));
-                	ParsingUtilities.mapper.writeValue(response.getWriter(), node);
-                } catch (IOException e) {
-                    logger.error("Error writing language labels to response stream");
                 }
-                response.getWriter().flush();
-                response.getWriter().close();
-                loaded = true;
-                break;
             }
         }
-        if (!loaded) {
+         
+        if (translations != null) {
+            try {
+                ObjectNode node = ParsingUtilities.mapper.createObjectNode();
+                node.put("dictionary", translations);
+                node.put("lang", new TextNode(langs[0]));
+            	respondJSON(response, node);
+            } catch (IOException e) {
+                logger.error("Error writing language labels to response stream");
+            }
+        } else {
         	logger.error("Failed to load any language files");
         }
     }
@@ -130,4 +135,29 @@ public class LoadLanguageCommand extends Command {
         }
         return null;
     }
+    
+    /**
+     * Perform a language fallback, server-side
+     * @param preferred
+     *      the JSON translation for the preferred language
+     * @param fallback
+     *      the JSON translation for the fallback language
+     * @return
+     *      a JSON object where values are from the preferred
+     *      language if available, and the fallback language otherwise
+     */
+    static ObjectNode mergeLanguages(ObjectNode preferred, ObjectNode fallback) {
+        ObjectNode results = ParsingUtilities.mapper.createObjectNode();
+        Iterator<Entry<String, JsonNode>> iterator = fallback.fields();
+        while(iterator.hasNext()) {
+            Entry<String,JsonNode> entry = iterator.next();
+            String code = entry.getKey();
+            JsonNode value = preferred.get(code);
+            if (value == null) {;
+                value = entry.getValue();
+            }
+            results.put(code, value);  
+        }
+        return results;
+    }
 }

main/tests/server/src/com/google/refine/commands/lang/LoadLanguageCommandTests.java

@@ -2,15 +2,9 @@ package com.google.refine.commands.lang;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.refine.RefineServlet;
-import com.google.refine.commands.CommandTestBase;
-import com.google.refine.util.ParsingUtilities;
-
-import edu.mit.simile.butterfly.ButterflyModule;
-
 import java.io.File;
 import java.io.IOException;
 
@@ -19,6 +13,16 @@ import javax.servlet.ServletException;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.refine.RefineServlet;
+import com.google.refine.commands.CommandTestBase;
+import com.google.refine.util.ParsingUtilities;
+
+import edu.mit.simile.butterfly.ButterflyModule;
+
 public class LoadLanguageCommandTests extends CommandTestBase {
 	
 	@BeforeMethod
@@ -26,11 +30,11 @@ public class LoadLanguageCommandTests extends CommandTestBase {
 		command = new LoadLanguageCommand();
 		ButterflyModule coreModule = mock(ButterflyModule.class);
 		
-        when(coreModule.getName()).thenReturn("core");
+		when(coreModule.getName()).thenReturn("core");
-        when(coreModule.getPath()).thenReturn(new File("webapp/modules/core"));
+		when(coreModule.getPath()).thenReturn(new File("webapp/modules/core"));
-        RefineServlet servlet = mock(RefineServlet.class);
+		RefineServlet servlet = mock(RefineServlet.class);
-        when(servlet.getModule("core")).thenReturn(coreModule);
+		when(servlet.getModule("core")).thenReturn(coreModule);
-        command.init(servlet);
+		command.init(servlet);
 	}
 	
 	@Test
@@ -44,5 +48,40 @@ public class LoadLanguageCommandTests extends CommandTestBase {
 		assertTrue(response.has("dictionary"));
 		assertTrue(response.has("lang"));
 	}
+	
+	@Test
+	public void testLoadUnknownLanguage() throws ServletException, IOException {
+		when(request.getParameter("module")).thenReturn("core");
+		when(request.getParameterValues("lang")).thenReturn(new String[] {"foobar"});
+		
+		command.doPost(request, response);
+		
+		JsonNode response = ParsingUtilities.mapper.readValue(writer.toString(), JsonNode.class);
+		assertTrue(response.has("dictionary"));
+		assertEquals(response.get("lang").asText(), "foobar");
+	}
+		
+	
+	@Test
+	public void testLanguageFallback() throws JsonParseException, JsonMappingException, IOException {
+		String fallbackJson = "{"
+				+ "\"foo\":\"hello\","
+				+ "\"bar\":\"world\""
+				+ "}";
+		String preferredJson = "{"
+				+ "\"foo\":\"hallo\""
+				+ "}";
+		String expectedJson = "{"
+				+ "\"foo\":\"hallo\","
+				+ "\"bar\":\"world\""
+				+ "}";
+		ObjectNode fallback = ParsingUtilities.mapper.readValue(fallbackJson, ObjectNode.class);
+		ObjectNode preferred = ParsingUtilities.mapper.readValue(preferredJson, ObjectNode.class);
+		ObjectNode expected = ParsingUtilities.mapper.readValue(expectedJson, ObjectNode.class);
+		
+		ObjectNode merged = LoadLanguageCommand.mergeLanguages(preferred, fallback);
+		
+		assertEquals(merged, expected);
+	}
 }