s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2210 from OpenRefine/issue-2209-language-fallback

Browse Source 
Add server-side language fallback.
This commit is contained in:
Antonin Delpeuch 2019-11-18 22:01:39 +00:00 committed by GitHub
commit b561824d04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 102 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
main/src/com/google/refine/commands/lang/LoadLanguageCommand.java
View File

@ -35,11 +35,14 @@ import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map.Entry;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import com.google.refine.ProjectManager;
@ -86,32 +89,34 @@ public class LoadLanguageCommand extends Command {
}
// Default language is English
langs = Arrays.copyOf(langs, langs.length+1);
langs[langs.length-1] = "en";
if (langs.length == 0 || langs[langs.length-1] != "en" ) {
langs = Arrays.copyOf(langs, langs.length+1);
langs[langs.length-1] = "en";
}
ObjectNode json = null;
boolean loaded = false;
for (String lang : langs) {
if (lang == null) continue;
json = loadLanguage(this.servlet, modname, lang);
ObjectNode translations = null;
for (int i = langs.length-1; i >= 0; i--) {
if (langs[i] == null) continue;
ObjectNode json = loadLanguage(this.servlet, modname, langs[i]);
if (json != null) {
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
try {
ObjectNode node = ParsingUtilities.mapper.createObjectNode();
node.put("dictionary", json);
node.put("lang", new TextNode(lang));
ParsingUtilities.mapper.writeValue(response.getWriter(), node);
} catch (IOException e) {
logger.error("Error writing language labels to response stream");
if (translations == null) {
translations = json;
} else {
translations = mergeLanguages(json, translations);
}
response.getWriter().flush();
response.getWriter().close();
loaded = true;
break;
}
}
if (!loaded) {
if (translations != null) {
try {
ObjectNode node = ParsingUtilities.mapper.createObjectNode();
node.put("dictionary", translations);
node.put("lang", new TextNode(langs[0]));
respondJSON(response, node);
} catch (IOException e) {
logger.error("Error writing language labels to response stream");
}
} else {
logger.error("Failed to load any language files");
}
}
@ -130,4 +135,29 @@ public class LoadLanguageCommand extends Command {
}
return null;
}
/**
* Perform a language fallback, server-side
* @param preferred
* the JSON translation for the preferred language
* @param fallback
* the JSON translation for the fallback language
* @return
* a JSON object where values are from the preferred
* language if available, and the fallback language otherwise
*/
static ObjectNode mergeLanguages(ObjectNode preferred, ObjectNode fallback) {
ObjectNode results = ParsingUtilities.mapper.createObjectNode();
Iterator<Entry<String, JsonNode>> iterator = fallback.fields();
while(iterator.hasNext()) {
Entry<String,JsonNode> entry = iterator.next();
String code = entry.getKey();
JsonNode value = preferred.get(code);
if (value == null) {;
value = entry.getValue();
}
results.put(code, value);
}
return results;
}
}

63
main/tests/server/src/com/google/refine/commands/lang/LoadLanguageCommandTests.java
View File

@ -2,15 +2,9 @@ package com.google.refine.commands.lang;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.refine.RefineServlet;
import com.google.refine.commands.CommandTestBase;
import com.google.refine.util.ParsingUtilities;
import edu.mit.simile.butterfly.ButterflyModule;
import java.io.File;
import java.io.IOException;
@ -19,6 +13,16 @@ import javax.servlet.ServletException;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.refine.RefineServlet;
import com.google.refine.commands.CommandTestBase;
import com.google.refine.util.ParsingUtilities;
import edu.mit.simile.butterfly.ButterflyModule;
public class LoadLanguageCommandTests extends CommandTestBase {
@BeforeMethod
@ -26,11 +30,11 @@ public class LoadLanguageCommandTests extends CommandTestBase {
command = new LoadLanguageCommand();
ButterflyModule coreModule = mock(ButterflyModule.class);
when(coreModule.getName()).thenReturn("core");
when(coreModule.getPath()).thenReturn(new File("webapp/modules/core"));
RefineServlet servlet = mock(RefineServlet.class);
when(servlet.getModule("core")).thenReturn(coreModule);
command.init(servlet);
when(coreModule.getName()).thenReturn("core");
when(coreModule.getPath()).thenReturn(new File("webapp/modules/core"));
RefineServlet servlet = mock(RefineServlet.class);
when(servlet.getModule("core")).thenReturn(coreModule);
command.init(servlet);
}
@Test
@ -44,5 +48,40 @@ public class LoadLanguageCommandTests extends CommandTestBase {
assertTrue(response.has("dictionary"));
assertTrue(response.has("lang"));
}
@Test
public void testLoadUnknownLanguage() throws ServletException, IOException {
when(request.getParameter("module")).thenReturn("core");
when(request.getParameterValues("lang")).thenReturn(new String[] {"foobar"});
command.doPost(request, response);
JsonNode response = ParsingUtilities.mapper.readValue(writer.toString(), JsonNode.class);
assertTrue(response.has("dictionary"));
assertEquals(response.get("lang").asText(), "foobar");
}
@Test
public void testLanguageFallback() throws JsonParseException, JsonMappingException, IOException {
String fallbackJson = "{"
+ "\"foo\":\"hello\","
+ "\"bar\":\"world\""
+ "}";
String preferredJson = "{"
+ "\"foo\":\"hallo\""
+ "}";
String expectedJson = "{"
+ "\"foo\":\"hallo\","
+ "\"bar\":\"world\""
+ "}";
ObjectNode fallback = ParsingUtilities.mapper.readValue(fallbackJson, ObjectNode.class);
ObjectNode preferred = ParsingUtilities.mapper.readValue(preferredJson, ObjectNode.class);
ObjectNode expected = ParsingUtilities.mapper.readValue(expectedJson, ObjectNode.class);
ObjectNode merged = LoadLanguageCommand.mergeLanguages(preferred, fallback);
assertEquals(merged, expected);
}
}