diff --git a/main/src/com/google/refine/io/FileProjectManager.java b/main/src/com/google/refine/io/FileProjectManager.java index 2204f37f2..75e4fa231 100644 --- a/main/src/com/google/refine/io/FileProjectManager.java +++ b/main/src/com/google/refine/io/FileProjectManager.java @@ -40,6 +40,8 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map.Entry; @@ -307,13 +309,14 @@ public class FileProjectManager extends ProjectManager { } protected boolean saveToFile(File file) throws IOException { - FileWriter writer = new FileWriter(file); + OutputStream stream = new FileOutputStream(file); boolean saveWasNeeded = saveNeeded(); try { - ParsingUtilities.defaultWriter.writeValue(writer, this); + // writeValue(OutputStream) is documented to use JsonEncoding.UTF8 + ParsingUtilities.defaultWriter.writeValue(stream, this); saveProjectMetadata(); } finally { - writer.close(); + stream.close(); } return saveWasNeeded; } diff --git a/main/src/com/google/refine/io/ProjectMetadataUtilities.java b/main/src/com/google/refine/io/ProjectMetadataUtilities.java index 61d5d227e..afddf947c 100644 --- a/main/src/com/google/refine/io/ProjectMetadataUtilities.java +++ b/main/src/com/google/refine/io/ProjectMetadataUtilities.java @@ -39,6 +39,7 @@ import java.io.FileReader; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; @@ -60,25 +61,33 @@ public class ProjectMetadataUtilities { public static void save(ProjectMetadata projectMeta, File projectDir) throws IOException { File tempFile = new File(projectDir, "metadata.temp.json"); saveToFile(projectMeta, tempFile); + if (tempFile.length() == 0) { + throw new IOException("Failed to save project metadata - keeping backups"); + } + + // TODO Do we want to make sure we can successfully deserialize the file too? File file = new File(projectDir, "metadata.json"); File oldFile = new File(projectDir, "metadata.old.json"); - if (oldFile.exists()) { - oldFile.delete(); - } - if (file.exists()) { + if(file.length() > 0) { + if (oldFile.exists()) { + oldFile.delete(); + } file.renameTo(oldFile); + } else { + file.delete(); + } } tempFile.renameTo(file); } protected static void saveToFile(ProjectMetadata projectMeta, File metadataFile) throws IOException { - Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile)); + Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile), StandardCharsets.UTF_8); try { - ParsingUtilities.defaultWriter.writeValue(writer, projectMeta); + ParsingUtilities.saveWriter.writeValue(writer, projectMeta); } finally { writer.close(); } @@ -121,7 +130,7 @@ public class ProjectMetadataUtilities { * creation and modification times based on whatever files are available. * * @param projectDir the project directory - * @param id the proejct id + * @param id the project id * @return */ static public ProjectMetadata recover(File projectDir, long id) { diff --git a/main/src/com/google/refine/preference/PreferenceStore.java b/main/src/com/google/refine/preference/PreferenceStore.java index 65bcfd69d..2f54b6204 100644 --- a/main/src/com/google/refine/preference/PreferenceStore.java +++ b/main/src/com/google/refine/preference/PreferenceStore.java @@ -37,6 +37,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.logging.Logger; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; @@ -100,6 +101,12 @@ public class PreferenceStore { if (entries.get(key) != null) { JsonNode o = entries.get(key); Object loaded = loadObject(o); + if (loaded == null) { + if ("scripting.starred-expressions".contentEquals(key)) { + // HACK to work around preferences corruption + loaded = new TopList(10); + } + } _prefs.put(key, loaded); } } diff --git a/main/tests/server/src/com/google/refine/io/FileProjectManagerTests.java b/main/tests/server/src/com/google/refine/io/FileProjectManagerTests.java index 582f5938b..367f26d30 100644 --- a/main/tests/server/src/com/google/refine/io/FileProjectManagerTests.java +++ b/main/tests/server/src/com/google/refine/io/FileProjectManagerTests.java @@ -27,6 +27,7 @@ package com.google.refine.io; import static org.mockito.Mockito.mock; +import static org.testng.Assert.assertEquals; import java.io.File; import java.io.IOException; @@ -50,7 +51,7 @@ public class FileProjectManagerTests { protected FileProjectManagerStub(File dir) { super(dir); - _projectsMetadata.put(1234L, mock(ProjectMetadata.class)); + _projectsMetadata.put(5555L, mock(ProjectMetadata.class)); } } @@ -69,11 +70,27 @@ public class FileProjectManagerTests { " \"class\" : \"com.google.refine.preference.TopList\",\n" + " \"list\" : [ ],\n" + " \"top\" : 2147483647\n" + - " }\n" + + " }\n" + " }\n" + " },\n" + - " \"projectIDs\" : [ 1234 ]\n" + + " \"projectIDs\" : [ 5555 ]\n" + " }"; TestUtils.isSerializedTo(manager, json); + } + + /** + * Test that we can save and restore non-ASCII characters. + * For best effectiveness, this should be run with a non-UTF8 + * default encoding for Java e.g. java -Dfile.encoding=cp1252 + * to simulate running on a Windows system + */ + @Test + public void saveReloadMultinationalCharacter () throws IOException { + FileProjectManager manager = new FileProjectManagerStub(workspaceDir); + manager.getPreferenceStore().put("testPref", "Refiné"); + manager.saveWorkspace(); + manager = new FileProjectManagerStub(workspaceDir); + assertEquals(manager.getPreferenceStore().get("testPref"), "Refiné"); + } } diff --git a/pom.xml b/pom.xml index dbbcb68cc..3fc3daed6 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ 3333 /tmp/refine 2.22.2 - + -Dfile.encoding=cp1252 UTF-8 2.11.0