diff --git a/main/src/com/google/refine/io/FileProjectManager.java b/main/src/com/google/refine/io/FileProjectManager.java
index 2204f37f2..75e4fa231 100644
--- a/main/src/com/google/refine/io/FileProjectManager.java
+++ b/main/src/com/google/refine/io/FileProjectManager.java
@@ -40,6 +40,8 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
@@ -307,13 +309,14 @@ public class FileProjectManager extends ProjectManager {
}
protected boolean saveToFile(File file) throws IOException {
- FileWriter writer = new FileWriter(file);
+ OutputStream stream = new FileOutputStream(file);
boolean saveWasNeeded = saveNeeded();
try {
- ParsingUtilities.defaultWriter.writeValue(writer, this);
+ // writeValue(OutputStream) is documented to use JsonEncoding.UTF8
+ ParsingUtilities.defaultWriter.writeValue(stream, this);
saveProjectMetadata();
} finally {
- writer.close();
+ stream.close();
}
return saveWasNeeded;
}
diff --git a/main/src/com/google/refine/io/ProjectMetadataUtilities.java b/main/src/com/google/refine/io/ProjectMetadataUtilities.java
index 61d5d227e..afddf947c 100644
--- a/main/src/com/google/refine/io/ProjectMetadataUtilities.java
+++ b/main/src/com/google/refine/io/ProjectMetadataUtilities.java
@@ -39,6 +39,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -60,25 +61,33 @@ public class ProjectMetadataUtilities {
public static void save(ProjectMetadata projectMeta, File projectDir) throws IOException {
File tempFile = new File(projectDir, "metadata.temp.json");
saveToFile(projectMeta, tempFile);
+ if (tempFile.length() == 0) {
+ throw new IOException("Failed to save project metadata - keeping backups");
+ }
+
+ // TODO Do we want to make sure we can successfully deserialize the file too?
File file = new File(projectDir, "metadata.json");
File oldFile = new File(projectDir, "metadata.old.json");
- if (oldFile.exists()) {
- oldFile.delete();
- }
-
if (file.exists()) {
+ if(file.length() > 0) {
+ if (oldFile.exists()) {
+ oldFile.delete();
+ }
file.renameTo(oldFile);
+ } else {
+ file.delete();
+ }
}
tempFile.renameTo(file);
}
protected static void saveToFile(ProjectMetadata projectMeta, File metadataFile) throws IOException {
- Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile));
+ Writer writer = new OutputStreamWriter(new FileOutputStream(metadataFile), StandardCharsets.UTF_8);
try {
- ParsingUtilities.defaultWriter.writeValue(writer, projectMeta);
+ ParsingUtilities.saveWriter.writeValue(writer, projectMeta);
} finally {
writer.close();
}
@@ -121,7 +130,7 @@ public class ProjectMetadataUtilities {
* creation and modification times based on whatever files are available.
*
* @param projectDir the project directory
- * @param id the proejct id
+ * @param id the project id
* @return
*/
static public ProjectMetadata recover(File projectDir, long id) {
diff --git a/main/src/com/google/refine/preference/PreferenceStore.java b/main/src/com/google/refine/preference/PreferenceStore.java
index 65bcfd69d..2f54b6204 100644
--- a/main/src/com/google/refine/preference/PreferenceStore.java
+++ b/main/src/com/google/refine/preference/PreferenceStore.java
@@ -37,6 +37,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import java.util.logging.Logger;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
@@ -100,6 +101,12 @@ public class PreferenceStore {
if (entries.get(key) != null) {
JsonNode o = entries.get(key);
Object loaded = loadObject(o);
+ if (loaded == null) {
+ if ("scripting.starred-expressions".contentEquals(key)) {
+ // HACK to work around preferences corruption
+ loaded = new TopList(10);
+ }
+ }
_prefs.put(key, loaded);
}
}
diff --git a/main/tests/server/src/com/google/refine/io/FileProjectManagerTests.java b/main/tests/server/src/com/google/refine/io/FileProjectManagerTests.java
index 582f5938b..367f26d30 100644
--- a/main/tests/server/src/com/google/refine/io/FileProjectManagerTests.java
+++ b/main/tests/server/src/com/google/refine/io/FileProjectManagerTests.java
@@ -27,6 +27,7 @@
package com.google.refine.io;
import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
@@ -50,7 +51,7 @@ public class FileProjectManagerTests {
protected FileProjectManagerStub(File dir) {
super(dir);
- _projectsMetadata.put(1234L, mock(ProjectMetadata.class));
+ _projectsMetadata.put(5555L, mock(ProjectMetadata.class));
}
}
@@ -69,11 +70,27 @@ public class FileProjectManagerTests {
" \"class\" : \"com.google.refine.preference.TopList\",\n" +
" \"list\" : [ ],\n" +
" \"top\" : 2147483647\n" +
- " }\n" +
+ " }\n" +
" }\n" +
" },\n" +
- " \"projectIDs\" : [ 1234 ]\n" +
+ " \"projectIDs\" : [ 5555 ]\n" +
" }";
TestUtils.isSerializedTo(manager, json);
+
}
+
+ /**
+ * Test that we can save and restore non-ASCII characters.
+ * For best effectiveness, this should be run with a non-UTF8
+ * default encoding for Java e.g. java -Dfile.encoding=cp1252
+ * to simulate running on a Windows system
+ */
+ @Test
+ public void saveReloadMultinationalCharacter () throws IOException {
+ FileProjectManager manager = new FileProjectManagerStub(workspaceDir);
+ manager.getPreferenceStore().put("testPref", "Refiné");
+ manager.saveWorkspace();
+ manager = new FileProjectManagerStub(workspaceDir);
+ assertEquals(manager.getPreferenceStore().get("testPref"), "Refiné");
+ }
}
diff --git a/pom.xml b/pom.xml
index dbbcb68cc..3fc3daed6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,7 +57,7 @@
3333
/tmp/refine
2.22.2
-
+ -Dfile.encoding=cp1252
UTF-8
2.11.0