From 44c43253a12eead74c4c0c49c8db0735d69d8353 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sun, 2 Dec 2018 12:58:05 +0900 Subject: [PATCH 1/3] Avoid localizing time for dates with unspecified time. Closes #1874. --- main/src/com/google/refine/expr/functions/ToDate.java | 5 ++++- .../tests/expr/functions/strings/ToFromConversionTests.java | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/main/src/com/google/refine/expr/functions/ToDate.java b/main/src/com/google/refine/expr/functions/ToDate.java index 2eecfe4b8..2466bd676 100644 --- a/main/src/com/google/refine/expr/functions/ToDate.java +++ b/main/src/com/google/refine/expr/functions/ToDate.java @@ -37,6 +37,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.OffsetDateTime; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Date; import java.util.GregorianCalendar; @@ -166,7 +167,9 @@ public class ToDate implements Function { return date; } else { try { - return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime().toInstant().atOffset(ZoneOffset.of("Z")); + return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime().toInstant() + .plusSeconds(ZonedDateTime.now().getOffset().getTotalSeconds()) + .atOffset(ZoneOffset.of("Z")); } catch (IllegalArgumentException e2) { return null; } diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java index cb7f467c4..2fb4ab2d9 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java @@ -49,6 +49,7 @@ import com.google.refine.expr.util.CalendarParserException; import com.google.refine.grel.ControlFunctionRegistry; import com.google.refine.grel.Function; import com.google.refine.tests.RefineTest; +import com.google.refine.util.ParsingUtilities; /** @@ -161,10 +162,10 @@ public class ToFromConversionTests extends RefineTest { Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parseAsOffsetDateTime("2013-06-01")); //if invalid format/locale strings are passed, ignore them - Assert.assertEquals(invoke("toDate", "2012-03-01","XXX"), CalendarParser.parseAsOffsetDateTime("2012-03-01")); + Assert.assertEquals(invoke("toDate", "2012-03-01","XXX"), invoke("toDate", "2012-03-01")); // If a long, convert to string - Assert.assertEquals(invoke("toDate", (long) 2012), CalendarParser.parseAsOffsetDateTime("2012-01-01")); + Assert.assertEquals(invoke("toDate", (long) 2012), invoke("toDate", "2012-01-01")); // If already a date, leave it alone Assert.assertEquals(invoke("toDate", CalendarParser.parseAsOffsetDateTime("2012-03-01")),CalendarParser.parseAsOffsetDateTime("2012-03-01")); From cf58092f2d76f99400cd288dbe87b496f55bf891 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sun, 2 Dec 2018 17:09:03 +0900 Subject: [PATCH 2/3] Fix stringToLocalDate for non-GMT timezones --- main/src/com/google/refine/util/ParsingUtilities.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main/src/com/google/refine/util/ParsingUtilities.java b/main/src/com/google/refine/util/ParsingUtilities.java index 7751bfc4d..b044b63c8 100644 --- a/main/src/com/google/refine/util/ParsingUtilities.java +++ b/main/src/com/google/refine/util/ParsingUtilities.java @@ -229,7 +229,8 @@ public class ParsingUtilities { if (parsed == null) { return null; } - return parsed.toLocalDateTime(); + return parsed.withOffsetSameInstant(OffsetDateTime.now().getOffset()) + .toLocalDateTime(); } static public String instantToString(Instant instant) { From be137fc53c056e1fcdcbf5528aec9106391a1750 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Thu, 6 Dec 2018 15:46:20 +0900 Subject: [PATCH 3/3] Add serialization tests in different timezone --- .../strings/ToFromConversionTests.java | 1 + .../refine/tests/io/ProjectMetadataTests.java | 40 ++++++++++++++----- .../tests/util/ParsingUtilitiesTests.java | 18 +++++++++ 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java index 2fb4ab2d9..1ca563373 100644 --- a/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java @@ -35,6 +35,7 @@ package com.google.refine.tests.expr.functions.strings; import java.time.OffsetDateTime; import java.util.Properties; +import java.util.TimeZone; import org.slf4j.LoggerFactory; import org.testng.Assert; diff --git a/main/tests/server/src/com/google/refine/tests/io/ProjectMetadataTests.java b/main/tests/server/src/com/google/refine/tests/io/ProjectMetadataTests.java index 8112523db..ba17e542c 100644 --- a/main/tests/server/src/com/google/refine/tests/io/ProjectMetadataTests.java +++ b/main/tests/server/src/com/google/refine/tests/io/ProjectMetadataTests.java @@ -2,24 +2,46 @@ package com.google.refine.tests.io; import java.io.IOException; import java.io.InputStream; +import java.util.TimeZone; import org.apache.commons.io.IOUtils; +import org.testng.annotations.BeforeSuite; import org.testng.annotations.Test; import com.google.refine.tests.util.TestUtils; import com.google.refine.util.ParsingUtilities; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.google.refine.ProjectMetadata; public class ProjectMetadataTests { + + private String jsonSaveMode = null; + private String jsonNonSaveMode = null; + + @BeforeSuite + public void setUpJson() throws IOException { + InputStream f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata.json"); + jsonNonSaveMode = IOUtils.toString(f); + f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata_save_mode.json"); + jsonSaveMode = IOUtils.toString(f); + } + @Test public void serializeProjectMetadata() throws IOException { - InputStream f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata.json"); - String json = IOUtils.toString(f); - f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata_save_mode.json"); - String fullJson = IOUtils.toString(f); - - f = ProjectMetadataTests.class.getClassLoader().getResourceAsStream("example_project_metadata_save_mode.json"); - ProjectMetadata metadata = ParsingUtilities.mapper.readValue(f, ProjectMetadata.class); - TestUtils.isSerializedTo(metadata, json); - TestUtils.isSerializedTo(metadata, fullJson, true); + ProjectMetadata metadata = ParsingUtilities.mapper.readValue(jsonSaveMode, ProjectMetadata.class); + TestUtils.isSerializedTo(metadata, jsonNonSaveMode); + TestUtils.isSerializedTo(metadata, jsonSaveMode, true); + } + + @Test + public void serializeProjectMetadataInDifferentTimezone() throws JsonParseException, JsonMappingException, IOException { + TimeZone.setDefault(TimeZone.getTimeZone("JST")); + try { + ProjectMetadata metadata = ParsingUtilities.mapper.readValue(jsonSaveMode, ProjectMetadata.class); + TestUtils.isSerializedTo(metadata, jsonNonSaveMode); + TestUtils.isSerializedTo(metadata, jsonSaveMode, true); + } finally { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + } } } diff --git a/main/tests/server/src/com/google/refine/tests/util/ParsingUtilitiesTests.java b/main/tests/server/src/com/google/refine/tests/util/ParsingUtilitiesTests.java index aff9e2a1f..609513b8a 100644 --- a/main/tests/server/src/com/google/refine/tests/util/ParsingUtilitiesTests.java +++ b/main/tests/server/src/com/google/refine/tests/util/ParsingUtilitiesTests.java @@ -37,6 +37,7 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.Map; +import java.util.TimeZone; import org.apache.commons.lang3.text.StrSubstitutor; import org.slf4j.LoggerFactory; @@ -89,6 +90,23 @@ public class ParsingUtilitiesTests extends RefineTest { Assert.assertEquals(2017, ParsingUtilities.stringToLocalDate("2017-04-03T08:09:43+00:00").getYear()); } + /** + * Converting between string and local time must be reversible, no matter the timezone. + */ + @Test + public void stringToLocalDateNonUTC() { + TimeZone.setDefault(TimeZone.getTimeZone("JST")); + try { + Assert.assertEquals(ParsingUtilities.stringToLocalDate("2001-08-12T00:00:00Z").getHour(), 9); + Assert.assertEquals(ParsingUtilities.localDateToString( + ParsingUtilities.stringToLocalDate("2001-08-12T00:00:00Z")), + "2001-08-12T00:00:00Z"); + + } finally { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + } + } + @Test public void parseProjectModifiedBeforeJDK8() { String modified = "2014-01-15T21:46:25Z";