From 7b857a4b24b2a85bc8445eff88a93afe4f7445c9 Mon Sep 17 00:00:00 2001 From: Jacky Date: Wed, 2 May 2018 22:02:35 -0400 Subject: [PATCH 1/2] implement nano date part --- .../refine/expr/functions/date/DatePart.java | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/main/src/com/google/refine/expr/functions/date/DatePart.java b/main/src/com/google/refine/expr/functions/date/DatePart.java index 126838c6d..8b305470c 100644 --- a/main/src/com/google/refine/expr/functions/date/DatePart.java +++ b/main/src/com/google/refine/expr/functions/date/DatePart.java @@ -36,6 +36,12 @@ package com.google.refine.expr.functions.date; import java.util.Calendar; import java.util.Date; import java.util.Properties; +import java.util.TimeZone; +import java.time.DayOfWeek; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.temporal.TemporalField; +import java.time.temporal.WeekFields; import org.json.JSONException; import org.json.JSONWriter; @@ -49,21 +55,61 @@ public class DatePart implements Function { @Override public Object call(Properties bindings, Object[] args) { if (args.length == 2 && - args[0] != null && (args[0] instanceof Calendar || args[0] instanceof Date) && + args[0] != null && (args[0] instanceof Calendar || args[0] instanceof Date || args[0] instanceof OffsetDateTime) && args[1] != null && args[1] instanceof String) { - String part = (String) args[1]; if (args[0] instanceof Calendar) { return getPart((Calendar) args[0], part); - } else { - Calendar c = Calendar.getInstance(); + } else if (args[0] instanceof Date) { + Calendar c = Calendar.getInstance(TimeZone.getTimeZone("UTC")); c.setTime((Date) args[0]); return getPart(c, part); + } else { + return getPart((OffsetDateTime) args[0], part); } } return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a date and a string"); } + private Object getPart(OffsetDateTime offsetDateTime, String part) { + if ("hours".equals(part) || "hour".equals(part) || "h".equals(part)) { + return offsetDateTime.getHour(); + } else if ("minutes".equals(part) || "minute".equals(part) || "min".equals(part)) { // avoid 'm' to avoid confusion with month + return offsetDateTime.getMinute(); + } else if ("seconds".equals(part) || "sec".equals(part) || "s".equals(part)) { + return offsetDateTime.getSecond(); + } else if ("milliseconds".equals(part) || "ms".equals(part) || "S".equals(part)) { + return Math.round(offsetDateTime.getNano() / 1000); + } else if ("nanos".equals(part) || "nano".equals(part) || "n".equals(part)) { + // JSR-310 is based on nanoseconds, not milliseconds. + return offsetDateTime.getNano(); + } else if ("years".equals(part) || "year".equals(part)) { + return offsetDateTime.getYear(); + } else if ("months".equals(part) || "month".equals(part)) { // avoid 'm' to avoid confusion with minute + return offsetDateTime.getMonth().getValue(); + } else if ("weeks".equals(part) || "week".equals(part) || "w".equals(part)) { + return getWeekOfMonth(offsetDateTime); + } else if ("days".equals(part) || "day".equals(part) || "d".equals(part)) { + return offsetDateTime.getDayOfMonth(); + } else if ("weekday".equals(part)) { + return offsetDateTime.getDayOfWeek().name(); + } else if ("time".equals(part)) { // get Time In Millis + return offsetDateTime.toInstant().toEpochMilli(); + } else { + return new EvalError("Date unit '" + part + "' not recognized."); + } + } + + private int getWeekOfMonth(OffsetDateTime offsetDateTime) { + LocalDate date = offsetDateTime.toLocalDate(); + DayOfWeek firstDayOfWeek = DayOfWeek.SUNDAY; + int minDays = 1; + WeekFields week = WeekFields.of(firstDayOfWeek, minDays); + TemporalField womField = week.weekOfMonth(); + + return date.get(womField); + } + static private String[] s_daysOfWeek = new String[] { "Saturday", "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; From 6787ab881efb57687c4f11ad35e40ce9ff6a8886 Mon Sep 17 00:00:00 2001 From: Jacky Date: Wed, 2 May 2018 22:02:57 -0400 Subject: [PATCH 2/2] implement nano date part UT --- .../expr/functions/date/DatePartTests.java | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 main/tests/server/src/com/google/refine/tests/expr/functions/date/DatePartTests.java diff --git a/main/tests/server/src/com/google/refine/tests/expr/functions/date/DatePartTests.java b/main/tests/server/src/com/google/refine/tests/expr/functions/date/DatePartTests.java new file mode 100644 index 000000000..f9830e037 --- /dev/null +++ b/main/tests/server/src/com/google/refine/tests/expr/functions/date/DatePartTests.java @@ -0,0 +1,244 @@ +package com.google.refine.tests.expr.functions.date; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.Calendar; +import java.util.Date; +import java.util.Properties; +import java.util.TimeZone; + +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import com.google.refine.expr.EvalError; +import com.google.refine.grel.ControlFunctionRegistry; +import com.google.refine.grel.Function; +import com.google.refine.tests.RefineTest; + + +public class DatePartTests extends RefineTest { + + static Properties bindings; + + @Override + @BeforeTest + public void init() { + logger = LoggerFactory.getLogger(this.getClass()); + } + + @BeforeMethod + public void SetUp() { + bindings = new Properties(); + } + + @AfterMethod + public void TearDown() { + bindings = null; + } + + /** + * Lookup a control function by name and invoke it with a variable number of args + */ + private static Object invoke(String name,Object... args) { + // registry uses static initializer, so no need to set it up + Function function = ControlFunctionRegistry.getFunction(name); + if (function == null) { + throw new IllegalArgumentException("Unknown function "+name); + } + if (args == null) { + return function.call(bindings,new Object[0]); + } else { + return function.call(bindings,args); + } + } + + @Test + public void testDateDatePart() throws ParseException { + Assert.assertTrue(invoke("datePart") instanceof EvalError); + + // 2018-4-30 23:55:44, cannot use new Date(2018 - 1900, 4 - 1, 30, 23, 55, 44). use below way to get a UTC date: + SimpleDateFormat isoFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + isoFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Date source = isoFormat.parse("2018-04-30T23:55:44"); + + // hours + Assert.assertEquals(invoke("datePart", source, "hours"), 23); + Assert.assertEquals(invoke("datePart", source, "hour"), 23); + Assert.assertEquals(invoke("datePart", source, "h"), 23); + + // minutes + Assert.assertEquals(invoke("datePart", source, "minutes"), 55); + Assert.assertEquals(invoke("datePart", source, "minute"), 55); + Assert.assertEquals(invoke("datePart", source, "min"), 55); + + // seconds + Assert.assertEquals(invoke("datePart", source, "seconds"), 44); + Assert.assertEquals(invoke("datePart", source, "sec"), 44); + Assert.assertEquals(invoke("datePart", source, "s"), 44); + + // milliseconds + Assert.assertEquals(invoke("datePart", source, "milliseconds"), 0); + Assert.assertEquals(invoke("datePart", source, "ms"), 0); + Assert.assertEquals(invoke("datePart", source, "S"), 0); + + // years + Assert.assertEquals(invoke("datePart", source, "years"), 2018); + Assert.assertEquals(invoke("datePart", source, "year"), 2018); + + // months + Assert.assertEquals(invoke("datePart", source, "months"), 4); + Assert.assertEquals(invoke("datePart", source, "month"), 4); + + // weeks + Assert.assertEquals(invoke("datePart", source, "weeks"), 5); + Assert.assertEquals(invoke("datePart", source, "week"), 5); + Assert.assertEquals(invoke("datePart", source, "w"), 5); + + // days, day, d + Assert.assertEquals(invoke("datePart", source, "days"), 30); + Assert.assertEquals(invoke("datePart", source, "day"), 30); + Assert.assertEquals(invoke("datePart", source, "d"), 30); + + // weekday + Assert.assertEquals(invoke("datePart", source, "weekday"), "Monday"); + + // time + Assert.assertEquals(invoke("datePart", source, "time"), 1525132544000l); + } + + @Test + public void testCalendarDatePart() throws ParseException { + // 2018-4-30 23:55:44 + SimpleDateFormat isoFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); + isoFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Date dt = isoFormat.parse("2018-04-30T23:55:44"); + Calendar source = dateToCalendar(dt); + source.set(Calendar.MILLISECOND, 789); + + // hours + Assert.assertEquals(invoke("datePart", source, "hours"), 23); + Assert.assertEquals(invoke("datePart", source, "hour"), 23); + Assert.assertEquals(invoke("datePart", source, "h"), 23); + + // minutes + Assert.assertEquals(invoke("datePart", source, "minutes"), 55); + Assert.assertEquals(invoke("datePart", source, "minute"), 55); + Assert.assertEquals(invoke("datePart", source, "min"), 55); + + // seconds + Assert.assertEquals(invoke("datePart", source, "seconds"), 44); + Assert.assertEquals(invoke("datePart", source, "sec"), 44); + Assert.assertEquals(invoke("datePart", source, "s"), 44); + + // milliseconds + Assert.assertEquals(invoke("datePart", source, "milliseconds"), 789); + Assert.assertEquals(invoke("datePart", source, "ms"), 789); + Assert.assertEquals(invoke("datePart", source, "S"), 789); + + // years + Assert.assertEquals(invoke("datePart", source, "years"), 2018); + Assert.assertEquals(invoke("datePart", source, "year"), 2018); + + // months + Assert.assertEquals(invoke("datePart", source, "months"), 4); + Assert.assertEquals(invoke("datePart", source, "month"), 4); + + // weeks + Assert.assertEquals(invoke("datePart", source, "weeks"), 5); + Assert.assertEquals(invoke("datePart", source, "week"), 5); + Assert.assertEquals(invoke("datePart", source, "w"), 5); + + // days, day, d + Assert.assertEquals(invoke("datePart", source, "days"), 30); + Assert.assertEquals(invoke("datePart", source, "day"), 30); + Assert.assertEquals(invoke("datePart", source, "d"), 30); + + // weekday + Assert.assertEquals(invoke("datePart", source, "weekday"), "Monday"); + + // time + Assert.assertEquals(invoke("datePart", source, "time"), 1525132544000l + 789); + } + + private DateTimeFormatter formatter = new DateTimeFormatterBuilder() + // here is the same as your code + .append(DateTimeFormatter.BASIC_ISO_DATE).appendLiteral('-') + // time (hour/minute/seconds) + .appendPattern("HH:mm:ss") + // optional nanos, with 9, 6 or 3 digits + .appendPattern("[.SSSSSSSSS][.SSSSSS][.SSS]") + // offset + .appendOffset("+HH:mm", "Z") + // create formatter + .toFormatter(); + + @Test + public void testOffsetDateTimeDatePart() { + // 2018-4-30 23:55:44 + OffsetDateTime source = OffsetDateTime.parse("20180430-23:55:44.000789000Z", + formatter); + + // hours + Assert.assertEquals(invoke("datePart", source, "hours"), 23); + Assert.assertEquals(invoke("datePart", source, "hour"), 23); + Assert.assertEquals(invoke("datePart", source, "h"), 23); + + // minutes + Assert.assertEquals(invoke("datePart", source, "minutes"), 55); + Assert.assertEquals(invoke("datePart", source, "minute"), 55); + Assert.assertEquals(invoke("datePart", source, "min"), 55); + + // seconds + Assert.assertEquals(invoke("datePart", source, "seconds"), 44); + Assert.assertEquals(invoke("datePart", source, "sec"), 44); + Assert.assertEquals(invoke("datePart", source, "s"), 44); + + // milliseconds + Assert.assertEquals(invoke("datePart", source, "milliseconds"), 789); + Assert.assertEquals(invoke("datePart", source, "ms"), 789); + Assert.assertEquals(invoke("datePart", source, "S"), 789); + + // nanos + Assert.assertEquals(invoke("datePart", source, "nanos"), 789000); + Assert.assertEquals(invoke("datePart", source, "nano"), 789000); + Assert.assertEquals(invoke("datePart", source, "n"), 789000); + + // years + Assert.assertEquals(invoke("datePart", source, "years"), 2018); + Assert.assertEquals(invoke("datePart", source, "year"), 2018); + + // months + Assert.assertEquals(invoke("datePart", source, "months"), 4); + Assert.assertEquals(invoke("datePart", source, "month"), 4); + + // weeks + Assert.assertEquals(invoke("datePart", source, "weeks"), 5); + Assert.assertEquals(invoke("datePart", source, "week"), 5); + Assert.assertEquals(invoke("datePart", source, "w"), 5); + + // days, day, d + Assert.assertEquals(invoke("datePart", source, "days"), 30); + Assert.assertEquals(invoke("datePart", source, "day"), 30); + Assert.assertEquals(invoke("datePart", source, "d"), 30); + + // weekday + Assert.assertEquals(invoke("datePart", source, "weekday"), "MONDAY"); + + // time + Assert.assertEquals(invoke("datePart", source, "time"), 1525132544000l); + } + + // Convert Date to Calendar + private Calendar dateToCalendar(Date date) { + Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + calendar.setTime(date); + return calendar; + } +} \ No newline at end of file