s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1826 from ostephens/fix-todate

Browse Source 
Fix toDate
This commit is contained in:
Antonin Delpeuch 2018-11-16 10:30:56 +00:00 committed by GitHub
commit b85d4d0cb8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 108 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
main/src/com/google/refine/expr/functions/ToDate.java
View File

@ -37,8 +37,10 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.GregorianCalendar; import java.util.GregorianCalendar;
import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Properties; import java.util.Properties;
import java.util.TimeZone; import java.util.TimeZone;
@ -59,10 +61,16 @@ public class ToDate implements Function {
@Override @Override
public Object call(Properties bindings, Object[] args) { public Object call(Properties bindings, Object[] args) {
String o1; String o1;
Boolean month_first = null;
List<String> formats = new ArrayList<String>();
OffsetDateTime date = null;
//Check there is at least one argument
if (args.length == 0) { if (args.length == 0) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects at least one argument"); return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects at least one argument");
} else { } else {
Object arg0 = args[0]; Object arg0 = args[0];
//check the first argument is something that can be parsed as a date
if (arg0 instanceof OffsetDateTime) { if (arg0 instanceof OffsetDateTime) {
return arg0; return arg0;
} else if (arg0 instanceof Long) { } else if (arg0 instanceof Long) {
@ -74,77 +82,96 @@ public class ToDate implements Function {
return new EvalError("Unable to parse as date"); return new EvalError("Unable to parse as date");
} }
} }
// "o, boolean month_first (optional)"
if (args.length == 1 || (args.length == 2 && args[1] instanceof Boolean)) {
boolean month_first = true;
if (args.length == 2) {
month_first = (Boolean) args[1];
}
try {
return CalendarParser.parseAsOffsetDateTime( o1, (month_first) ? CalendarParser.MM_DD_YY : CalendarParser.DD_MM_YY);
} catch (CalendarParserException e) {
OffsetDateTime d = ParsingUtilities.stringToDate(o1);
if (d != null) {
return d;
} else {
try {
return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime().toInstant().atOffset(ZoneOffset.of("Z"));
} catch (IllegalArgumentException e2) {
}
}
return new EvalError("Unable to parse as date");
}
} else if (args.length>=2) {
// "o, format1, format2 (optional), ..."
Locale locale = Locale.getDefault();
for (int i=1;i<args.length;i++) { if(args.length==1) {
date = parse(o1,true,formats);
} else if (args.length > 1) {
if(args[1] instanceof Boolean) {
month_first = (Boolean) args[1];
} else if (args[1] instanceof String) {
formats.add(StringUtils.trim((String) args[1]));
} else {
return new EvalError("Invalid argument");
}
for(int i=2;i<args.length;i++) {
if (!(args[i] instanceof String)) { if (!(args[i] instanceof String)) {
// skip formats that aren't strings // skip formats that aren't strings
continue; continue;
} }
String format = StringUtils.trim((String) args[i]); formats.add(StringUtils.trim((String) args[i]));
DateFormat formatter; }
// Attempt to parse first string as a language tag if(month_first != null) {
if (i == 1) { date = parse(o1,month_first,formats);
Locale possibleLocale = Locale.forLanguageTag(format); // Java 1.7+ } else {
boolean valid = false; date = parse(o1,formats);
for (Locale l : DateFormat.getAvailableLocales()) { }
if (l.equals(possibleLocale)) {
locale = possibleLocale; }
valid = true; if(date != null) {
break; return date;
} }
} return new EvalError("Unable to convert to a date");
if (valid) { // If we got a valid locale }
if (args.length == 2) { // No format strings to try, process using default
formatter = DateFormat.getDateInstance(DateFormat.DEFAULT, locale); private OffsetDateTime parse(String o1, Boolean month_first, List<String> formats) {
OffsetDateTime date = parse(o1, formatter); if(month_first != null) {
if (date != null) { try {
return date; return CalendarParser.parseAsOffsetDateTime( o1, (month_first) ? CalendarParser.MM_DD_YY : CalendarParser.DD_MM_YY);
} else { } catch (CalendarParserException e) {
return new EvalError("Unable to parse as date"); }
} }
} return parse(o1,formats);
continue; // Don't try to process locale string as a format string if it was valid }
}
} private OffsetDateTime parse(String o1, List<String> formats) {
if(formats.size()>0) {
String f1 = formats.get(0);
formats.remove(0);
return parse(o1,f1,formats);
} else {
return parse(o1,Locale.getDefault(),formats);
}
}
private OffsetDateTime parse(String o1, String f1, List<String> formats) {
Locale locale = Locale.getDefault();
Locale possibleLocale = Locale.forLanguageTag(f1); // Java 1.7+
for (Locale l : DateFormat.getAvailableLocales()) {
if (l.equals(possibleLocale)) {
locale = possibleLocale;
} else {
formats.add(0,f1);
}
}
return parse(o1,locale,formats);
}
private OffsetDateTime parse(String o1, Locale locale, List<String> formats) {
DateFormat formatter;
OffsetDateTime date;
//need to try using each format in the formats list!
if(formats.size()>0) {
for(int i=0;i<formats.size();i++) {
try { try {
formatter = new SimpleDateFormat(format,locale); formatter = new SimpleDateFormat(formats.get(i),locale);
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
return new EvalError("Unknown date format"); continue;
} }
formatter.setLenient(true); date = parse(o1, formatter);
OffsetDateTime date = parse(o1, formatter);
if (date != null) { if (date != null) {
return date; return date;
} }
} }
return new EvalError("Unable to parse as date"); }
date = ParsingUtilities.stringToDate(o1);
if (date != null) {
return date;
} else { } else {
return new EvalError("Unable to parse as date"); try {
return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime().toInstant().atOffset(ZoneOffset.of("Z"));
} catch (IllegalArgumentException e2) {
return null;
}
} }
} }

27
main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java
View File

@ -138,17 +138,36 @@ public class ToFromConversionTests extends RefineTest {
Assert.assertTrue(invoke("toDate", (Object) null) instanceof EvalError); Assert.assertTrue(invoke("toDate", (Object) null) instanceof EvalError);
Assert.assertTrue(invoke("toDate", "") instanceof EvalError); Assert.assertTrue(invoke("toDate", "") instanceof EvalError);
Assert.assertTrue(invoke("toDate", 1.0) instanceof EvalError); Assert.assertTrue(invoke("toDate", 1.0) instanceof EvalError);
Assert.assertTrue(invoke("toDate", "2012-03-01","xxx") instanceof EvalError); // bad format string //Assert.assertTrue(invoke("toDate", "2012-03-01","xxx") instanceof EvalError); // bad format string
Assert.assertTrue(invoke("toDate", "2012-03-01") instanceof OffsetDateTime); Assert.assertTrue(invoke("toDate", "2012-03-01") instanceof OffsetDateTime);
Assert.assertEquals(invoke("toDate", "2012-03-01"),CalendarParser.parseAsOffsetDateTime("2012-03-01")); Assert.assertEquals(invoke("toDate", "2012-03-01"),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
//parse as 'month first' date with and without explicit 'true' parameter
Assert.assertEquals(invoke("toDate", "01/03/2012"),CalendarParser.parseAsOffsetDateTime("2012-01-03"));
Assert.assertEquals(invoke("toDate", "01/03/2012",true),CalendarParser.parseAsOffsetDateTime("2012-01-03"));
//parse as 'month first' date with 'false' parameter
Assert.assertEquals(invoke("toDate", "01/03/2012",false),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
//parse as 'month first' date without 'false' parameter but with format specified
Assert.assertEquals(invoke("toDate", "01/03/2012","dd/MM/yyyy"),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parseAsOffsetDateTime("2012-03-01")); Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
//Two digit year
Assert.assertEquals(invoke("toDate", "02-02-01"),CalendarParser.parseAsOffsetDateTime("2001-02-02"));
// Multiple format strings should get tried sequentially until one succeeds or all are exhausted // Multiple format strings should get tried sequentially until one succeeds or all are exhausted
Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parseAsOffsetDateTime("2012-03-01")); Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parseAsOffsetDateTime("2012-03-01"));
// Boolean argument combined with Multiple format strings
Assert.assertEquals(invoke("toDate", "01/03/2012",false, "MMM","yyyy-MM-dd","MM/dd/yyyy"), CalendarParser.parseAsOffsetDateTime("2012-03-01"));
// First string can be a locale identifier instead of a format string // First string can be a locale identifier instead of a format string
Assert.assertEquals(invoke("toDate", "2013-06-01","zh"), CalendarParser.parseAsOffsetDateTime("2013-06-01")); Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parseAsOffsetDateTime("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parseAsOffsetDateTime("2013-06-01"));
//if invalid format/locale strings are passed, ignore them
Assert.assertEquals(invoke("toDate", "2012-03-01","XXX"), CalendarParser.parseAsOffsetDateTime("2012-03-01"));
// If a long, convert to string
Assert.assertEquals(invoke("toDate", (long) 2012), CalendarParser.parseAsOffsetDateTime("2012-01-01"));
// If already a date, leave it alone
Assert.assertEquals(invoke("toDate", CalendarParser.parseAsOffsetDateTime("2012-03-01")),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
} }
@Test @Test