s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial attempt to fix toDate function

Browse Source
This commit is contained in:
Owen Stephens 2018-11-08 09:11:49 +00:00
parent d674a99142
commit cc12098828
2 changed files with 106 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
main/src/com/google/refine/expr/functions/ToDate.java
View File

@ -59,10 +59,18 @@ public class ToDate implements Function {
@Override @Override
public Object call(Properties bindings, Object[] args) { public Object call(Properties bindings, Object[] args) {
String o1; String o1;
Boolean month_first = null;
Locale locale = Locale.getDefault();
Integer arg_pointer = 0; //pointer used to keep track of which argument we are parsing
DateFormat formatter;
OffsetDateTime date;
//Check there is at least one argument
if (args.length == 0) { if (args.length == 0) {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects at least one argument"); return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects at least one argument");
} else { } else {
Object arg0 = args[0]; Object arg0 = args[arg_pointer];
//check the first argument is something that can be parsed as a date
if (arg0 instanceof OffsetDateTime) { if (arg0 instanceof OffsetDateTime) {
return arg0; return arg0;
} else if (arg0 instanceof Long) { } else if (arg0 instanceof Long) {
@ -73,79 +81,95 @@ public class ToDate implements Function {
// ignore cell values that aren't Date, Calendar, Long or String // ignore cell values that aren't Date, Calendar, Long or String
return new EvalError("Unable to parse as date"); return new EvalError("Unable to parse as date");
} }
arg_pointer++; //increment arg_pointer to 1
} }
// "o, boolean month_first (optional)" //
if (args.length == 1 || (args.length == 2 && args[1] instanceof Boolean)) { if(args.length==arg_pointer) {
boolean month_first = true; //if there is just one valid argument, we treat as if month_first set to true
if (args.length == 2) { month_first = true;
month_first = (Boolean) args[1]; }
//if there are two or more arguments work out what type of arguments they are
if (args.length>arg_pointer) {
//is the first argument a boolean? If so use it as the month_first option
if(args[arg_pointer] instanceof Boolean) {
month_first = (Boolean) args[arg_pointer];
arg_pointer++; //increment arg_pointer to 2
} }
//if first argument isn't Boolean, do nothing
}
//month first helper maybe set by now so try a parse
if(month_first != null) {
try { try {
return CalendarParser.parseAsOffsetDateTime( o1, (month_first) ? CalendarParser.MM_DD_YY : CalendarParser.DD_MM_YY); return CalendarParser.parseAsOffsetDateTime( o1, (month_first) ? CalendarParser.MM_DD_YY : CalendarParser.DD_MM_YY);
} catch (CalendarParserException e) { } catch (CalendarParserException e) {
OffsetDateTime d = ParsingUtilities.stringToDate(o1); if(args.length-arg_pointer<1) { //no more arguments to try
if (d != null) { OffsetDateTime d = ParsingUtilities.stringToDate(o1);
return d; if (d != null) {
} else { return d;
try { } else {
return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime().toInstant().atOffset(ZoneOffset.of("Z")); try {
} catch (IllegalArgumentException e2) { return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime().toInstant().atOffset(ZoneOffset.of("Z"));
} } catch (IllegalArgumentException e2) {
} return new EvalError("Unable to parse as date");
return new EvalError("Unable to parse as date"); }
} }
} else if (args.length>=2) { // if arguments >2 do nothing - there are still things we can try
// "o, format1, format2 (optional), ..." }
Locale locale = Locale.getDefault(); }
}
for (int i=1;i<args.length;i++) { //check if we have still have arguments to parse
if (!(args[i] instanceof String)) { if (args.length-arg_pointer>=1) {
// skip formats that aren't strings if(args[arg_pointer] instanceof String);
continue; String localeString = StringUtils.trim((String) args[arg_pointer]);
Locale possibleLocale = Locale.forLanguageTag(localeString); // Java 1.7+
for (Locale l : DateFormat.getAvailableLocales()) {
if (l.equals(possibleLocale.toLanguageTag())) {
locale = possibleLocale;
arg_pointer++;
break;
} }
String format = StringUtils.trim((String) args[i]);
DateFormat formatter;
// Attempt to parse first string as a language tag
if (i == 1) {
Locale possibleLocale = Locale.forLanguageTag(format); // Java 1.7+
boolean valid = false;
for (Locale l : DateFormat.getAvailableLocales()) {
if (l.equals(possibleLocale)) {
locale = possibleLocale;
valid = true;
break;
}
}
if (valid) { // If we got a valid locale
if (args.length == 2) { // No format strings to try, process using default
formatter = DateFormat.getDateInstance(DateFormat.DEFAULT, locale);
OffsetDateTime date = parse(o1, formatter);
if (date != null) {
return date;
} else {
return new EvalError("Unable to parse as date");
}
}
continue; // Don't try to process locale string as a format string if it was valid
}
}
try {
formatter = new SimpleDateFormat(format,locale);
} catch (IllegalArgumentException e) {
return new EvalError("Unknown date format");
}
formatter.setLenient(true);
OffsetDateTime date = parse(o1, formatter);
if (date != null) {
return date;
}
} }
return new EvalError("Unable to parse as date");
} else {
return new EvalError("Unable to parse as date");
} }
while (arg_pointer < args.length) {
if (!(args[arg_pointer] instanceof String)) {
// skip formats that aren't strings
continue;
}
String format = StringUtils.trim((String) args[arg_pointer]);
try {
formatter = new SimpleDateFormat(format,locale);
} catch (IllegalArgumentException e) {
return new EvalError("Unknown date format");
}
date = parse(o1, formatter);
if (date != null) {
return date;
}
arg_pointer++;
}
//no formats, or not managed to convert to date using provided formats, so try default format
formatter = DateFormat.getDateInstance(DateFormat.DEFAULT, locale);
date = parse(o1, formatter);
if (date != null) {
return date;
}
//if we get here without successfully getting a date, try just a basic parse
date = ParsingUtilities.stringToDate(o1);
if (date != null) {
return date;
} else { //trying one last way
try {
return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime().toInstant().atOffset(ZoneOffset.of("Z"));
} catch (IllegalArgumentException e2) {
return new EvalError("Unable to parse as date");
}
}
} }
private OffsetDateTime parse(String o1, DateFormat formatter) { private OffsetDateTime parse(String o1, DateFormat formatter) {

11
main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java
View File

@ -141,14 +141,25 @@ public class ToFromConversionTests extends RefineTest {
Assert.assertTrue(invoke("toDate", "2012-03-01","xxx") instanceof EvalError); // bad format string Assert.assertTrue(invoke("toDate", "2012-03-01","xxx") instanceof EvalError); // bad format string
Assert.assertTrue(invoke("toDate", "2012-03-01") instanceof OffsetDateTime); Assert.assertTrue(invoke("toDate", "2012-03-01") instanceof OffsetDateTime);
Assert.assertEquals(invoke("toDate", "2012-03-01"),CalendarParser.parseAsOffsetDateTime("2012-03-01")); Assert.assertEquals(invoke("toDate", "2012-03-01"),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
//parse as 'month first' date with and without explicit 'true' parameter
Assert.assertEquals(invoke("toDate", "01/03/2012"),CalendarParser.parseAsOffsetDateTime("2012-01-03"));
Assert.assertEquals(invoke("toDate", "01/03/2012",true),CalendarParser.parseAsOffsetDateTime("2012-01-03"));
//parse as 'month first' date with 'false' parameter
Assert.assertEquals(invoke("toDate", "01/03/2012",false),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
//parse as 'month first' date without 'false' parameter but with format specified
Assert.assertEquals(invoke("toDate", "01/03/2012","dd/MM/yyyy"),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parseAsOffsetDateTime("2012-03-01")); Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
// Multiple format strings should get tried sequentially until one succeeds or all are exhausted // Multiple format strings should get tried sequentially until one succeeds or all are exhausted
Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parseAsOffsetDateTime("2012-03-01")); Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parseAsOffsetDateTime("2012-03-01"));
// Boolean argument combined with Multiple format strings
Assert.assertEquals(invoke("toDate", "01/03/2012",false, "MMM","yyyy-MM-dd","MM/dd/yyyy"), CalendarParser.parseAsOffsetDateTime("2012-03-01"));
// First string can be a locale identifier instead of a format string // First string can be a locale identifier instead of a format string
Assert.assertEquals(invoke("toDate", "2013-06-01","zh"), CalendarParser.parseAsOffsetDateTime("2013-06-01")); Assert.assertEquals(invoke("toDate", "2013-06-01","zh"), CalendarParser.parseAsOffsetDateTime("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parseAsOffsetDateTime("2013-06-01")); Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parseAsOffsetDateTime("2013-06-01"));
} }
@Test @Test