s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1597 from OpenRefine/issue/1588-2

Browse Source 
unify the internal date type
This commit is contained in:
Antonin Delpeuch 2018-05-18 22:03:14 +02:00 committed by GitHub
commit b01177c0d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 412 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
main/src/com/google/refine/browsing/filters/ExpressionTimeComparisonRowFilter.java
View File

@ -33,8 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing.filters;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import com.google.refine.browsing.util.RowEvaluable;
import com.google.refine.expr.ExpressionUtils;
@ -67,11 +66,8 @@ abstract public class ExpressionTimeComparisonRowFilter extends ExpressionNumber
if (ExpressionUtils.isError(v)) {
return _selectError;
} else if (ExpressionUtils.isNonBlankData(v)) {
if (v instanceof Date) {
long time = ((Date) v).getTime();
return _selectTime && checkValue(time);
} else if (v instanceof Calendar) {
long time = ((Calendar) v).getTime().getTime();
if (v instanceof OffsetDateTime) {
long time = ((OffsetDateTime) v).toInstant().toEpochMilli();
return _selectTime && checkValue(time);
} else {
return _selectNonTime;

11
main/src/com/google/refine/browsing/util/ExpressionTimeValueBinner.java
View File

@ -33,9 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing.util;
import java.util.Calendar;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.Date;
import java.util.Properties;
import com.google.refine.browsing.RecordVisitor;
@ -162,12 +161,8 @@ public class ExpressionTimeValueBinner implements RowVisitor, RecordVisitor {
if (ExpressionUtils.isError(value)) {
hasError = true;
} else if (ExpressionUtils.isNonBlankData(value)) {
if (value instanceof Calendar) {
value = ((Calendar) value).getTime();
}
if (value instanceof Date) {
long t = ((Date) value).getTime();
if (value instanceof OffsetDateTime) {
long t = ((OffsetDateTime) value).toInstant().toEpochMilli();
hasTime = true;
int bin = (int) Math.floor((double) (t - _index.getMin()) / (double) _index.getStep());

27
main/src/com/google/refine/browsing/util/TimeBinIndex.java
View File

@ -33,10 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.browsing.util;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Properties;
@ -181,13 +180,9 @@ abstract public class TimeBinIndex {
if (ExpressionUtils.isError(v)) {
_hasError = true;
} else if (ExpressionUtils.isNonBlankData(v)) {
if (v instanceof Calendar) {
v = ((Calendar) v).getTime();
}
if (v instanceof Date) {
if (v instanceof OffsetDateTime) {
_hasTime = true;
processValue(((Date) v).getTime(), allValues);
processValue(((OffsetDateTime) v).toInstant().toEpochMilli(), allValues);
} else {
_hasNonTime = true;
}
@ -202,13 +197,9 @@ abstract public class TimeBinIndex {
if (ExpressionUtils.isError(v)) {
_hasError = true;
} else if (ExpressionUtils.isNonBlankData(v)) {
if (v instanceof Calendar) {
v = ((Calendar) v).getTime();
}
if (v instanceof Date) {
if (v instanceof OffsetDateTime) {
_hasTime = true;
processValue(((Date) v).getTime(), allValues);
processValue(((OffsetDateTime) v).toInstant().toEpochMilli(), allValues);
} else {
_hasNonTime = true;
}
@ -219,13 +210,9 @@ abstract public class TimeBinIndex {
} else {
_totalValueCount++;
if (value instanceof Calendar) {
value = ((Calendar) value).getTime();
}
if (value instanceof Date) {
if (value instanceof OffsetDateTime) {
_hasTime = true;
processValue(((Date) value).getTime(), allValues);
processValue(((OffsetDateTime) value).toInstant().toEpochMilli(), allValues);
} else {
_hasNonTime = true;
}

10
main/src/com/google/refine/commands/expr/PreviewExpressionCommand.java
View File

@ -35,10 +35,7 @@ package com.google.refine.commands.expr;
import java.io.IOException;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.OffsetDateTime;
import java.util.Calendar;
import java.util.List;
import java.util.Properties;
@ -210,12 +207,7 @@ public class PreviewExpressionCommand extends Command {
sb.append(" ]");
} else if (v instanceof HasFields) {
sb.append("[object " + v.getClass().getSimpleName() + "]");
} else if (v instanceof Calendar) {
Calendar c = (Calendar) v;
sb.append("[date " +
ParsingUtilities.dateToString(OffsetDateTime.ofInstant(c.toInstant(), ZoneId.systemDefault())) +"]");
} else if (v instanceof LocalDateTime) {
} else if (v instanceof OffsetDateTime) {
sb.append("[date " +
ParsingUtilities.dateToString((OffsetDateTime) v) +"]");
} else if (v instanceof String) {

10
main/src/com/google/refine/exporters/CustomizableTabularExporterUtilities.java
View File

@ -35,9 +35,9 @@ package com.google.refine.exporters;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -348,10 +348,8 @@ abstract public class CustomizableTabularExporterUtilities {
if (text == null) {
if (value instanceof String) {
text = (String) value;
} else if (value instanceof Calendar) {
text = dateFormatter.format(((Calendar) value).getTime());
} else if (value instanceof Date) {
text = dateFormatter.format((Date) value);
} else if (value instanceof OffsetDateTime) {
text = ((OffsetDateTime) value).format(DateTimeFormatter.ISO_INSTANT);
} else {
text = value.toString();
}

13
main/src/com/google/refine/exporters/OdsExporter.java
View File

@ -35,8 +35,7 @@ package com.google.refine.exporters;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Properties;
@ -49,6 +48,7 @@ import org.odftoolkit.odfdom.doc.table.OdfTableRow;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities;
public class OdsExporter implements StreamExporter {
@ -97,12 +97,9 @@ public class OdsExporter implements StreamExporter {
c.setDoubleValue(((Number) v).doubleValue());
} else if (v instanceof Boolean) {
c.setBooleanValue(((Boolean) v).booleanValue());
} else if (v instanceof Date) {
Calendar cal = Calendar.getInstance();
cal.setTime((Date) v);
c.setDateValue(cal);
} else if (v instanceof Calendar) {
c.setDateValue((Calendar) v);
} else if (v instanceof OffsetDateTime) {
OffsetDateTime odt = (OffsetDateTime)v;
c.setDateValue(ParsingUtilities.offsetDateTimeToCalendar(odt));
} else {
c.setStringValue(cellData.text);
}

13
main/src/com/google/refine/exporters/XlsExporter.java
View File

@ -35,8 +35,7 @@ package com.google.refine.exporters;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Properties;
@ -54,6 +53,7 @@ import org.json.JSONObject;
import com.google.refine.ProjectManager;
import com.google.refine.browsing.Engine;
import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities;
public class XlsExporter implements StreamExporter {
final private boolean xml;
@ -112,11 +112,9 @@ public class XlsExporter implements StreamExporter {
c.setCellValue(((Number) v).doubleValue());
} else if (v instanceof Boolean) {
c.setCellValue(((Boolean) v).booleanValue());
} else if (v instanceof Date) {
c.setCellValue((Date) v);
c.setCellStyle(dateStyle);
} else if (v instanceof Calendar) {
c.setCellValue((Calendar) v);
} else if (v instanceof OffsetDateTime) {
OffsetDateTime odt = (OffsetDateTime)v;
c.setCellValue(ParsingUtilities.offsetDateTimeToCalendar(odt));
c.setCellStyle(dateStyle);
} else {
String s = cellData.text;
@ -133,7 +131,6 @@ public class XlsExporter implements StreamExporter {
hl.setAddress(cellData.link);
}
}
}
}
}

6
main/src/com/google/refine/expr/ExpressionUtils.java
View File

@ -34,9 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr;
import java.io.Serializable;
import java.util.Calendar;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
@ -138,8 +137,7 @@ public class ExpressionUtils {
v instanceof Number ||
v instanceof String ||
v instanceof Boolean ||
v instanceof Date ||
v instanceof Calendar ||
v instanceof OffsetDateTime ||
v instanceof EvalError;
}

42
main/src/com/google/refine/expr/functions/ToDate.java
View File

@ -36,21 +36,22 @@ package com.google.refine.expr.functions;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.OffsetDateTime;
import java.util.Calendar;
import java.time.ZoneOffset;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Locale;
import java.util.Properties;
import java.util.TimeZone;
import org.apache.commons.lang3.StringUtils;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.util.CalendarParser;
import com.google.refine.expr.util.CalendarParserException;
import com.google.refine.grel.Function;
import com.google.refine.expr.util.CalendarParser;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.util.ParsingUtilities;
public class ToDate implements Function {
@ -62,10 +63,8 @@ public class ToDate implements Function {
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects at least one argument");
} else {
Object arg0 = args[0];
if (arg0 instanceof Date) {
if (arg0 instanceof OffsetDateTime) {
return arg0;
} else if (arg0 instanceof Calendar) {
return ((Calendar) arg0).getTime();
} else if (arg0 instanceof Long) {
o1 = ((Long) arg0).toString(); // treat integers as years
} else if (arg0 instanceof String && arg0.toString().trim().length() > 0) {
@ -83,21 +82,16 @@ public class ToDate implements Function {
month_first = (Boolean) args[1];
}
try {
return CalendarParser.parse( o1, (month_first) ? CalendarParser.MM_DD_YY : CalendarParser.DD_MM_YY);
return CalendarParser.parseAsOffsetDateTime( o1, (month_first) ? CalendarParser.MM_DD_YY : CalendarParser.DD_MM_YY);
} catch (CalendarParserException e) {
OffsetDateTime d = ParsingUtilities.stringToDate(o1);
if (d != null) {
return d;
} else {
try {
return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime();
return javax.xml.bind.DatatypeConverter.parseDateTime(o1).getTime().toInstant().atOffset(ZoneOffset.of("Z"));
} catch (IllegalArgumentException e2) {
}
// alternate implementation which may be useful on some JVMs?
// try {
// return javax.xml.datatype.DatatypeFactory.newInstance().newXMLGregorianCalendar(o1).toGregorianCalendar().getTime();
// } catch (DatatypeConfigurationException e2) {
// }
}
return new EvalError("Unable to parse as date");
}
@ -114,14 +108,7 @@ public class ToDate implements Function {
DateFormat formatter;
// Attempt to parse first string as a language tag
if (i == 1) {
// Locale possibleLocale = Locale.forLanguageTag(format); // Java 1.7+ only
Locale possibleLocale;
int c = format.indexOf('_');
if (c > 0) {
possibleLocale = new Locale(format.substring(0, c),format.substring(c+1));
} else {
possibleLocale = new Locale(format);
}
Locale possibleLocale = Locale.forLanguageTag(format); // Java 1.7+
boolean valid = false;
for (Locale l : DateFormat.getAvailableLocales()) {
if (l.equals(possibleLocale)) {
@ -133,8 +120,7 @@ public class ToDate implements Function {
if (valid) { // If we got a valid locale
if (args.length == 2) { // No format strings to try, process using default
formatter = DateFormat.getDateInstance(DateFormat.DEFAULT, locale);
formatter.setLenient(true);
GregorianCalendar date = parse(o1, formatter);
OffsetDateTime date = parse(o1, formatter);
if (date != null) {
return date;
} else {
@ -150,10 +136,11 @@ public class ToDate implements Function {
return new EvalError("Unknown date format");
}
formatter.setLenient(true);
GregorianCalendar date = parse(o1, formatter);
OffsetDateTime date = parse(o1, formatter);
if (date != null) {
return date;
}
}
return new EvalError("Unable to parse as date");
} else {
@ -161,19 +148,18 @@ public class ToDate implements Function {
}
}
private GregorianCalendar parse(String o1, DateFormat formatter) {
private OffsetDateTime parse(String o1, DateFormat formatter) {
try {
formatter.setTimeZone(TimeZone.getTimeZone("Z"));
Date date = formatter.parse(o1);
GregorianCalendar c = new GregorianCalendar();
c.setTime(date);
return c;
return ParsingUtilities.calendarToOffsetDateTime(c);
} catch (java.text.ParseException e) {
return null;
}
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {

12
main/src/com/google/refine/expr/functions/ToString.java
View File

@ -33,10 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Properties;
import org.json.JSONException;
@ -55,9 +53,9 @@ public class ToString implements Function {
Object o1 = args[0];
if (args.length == 2 && args[1] instanceof String) {
Object o2 = args[1];
if (o1 instanceof Calendar || o1 instanceof Date) {
DateFormat formatter = new SimpleDateFormat((String) o2);
return formatter.format(o1 instanceof Date ? ((Date) o1) : ((Calendar) o1).getTime());
if (o1 instanceof OffsetDateTime) {
OffsetDateTime odt = (OffsetDateTime)o1;
return odt.format(DateTimeFormatter.ofPattern((String)o2));
} else if (o1 instanceof Number) {
return String.format((String) o2, (Number) o1);
}

5
main/src/com/google/refine/expr/functions/Type.java
View File

@ -33,8 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Properties;
@ -55,7 +54,7 @@ public class Type implements Function {
if (v != null) {
if (v instanceof String) {
return "string";
} else if (v instanceof Calendar || v instanceof Date) {
} else if (v instanceof OffsetDateTime) {
return "date";
} else if (v instanceof Number) {
return "number";

39
main/src/com/google/refine/expr/functions/date/Inc.java
View File

@ -33,8 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions.date;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalUnit;
import java.util.Properties;
import org.json.JSONException;
@ -49,42 +50,38 @@ public class Inc implements Function {
@Override
public Object call(Properties bindings, Object[] args) {
if (args.length == 3 &&
args[0] != null && (args[0] instanceof Calendar || args[0] instanceof Date) &&
args[0] != null && (args[0] instanceof OffsetDateTime) &&
args[1] != null && args[1] instanceof Number &&
args[2] != null && args[2] instanceof String) {
Calendar date;
if (args[0] instanceof Calendar) {
date = (Calendar) ((Calendar) args[0]).clone(); // must copy so not to modify original
} else {
date = Calendar.getInstance();
date.setTime((Date) args[0]);
}
OffsetDateTime date = (OffsetDateTime)args[0];
int amount = ((Number) args[1]).intValue();
String unit = (String) args[2];
date.add(getField(unit), amount);
return date;
return date.plus(amount, getField(unit));
}
return new EvalError(ControlFunctionRegistry.getFunctionName(this) + " expects a date, a number and a string");
}
private int getField(String unit) {
private TemporalUnit getField(String unit) {
if ("hours".equals(unit) || "hour".equals(unit) || "h".equals(unit)) {
return Calendar.HOUR;
return ChronoUnit.HOURS;
} else if ("days".equals(unit) || "day".equals(unit) || "d".equals(unit)) {
return Calendar.DAY_OF_MONTH;
return ChronoUnit.DAYS;
} else if ("years".equals(unit) || "year".equals(unit)) {
return Calendar.YEAR;
return ChronoUnit.YEARS;
} else if ("months".equals(unit) || "month".equals(unit)) { // avoid 'm' to avoid confusion with minute
return Calendar.MONTH;
return ChronoUnit.MONTHS;
} else if ("minutes".equals(unit) || "minute".equals(unit) || "min".equals(unit)) { // avoid 'm' to avoid confusion with month
return Calendar.MINUTE;
return ChronoUnit.MINUTES;
} else if ("weeks".equals(unit) || "week".equals(unit) || "w".equals(unit)) {
return Calendar.WEEK_OF_MONTH;
return ChronoUnit.WEEKS;
} else if ("seconds".equals(unit) || "sec".equals(unit) || "s".equals(unit)) {
return Calendar.SECOND;
return ChronoUnit.SECONDS;
} else if ("milliseconds".equals(unit) || "ms".equals(unit) || "S".equals(unit)) {
return ChronoUnit.MILLIS;
} else if ("nanos".equals(unit) || "nano".equals(unit) || "n".equals(unit)) {
return ChronoUnit.NANOS;
} else {
throw new RuntimeException("Unit '" + unit + "' not recognized.");
}

5
main/src/com/google/refine/expr/functions/date/Now.java
View File

@ -33,7 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions.date;
import java.util.Calendar;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.Properties;
import org.json.JSONException;
@ -46,7 +47,7 @@ public class Now implements Function {
@Override
public Object call(Properties bindings, Object[] args) {
if (args.length == 0) {
return Calendar.getInstance();
return OffsetDateTime.now(ZoneId.of("Z"));
}
return null;
}

37
main/src/com/google/refine/expr/functions/strings/Diff.java
View File

@ -33,8 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.expr.functions.strings;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
@ -42,8 +41,6 @@ import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.util.CalendarParser;
import com.google.refine.expr.util.CalendarParserException;
import com.google.refine.grel.Function;
public class Diff implements Function {
@ -56,21 +53,24 @@ public class Diff implements Function {
if (o1 != null && o2 != null) {
if (args.length == 2 && o1 instanceof String && o2 instanceof String) {
return StringUtils.difference((String) o1,(String) o2);
} else if ((o1 instanceof Date || o1 instanceof Calendar) && args.length == 3) {
} else if (o1 instanceof OffsetDateTime && o2 instanceof OffsetDateTime && args.length == 3) {
Object o3 = args[2];
if (o3 != null && o3 instanceof String) {
try {
String unit = ((String) o3).toLowerCase();
Date c1 = (o1 instanceof Date) ? (Date) o1 : ((Calendar) o1).getTime();
Date c2;
if (o2 instanceof Date) {
c2 = (Date) o2;
} else if (o2 instanceof Calendar) {
c2 = ((Calendar) o2).getTime();
} else {
c2 = CalendarParser.parse((o2 instanceof String) ? (String) o2 : o2.toString()).getTime();
OffsetDateTime c1 = (OffsetDateTime)o1;
OffsetDateTime c2 = (OffsetDateTime)o2;
long delta = getNano(c1) - getNano(c2);
if ("nanos".equals(unit)) {
return delta;
}
long delta = (c1.getTime() - c2.getTime()) / 1000;
delta /= 1000;
if ("milliseconds".equals(unit)) {
return delta;
}
delta /= 1000000;
if ("seconds".equals(unit)) {
return delta;
}
@ -96,9 +96,6 @@ public class Diff implements Function {
return days / 365;
}
return new EvalError("Unknown time unit " + unit);
} catch (CalendarParserException e) {
return new EvalError(e);
}
}
}
}
@ -114,4 +111,8 @@ public class Diff implements Function {
writer.key("returns"); writer.value("string for strings, number for dates");
writer.endObject();
}
private long getNano(OffsetDateTime odt) {
return odt.toEpochSecond() * 1000000000l + odt.toInstant().getNano();
}
}

20
main/src/com/google/refine/expr/util/CalendarParser.java
View File

@ -35,12 +35,15 @@ package com.google.refine.expr.util;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.time.OffsetDateTime;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.refine.util.ParsingUtilities;
// Taken from http://icecube.wisc.edu/~dglo/software/calparse/index.html
// Copyright Dave Glowacki. Released under the BSD license.
@ -866,6 +869,10 @@ public class CalendarParser {
return parse(dateStr, YY_MM_DD);
}
public static final OffsetDateTime parseAsOffsetDateTime(String dateStr) throws CalendarParserException {
return ParsingUtilities.calendarToOffsetDateTime(parse(dateStr));
}
/**
* Extract a date from a string.
*
@ -885,6 +892,11 @@ public class CalendarParser {
return parse(dateStr, order, true);
}
public static final OffsetDateTime parseAsOffsetDateTime(String dateStr, int order)
throws CalendarParserException {
return ParsingUtilities.calendarToOffsetDateTime(parse(dateStr, order));
}
/**
* Extract a date from a string.
*
@ -911,6 +923,11 @@ public class CalendarParser {
return parseString(dateStr, order, ignoreChanges);
}
public static final OffsetDateTime parseAsOffsetDateTime(String dateStr, int order,
boolean ignoreChanges) throws CalendarParserException {
return ParsingUtilities.calendarToOffsetDateTime(parse(dateStr, order, ignoreChanges));
}
/**
* Parse a non-numeric token from the date string.
*
@ -1620,7 +1637,7 @@ public class CalendarParser {
state.setYear(tmpYear + (CENTURY_OFFSET - 100));
}
GregorianCalendar cal = new GregorianCalendar();
GregorianCalendar cal = new GregorianCalendar(TimeZone.getTimeZone("Z"));
state.setCalendar(cal, ignoreChanges);
@ -1631,6 +1648,7 @@ public class CalendarParser {
+ state.getMillisecond() + " => " + toString(cal));
}
// return cal.toInstant().atOffset(ZoneOffset.of("Z"));
return cal;
}

2
main/src/com/google/refine/expr/util/CalendarParserException.java
View File

@ -37,7 +37,7 @@ package com.google.refine.expr.util;
// Copyright Dave Glowacki. Released under the BSD license.
/**
* Thrown when an invalid date is encountered in <tt>CalendarParser</tt>.
* Thrown when an invalid date is encountered in <tt>OffsetDateTimeParser</tt>.
*/
public class CalendarParserException extends Exception {

8
main/src/com/google/refine/model/Cell.java
View File

@ -39,8 +39,6 @@ import java.time.Instant;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Calendar;
import java.util.Date;
import java.util.Properties;
import org.json.JSONException;
@ -92,11 +90,7 @@ public class Cell implements HasFields, Jsonizable {
writer.key("v");
if (value != null) {
Instant instant = null;
if (value instanceof Calendar) {
instant = ((Calendar)value).toInstant();
} else if (value instanceof Date) {
instant = ((Date)value).toInstant();
} else if (value instanceof OffsetDateTime) {
if (value instanceof OffsetDateTime) {
instant = ((OffsetDateTime)value).toInstant();
} else if (value instanceof LocalDateTime) {
instant = ((LocalDateTime)value).toInstant(ZoneOffset.of("Z"));

9
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -38,11 +38,8 @@ import java.io.InputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.time.ZoneId;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
@ -307,11 +304,7 @@ public class StandardReconConfig extends ReconConfig {
jsonWriter.key("id"); jsonWriter.value(cell2.recon.match.id);
jsonWriter.key("name"); jsonWriter.value(cell2.recon.match.name);
jsonWriter.endObject();
} else if (cell2.value instanceof Calendar) {
Calendar calendar = (Calendar) cell2.value;
OffsetDateTime d = OffsetDateTime.ofInstant(calendar.toInstant(), ZoneId.of("Z"));
jsonWriter.value(ParsingUtilities.dateToString(d));
} else if (cell2.value instanceof Date) {
} else if (cell2.value instanceof OffsetDateTime) {
jsonWriter.value(ParsingUtilities.dateToString((OffsetDateTime) cell2.value));
} else {
jsonWriter.value(cell2.value.toString());

12
main/src/com/google/refine/sorting/DateCriterion.java
View File

@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.sorting;
import java.util.Calendar;
import java.util.Date;
import java.time.Instant;
import java.time.OffsetDateTime;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils;
@ -48,10 +48,8 @@ public class DateCriterion extends Criterion {
@Override
protected Object makeKey(Object value) {
if (ExpressionUtils.isNonBlankData(value)) {
if (value instanceof Date) {
return value;
} else if (value instanceof Calendar) {
return ((Calendar) value).getTime();
if (value instanceof OffsetDateTime) {
return ((OffsetDateTime) value).toInstant();
} else {
return s_error;
}
@ -61,7 +59,7 @@ public class DateCriterion extends Criterion {
@Override
public int compareKeys(Object key1, Object key2) {
return ((Date) key1).compareTo((Date) key2);
return ((Instant) key1).compareTo((Instant) key2);
}
};
}

9
main/src/com/google/refine/sorting/NumberCriterion.java
View File

@ -33,8 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.sorting;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.ExpressionUtils;
@ -53,10 +52,8 @@ public class NumberCriterion extends Criterion {
return value;
} else if (value instanceof Boolean) {
return ((Boolean) value).booleanValue() ? 1 : 0;
} else if (value instanceof Date) {
return ((Date) value).getTime();
} else if (value instanceof Calendar) {
return ((Calendar) value).getTime().getTime();
} else if (value instanceof OffsetDateTime) {
return ((OffsetDateTime) value).toInstant().toEpochMilli();
} else if (value instanceof String) {
try {
double d = Double.parseDouble((String) value);

3
main/src/com/google/refine/util/JSONUtilities.java
View File

@ -40,7 +40,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
@ -197,8 +196,6 @@ public class JSONUtilities {
obj.put(key, ((Double) value).doubleValue());
} else if (value instanceof Boolean) {
obj.put(key, value);
} else if (value instanceof Date) {
obj.put(key, ParsingUtilities.dateToString((OffsetDateTime) value));
} else if (value instanceof Calendar) {
obj.put(key, ParsingUtilities.dateToString(OffsetDateTime.ofInstant(((Calendar)value).toInstant(), ZoneId.of("Z"))));
} else if (value instanceof String) {

13
main/src/com/google/refine/util/ParsingUtilities.java
View File

@ -45,7 +45,10 @@ import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.Properties;
import java.util.TimeZone;
import javax.servlet.http.HttpServletRequest;
@ -223,4 +226,14 @@ public class ParsingUtilities {
static public String instantToLocalDateTimeString(Instant instant) {
return LocalDateTime.ofInstant(instant, ZoneId.systemDefault()).format(ISO8601);
}
static public OffsetDateTime calendarToOffsetDateTime(Calendar calendar) {
return calendar.toInstant().atOffset(ZoneOffset.of("Z"));
}
static public Calendar offsetDateTimeToCalendar(OffsetDateTime offsetDateTime) {
Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("Z"));
cal.setTimeInMillis(offsetDateTime.toInstant().toEpochMilli());
return cal;
}
}

16
main/src/com/google/refine/util/StringUtils.java
View File

@ -1,21 +1,21 @@
package com.google.refine.util;
import java.text.DateFormat;
import java.util.Calendar;
import java.util.Date;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
public class StringUtils {
private static String DEFAULT_PATTERN = "dd-MMM-yyyy";
/**
* String formatting method that knows how to format dates (using the defaul locale's date formatter)
* String formatting method that knows how to format dates (using the default locale's date formatter)
* @param o object to be converted to a string
* @return string representing object
*/
public static String toString(Object o) {
if (o instanceof Calendar || o instanceof Date) {
DateFormat formatter = DateFormat.getDateInstance();
return formatter.format(o instanceof Date ? ((Date) o) : ((Calendar) o).getTime());
// to replace the DateFormat with java.time.format.DateTimeFormatter
if (o instanceof OffsetDateTime) {
OffsetDateTime odt = (OffsetDateTime)o;
return odt.format(DateTimeFormatter.ofPattern(DEFAULT_PATTERN));
} else if (o == null) {
return "null";
} else {

7
main/tests/server/src/com/google/refine/tests/exporters/CsvExporterTests.java
View File

@ -40,8 +40,6 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Calendar;
import java.util.Date;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
@ -60,7 +58,6 @@ import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.tests.RefineTest;
import com.google.refine.util.ParsingUtilities;
public class CsvExporterTests extends RefineTest {
@ -198,6 +195,9 @@ public class CsvExporterTests extends RefineTest {
",row2cell1,row2cell2\n");
}
// all date type cells are in unified format
/**
@Ignore
@Test
public void exportDateColumnsPreVersion28(){
CreateGrid(1,2);
@ -219,6 +219,7 @@ public class CsvExporterTests extends RefineTest {
Assert.assertEquals(writer.toString(), expectedOutput);
}
*/
//helper methods
protected void CreateColumns(int noOfColumns){

27
main/tests/server/src/com/google/refine/tests/exporters/XlsExporterTests.java
View File

@ -40,6 +40,7 @@ import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.Properties;
import org.slf4j.LoggerFactory;
@ -121,6 +122,20 @@ public class XlsExporterTests extends RefineTest {
}
@Test
public void exportDateType() throws IOException{
OffsetDateTime odt = OffsetDateTime.now();
createDateGrid(2, 2, odt);
try {
SUT.export(project, options, engine, stream);
} catch (IOException e) {
Assert.fail();
}
Assert.assertEquals(stream.size(),4096);
}
@Test(enabled=false)
public void exportSimpleXlsNoHeader(){
CreateGrid(2, 2);
@ -179,4 +194,16 @@ public class XlsExporterTests extends RefineTest {
project.rows.add(row);
}
}
private void createDateGrid(int noOfRows, int noOfColumns, OffsetDateTime now){
CreateColumns(noOfColumns);
for(int i = 0; i < noOfRows; i++){
Row row = new Row(noOfColumns);
for(int j = 0; j < noOfColumns; j++){
row.cells.add(new Cell(now, null));
}
project.rows.add(row);
}
}
}

2
main/tests/server/src/com/google/refine/tests/expr/functions/CoalesceTests.java
View File

@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.refine.tests.expr.functions.strings;
package com.google.refine.tests.expr.functions;
import java.util.Properties;

118
main/tests/server/src/com/google/refine/tests/expr/functions/date/DatePartTests.java
View File

@ -1,7 +1,5 @@
package com.google.refine.tests.expr.functions.date;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
@ -17,7 +15,6 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.tests.RefineTest;
@ -59,124 +56,11 @@ public class DatePartTests extends RefineTest {
}
}
@Test
public void testDateDatePart() throws ParseException {
Assert.assertTrue(invoke("datePart") instanceof EvalError);
// 2018-4-30 23:55:44, cannot use new Date(2018 - 1900, 4 - 1, 30, 23, 55, 44). use below way to get a UTC date:
SimpleDateFormat isoFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
isoFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
Date source = isoFormat.parse("2018-04-30T23:55:44");
// hours
Assert.assertEquals(invoke("datePart", source, "hours"), 23);
Assert.assertEquals(invoke("datePart", source, "hour"), 23);
Assert.assertEquals(invoke("datePart", source, "h"), 23);
// minutes
Assert.assertEquals(invoke("datePart", source, "minutes"), 55);
Assert.assertEquals(invoke("datePart", source, "minute"), 55);
Assert.assertEquals(invoke("datePart", source, "min"), 55);
// seconds
Assert.assertEquals(invoke("datePart", source, "seconds"), 44);
Assert.assertEquals(invoke("datePart", source, "sec"), 44);
Assert.assertEquals(invoke("datePart", source, "s"), 44);
// milliseconds
Assert.assertEquals(invoke("datePart", source, "milliseconds"), 0);
Assert.assertEquals(invoke("datePart", source, "ms"), 0);
Assert.assertEquals(invoke("datePart", source, "S"), 0);
// years
Assert.assertEquals(invoke("datePart", source, "years"), 2018);
Assert.assertEquals(invoke("datePart", source, "year"), 2018);
// months
Assert.assertEquals(invoke("datePart", source, "months"), 4);
Assert.assertEquals(invoke("datePart", source, "month"), 4);
// weeks
Assert.assertEquals(invoke("datePart", source, "weeks"), 5);
Assert.assertEquals(invoke("datePart", source, "week"), 5);
Assert.assertEquals(invoke("datePart", source, "w"), 5);
// days, day, d
Assert.assertEquals(invoke("datePart", source, "days"), 30);
Assert.assertEquals(invoke("datePart", source, "day"), 30);
Assert.assertEquals(invoke("datePart", source, "d"), 30);
// weekday
Assert.assertEquals(invoke("datePart", source, "weekday"), "Monday");
// time
Assert.assertEquals(invoke("datePart", source, "time"), 1525132544000l);
}
@Test
public void testCalendarDatePart() throws ParseException {
// 2018-4-30 23:55:44
SimpleDateFormat isoFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
isoFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
Date dt = isoFormat.parse("2018-04-30T23:55:44");
Calendar source = dateToCalendar(dt);
source.set(Calendar.MILLISECOND, 789);
// hours
Assert.assertEquals(invoke("datePart", source, "hours"), 23);
Assert.assertEquals(invoke("datePart", source, "hour"), 23);
Assert.assertEquals(invoke("datePart", source, "h"), 23);
// minutes
Assert.assertEquals(invoke("datePart", source, "minutes"), 55);
Assert.assertEquals(invoke("datePart", source, "minute"), 55);
Assert.assertEquals(invoke("datePart", source, "min"), 55);
// seconds
Assert.assertEquals(invoke("datePart", source, "seconds"), 44);
Assert.assertEquals(invoke("datePart", source, "sec"), 44);
Assert.assertEquals(invoke("datePart", source, "s"), 44);
// milliseconds
Assert.assertEquals(invoke("datePart", source, "milliseconds"), 789);
Assert.assertEquals(invoke("datePart", source, "ms"), 789);
Assert.assertEquals(invoke("datePart", source, "S"), 789);
// years
Assert.assertEquals(invoke("datePart", source, "years"), 2018);
Assert.assertEquals(invoke("datePart", source, "year"), 2018);
// months
Assert.assertEquals(invoke("datePart", source, "months"), 4);
Assert.assertEquals(invoke("datePart", source, "month"), 4);
// weeks
Assert.assertEquals(invoke("datePart", source, "weeks"), 5);
Assert.assertEquals(invoke("datePart", source, "week"), 5);
Assert.assertEquals(invoke("datePart", source, "w"), 5);
// days, day, d
Assert.assertEquals(invoke("datePart", source, "days"), 30);
Assert.assertEquals(invoke("datePart", source, "day"), 30);
Assert.assertEquals(invoke("datePart", source, "d"), 30);
// weekday
Assert.assertEquals(invoke("datePart", source, "weekday"), "Monday");
// time
Assert.assertEquals(invoke("datePart", source, "time"), 1525132544000l + 789);
}
private DateTimeFormatter formatter = new DateTimeFormatterBuilder()
// here is the same as your code
.append(DateTimeFormatter.BASIC_ISO_DATE).appendLiteral('-')
// time (hour/minute/seconds)
.appendPattern("HH:mm:ss")
// optional nanos, with 9, 6 or 3 digits
.appendPattern("[.SSSSSSSSS][.SSSSSS][.SSS]")
// offset
.appendPattern("[.SSSSSSSSS][.SSSSSS][.SSS]") // optional nanos, with 9, 6 or 3 digits
.appendOffset("+HH:mm", "Z")
// create formatter
.toFormatter();
@Test

118
main/tests/server/src/com/google/refine/tests/expr/functions/date/IncTests.java Normal file
View File

@ -0,0 +1,118 @@
package com.google.refine.tests.expr.functions.date;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.temporal.ChronoUnit;
import java.util.Properties;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.expr.EvalError;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.tests.RefineTest;
public class IncTests extends RefineTest {
private static Properties bindings;
private DateTimeFormatter formatter = new DateTimeFormatterBuilder()
.append(DateTimeFormatter.BASIC_ISO_DATE).appendLiteral('-')
.appendPattern("HH:mm:ss")
.appendPattern("[.SSSSSSSSS][.SSSSSS][.SSS]") // optional nanos, with 9, 6 or 3 digits
.appendOffset("+HH:mm", "Z")
.toFormatter();
@Override
@BeforeTest
public void init() {
logger = LoggerFactory.getLogger(this.getClass());
}
@BeforeMethod
public void setUp() {
bindings = new Properties();
}
@AfterMethod
public void tearDown() {
bindings = null;
}
/**
* Lookup a control function by name and invoke it with a variable number of args
*/
private static Object invoke(String name,Object... args) {
// registry uses static initializer, so no need to set it up
Function function = ControlFunctionRegistry.getFunction(name);
if (function == null) {
throw new IllegalArgumentException("Unknown function "+name);
}
if (args == null) {
return function.call(bindings,new Object[0]);
} else {
return function.call(bindings,args);
}
}
@Test
public void testInc() {
OffsetDateTime source = OffsetDateTime.parse("20180510-23:55:44.000789000Z",
formatter);
// add hours
Assert.assertTrue(invoke("inc", source, 2, "hours") instanceof OffsetDateTime);
Assert.assertEquals(invoke("inc", source, 2, "hours"), source.plus(2, ChronoUnit.HOURS));
Assert.assertEquals(invoke("inc", source, 2, "hour"), source.plus(2, ChronoUnit.HOURS));
Assert.assertEquals(invoke("inc", source, 2, "h"), source.plus(2, ChronoUnit.HOURS));
// add years
Assert.assertTrue(invoke("inc", source, 2, "year") instanceof OffsetDateTime);
Assert.assertEquals(invoke("inc", source, 2, "years"), source.plus(2, ChronoUnit.YEARS));
Assert.assertEquals(invoke("inc", source, 2, "year"), source.plus(2, ChronoUnit.YEARS));
// add months
Assert.assertTrue(invoke("inc", source, 2, "months") instanceof OffsetDateTime);
Assert.assertEquals(invoke("inc", source, 2, "months"), source.plus(2, ChronoUnit.MONTHS));
Assert.assertEquals(invoke("inc", source, 2, "month"), source.plus(2, ChronoUnit.MONTHS));
// add minutes
Assert.assertTrue(invoke("inc", source, 2, "minutes") instanceof OffsetDateTime);
Assert.assertEquals(invoke("inc", source, 2, "minutes"), source.plus(2, ChronoUnit.MINUTES));
Assert.assertEquals(invoke("inc", source, 2, "minute"), source.plus(2, ChronoUnit.MINUTES));
Assert.assertEquals(invoke("inc", source, 2, "min"), source.plus(2, ChronoUnit.MINUTES));
// add weeks
Assert.assertTrue(invoke("inc", source, 2, "weeks") instanceof OffsetDateTime);
Assert.assertEquals(invoke("inc", source, 2, "weeks"), source.plus(2, ChronoUnit.WEEKS));
Assert.assertEquals(invoke("inc", source, 2, "week"), source.plus(2, ChronoUnit.WEEKS));
Assert.assertEquals(invoke("inc", source, 2, "w"), source.plus(2, ChronoUnit.WEEKS));
// add seconds
Assert.assertTrue(invoke("inc", source, 2, "seconds") instanceof OffsetDateTime);
Assert.assertEquals(invoke("inc", source, 2, "seconds"), source.plus(2, ChronoUnit.SECONDS));
Assert.assertEquals(invoke("inc", source, 2, "sec"), source.plus(2, ChronoUnit.SECONDS));
Assert.assertEquals(invoke("inc", source, 2, "s"), source.plus(2, ChronoUnit.SECONDS));
// add milliseconds
Assert.assertTrue(invoke("inc", source, 2, "milliseconds") instanceof OffsetDateTime);
Assert.assertEquals(invoke("inc", source, 2, "milliseconds"), source.plus(2, ChronoUnit.MILLIS));
Assert.assertEquals(invoke("inc", source, 2, "ms"), source.plus(2, ChronoUnit.MILLIS));
Assert.assertEquals(invoke("inc", source, 2, "S"), source.plus(2, ChronoUnit.MILLIS));
// add nanos
Assert.assertTrue(invoke("inc", source, 2, "nanos") instanceof OffsetDateTime);
Assert.assertEquals(invoke("inc", source, 2, "nanos"), source.plus(2, ChronoUnit.NANOS));
Assert.assertEquals(invoke("inc", source, 2, "nano"), source.plus(2, ChronoUnit.NANOS));
Assert.assertEquals(invoke("inc", source, 2, "n"), source.plus(2, ChronoUnit.NANOS));
// exception
Assert.assertTrue(invoke("inc", source, 99) instanceof EvalError);
Assert.assertTrue(invoke("inc", source.toInstant().toEpochMilli(), 99, "h") instanceof EvalError);
}
}

70
main/tests/server/src/com/google/refine/tests/expr/functions/date/NowTests.java Normal file
View File

@ -0,0 +1,70 @@
package com.google.refine.tests.expr.functions.date;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.util.Properties;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.tests.RefineTest;
public class NowTests extends RefineTest {
private static Properties bindings;
private DateTimeFormatter formatter = new DateTimeFormatterBuilder()
.append(DateTimeFormatter.BASIC_ISO_DATE).appendLiteral('-')
.appendPattern("HH:mm:ss")
.appendPattern("[.SSSSSSSSS][.SSSSSS][.SSS]") // optional nanos, with 9, 6 or 3 digits
.appendOffset("+HH:mm", "Z")
.toFormatter();
@Override
@BeforeTest
public void init() {
logger = LoggerFactory.getLogger(this.getClass());
}
@BeforeMethod
public void setUp() {
bindings = new Properties();
}
@AfterMethod
public void tearDown() {
bindings = null;
}
/**
* Lookup a control function by name and invoke it with a variable number of args
*/
private static Object invoke(String name,Object... args) {
// registry uses static initializer, so no need to set it up
Function function = ControlFunctionRegistry.getFunction(name);
if (function == null) {
throw new IllegalArgumentException("Unknown function "+name);
}
if (args == null) {
return function.call(bindings,new Object[0]);
} else {
return function.call(bindings,args);
}
}
@Test
public void testNow() {
// 2018-4-30 23:55:44
OffsetDateTime source = OffsetDateTime.parse("20180430-23:55:44.000789000Z",
formatter);
Assert.assertTrue(invoke("now") instanceof OffsetDateTime);
Assert.assertTrue(((OffsetDateTime)invoke("now")).isAfter(source));
}
}

40
main/tests/server/src/com/google/refine/tests/expr/functions/strings/DiffTests.java
View File

@ -1,6 +1,7 @@
package com.google.refine.tests.expr.functions.strings;
import java.util.Calendar;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Properties;
import org.slf4j.LoggerFactory;
@ -11,8 +12,6 @@ import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.util.CalendarParser;
import com.google.refine.expr.util.CalendarParserException;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.tests.RefineTest;
@ -21,21 +20,16 @@ import com.google.refine.tests.RefineTest;
public class DiffTests extends RefineTest {
static Properties bindings;
private Calendar date1;
private Calendar date2;
private OffsetDateTime odt1;
private OffsetDateTime odt2;
@Override
@BeforeTest
public void init() {
logger = LoggerFactory.getLogger(this.getClass());
try {
date1 = CalendarParser.parse("2012-08-02");
date2 = CalendarParser.parse("2012-10-02");
} catch (CalendarParserException e) {
// TODO Auto-generated catch block
e.printStackTrace();
Assert.fail();
}
odt1 = OffsetDateTime.parse("2011-09-01T10:15:30.123456+01:00", DateTimeFormatter.ISO_OFFSET_DATE_TIME);
odt2 = OffsetDateTime.parse("2011-12-02T10:16:30.123467+01:00", DateTimeFormatter.ISO_OFFSET_DATE_TIME);
}
@BeforeMethod
@ -68,18 +62,22 @@ public class DiffTests extends RefineTest {
public void testDiffInvalidParams() {
Assert.assertTrue(invoke("diff") instanceof EvalError);
Assert.assertTrue(invoke("diff", "one","two","three") instanceof EvalError);
Assert.assertTrue(invoke("diff", date1,date2) instanceof EvalError);
Assert.assertTrue(invoke("diff", date1,date2,"foo") instanceof EvalError);
}
@Test
public void testDiff() {
public void testDiffString() {
Assert.assertEquals((String)(invoke("diff", "onetwo","onetwothree")),"three");
Assert.assertEquals(invoke("diff",date2,date1,"days"),Long.valueOf(61));
Assert.assertEquals(invoke("diff",date2,date1,"weeks"),Long.valueOf(8));
Assert.assertEquals(invoke("diff",date2,date1,"months"),Long.valueOf(2));
Assert.assertEquals(invoke("diff",date2,date1,"hours"),Long.valueOf(1464));
Assert.assertEquals(invoke("diff",date2,date1,"seconds"),Long.valueOf(5270400));
}
@Test
public void testDiffOffsetDateTime() {
// OffsetDateTime diff:
Assert.assertEquals(invoke("diff",odt2,odt1,"days"),Long.valueOf(92));
Assert.assertEquals(invoke("diff",odt2,odt1,"weeks"),Long.valueOf(13));
Assert.assertEquals(invoke("diff",odt2,odt1,"months"),Long.valueOf(3));
Assert.assertEquals(invoke("diff",odt2,odt1,"hours"),Long.valueOf(2208));
Assert.assertEquals(invoke("diff",odt2,odt1,"seconds"),Long.valueOf(7948860));
Assert.assertEquals(invoke("diff",odt2,odt1,"milliseconds"),Long.valueOf(7948860000011l));
Assert.assertEquals(invoke("diff",odt2,odt1,"nanos"),Long.valueOf(7948860000011000l));
}
}

47
main/tests/server/src/com/google/refine/tests/expr/functions/strings/ToFromConversionTests.java
View File

@ -33,8 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.tests.expr.functions.strings;
import java.text.DateFormat;
import java.util.GregorianCalendar;
import java.time.OffsetDateTime;
import java.util.Properties;
import org.slf4j.LoggerFactory;
@ -45,8 +44,8 @@ import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.refine.expr.EvalError;
import com.google.refine.expr.util.CalendarParser;
import com.google.refine.expr.util.CalendarParserException;
import com.google.refine.expr.util.CalendarParser;
import com.google.refine.grel.ControlFunctionRegistry;
import com.google.refine.grel.Function;
import com.google.refine.tests.RefineTest;
@ -118,11 +117,20 @@ public class ToFromConversionTests extends RefineTest {
Assert.assertEquals(invoke("toString", Double.valueOf(100.0)),"100.0");
Assert.assertEquals(invoke("toString", Double.valueOf(100.0),"%.0f"),"100");
String expectedDate = DateFormat.getDateInstance().format(new GregorianCalendar(2013,5,1).getTime());
Assert.assertEquals(invoke("toString", CalendarParser.parse("2013-06-01")), expectedDate);
Assert.assertEquals(invoke("toString", CalendarParser.parse("2013-06-01").getTime()), expectedDate);
Assert.assertEquals(invoke("toString", CalendarParser.parse("2013-06-01"),"yyyy"),"2013");
Assert.assertEquals(invoke("toString", CalendarParser.parse("2013-06-01"),"yyyy-MM-dd"),"2013-06-01");
String inputDate = "2013-06-01";
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDate)),
"01-Jun-2013");
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDate), "yyyy-MM-dd"),
"2013-06-01");
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDate), "yyyy/dd/MM"), "2013/01/06");
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDate), "yyyy-MMM"), "2013-Jun");
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDate), "yyyy-MM-dd hh:mm:ss"), "2013-06-01 12:00:00");
String inputDateTime = "2013-06-01 13:12:11";
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDateTime)), "01-Jun-2013");
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDateTime), "yyyy-MM-dd"), "2013-06-01");
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDateTime), "yyyy-MM-dd hh:mm:ss"),"2013-06-01 01:12:11");
Assert.assertEquals(invoke("toString", CalendarParser.parseAsOffsetDateTime(inputDateTime), "yyyy-MM-dd HH:mm:ss"),"2013-06-01 13:12:11");
}
@Test
@ -132,21 +140,16 @@ public class ToFromConversionTests extends RefineTest {
Assert.assertTrue(invoke("toDate", "") instanceof EvalError);
Assert.assertTrue(invoke("toDate", 1.0) instanceof EvalError);
Assert.assertTrue(invoke("toDate", "2012-03-01","xxx") instanceof EvalError); // bad format string
Assert.assertTrue(invoke("toDate", "2012-03-01") instanceof GregorianCalendar);
Assert.assertEquals(invoke("toDate", "2012-03-01"),CalendarParser.parse("2012-03-01"));
Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parse("2012-03-01"));
// Multiple format strings should get tried sequentially until one succeeds or all are exhausted
Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parse("2012-03-01"));
// First string can be a locale identifier instead of a format string
Assert.assertEquals(invoke("toDate", "2013-06-01","zh"), CalendarParser.parse("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_HK","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_TW","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh_CN","dd-MMM-yyyy"), CalendarParser.parse("2013-06-01"));
Assert.assertTrue(invoke("toDate", "2012-03-01") instanceof OffsetDateTime);
Assert.assertEquals(invoke("toDate", "2012-03-01"),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
Assert.assertEquals(invoke("toDate", "2012-03-01","yyyy-MM-dd"),CalendarParser.parseAsOffsetDateTime("2012-03-01"));
// Date
// Calendar
// String
// Multiple format strings should get tried sequentially until one succeeds or all are exhausted
Assert.assertEquals(invoke("toDate", "2012-03-01","MMM","yyyy-MM-dd"), CalendarParser.parseAsOffsetDateTime("2012-03-01"));
// First string can be a locale identifier instead of a format string
Assert.assertEquals(invoke("toDate", "2013-06-01","zh"), CalendarParser.parseAsOffsetDateTime("2013-06-01"));
Assert.assertEquals(invoke("toDate", "01-六月-2013","zh","dd-MMM-yyyy"), CalendarParser.parseAsOffsetDateTime("2013-06-01"));
}
@Test