""" test to_datetime """
import calendar
from collections import deque
from datetime import (
date,
datetime,
timedelta,
timezone,
)
from decimal import Decimal
import locale
from dateutil.parser import parse
from dateutil.tz.tz import tzoffset
import numpy as np
import pytest
import pytz
from pandas._libs import tslib
from pandas._libs.tslibs import (
iNaT,
parsing,
)
from pandas.errors import (
OutOfBoundsDatetime,
OutOfBoundsTimedelta,
)
import pandas.util._test_decorators as td
from pandas.core.dtypes.common import is_datetime64_ns_dtype
import pandas as pd
from pandas import (
DataFrame,
DatetimeIndex,
Index,
NaT,
Series,
Timestamp,
date_range,
isna,
to_datetime,
)
import pandas._testing as tm
from pandas.core.arrays import DatetimeArray
from pandas.core.tools import datetimes as tools
from pandas.core.tools.datetimes import start_caching_at
PARSING_ERR_MSG = (
r"You might want to try:\n"
r" - passing `format` if your strings have a consistent format;\n"
r" - passing `format=\'ISO8601\'` if your strings are all ISO8601 "
r"but not necessarily in exactly the same format;\n"
r" - passing `format=\'mixed\'`, and the format will be inferred "
r"for each element individually. You might want to use `dayfirst` "
r"alongside this."
)
pytestmark = pytest.mark.filterwarnings(
"ignore:errors='ignore' is deprecated:FutureWarning"
)
@pytest.fixture(params=[True, False])
def cache(request):
"""
cache keyword to pass to to_datetime.
"""
return request.param
class TestTimeConversionFormats:
@pytest.mark.parametrize("readonly", [True, False])
def test_to_datetime_readonly(self, readonly):
# GH#34857
arr = np.array([], dtype=object)
if readonly:
arr.setflags(write=False)
result = to_datetime(arr)
expected = to_datetime([])
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"format, expected",
[
[
"%d/%m/%Y",
[Timestamp("20000101"), Timestamp("20000201"), Timestamp("20000301")],
],
[
"%m/%d/%Y",
[Timestamp("20000101"), Timestamp("20000102"), Timestamp("20000103")],
],
],
)
def test_to_datetime_format(self, cache, index_or_series, format, expected):
values = index_or_series(["1/1/2000", "1/2/2000", "1/3/2000"])
result = to_datetime(values, format=format, cache=cache)
expected = index_or_series(expected)
tm.assert_equal(result, expected)
@pytest.mark.parametrize(
"arg, expected, format",
[
["1/1/2000", "20000101", "%d/%m/%Y"],
["1/1/2000", "20000101", "%m/%d/%Y"],
["1/2/2000", "20000201", "%d/%m/%Y"],
["1/2/2000", "20000102", "%m/%d/%Y"],
["1/3/2000", "20000301", "%d/%m/%Y"],
["1/3/2000", "20000103", "%m/%d/%Y"],
],
)
def test_to_datetime_format_scalar(self, cache, arg, expected, format):
result = to_datetime(arg, format=format, cache=cache)
expected = Timestamp(expected)
assert result == expected
def test_to_datetime_format_YYYYMMDD(self, cache):
ser = Series([19801222, 19801222] + [19810105] * 5)
expected = Series([Timestamp(x) for x in ser.apply(str)])
result = to_datetime(ser, format="%Y%m%d", cache=cache)
tm.assert_series_equal(result, expected)
result = to_datetime(ser.apply(str), format="%Y%m%d", cache=cache)
tm.assert_series_equal(result, expected)
def test_to_datetime_format_YYYYMMDD_with_nat(self, cache):
# Explicit cast to float to explicit cast when setting np.nan
ser = Series([19801222, 19801222] + [19810105] * 5, dtype="float")
# with NaT
expected = Series(
[Timestamp("19801222"), Timestamp("19801222")] + [Timestamp("19810105")] * 5
)
expected[2] = np.nan
ser[2] = np.nan
result = to_datetime(ser, format="%Y%m%d", cache=cache)
tm.assert_series_equal(result, expected)
# string with NaT
ser2 = ser.apply(str)
ser2[2] = "nat"
with pytest.raises(
ValueError,
match=(
'unconverted data remains when parsing with format "%Y%m%d": ".0", '
"at position 0"
),
):
# https://github.com/pandas-dev/pandas/issues/50051
to_datetime(ser2, format="%Y%m%d", cache=cache)
def test_to_datetime_format_YYYYMM_with_nat(self, cache):
# https://github.com/pandas-dev/pandas/issues/50237
# Explicit cast to float to explicit cast when setting np.nan
ser = Series([198012, 198012] + [198101] * 5, dtype="float")
expected = Series(
[Timestamp("19801201"), Timestamp("19801201")] + [Timestamp("19810101")] * 5
)
expected[2] = np.nan
ser[2] = np.nan
result = to_datetime(ser, format="%Y%m", cache=cache)
tm.assert_series_equal(result, expected)
def test_to_datetime_format_YYYYMMDD_ignore(self, cache):
# coercion
# GH 7930, GH 14487
ser = Series([20121231, 20141231, 99991231])
result = to_datetime(ser, format="%Y%m%d", errors="ignore", cache=cache)
expected = Series(
[20121231, 20141231, 99991231],
dtype=object,
)
tm.assert_series_equal(result, expected)
def test_to_datetime_format_YYYYMMDD_ignore_with_outofbounds(self, cache):
# https://github.com/pandas-dev/pandas/issues/26493
result = to_datetime(
["15010101", "20150101", np.nan],
format="%Y%m%d",
errors="ignore",
cache=cache,
)
expected = Index(["15010101", "20150101", np.nan], dtype=object)
tm.assert_index_equal(result, expected)
def test_to_datetime_format_YYYYMMDD_coercion(self, cache):
# coercion
# GH 7930
ser = Series([20121231, 20141231, 99991231])
result = to_datetime(ser, format="%Y%m%d", errors="coerce", cache=cache)
expected = Series(["20121231", "20141231", "NaT"], dtype="M8[ns]")
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"input_s",
[
# Null values with Strings
["19801222", "20010112", None],
["19801222", "20010112", np.nan],
["19801222", "20010112", NaT],
["19801222", "20010112", "NaT"],
# Null values with Integers
[19801222, 20010112, None],
[19801222, 20010112, np.nan],
[19801222, 20010112, NaT],
[19801222, 20010112, "NaT"],
],
)
def test_to_datetime_format_YYYYMMDD_with_none(self, input_s):
# GH 30011
# format='%Y%m%d'
# with None
expected = Series([Timestamp("19801222"), Timestamp("20010112"), NaT])
result = Series(to_datetime(input_s, format="%Y%m%d"))
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"input_s, expected",
[
# NaN before strings with invalid date values
[
Series(["19801222", np.nan, "20010012", "10019999"]),
Series([Timestamp("19801222"), np.nan, np.nan, np.nan]),
],
# NaN after strings with invalid date values
[
Series(["19801222", "20010012", "10019999", np.nan]),
Series([Timestamp("19801222"), np.nan, np.nan, np.nan]),
],
# NaN before integers with invalid date values
[
Series([20190813, np.nan, 20010012, 20019999]),
Series([Timestamp("20190813"), np.nan, np.nan, np.nan]),
],
# NaN after integers with invalid date values
[
Series([20190813, 20010012, np.nan, 20019999]),
Series([Timestamp("20190813"), np.nan, np.nan, np.nan]),
],
],
)
def test_to_datetime_format_YYYYMMDD_overflow(self, input_s, expected):
# GH 25512
# format='%Y%m%d', errors='coerce'
result = to_datetime(input_s, format="%Y%m%d", errors="coerce")
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"data, format, expected",
[
([pd.NA], "%Y%m%d%H%M%S", DatetimeIndex(["NaT"])),
([pd.NA], None, DatetimeIndex(["NaT"])),
(
[pd.NA, "20210202202020"],
"%Y%m%d%H%M%S",
DatetimeIndex(["NaT", "2021-02-02 20:20:20"]),
),
(["201010", pd.NA], "%y%m%d", DatetimeIndex(["2020-10-10", "NaT"])),
(["201010", pd.NA], "%d%m%y", DatetimeIndex(["2010-10-20", "NaT"])),
([None, np.nan, pd.NA], None, DatetimeIndex(["NaT", "NaT", "NaT"])),
([None, np.nan, pd.NA], "%Y%m%d", DatetimeIndex(["NaT", "NaT", "NaT"])),
],
)
def test_to_datetime_with_NA(self, data, format, expected):
# GH#42957
result = to_datetime(data, format=format)
tm.assert_index_equal(result, expected)
def test_to_datetime_with_NA_with_warning(self):
# GH#42957
result = to_datetime(["201010", pd.NA])
expected = DatetimeIndex(["2010-10-20", "NaT"])
tm.assert_index_equal(result, expected)
def test_to_datetime_format_integer(self, cache):
# GH 10178
ser = Series([2000, 2001, 2002])
expected = Series([Timestamp(x) for x in ser.apply(str)])
result = to_datetime(ser, format="%Y", cache=cache)
tm.assert_series_equal(result, expected)
ser = Series([200001, 200105, 200206])
expected = Series([Timestamp(x[:4] + "-" + x[4:]) for x in ser.apply(str)])
result = to_datetime(ser, format="%Y%m", cache=cache)
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"int_date, expected",
[
# valid date, length == 8
[20121030, datetime(2012, 10, 30)],
# short valid date, length == 6
[199934, datetime(1999, 3, 4)],
# long integer date partially parsed to datetime(2012,1,1), length > 8
[2012010101, 2012010101],
# invalid date partially parsed to datetime(2012,9,9), length == 8
[20129930, 20129930],
# short integer date partially parsed to datetime(2012,9,9), length < 8
[2012993, 2012993],
# short invalid date, length == 4
[2121, 2121],
],
)
def test_int_to_datetime_format_YYYYMMDD_typeerror(self, int_date, expected):
# GH 26583
result = to_datetime(int_date, format="%Y%m%d", errors="ignore")
assert result == expected
def test_to_datetime_format_microsecond(self, cache):
month_abbr = calendar.month_abbr[4]
val = f"01-{month_abbr}-2011 00:00:01.978"
format = "%d-%b-%Y %H:%M:%S.%f"
result = to_datetime(val, format=format, cache=cache)
exp = datetime.strptime(val, format)
assert result == exp
@pytest.mark.parametrize(
"value, format, dt",
[
["01/10/2010 15:20", "%m/%d/%Y %H:%M", Timestamp("2010-01-10 15:20")],
["01/10/2010 05:43", "%m/%d/%Y %I:%M", Timestamp("2010-01-10 05:43")],
[
"01/10/2010 13:56:01",
"%m/%d/%Y %H:%M:%S",
Timestamp("2010-01-10 13:56:01"),
],
# The 3 tests below are locale-dependent.
# They pass, except when the machine locale is zh_CN or it_IT .
pytest.param(
"01/10/2010 08:14 PM",
"%m/%d/%Y %I:%M %p",
Timestamp("2010-01-10 20:14"),
marks=pytest.mark.xfail(
locale.getlocale()[0] in ("zh_CN", "it_IT"),
reason="fail on a CI build with LC_ALL=zh_CN.utf8/it_IT.utf8",
strict=False,
),
),
pytest.param(
"01/10/2010 07:40 AM",
"%m/%d/%Y %I:%M %p",
Timestamp("2010-01-10 07:40"),
marks=pytest.mark.xfail(
locale.getlocale()[0] in ("zh_CN", "it_IT"),
reason="fail on a CI build with LC_ALL=zh_CN.utf8/it_IT.utf8",
strict=False,
),
),
pytest.param(
"01/10/2010 09:12:56 AM",
"%m/%d/%Y %I:%M:%S %p",
Timestamp("2010-01-10 09:12:56"),
marks=pytest.mark.xfail(
locale.getlocale()[0] in ("zh_CN", "it_IT"),
reason="fail on a CI build with LC_ALL=zh_CN.utf8/it_IT.utf8",
strict=False,
),
),
],
)
def test_to_datetime_format_time(self, cache, value, format, dt):
assert to_datetime(value, format=format, cache=cache) == dt
@td.skip_if_not_us_locale
def test_to_datetime_with_non_exact(self, cache):
# GH 10834
# 8904
# exact kw
ser = Series(
["19MAY11", "foobar19MAY11", "19MAY11:00:00:00", "19MAY11 00:00:00Z"]
)
result = to_datetime(ser, format="%d%b%y", exact=False, cache=cache)
expected = to_datetime(
ser.str.extract(r"(\d+\w+\d+)", expand=False), format="%d%b%y", cache=cache
)
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"format, expected",
[
("%Y-%m-%d", Timestamp(2000, 1, 3)),
("%Y-%d-%m", Timestamp(2000, 3, 1)),
("%Y-%m-%d %H", Timestamp(2000, 1, 3, 12)),
("%Y-%d-%m %H", Timestamp(2000, 3, 1, 12)),
("%Y-%m-%d %H:%M", Timestamp(2000, 1, 3, 12, 34)),
("%Y-%d-%m %H:%M", Timestamp(2000, 3, 1, 12, 34)),
("%Y-%m-%d %H:%M:%S", Timestamp(2000, 1, 3, 12, 34, 56)),
("%Y-%d-%m %H:%M:%S", Timestamp(2000, 3, 1, 12, 34, 56)),
("%Y-%m-%d %H:%M:%S.%f", Timestamp(2000, 1, 3, 12, 34, 56, 123456)),
("%Y-%d-%m %H:%M:%S.%f", Timestamp(2000, 3, 1, 12, 34, 56, 123456)),
(
"%Y-%m-%d %H:%M:%S.%f%z",
Timestamp(2000, 1, 3, 12, 34, 56, 123456, tz="UTC+01:00"),
),
(
"%Y-%d-%m %H:%M:%S.%f%z",
Timestamp(2000, 3, 1, 12, 34, 56, 123456, tz="UTC+01:00"),
),
],
)
def test_non_exact_doesnt_parse_whole_string(self, cache, format, expected):
# https://github.com/pandas-dev/pandas/issues/50412
# the formats alternate between ISO8601 and non-ISO8601 to check both paths
result = to_datetime(
"2000-01-03 12:34:56.123456+01:00", format=format, exact=False
)
assert result == expected
@pytest.mark.parametrize(
"arg",
[
"2012-01-01 09:00:00.000000001",
"2012-01-01 09:00:00.000001",
"2012-01-01 09:00:00.001",
"2012-01-01 09:00:00.001000",
"2012-01-01 09:00:00.001000000",
],
)
def test_parse_nanoseconds_with_formula(self, cache, arg):
# GH8989
# truncating the nanoseconds when a format was provided
expected = to_datetime(arg, cache=cache)
result = to_datetime(arg, format="%Y-%m-%d %H:%M:%S.%f", cache=cache)
assert result == expected
@pytest.mark.parametrize(
"value,fmt,expected",
[
["2009324", "%Y%W%w", Timestamp("2009-08-13")],
["2013020", "%Y%U%w", Timestamp("2013-01-13")],
],
)
def test_to_datetime_format_weeks(self, value, fmt, expected, cache):
assert to_datetime(value, format=fmt, cache=cache) == expected
@pytest.mark.parametrize(
"fmt,dates,expected_dates",
[
[
"%Y-%m-%d %H:%M:%S %Z",
["2010-01-01 12:00:00 UTC"] * 2,
[Timestamp("2010-01-01 12:00:00", tz="UTC")] * 2,
],
[
"%Y-%m-%d %H:%M:%S%z",
["2010-01-01 12:00:00+0100"] * 2,
[
Timestamp(
"2010-01-01 12:00:00", tzinfo=timezone(timedelta(minutes=60))
)
]
* 2,
],
[
"%Y-%m-%d %H:%M:%S %z",
["2010-01-01 12:00:00 +0100"] * 2,
[
Timestamp(
"2010-01-01 12:00:00", tzinfo=timezone(timedelta(minutes=60))
)
]
* 2,
],
[
"%Y-%m-%d %H:%M:%S %z",
["2010-01-01 12:00:00 Z", "2010-01-01 12:00:00 Z"],
[
Timestamp(
"2010-01-01 12:00:00", tzinfo=pytz.FixedOffset(0)
), # pytz coerces to UTC
Timestamp("2010-01-01 12:00:00", tzinfo=pytz.FixedOffset(0)),
],
],
],
)
def test_to_datetime_parse_tzname_or_tzoffset(self, fmt, dates, expected_dates):
# GH 13486
result = to_datetime(dates, format=fmt)
expected = Index(expected_dates)
tm.assert_equal(result, expected)
@pytest.mark.parametrize(
"fmt,dates,expected_dates",
[
[
"%Y-%m-%d %H:%M:%S %Z",
[
"2010-01-01 12:00:00 UTC",
"2010-01-01 12:00:00 GMT",
"2010-01-01 12:00:00 US/Pacific",
],
[
Timestamp("2010-01-01 12:00:00", tz="UTC"),
Timestamp("2010-01-01 12:00:00", tz="GMT"),
Timestamp("2010-01-01 12:00:00", tz="US/Pacific"),
],
],
[
"%Y-%m-%d %H:%M:%S %z",
["2010-01-01 12:00:00 +0100", "2010-01-01 12:00:00 -0100"],
[
Timestamp(
"2010-01-01 12:00:00", tzinfo=timezone(timedelta(minutes=60))
),
Timestamp(
"2010-01-01 12:00:00", tzinfo=timezone(timedelta(minutes=-60))
),
],
],
],
)
def test_to_datetime_parse_tzname_or_tzoffset_utc_false_deprecated(
self, fmt, dates, expected_dates
):
# GH 13486, 50887
msg = "parsing datetimes with mixed time zones will raise an error"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = to_datetime(dates, format=fmt)
expected = Index(expected_dates)
tm.assert_equal(result, expected)
def test_to_datetime_parse_tzname_or_tzoffset_different_tz_to_utc(self):
# GH 32792
dates = [
"2010-01-01 12:00:00 +0100",
"2010-01-01 12:00:00 -0100",
"2010-01-01 12:00:00 +0300",
"2010-01-01 12:00:00 +0400",
]
expected_dates = [
"2010-01-01 11:00:00+00:00",
"2010-01-01 13:00:00+00:00",
"2010-01-01 09:00:00+00:00",
"2010-01-01 08:00:00+00:00",
]
fmt = "%Y-%m-%d %H:%M:%S %z"
result = to_datetime(dates, format=fmt, utc=True)
expected = DatetimeIndex(expected_dates)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"offset", ["+0", "-1foo", "UTCbar", ":10", "+01:000:01", ""]
)
def test_to_datetime_parse_timezone_malformed(self, offset):
fmt = "%Y-%m-%d %H:%M:%S %z"
date = "2010-01-01 12:00:00 " + offset
msg = "|".join(
[
r'^time data ".*" doesn\'t match format ".*", at position 0. '
f"{PARSING_ERR_MSG}$",
r'^unconverted data remains when parsing with format ".*": ".*", '
f"at position 0. {PARSING_ERR_MSG}$",
]
)
with pytest.raises(ValueError, match=msg):
to_datetime([date], format=fmt)
def test_to_datetime_parse_timezone_keeps_name(self):
# GH 21697
fmt = "%Y-%m-%d %H:%M:%S %z"
arg = Index(["2010-01-01 12:00:00 Z"], name="foo")
result = to_datetime(arg, format=fmt)
expected = DatetimeIndex(["2010-01-01 12:00:00"], tz="UTC", name="foo")
tm.assert_index_equal(result, expected)
class TestToDatetime:
@pytest.mark.filterwarnings("ignore:Could not infer format")
def test_to_datetime_overflow(self):
# we should get an OutOfBoundsDatetime, NOT OverflowError
# TODO: Timestamp raises ValueError("could not convert string to Timestamp")
# can we make these more consistent?
arg = "08335394550"
msg = 'Parsing "08335394550" to datetime overflows, at position 0'
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(arg)
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime([arg])
res = to_datetime(arg, errors="coerce")
assert res is NaT
res = to_datetime([arg], errors="coerce")
tm.assert_index_equal(res, Index([NaT]))
res = to_datetime(arg, errors="ignore")
assert isinstance(res, str) and res == arg
res = to_datetime([arg], errors="ignore")
tm.assert_index_equal(res, Index([arg], dtype=object))
def test_to_datetime_mixed_datetime_and_string(self):
# GH#47018 adapted old doctest with new behavior
d1 = datetime(2020, 1, 1, 17, tzinfo=timezone(-timedelta(hours=1)))
d2 = datetime(2020, 1, 1, 18, tzinfo=timezone(-timedelta(hours=1)))
res = to_datetime(["2020-01-01 17:00 -0100", d2])
expected = to_datetime([d1, d2]).tz_convert(timezone(timedelta(minutes=-60)))
tm.assert_index_equal(res, expected)
def test_to_datetime_mixed_string_and_numeric(self):
# GH#55780 np.array(vals) would incorrectly cast the number to str
vals = ["2016-01-01", 0]
expected = DatetimeIndex([Timestamp(x) for x in vals])
result = to_datetime(vals, format="mixed")
result2 = to_datetime(vals[::-1], format="mixed")[::-1]
result3 = DatetimeIndex(vals)
result4 = DatetimeIndex(vals[::-1])[::-1]
tm.assert_index_equal(result, expected)
tm.assert_index_equal(result2, expected)
tm.assert_index_equal(result3, expected)
tm.assert_index_equal(result4, expected)
@pytest.mark.parametrize(
"format", ["%Y-%m-%d", "%Y-%d-%m"], ids=["ISO8601", "non-ISO8601"]
)
def test_to_datetime_mixed_date_and_string(self, format):
# https://github.com/pandas-dev/pandas/issues/50108
d1 = date(2020, 1, 2)
res = to_datetime(["2020-01-01", d1], format=format)
expected = DatetimeIndex(["2020-01-01", "2020-01-02"], dtype="M8[ns]")
tm.assert_index_equal(res, expected)
@pytest.mark.parametrize(
"fmt",
["%Y-%d-%m %H:%M:%S%z", "%Y-%m-%d %H:%M:%S%z"],
ids=["non-ISO8601 format", "ISO8601 format"],
)
@pytest.mark.parametrize(
"utc, args, expected",
[
pytest.param(
True,
["2000-01-01 01:00:00-08:00", "2000-01-01 02:00:00-08:00"],
DatetimeIndex(
["2000-01-01 09:00:00+00:00", "2000-01-01 10:00:00+00:00"],
dtype="datetime64[ns, UTC]",
),
id="all tz-aware, with utc",
),
pytest.param(
False,
["2000-01-01 01:00:00+00:00", "2000-01-01 02:00:00+00:00"],
DatetimeIndex(
["2000-01-01 01:00:00+00:00", "2000-01-01 02:00:00+00:00"],
),
id="all tz-aware, without utc",
),
pytest.param(
True,
["2000-01-01 01:00:00-08:00", "2000-01-01 02:00:00+00:00"],
DatetimeIndex(
["2000-01-01 09:00:00+00:00", "2000-01-01 02:00:00+00:00"],
dtype="datetime64[ns, UTC]",
),
id="all tz-aware, mixed offsets, with utc",
),
pytest.param(
True,
["2000-01-01 01:00:00", "2000-01-01 02:00:00+00:00"],
DatetimeIndex(
["2000-01-01 01:00:00+00:00", "2000-01-01 02:00:00+00:00"],
dtype="datetime64[ns, UTC]",
),
id="tz-aware string, naive pydatetime, with utc",
),
],
)
@pytest.mark.parametrize(
"constructor",
[Timestamp, lambda x: Timestamp(x).to_pydatetime()],
)
def test_to_datetime_mixed_datetime_and_string_with_format(
self, fmt, utc, args, expected, constructor
):
# https://github.com/pandas-dev/pandas/issues/49298
# https://github.com/pandas-dev/pandas/issues/50254
# note: ISO8601 formats go down a fastpath, so we need to check both
# a ISO8601 format and a non-ISO8601 one
ts1 = constructor(args[0])
ts2 = args[1]
result = to_datetime([ts1, ts2], format=fmt, utc=utc)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"fmt",
["%Y-%d-%m %H:%M:%S%z", "%Y-%m-%d %H:%M:%S%z"],
ids=["non-ISO8601 format", "ISO8601 format"],
)
@pytest.mark.parametrize(
"constructor",
[Timestamp, lambda x: Timestamp(x).to_pydatetime()],
)
def test_to_datetime_mixed_datetime_and_string_with_format_mixed_offsets_utc_false(
self, fmt, constructor
):
# https://github.com/pandas-dev/pandas/issues/49298
# https://github.com/pandas-dev/pandas/issues/50254
# note: ISO8601 formats go down a fastpath, so we need to check both
# a ISO8601 format and a non-ISO8601 one
args = ["2000-01-01 01:00:00", "2000-01-01 02:00:00+00:00"]
ts1 = constructor(args[0])
ts2 = args[1]
msg = "parsing datetimes with mixed time zones will raise an error"
expected = Index(
[
Timestamp("2000-01-01 01:00:00"),
Timestamp("2000-01-01 02:00:00+0000", tz="UTC"),
],
)
with tm.assert_produces_warning(FutureWarning, match=msg):
result = to_datetime([ts1, ts2], format=fmt, utc=False)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"fmt, expected",
[
pytest.param(
"%Y-%m-%d %H:%M:%S%z",
Index(
[
Timestamp("2000-01-01 09:00:00+0100", tz="UTC+01:00"),
Timestamp("2000-01-02 02:00:00+0200", tz="UTC+02:00"),
NaT,
]
),
id="ISO8601, non-UTC",
),
pytest.param(
"%Y-%d-%m %H:%M:%S%z",
Index(
[
Timestamp("2000-01-01 09:00:00+0100", tz="UTC+01:00"),
Timestamp("2000-02-01 02:00:00+0200", tz="UTC+02:00"),
NaT,
]
),
id="non-ISO8601, non-UTC",
),
],
)
def test_to_datetime_mixed_offsets_with_none_tz(self, fmt, expected):
# https://github.com/pandas-dev/pandas/issues/50071
msg = "parsing datetimes with mixed time zones will raise an error"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = to_datetime(
["2000-01-01 09:00:00+01:00", "2000-01-02 02:00:00+02:00", None],
format=fmt,
utc=False,
)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"fmt, expected",
[
pytest.param(
"%Y-%m-%d %H:%M:%S%z",
DatetimeIndex(
["2000-01-01 08:00:00+00:00", "2000-01-02 00:00:00+00:00", "NaT"],
dtype="datetime64[ns, UTC]",
),
id="ISO8601, UTC",
),
pytest.param(
"%Y-%d-%m %H:%M:%S%z",
DatetimeIndex(
["2000-01-01 08:00:00+00:00", "2000-02-01 00:00:00+00:00", "NaT"],
dtype="datetime64[ns, UTC]",
),
id="non-ISO8601, UTC",
),
],
)
def test_to_datetime_mixed_offsets_with_none(self, fmt, expected):
# https://github.com/pandas-dev/pandas/issues/50071
result = to_datetime(
["2000-01-01 09:00:00+01:00", "2000-01-02 02:00:00+02:00", None],
format=fmt,
utc=True,
)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"fmt",
["%Y-%d-%m %H:%M:%S%z", "%Y-%m-%d %H:%M:%S%z"],
ids=["non-ISO8601 format", "ISO8601 format"],
)
@pytest.mark.parametrize(
"args",
[
pytest.param(
["2000-01-01 01:00:00-08:00", "2000-01-01 02:00:00-07:00"],
id="all tz-aware, mixed timezones, without utc",
),
],
)
@pytest.mark.parametrize(
"constructor",
[Timestamp, lambda x: Timestamp(x).to_pydatetime()],
)
def test_to_datetime_mixed_datetime_and_string_with_format_raises(
self, fmt, args, constructor
):
# https://github.com/pandas-dev/pandas/issues/49298
# note: ISO8601 formats go down a fastpath, so we need to check both
# a ISO8601 format and a non-ISO8601 one
ts1 = constructor(args[0])
ts2 = constructor(args[1])
with pytest.raises(
ValueError, match="cannot be converted to datetime64 unless utc=True"
):
to_datetime([ts1, ts2], format=fmt, utc=False)
def test_to_datetime_np_str(self):
# GH#32264
# GH#48969
value = np.str_("2019-02-04 10:18:46.297000+0000")
ser = Series([value])
exp = Timestamp("2019-02-04 10:18:46.297000", tz="UTC")
assert to_datetime(value) == exp
assert to_datetime(ser.iloc[0]) == exp
res = to_datetime([value])
expected = Index([exp])
tm.assert_index_equal(res, expected)
res = to_datetime(ser)
expected = Series(expected)
tm.assert_series_equal(res, expected)
@pytest.mark.parametrize(
"s, _format, dt",
[
["2015-1-1", "%G-%V-%u", datetime(2014, 12, 29, 0, 0)],
["2015-1-4", "%G-%V-%u", datetime(2015, 1, 1, 0, 0)],
["2015-1-7", "%G-%V-%u", datetime(2015, 1, 4, 0, 0)],
],
)
def test_to_datetime_iso_week_year_format(self, s, _format, dt):
# See GH#16607
assert to_datetime(s, format=_format) == dt
@pytest.mark.parametrize(
"msg, s, _format",
[
[
"ISO week directive '%V' is incompatible with the year directive "
"'%Y'. Use the ISO year '%G' instead.",
"1999 50",
"%Y %V",
],
[
"ISO year directive '%G' must be used with the ISO week directive "
"'%V' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 51",
"%G %V",
],
[
"ISO year directive '%G' must be used with the ISO week directive "
"'%V' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 Monday",
"%G %A",
],
[
"ISO year directive '%G' must be used with the ISO week directive "
"'%V' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 Mon",
"%G %a",
],
[
"ISO year directive '%G' must be used with the ISO week directive "
"'%V' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 6",
"%G %w",
],
[
"ISO year directive '%G' must be used with the ISO week directive "
"'%V' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 6",
"%G %u",
],
[
"ISO year directive '%G' must be used with the ISO week directive "
"'%V' and a weekday directive '%A', '%a', '%w', or '%u'.",
"2051",
"%G",
],
[
"Day of the year directive '%j' is not compatible with ISO year "
"directive '%G'. Use '%Y' instead.",
"1999 51 6 256",
"%G %V %u %j",
],
[
"ISO week directive '%V' is incompatible with the year directive "
"'%Y'. Use the ISO year '%G' instead.",
"1999 51 Sunday",
"%Y %V %A",
],
[
"ISO week directive '%V' is incompatible with the year directive "
"'%Y'. Use the ISO year '%G' instead.",
"1999 51 Sun",
"%Y %V %a",
],
[
"ISO week directive '%V' is incompatible with the year directive "
"'%Y'. Use the ISO year '%G' instead.",
"1999 51 1",
"%Y %V %w",
],
[
"ISO week directive '%V' is incompatible with the year directive "
"'%Y'. Use the ISO year '%G' instead.",
"1999 51 1",
"%Y %V %u",
],
[
"ISO week directive '%V' must be used with the ISO year directive "
"'%G' and a weekday directive '%A', '%a', '%w', or '%u'.",
"20",
"%V",
],
[
"ISO week directive '%V' must be used with the ISO year directive "
"'%G' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 51 Sunday",
"%V %A",
],
[
"ISO week directive '%V' must be used with the ISO year directive "
"'%G' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 51 Sun",
"%V %a",
],
[
"ISO week directive '%V' must be used with the ISO year directive "
"'%G' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 51 1",
"%V %w",
],
[
"ISO week directive '%V' must be used with the ISO year directive "
"'%G' and a weekday directive '%A', '%a', '%w', or '%u'.",
"1999 51 1",
"%V %u",
],
[
"Day of the year directive '%j' is not compatible with ISO year "
"directive '%G'. Use '%Y' instead.",
"1999 50",
"%G %j",
],
[
"ISO week directive '%V' must be used with the ISO year directive "
"'%G' and a weekday directive '%A', '%a', '%w', or '%u'.",
"20 Monday",
"%V %A",
],
],
)
@pytest.mark.parametrize("errors", ["raise", "coerce", "ignore"])
def test_error_iso_week_year(self, msg, s, _format, errors):
# See GH#16607, GH#50308
# This test checks for errors thrown when giving the wrong format
# However, as discussed on PR#25541, overriding the locale
# causes a different error to be thrown due to the format being
# locale specific, but the test data is in english.
# Therefore, the tests only run when locale is not overwritten,
# as a sort of solution to this problem.
if locale.getlocale() != ("zh_CN", "UTF-8") and locale.getlocale() != (
"it_IT",
"UTF-8",
):
with pytest.raises(ValueError, match=msg):
to_datetime(s, format=_format, errors=errors)
@pytest.mark.parametrize("tz", [None, "US/Central"])
def test_to_datetime_dtarr(self, tz):
# DatetimeArray
dti = date_range("1965-04-03", periods=19, freq="2W", tz=tz)
arr = dti._data
result = to_datetime(arr)
assert result is arr
# Doesn't work on Windows since tzpath not set correctly
@td.skip_if_windows
@pytest.mark.parametrize("arg_class", [Series, Index])
@pytest.mark.parametrize("utc", [True, False])
@pytest.mark.parametrize("tz", [None, "US/Central"])
def test_to_datetime_arrow(self, tz, utc, arg_class):
pa = pytest.importorskip("pyarrow")
dti = date_range("1965-04-03", periods=19, freq="2W", tz=tz)
dti = arg_class(dti)
dti_arrow = dti.astype(pd.ArrowDtype(pa.timestamp(unit="ns", tz=tz)))
result = to_datetime(dti_arrow, utc=utc)
expected = to_datetime(dti, utc=utc).astype(
pd.ArrowDtype(pa.timestamp(unit="ns", tz=tz if not utc else "UTC"))
)
if not utc and arg_class is not Series:
# Doesn't hold for utc=True, since that will astype
# to_datetime also returns a new object for series
assert result is dti_arrow
if arg_class is Series:
tm.assert_series_equal(result, expected)
else:
tm.assert_index_equal(result, expected)
def test_to_datetime_pydatetime(self):
actual = to_datetime(datetime(2008, 1, 15))
assert actual == datetime(2008, 1, 15)
def test_to_datetime_YYYYMMDD(self):
actual = to_datetime("20080115")
assert actual == datetime(2008, 1, 15)
def test_to_datetime_unparsable_ignore(self):
# unparsable
ser = "Month 1, 1999"
assert to_datetime(ser, errors="ignore") == ser
@td.skip_if_windows # `tm.set_timezone` does not work in windows
def test_to_datetime_now(self):
# See GH#18666
with tm.set_timezone("US/Eastern"):
# GH#18705
now = Timestamp("now").as_unit("ns")
pdnow = to_datetime("now")
pdnow2 = to_datetime(["now"])[0]
# These should all be equal with infinite perf; this gives
# a generous margin of 10 seconds
assert abs(pdnow._value - now._value) < 1e10
assert abs(pdnow2._value - now._value) < 1e10
assert pdnow.tzinfo is None
assert pdnow2.tzinfo is None
@td.skip_if_windows # `tm.set_timezone` does not work in windows
@pytest.mark.parametrize("tz", ["Pacific/Auckland", "US/Samoa"])
def test_to_datetime_today(self, tz):
# See GH#18666
# Test with one timezone far ahead of UTC and another far behind, so
# one of these will _almost_ always be in a different day from UTC.
# Unfortunately this test between 12 and 1 AM Samoa time
# this both of these timezones _and_ UTC will all be in the same day,
# so this test will not detect the regression introduced in #18666.
with tm.set_timezone(tz):
nptoday = np.datetime64("today").astype("datetime64[ns]").astype(np.int64)
pdtoday = to_datetime("today")
pdtoday2 = to_datetime(["today"])[0]
tstoday = Timestamp("today").as_unit("ns")
tstoday2 = Timestamp.today().as_unit("ns")
# These should all be equal with infinite perf; this gives
# a generous margin of 10 seconds
assert abs(pdtoday.normalize()._value - nptoday) < 1e10
assert abs(pdtoday2.normalize()._value - nptoday) < 1e10
assert abs(pdtoday._value - tstoday._value) < 1e10
assert abs(pdtoday._value - tstoday2._value) < 1e10
assert pdtoday.tzinfo is None
assert pdtoday2.tzinfo is None
@pytest.mark.parametrize("arg", ["now", "today"])
def test_to_datetime_today_now_unicode_bytes(self, arg):
to_datetime([arg])
@pytest.mark.parametrize(
"format, expected_ds",
[
("%Y-%m-%d %H:%M:%S%z", "2020-01-03"),
("%Y-%d-%m %H:%M:%S%z", "2020-03-01"),
(None, "2020-01-03"),
],
)
@pytest.mark.parametrize(
"string, attribute",
[
("now", "utcnow"),
("today", "today"),
],
)
def test_to_datetime_now_with_format(self, format, expected_ds, string, attribute):
# https://github.com/pandas-dev/pandas/issues/50359
result = to_datetime(["2020-01-03 00:00:00Z", string], format=format, utc=True)
expected = DatetimeIndex(
[expected_ds, getattr(Timestamp, attribute)()], dtype="datetime64[ns, UTC]"
)
assert (expected - result).max().total_seconds() < 1
@pytest.mark.parametrize(
"dt", [np.datetime64("2000-01-01"), np.datetime64("2000-01-02")]
)
def test_to_datetime_dt64s(self, cache, dt):
assert to_datetime(dt, cache=cache) == Timestamp(dt)
@pytest.mark.parametrize(
"arg, format",
[
("2001-01-01", "%Y-%m-%d"),
("01-01-2001", "%d-%m-%Y"),
],
)
def test_to_datetime_dt64s_and_str(self, arg, format):
# https://github.com/pandas-dev/pandas/issues/50036
result = to_datetime([arg, np.datetime64("2020-01-01")], format=format)
expected = DatetimeIndex(["2001-01-01", "2020-01-01"])
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"dt", [np.datetime64("1000-01-01"), np.datetime64("5000-01-02")]
)
@pytest.mark.parametrize("errors", ["raise", "ignore", "coerce"])
def test_to_datetime_dt64s_out_of_ns_bounds(self, cache, dt, errors):
# GH#50369 We cast to the nearest supported reso, i.e. "s"
ts = to_datetime(dt, errors=errors, cache=cache)
assert isinstance(ts, Timestamp)
assert ts.unit == "s"
assert ts.asm8 == dt
ts = Timestamp(dt)
assert ts.unit == "s"
assert ts.asm8 == dt
@pytest.mark.skip_ubsan
def test_to_datetime_dt64d_out_of_bounds(self, cache):
dt64 = np.datetime64(np.iinfo(np.int64).max, "D")
msg = "Out of bounds second timestamp: 25252734927768524-07-27"
with pytest.raises(OutOfBoundsDatetime, match=msg):
Timestamp(dt64)
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(dt64, errors="raise", cache=cache)
assert to_datetime(dt64, errors="coerce", cache=cache) is NaT
@pytest.mark.parametrize("unit", ["s", "D"])
def test_to_datetime_array_of_dt64s(self, cache, unit):
# https://github.com/pandas-dev/pandas/issues/31491
# Need at least 50 to ensure cache is used.
dts = [
np.datetime64("2000-01-01", unit),
np.datetime64("2000-01-02", unit),
] * 30
# Assuming all datetimes are in bounds, to_datetime() returns
# an array that is equal to Timestamp() parsing
result = to_datetime(dts, cache=cache)
if cache:
# FIXME: behavior should not depend on cache
expected = DatetimeIndex([Timestamp(x).asm8 for x in dts], dtype="M8[s]")
else:
expected = DatetimeIndex([Timestamp(x).asm8 for x in dts], dtype="M8[ns]")
tm.assert_index_equal(result, expected)
# A list of datetimes where the last one is out of bounds
dts_with_oob = dts + [np.datetime64("9999-01-01")]
# As of GH#51978 we do not raise in this case
to_datetime(dts_with_oob, errors="raise")
result = to_datetime(dts_with_oob, errors="coerce", cache=cache)
if not cache:
# FIXME: shouldn't depend on cache!
expected = DatetimeIndex(
[Timestamp(dts_with_oob[0]).asm8, Timestamp(dts_with_oob[1]).asm8] * 30
+ [NaT],
)
else:
expected = DatetimeIndex(np.array(dts_with_oob, dtype="M8[s]"))
tm.assert_index_equal(result, expected)
# With errors='ignore', out of bounds datetime64s
# are converted to their .item(), which depending on the version of
# numpy is either a python datetime.datetime or datetime.date
result = to_datetime(dts_with_oob, errors="ignore", cache=cache)
if not cache:
# FIXME: shouldn't depend on cache!
expected = Index(dts_with_oob)
tm.assert_index_equal(result, expected)
def test_out_of_bounds_errors_ignore(self):
# https://github.com/pandas-dev/pandas/issues/50587
result = to_datetime(np.datetime64("9999-01-01"), errors="ignore")
expected = np.datetime64("9999-01-01")
assert result == expected
def test_out_of_bounds_errors_ignore2(self):
# GH#12424
msg = "errors='ignore' is deprecated"
with tm.assert_produces_warning(FutureWarning, match=msg):
res = to_datetime(
Series(["2362-01-01", np.nan], dtype=object), errors="ignore"
)
exp = Series(["2362-01-01", np.nan], dtype=object)
tm.assert_series_equal(res, exp)
def test_to_datetime_tz(self, cache):
# xref 8260
# uniform returns a DatetimeIndex
arr = [
Timestamp("2013-01-01 13:00:00-0800", tz="US/Pacific"),
Timestamp("2013-01-02 14:00:00-0800", tz="US/Pacific"),
]
result = to_datetime(arr, cache=cache)
expected = DatetimeIndex(
["2013-01-01 13:00:00", "2013-01-02 14:00:00"], tz="US/Pacific"
)
tm.assert_index_equal(result, expected)
def test_to_datetime_tz_mixed(self, cache):
# mixed tzs will raise if errors='raise'
# https://github.com/pandas-dev/pandas/issues/50585
arr = [
Timestamp("2013-01-01 13:00:00", tz="US/Pacific"),
Timestamp("2013-01-02 14:00:00", tz="US/Eastern"),
]
msg = (
"Tz-aware datetime.datetime cannot be "
"converted to datetime64 unless utc=True"
)
with pytest.raises(ValueError, match=msg):
to_datetime(arr, cache=cache)
depr_msg = "errors='ignore' is deprecated"
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
result = to_datetime(arr, cache=cache, errors="ignore")
expected = Index(
[
Timestamp("2013-01-01 13:00:00-08:00"),
Timestamp("2013-01-02 14:00:00-05:00"),
],
dtype="object",
)
tm.assert_index_equal(result, expected)
result = to_datetime(arr, cache=cache, errors="coerce")
expected = DatetimeIndex(
["2013-01-01 13:00:00-08:00", "NaT"], dtype="datetime64[ns, US/Pacific]"
)
tm.assert_index_equal(result, expected)
def test_to_datetime_different_offsets(self, cache):
# inspired by asv timeseries.ToDatetimeNONISO8601 benchmark
# see GH-26097 for more
ts_string_1 = "March 1, 2018 12:00:00+0400"
ts_string_2 = "March 1, 2018 12:00:00+0500"
arr = [ts_string_1] * 5 + [ts_string_2] * 5
expected = Index([parse(x) for x in arr])
msg = "parsing datetimes with mixed time zones will raise an error"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = to_datetime(arr, cache=cache)
tm.assert_index_equal(result, expected)
def test_to_datetime_tz_pytz(self, cache):
# see gh-8260
us_eastern = pytz.timezone("US/Eastern")
arr = np.array(
[
us_eastern.localize(
datetime(year=2000, month=1, day=1, hour=3, minute=0)
),
us_eastern.localize(
datetime(year=2000, month=6, day=1, hour=3, minute=0)
),
],
dtype=object,
)
result = to_datetime(arr, utc=True, cache=cache)
expected = DatetimeIndex(
["2000-01-01 08:00:00+00:00", "2000-06-01 07:00:00+00:00"],
dtype="datetime64[ns, UTC]",
freq=None,
)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"init_constructor, end_constructor",
[
(Index, DatetimeIndex),
(list, DatetimeIndex),
(np.array, DatetimeIndex),
(Series, Series),
],
)
def test_to_datetime_utc_true(self, cache, init_constructor, end_constructor):
# See gh-11934 & gh-6415
data = ["20100102 121314", "20100102 121315"]
expected_data = [
Timestamp("2010-01-02 12:13:14", tz="utc"),
Timestamp("2010-01-02 12:13:15", tz="utc"),
]
result = to_datetime(
init_constructor(data), format="%Y%m%d %H%M%S", utc=True, cache=cache
)
expected = end_constructor(expected_data)
tm.assert_equal(result, expected)
@pytest.mark.parametrize(
"scalar, expected",
[
["20100102 121314", Timestamp("2010-01-02 12:13:14", tz="utc")],
["20100102 121315", Timestamp("2010-01-02 12:13:15", tz="utc")],
],
)
def test_to_datetime_utc_true_scalar(self, cache, scalar, expected):
# Test scalar case as well
result = to_datetime(scalar, format="%Y%m%d %H%M%S", utc=True, cache=cache)
assert result == expected
def test_to_datetime_utc_true_with_series_single_value(self, cache):
# GH 15760 UTC=True with Series
ts = 1.5e18
result = to_datetime(Series([ts]), utc=True, cache=cache)
expected = Series([Timestamp(ts, tz="utc")])
tm.assert_series_equal(result, expected)
def test_to_datetime_utc_true_with_series_tzaware_string(self, cache):
ts = "2013-01-01 00:00:00-01:00"
expected_ts = "2013-01-01 01:00:00"
data = Series([ts] * 3)
result = to_datetime(data, utc=True, cache=cache)
expected = Series([Timestamp(expected_ts, tz="utc")] * 3)
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"date, dtype",
[
("2013-01-01 01:00:00", "datetime64[ns]"),
("2013-01-01 01:00:00", "datetime64[ns, UTC]"),
],
)
def test_to_datetime_utc_true_with_series_datetime_ns(self, cache, date, dtype):
expected = Series(
[Timestamp("2013-01-01 01:00:00", tz="UTC")], dtype="M8[ns, UTC]"
)
result = to_datetime(Series([date], dtype=dtype), utc=True, cache=cache)
tm.assert_series_equal(result, expected)
def test_to_datetime_tz_psycopg2(self, request, cache):
# xref 8260
psycopg2_tz = pytest.importorskip("psycopg2.tz")
# misc cases
tz1 = psycopg2_tz.FixedOffsetTimezone(offset=-300, name=None)
tz2 = psycopg2_tz.FixedOffsetTimezone(offset=-240, name=None)
arr = np.array(
[
datetime(2000, 1, 1, 3, 0, tzinfo=tz1),
datetime(2000, 6, 1, 3, 0, tzinfo=tz2),
],
dtype=object,
)
result = to_datetime(arr, errors="coerce", utc=True, cache=cache)
expected = DatetimeIndex(
["2000-01-01 08:00:00+00:00", "2000-06-01 07:00:00+00:00"],
dtype="datetime64[ns, UTC]",
freq=None,
)
tm.assert_index_equal(result, expected)
# dtype coercion
i = DatetimeIndex(
["2000-01-01 08:00:00"],
tz=psycopg2_tz.FixedOffsetTimezone(offset=-300, name=None),
)
assert is_datetime64_ns_dtype(i)
# tz coercion
result = to_datetime(i, errors="coerce", cache=cache)
tm.assert_index_equal(result, i)
result = to_datetime(i, errors="coerce", utc=True, cache=cache)
expected = DatetimeIndex(["2000-01-01 13:00:00"], dtype="datetime64[ns, UTC]")
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize("arg", [True, False])
def test_datetime_bool(self, cache, arg):
# GH13176
msg = r"dtype bool cannot be converted to datetime64\[ns\]"
with pytest.raises(TypeError, match=msg):
to_datetime(arg)
assert to_datetime(arg, errors="coerce", cache=cache) is NaT
assert to_datetime(arg, errors="ignore", cache=cache) is arg
def test_datetime_bool_arrays_mixed(self, cache):
msg = f"{type(cache)} is not convertible to datetime"
with pytest.raises(TypeError, match=msg):
to_datetime([False, datetime.today()], cache=cache)
with pytest.raises(
ValueError,
match=(
r'^time data "True" doesn\'t match format "%Y%m%d", '
f"at position 1. {PARSING_ERR_MSG}$"
),
):
to_datetime(["20130101", True], cache=cache)
tm.assert_index_equal(
to_datetime([0, False, NaT, 0.0], errors="coerce", cache=cache),
DatetimeIndex(
[to_datetime(0, cache=cache), NaT, NaT, to_datetime(0, cache=cache)]
),
)
@pytest.mark.parametrize("arg", [bool, to_datetime])
def test_datetime_invalid_datatype(self, arg):
# GH13176
msg = "is not convertible to datetime"
with pytest.raises(TypeError, match=msg):
to_datetime(arg)
@pytest.mark.parametrize("errors", ["coerce", "raise", "ignore"])
def test_invalid_format_raises(self, errors):
# https://github.com/pandas-dev/pandas/issues/50255
with pytest.raises(
ValueError, match="':' is a bad directive in format 'H%:M%:S%"
):
to_datetime(["00:00:00"], format="H%:M%:S%", errors=errors)
@pytest.mark.parametrize("value", ["a", "00:01:99"])
@pytest.mark.parametrize("format", [None, "%H:%M:%S"])
def test_datetime_invalid_scalar(self, value, format):
# GH24763
res = to_datetime(value, errors="ignore", format=format)
assert res == value
res = to_datetime(value, errors="coerce", format=format)
assert res is NaT
msg = "|".join(
[
r'^time data "a" doesn\'t match format "%H:%M:%S", at position 0. '
f"{PARSING_ERR_MSG}$",
r'^Given date string "a" not likely a datetime, at position 0$',
r'^unconverted data remains when parsing with format "%H:%M:%S": "9", '
f"at position 0. {PARSING_ERR_MSG}$",
r"^second must be in 0..59: 00:01:99, at position 0$",
]
)
with pytest.raises(ValueError, match=msg):
to_datetime(value, errors="raise", format=format)
@pytest.mark.parametrize("value", ["3000/12/11 00:00:00"])
@pytest.mark.parametrize("format", [None, "%H:%M:%S"])
def test_datetime_outofbounds_scalar(self, value, format):
# GH24763
res = to_datetime(value, errors="ignore", format=format)
assert res == value
res = to_datetime(value, errors="coerce", format=format)
assert res is NaT
if format is not None:
msg = r'^time data ".*" doesn\'t match format ".*", at position 0.'
with pytest.raises(ValueError, match=msg):
to_datetime(value, errors="raise", format=format)
else:
msg = "^Out of bounds .*, at position 0$"
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(value, errors="raise", format=format)
@pytest.mark.parametrize(
("values"), [(["a"]), (["00:01:99"]), (["a", "b", "99:00:00"])]
)
@pytest.mark.parametrize("format", [(None), ("%H:%M:%S")])
def test_datetime_invalid_index(self, values, format):
# GH24763
# Not great to have logic in tests, but this one's hard to
# parametrise over
if format is None and len(values) > 1:
warn = UserWarning
else:
warn = None
with tm.assert_produces_warning(
warn, match="Could not infer format", raise_on_extra_warnings=False
):
res = to_datetime(values, errors="ignore", format=format)
tm.assert_index_equal(res, Index(values, dtype=object))
with tm.assert_produces_warning(
warn, match="Could not infer format", raise_on_extra_warnings=False
):
res = to_datetime(values, errors="coerce", format=format)
tm.assert_index_equal(res, DatetimeIndex([NaT] * len(values)))
msg = "|".join(
[
r'^Given date string "a" not likely a datetime, at position 0$',
r'^time data "a" doesn\'t match format "%H:%M:%S", at position 0. '
f"{PARSING_ERR_MSG}$",
r'^unconverted data remains when parsing with format "%H:%M:%S": "9", '
f"at position 0. {PARSING_ERR_MSG}$",
r"^second must be in 0..59: 00:01:99, at position 0$",
]
)
with pytest.raises(ValueError, match=msg):
with tm.assert_produces_warning(
warn, match="Could not infer format", raise_on_extra_warnings=False
):
to_datetime(values, errors="raise", format=format)
@pytest.mark.parametrize("utc", [True, None])
@pytest.mark.parametrize("format", ["%Y%m%d %H:%M:%S", None])
@pytest.mark.parametrize("constructor", [list, tuple, np.array, Index, deque])
def test_to_datetime_cache(self, utc, format, constructor):
date = "20130101 00:00:00"
test_dates = [date] * 10**5
data = constructor(test_dates)
result = to_datetime(data, utc=utc, format=format, cache=True)
expected = to_datetime(data, utc=utc, format=format, cache=False)
tm.assert_index_equal(result, expected)
def test_to_datetime_from_deque(self):
# GH 29403
result = to_datetime(deque([Timestamp("2010-06-02 09:30:00")] * 51))
expected = to_datetime([Timestamp("2010-06-02 09:30:00")] * 51)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize("utc", [True, None])
@pytest.mark.parametrize("format", ["%Y%m%d %H:%M:%S", None])
def test_to_datetime_cache_series(self, utc, format):
date = "20130101 00:00:00"
test_dates = [date] * 10**5
data = Series(test_dates)
result = to_datetime(data, utc=utc, format=format, cache=True)
expected = to_datetime(data, utc=utc, format=format, cache=False)
tm.assert_series_equal(result, expected)
def test_to_datetime_cache_scalar(self):
date = "20130101 00:00:00"
result = to_datetime(date, cache=True)
expected = Timestamp("20130101 00:00:00")
assert result == expected
@pytest.mark.parametrize(
"datetimelikes,expected_values",
(
(
(None, np.nan) + (NaT,) * start_caching_at,
(NaT,) * (start_caching_at + 2),
),
(
(None, Timestamp("2012-07-26")) + (NaT,) * start_caching_at,
(NaT, Timestamp("2012-07-26")) + (NaT,) * start_caching_at,
),
(
(None,)
+ (NaT,) * start_caching_at
+ ("2012 July 26", Timestamp("2012-07-26")),
(NaT,) * (start_caching_at + 1)
+ (Timestamp("2012-07-26"), Timestamp("2012-07-26")),
),
),
)
def test_convert_object_to_datetime_with_cache(
self, datetimelikes, expected_values
):
# GH#39882
ser = Series(
datetimelikes,
dtype="object",
)
result_series = to_datetime(ser, errors="coerce")
expected_series = Series(
expected_values,
dtype="datetime64[ns]",
)
tm.assert_series_equal(result_series, expected_series)
@pytest.mark.parametrize("cache", [True, False])
@pytest.mark.parametrize(
"input",
[
Series([NaT] * 20 + [None] * 20, dtype="object"),
Series([NaT] * 60 + [None] * 60, dtype="object"),
Series([None] * 20),
Series([None] * 60),
Series([""] * 20),
Series([""] * 60),
Series([pd.NA] * 20),
Series([pd.NA] * 60),
Series([np.nan] * 20),
Series([np.nan] * 60),
],
)
def test_to_datetime_converts_null_like_to_nat(self, cache, input):
# GH35888
expected = Series([NaT] * len(input), dtype="M8[ns]")
result = to_datetime(input, cache=cache)
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"date, format",
[
("2017-20", "%Y-%W"),
("20 Sunday", "%W %A"),
("20 Sun", "%W %a"),
("2017-21", "%Y-%U"),
("20 Sunday", "%U %A"),
("20 Sun", "%U %a"),
],
)
def test_week_without_day_and_calendar_year(self, date, format):
# GH16774
msg = "Cannot use '%W' or '%U' without day and year"
with pytest.raises(ValueError, match=msg):
to_datetime(date, format=format)
def test_to_datetime_coerce(self):
# GH 26122
ts_strings = [
"March 1, 2018 12:00:00+0400",
"March 1, 2018 12:00:00+0500",
"20100240",
]
msg = "parsing datetimes with mixed time zones will raise an error"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = to_datetime(ts_strings, errors="coerce")
expected = Index(
[
datetime(2018, 3, 1, 12, 0, tzinfo=tzoffset(None, 14400)),
datetime(2018, 3, 1, 12, 0, tzinfo=tzoffset(None, 18000)),
NaT,
]
)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"string_arg, format",
[("March 1, 2018", "%B %d, %Y"), ("2018-03-01", "%Y-%m-%d")],
)
@pytest.mark.parametrize(
"outofbounds",
[
datetime(9999, 1, 1),
date(9999, 1, 1),
np.datetime64("9999-01-01"),
"January 1, 9999",
"9999-01-01",
],
)
def test_to_datetime_coerce_oob(self, string_arg, format, outofbounds):
# https://github.com/pandas-dev/pandas/issues/50255
ts_strings = [string_arg, outofbounds]
result = to_datetime(ts_strings, errors="coerce", format=format)
expected = DatetimeIndex([datetime(2018, 3, 1), NaT])
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"errors, expected",
[
("coerce", Index([NaT, NaT])),
("ignore", Index(["200622-12-31", "111111-24-11"], dtype=object)),
],
)
def test_to_datetime_malformed_no_raise(self, errors, expected):
# GH 28299
# GH 48633
ts_strings = ["200622-12-31", "111111-24-11"]
with tm.assert_produces_warning(
UserWarning, match="Could not infer format", raise_on_extra_warnings=False
):
result = to_datetime(ts_strings, errors=errors)
tm.assert_index_equal(result, expected)
def test_to_datetime_malformed_raise(self):
# GH 48633
ts_strings = ["200622-12-31", "111111-24-11"]
msg = (
'Parsed string "200622-12-31" gives an invalid tzoffset, which must '
r"be between -timedelta\(hours=24\) and timedelta\(hours=24\), "
"at position 0"
)
with pytest.raises(
ValueError,
match=msg,
):
with tm.assert_produces_warning(
UserWarning, match="Could not infer format"
):
to_datetime(
ts_strings,
errors="raise",
)
def test_iso_8601_strings_with_same_offset(self):
# GH 17697, 11736
ts_str = "2015-11-18 15:30:00+05:30"
result = to_datetime(ts_str)
expected = Timestamp(ts_str)
assert result == expected
expected = DatetimeIndex([Timestamp(ts_str)] * 2)
result = to_datetime([ts_str] * 2)
tm.assert_index_equal(result, expected)
result = DatetimeIndex([ts_str] * 2)
tm.assert_index_equal(result, expected)
def test_iso_8601_strings_with_different_offsets(self):
# GH 17697, 11736, 50887
ts_strings = ["2015-11-18 15:30:00+05:30", "2015-11-18 16:30:00+06:30", NaT]
msg = "parsing datetimes with mixed time zones will raise an error"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = to_datetime(ts_strings)
expected = np.array(
[
datetime(2015, 11, 18, 15, 30, tzinfo=tzoffset(None, 19800)),
datetime(2015, 11, 18, 16, 30, tzinfo=tzoffset(None, 23400)),
NaT,
],
dtype=object,
)
# GH 21864
expected = Index(expected)
tm.assert_index_equal(result, expected)
def test_iso_8601_strings_with_different_offsets_utc(self):
ts_strings = ["2015-11-18 15:30:00+05:30", "2015-11-18 16:30:00+06:30", NaT]
result = to_datetime(ts_strings, utc=True)
expected = DatetimeIndex(
[Timestamp(2015, 11, 18, 10), Timestamp(2015, 11, 18, 10), NaT], tz="UTC"
)
tm.assert_index_equal(result, expected)
def test_mixed_offsets_with_native_datetime_raises(self):
# GH 25978
vals = [
"nan",
Timestamp("1990-01-01"),
"2015-03-14T16:15:14.123-08:00",
"2019-03-04T21:56:32.620-07:00",
None,
"today",
"now",
]
ser = Series(vals)
assert all(ser[i] is vals[i] for i in range(len(vals))) # GH#40111
now = Timestamp("now")
today = Timestamp("today")
msg = "parsing datetimes with mixed time zones will raise an error"
with tm.assert_produces_warning(FutureWarning, match=msg):
mixed = to_datetime(ser)
expected = Series(
[
"NaT",
Timestamp("1990-01-01"),
Timestamp("2015-03-14T16:15:14.123-08:00").to_pydatetime(),
Timestamp("2019-03-04T21:56:32.620-07:00").to_pydatetime(),
None,
],
dtype=object,
)
tm.assert_series_equal(mixed[:-2], expected)
# we'll check mixed[-1] and mixed[-2] match now and today to within
# call-timing tolerances
assert (now - mixed.iloc[-1]).total_seconds() <= 0.1
assert (today - mixed.iloc[-2]).total_seconds() <= 0.1
with pytest.raises(ValueError, match="Tz-aware datetime.datetime"):
to_datetime(mixed)
def test_non_iso_strings_with_tz_offset(self):
result = to_datetime(["March 1, 2018 12:00:00+0400"] * 2)
expected = DatetimeIndex(
[datetime(2018, 3, 1, 12, tzinfo=timezone(timedelta(minutes=240)))] * 2
)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"ts, expected",
[
(Timestamp("2018-01-01"), Timestamp("2018-01-01", tz="UTC")),
(
Timestamp("2018-01-01", tz="US/Pacific"),
Timestamp("2018-01-01 08:00", tz="UTC"),
),
],
)
def test_timestamp_utc_true(self, ts, expected):
# GH 24415
result = to_datetime(ts, utc=True)
assert result == expected
@pytest.mark.parametrize("dt_str", ["00010101", "13000101", "30000101", "99990101"])
def test_to_datetime_with_format_out_of_bounds(self, dt_str):
# GH 9107
msg = "Out of bounds nanosecond timestamp"
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(dt_str, format="%Y%m%d")
def test_to_datetime_utc(self):
arr = np.array([parse("2012-06-13T01:39:00Z")], dtype=object)
result = to_datetime(arr, utc=True)
assert result.tz is timezone.utc
def test_to_datetime_fixed_offset(self):
from pandas.tests.indexes.datetimes.test_timezones import FixedOffset
fixed_off = FixedOffset(-420, "-07:00")
dates = [
datetime(2000, 1, 1, tzinfo=fixed_off),
datetime(2000, 1, 2, tzinfo=fixed_off),
datetime(2000, 1, 3, tzinfo=fixed_off),
]
result = to_datetime(dates)
assert result.tz == fixed_off
@pytest.mark.parametrize(
"date",
[
["2020-10-26 00:00:00+06:00", "2020-10-26 00:00:00+01:00"],
["2020-10-26 00:00:00+06:00", Timestamp("2018-01-01", tz="US/Pacific")],
[
"2020-10-26 00:00:00+06:00",
datetime(2020, 1, 1, 18, tzinfo=pytz.timezone("Australia/Melbourne")),
],
],
)
def test_to_datetime_mixed_offsets_with_utc_false_deprecated(self, date):
# GH 50887
msg = "parsing datetimes with mixed time zones will raise an error"
with tm.assert_produces_warning(FutureWarning, match=msg):
to_datetime(date, utc=False)
class TestToDatetimeUnit:
@pytest.mark.parametrize("unit", ["Y", "M"])
@pytest.mark.parametrize("item", [150, float(150)])
def test_to_datetime_month_or_year_unit_int(self, cache, unit, item, request):
# GH#50870 Note we have separate tests that pd.Timestamp gets these right
ts = Timestamp(item, unit=unit)
expected = DatetimeIndex([ts], dtype="M8[ns]")
result = to_datetime([item], unit=unit, cache=cache)
tm.assert_index_equal(result, expected)
result = to_datetime(np.array([item], dtype=object), unit=unit, cache=cache)
tm.assert_index_equal(result, expected)
result = to_datetime(np.array([item]), unit=unit, cache=cache)
tm.assert_index_equal(result, expected)
# with a nan!
result = to_datetime(np.array([item, np.nan]), unit=unit, cache=cache)
assert result.isna()[1]
tm.assert_index_equal(result[:1], expected)
@pytest.mark.parametrize("unit", ["Y", "M"])
def test_to_datetime_month_or_year_unit_non_round_float(self, cache, unit):
# GH#50301
# Match Timestamp behavior in disallowing non-round floats with
# Y or M unit
warn_msg = "strings will be parsed as datetime strings"
msg = f"Conversion of non-round float with unit={unit} is ambiguous"
with pytest.raises(ValueError, match=msg):
to_datetime([1.5], unit=unit, errors="raise")
with pytest.raises(ValueError, match=msg):
to_datetime(np.array([1.5]), unit=unit, errors="raise")
with pytest.raises(ValueError, match=msg):
with tm.assert_produces_warning(FutureWarning, match=warn_msg):
to_datetime(["1.5"], unit=unit, errors="raise")
# with errors="ignore" we also end up raising within the Timestamp
# constructor; this may not be ideal
with pytest.raises(ValueError, match=msg):
to_datetime([1.5], unit=unit, errors="ignore")
res = to_datetime([1.5], unit=unit, errors="coerce")
expected = Index([NaT], dtype="M8[ns]")
tm.assert_index_equal(res, expected)
with tm.assert_produces_warning(FutureWarning, match=warn_msg):
res = to_datetime(["1.5"], unit=unit, errors="coerce")
tm.assert_index_equal(res, expected)
# round floats are OK
res = to_datetime([1.0], unit=unit)
expected = to_datetime([1], unit=unit)
tm.assert_index_equal(res, expected)
def test_unit(self, cache):
# GH 11758
# test proper behavior with errors
msg = "cannot specify both format and unit"
with pytest.raises(ValueError, match=msg):
to_datetime([1], unit="D", format="%Y%m%d", cache=cache)
def test_unit_str(self, cache):
# GH 57051
# Test that strs aren't dropping precision to 32-bit accidentally.
with tm.assert_produces_warning(FutureWarning):
res = to_datetime(["1704660000"], unit="s", origin="unix")
expected = to_datetime([1704660000], unit="s", origin="unix")
tm.assert_index_equal(res, expected)
def test_unit_array_mixed_nans(self, cache):
values = [11111111111111111, 1, 1.0, iNaT, NaT, np.nan, "NaT", ""]
result = to_datetime(values, unit="D", errors="ignore", cache=cache)
expected = Index(
[
11111111111111111,
Timestamp("1970-01-02"),
Timestamp("1970-01-02"),
NaT,
NaT,
NaT,
NaT,
NaT,
],
dtype=object,
)
tm.assert_index_equal(result, expected)
result = to_datetime(values, unit="D", errors="coerce", cache=cache)
expected = DatetimeIndex(
["NaT", "1970-01-02", "1970-01-02", "NaT", "NaT", "NaT", "NaT", "NaT"],
dtype="M8[ns]",
)
tm.assert_index_equal(result, expected)
msg = "cannot convert input 11111111111111111 with the unit 'D'"
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(values, unit="D", errors="raise", cache=cache)
def test_unit_array_mixed_nans_large_int(self, cache):
values = [1420043460000000000000000, iNaT, NaT, np.nan, "NaT"]
result = to_datetime(values, errors="ignore", unit="s", cache=cache)
expected = Index([1420043460000000000000000, NaT, NaT, NaT, NaT], dtype=object)
tm.assert_index_equal(result, expected)
result = to_datetime(values, errors="coerce", unit="s", cache=cache)
expected = DatetimeIndex(["NaT", "NaT", "NaT", "NaT", "NaT"], dtype="M8[ns]")
tm.assert_index_equal(result, expected)
msg = "cannot convert input 1420043460000000000000000 with the unit 's'"
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(values, errors="raise", unit="s", cache=cache)
def test_to_datetime_invalid_str_not_out_of_bounds_valuerror(self, cache):
# if we have a string, then we raise a ValueError
# and NOT an OutOfBoundsDatetime
msg = "non convertible value foo with the unit 's'"
with pytest.raises(ValueError, match=msg):
to_datetime("foo", errors="raise", unit="s", cache=cache)
@pytest.mark.parametrize("error", ["raise", "coerce", "ignore"])
def test_unit_consistency(self, cache, error):
# consistency of conversions
expected = Timestamp("1970-05-09 14:25:11")
result = to_datetime(11111111, unit="s", errors=error, cache=cache)
assert result == expected
assert isinstance(result, Timestamp)
@pytest.mark.parametrize("errors", ["ignore", "raise", "coerce"])
@pytest.mark.parametrize("dtype", ["float64", "int64"])
def test_unit_with_numeric(self, cache, errors, dtype):
# GH 13180
# coercions from floats/ints are ok
expected = DatetimeIndex(
["2015-06-19 05:33:20", "2015-05-27 22:33:20"], dtype="M8[ns]"
)
arr = np.array([1.434692e18, 1.432766e18]).astype(dtype)
result = to_datetime(arr, errors=errors, cache=cache)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"exp, arr, warning",
[
[
["NaT", "2015-06-19 05:33:20", "2015-05-27 22:33:20"],
["foo", 1.434692e18, 1.432766e18],
UserWarning,
],
[
["2015-06-19 05:33:20", "2015-05-27 22:33:20", "NaT", "NaT"],
[1.434692e18, 1.432766e18, "foo", "NaT"],
None,
],
],
)
def test_unit_with_numeric_coerce(self, cache, exp, arr, warning):
# but we want to make sure that we are coercing
# if we have ints/strings
expected = DatetimeIndex(exp, dtype="M8[ns]")
with tm.assert_produces_warning(warning, match="Could not infer format"):
result = to_datetime(arr, errors="coerce", cache=cache)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize(
"arr",
[
[Timestamp("20130101"), 1.434692e18, 1.432766e18],
[1.434692e18, 1.432766e18, Timestamp("20130101")],
],
)
def test_unit_mixed(self, cache, arr):
# GH#50453 pre-2.0 with mixed numeric/datetimes and errors="coerce"
# the numeric entries would be coerced to NaT, was never clear exactly
# why.
# mixed integers/datetimes
expected = Index([Timestamp(x) for x in arr], dtype="M8[ns]")
result = to_datetime(arr, errors="coerce", cache=cache)
tm.assert_index_equal(result, expected)
# GH#49037 pre-2.0 this raised, but it always worked with Series,
# was never clear why it was disallowed
result = to_datetime(arr, errors="raise", cache=cache)
tm.assert_index_equal(result, expected)
result = DatetimeIndex(arr)
tm.assert_index_equal(result, expected)
def test_unit_rounding(self, cache):
# GH 14156 & GH 20445: argument will incur floating point errors
# but no premature rounding
value = 1434743731.8770001
result = to_datetime(value, unit="s", cache=cache)
expected = Timestamp("2015-06-19 19:55:31.877000093")
assert result == expected
alt = Timestamp(value, unit="s")
assert alt == result
def test_unit_ignore_keeps_name(self, cache):
# GH 21697
expected = Index([15e9] * 2, name="name")
result = to_datetime(expected, errors="ignore", unit="s", cache=cache)
tm.assert_index_equal(result, expected)
def test_to_datetime_errors_ignore_utc_true(self):
# GH#23758
result = to_datetime([1], unit="s", utc=True, errors="ignore")
expected = DatetimeIndex(["1970-01-01 00:00:01"], dtype="M8[ns, UTC]")
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize("dtype", [int, float])
def test_to_datetime_unit(self, dtype):
epoch = 1370745748
ser = Series([epoch + t for t in range(20)]).astype(dtype)
result = to_datetime(ser, unit="s")
expected = Series(
[
Timestamp("2013-06-09 02:42:28") + timedelta(seconds=t)
for t in range(20)
],
dtype="M8[ns]",
)
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize("null", [iNaT, np.nan])
def test_to_datetime_unit_with_nulls(self, null):
epoch = 1370745748
ser = Series([epoch + t for t in range(20)] + [null])
result = to_datetime(ser, unit="s")
expected = Series(
[Timestamp("2013-06-09 02:42:28") + timedelta(seconds=t) for t in range(20)]
+ [NaT],
dtype="M8[ns]",
)
tm.assert_series_equal(result, expected)
def test_to_datetime_unit_fractional_seconds(self):
# GH13834
epoch = 1370745748
ser = Series([epoch + t for t in np.arange(0, 2, 0.25)] + [iNaT]).astype(float)
result = to_datetime(ser, unit="s")
expected = Series(
[
Timestamp("2013-06-09 02:42:28") + timedelta(seconds=t)
for t in np.arange(0, 2, 0.25)
]
+ [NaT],
dtype="M8[ns]",
)
# GH20455 argument will incur floating point errors but no premature rounding
result = result.round("ms")
tm.assert_series_equal(result, expected)
def test_to_datetime_unit_na_values(self):
result = to_datetime([1, 2, "NaT", NaT, np.nan], unit="D")
expected = DatetimeIndex(
[Timestamp("1970-01-02"), Timestamp("1970-01-03")] + ["NaT"] * 3,
dtype="M8[ns]",
)
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize("bad_val", ["foo", 111111111])
def test_to_datetime_unit_invalid(self, bad_val):
msg = f"{bad_val} with the unit 'D'"
with pytest.raises(ValueError, match=msg):
to_datetime([1, 2, bad_val], unit="D")
@pytest.mark.parametrize("bad_val", ["foo", 111111111])
def test_to_timestamp_unit_coerce(self, bad_val):
# coerce we can process
expected = DatetimeIndex(
[Timestamp("1970-01-02"), Timestamp("1970-01-03")] + ["NaT"] * 1,
dtype="M8[ns]",
)
result = to_datetime([1, 2, bad_val], unit="D", errors="coerce")
tm.assert_index_equal(result, expected)
def test_float_to_datetime_raise_near_bounds(self):
# GH50183
msg = "cannot convert input with unit 'D'"
oneday_in_ns = 1e9 * 60 * 60 * 24
tsmax_in_days = 2**63 / oneday_in_ns # 2**63 ns, in days
# just in bounds
should_succeed = Series(
[0, tsmax_in_days - 0.005, -tsmax_in_days + 0.005], dtype=float
)
expected = (should_succeed * oneday_in_ns).astype(np.int64)
for error_mode in ["raise", "coerce", "ignore"]:
result1 = to_datetime(should_succeed, unit="D", errors=error_mode)
# Cast to `np.float64` so that `rtol` and inexact checking kick in
# (`check_exact` doesn't take place for integer dtypes)
tm.assert_almost_equal(
result1.astype(np.int64).astype(np.float64),
expected.astype(np.float64),
rtol=1e-10,
)
# just out of bounds
should_fail1 = Series([0, tsmax_in_days + 0.005], dtype=float)
should_fail2 = Series([0, -tsmax_in_days - 0.005], dtype=float)
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(should_fail1, unit="D", errors="raise")
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(should_fail2, unit="D", errors="raise")
class TestToDatetimeDataFrame:
@pytest.fixture
def df(self):
return DataFrame(
{
"year": [2015, 2016],
"month": [2, 3],
"day": [4, 5],
"hour": [6, 7],
"minute": [58, 59],
"second": [10, 11],
"ms": [1, 1],
"us": [2, 2],
"ns": [3, 3],
}
)
def test_dataframe(self, df, cache):
result = to_datetime(
{"year": df["year"], "month": df["month"], "day": df["day"]}, cache=cache
)
expected = Series(
[Timestamp("20150204 00:00:00"), Timestamp("20160305 00:0:00")]
)
tm.assert_series_equal(result, expected)
# dict-like
result = to_datetime(df[["year", "month", "day"]].to_dict(), cache=cache)
tm.assert_series_equal(result, expected)
def test_dataframe_dict_with_constructable(self, df, cache):
# dict but with constructable
df2 = df[["year", "month", "day"]].to_dict()
df2["month"] = 2
result = to_datetime(df2, cache=cache)
expected2 = Series(
[Timestamp("20150204 00:00:00"), Timestamp("20160205 00:0:00")]
)
tm.assert_series_equal(result, expected2)
@pytest.mark.parametrize(
"unit",
[
{
"year": "years",
"month": "months",
"day": "days",
"hour": "hours",
"minute": "minutes",
"second": "seconds",
},
{
"year": "year",
"month": "month",
"day": "day",
"hour": "hour",
"minute": "minute",
"second": "second",
},
],
)
def test_dataframe_field_aliases_column_subset(self, df, cache, unit):
# unit mappings
result = to_datetime(df[list(unit.keys())].rename(columns=unit), cache=cache)
expected = Series(
[Timestamp("20150204 06:58:10"), Timestamp("20160305 07:59:11")],
dtype="M8[ns]",
)
tm.assert_series_equal(result, expected)
def test_dataframe_field_aliases(self, df, cache):
d = {
"year": "year",
"month": "month",
"day": "day",
"hour": "hour",
"minute": "minute",
"second": "second",
"ms": "ms",
"us": "us",
"ns": "ns",
}
result = to_datetime(df.rename(columns=d), cache=cache)
expected = Series(
[
Timestamp("20150204 06:58:10.001002003"),
Timestamp("20160305 07:59:11.001002003"),
]
)
tm.assert_series_equal(result, expected)
def test_dataframe_str_dtype(self, df, cache):
# coerce back to int
result = to_datetime(df.astype(str), cache=cache)
expected = Series(
[
Timestamp("20150204 06:58:10.001002003"),
Timestamp("20160305 07:59:11.001002003"),
]
)
tm.assert_series_equal(result, expected)
def test_dataframe_coerce(self, cache):
# passing coerce
df2 = DataFrame({"year": [2015, 2016], "month": [2, 20], "day": [4, 5]})
msg = (
r'^cannot assemble the datetimes: time data ".+" doesn\'t '
r'match format "%Y%m%d", at position 1\.'
)
with pytest.raises(ValueError, match=msg):
to_datetime(df2, cache=cache)
result = to_datetime(df2, errors="coerce", cache=cache)
expected = Series([Timestamp("20150204 00:00:00"), NaT])
tm.assert_series_equal(result, expected)
def test_dataframe_extra_keys_raisesm(self, df, cache):
# extra columns
msg = r"extra keys have been passed to the datetime assemblage: \[foo\]"
with pytest.raises(ValueError, match=msg):
df2 = df.copy()
df2["foo"] = 1
to_datetime(df2, cache=cache)
@pytest.mark.parametrize(
"cols",
[
["year"],
["year", "month"],
["year", "month", "second"],
["month", "day"],
["year", "day", "second"],
],
)
def test_dataframe_missing_keys_raises(self, df, cache, cols):
# not enough
msg = (
r"to assemble mappings requires at least that \[year, month, "
r"day\] be specified: \[.+\] is missing"
)
with pytest.raises(ValueError, match=msg):
to_datetime(df[cols], cache=cache)
def test_dataframe_duplicate_columns_raises(self, cache):
# duplicates
msg = "cannot assemble with duplicate keys"
df2 = DataFrame({"year": [2015, 2016], "month": [2, 20], "day": [4, 5]})
df2.columns = ["year", "year", "day"]
with pytest.raises(ValueError, match=msg):
to_datetime(df2, cache=cache)
df2 = DataFrame(
{"year": [2015, 2016], "month": [2, 20], "day": [4, 5], "hour": [4, 5]}
)
df2.columns = ["year", "month", "day", "day"]
with pytest.raises(ValueError, match=msg):
to_datetime(df2, cache=cache)
def test_dataframe_int16(self, cache):
# GH#13451
df = DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
# int16
result = to_datetime(df.astype("int16"), cache=cache)
expected = Series(
[Timestamp("20150204 00:00:00"), Timestamp("20160305 00:00:00")]
)
tm.assert_series_equal(result, expected)
def test_dataframe_mixed(self, cache):
# mixed dtypes
df = DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
df["month"] = df["month"].astype("int8")
df["day"] = df["day"].astype("int8")
result = to_datetime(df, cache=cache)
expected = Series(
[Timestamp("20150204 00:00:00"), Timestamp("20160305 00:00:00")]
)
tm.assert_series_equal(result, expected)
def test_dataframe_float(self, cache):
# float
df = DataFrame({"year": [2000, 2001], "month": [1.5, 1], "day": [1, 1]})
msg = (
r"^cannot assemble the datetimes: unconverted data remains when parsing "
r'with format ".*": "1", at position 0.'
)
with pytest.raises(ValueError, match=msg):
to_datetime(df, cache=cache)
def test_dataframe_utc_true(self):
# GH#23760
df = DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
result = to_datetime(df, utc=True)
expected = Series(
np.array(["2015-02-04", "2016-03-05"], dtype="datetime64[ns]")
).dt.tz_localize("UTC")
tm.assert_series_equal(result, expected)
class TestToDatetimeMisc:
def test_to_datetime_barely_out_of_bounds(self):
# GH#19529
# GH#19382 close enough to bounds that dropping nanos would result
# in an in-bounds datetime
arr = np.array(["2262-04-11 23:47:16.854775808"], dtype=object)
msg = "^Out of bounds nanosecond timestamp: .*, at position 0"
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime(arr)
@pytest.mark.parametrize(
"arg, exp_str",
[
["2012-01-01 00:00:00", "2012-01-01 00:00:00"],
["20121001", "2012-10-01"], # bad iso 8601
],
)
def test_to_datetime_iso8601(self, cache, arg, exp_str):
result = to_datetime([arg], cache=cache)
exp = Timestamp(exp_str)
assert result[0] == exp
@pytest.mark.parametrize(
"input, format",
[
("2012", "%Y-%m"),
("2012-01", "%Y-%m-%d"),
("2012-01-01", "%Y-%m-%d %H"),
("2012-01-01 10", "%Y-%m-%d %H:%M"),
("2012-01-01 10:00", "%Y-%m-%d %H:%M:%S"),
("2012-01-01 10:00:00", "%Y-%m-%d %H:%M:%S.%f"),
("2012-01-01 10:00:00.123", "%Y-%m-%d %H:%M:%S.%f%z"),
(0, "%Y-%m-%d"),
],
)
@pytest.mark.parametrize("exact", [True, False])
def test_to_datetime_iso8601_fails(self, input, format, exact):
# https://github.com/pandas-dev/pandas/issues/12649
# `format` is longer than the string, so this fails regardless of `exact`
with pytest.raises(
ValueError,
match=(
rf"time data \"{input}\" doesn't match format "
rf"\"{format}\", at position 0"
),
):
to_datetime(input, format=format, exact=exact)
@pytest.mark.parametrize(
"input, format",
[
("2012-01-01", "%Y-%m"),
("2012-01-01 10", "%Y-%m-%d"),
("2012-01-01 10:00", "%Y-%m-%d %H"),
("2012-01-01 10:00:00", "%Y-%m-%d %H:%M"),
(0, "%Y-%m-%d"),
],
)
def test_to_datetime_iso8601_exact_fails(self, input, format):
# https://github.com/pandas-dev/pandas/issues/12649
# `format` is shorter than the date string, so only fails with `exact=True`
msg = "|".join(
[
'^unconverted data remains when parsing with format ".*": ".*"'
f", at position 0. {PARSING_ERR_MSG}$",
f'^time data ".*" doesn\'t match format ".*", at position 0. '
f"{PARSING_ERR_MSG}$",
]
)
with pytest.raises(
ValueError,
match=(msg),
):
to_datetime(input, format=format)
@pytest.mark.parametrize(
"input, format",
[
("2012-01-01", "%Y-%m"),
("2012-01-01 00", "%Y-%m-%d"),
("2012-01-01 00:00", "%Y-%m-%d %H"),
("2012-01-01 00:00:00", "%Y-%m-%d %H:%M"),
],
)
def test_to_datetime_iso8601_non_exact(self, input, format):
# https://github.com/pandas-dev/pandas/issues/12649
expected = Timestamp(2012, 1, 1)
result = to_datetime(input, format=format, exact=False)
assert result == expected
@pytest.mark.parametrize(
"input, format",
[
("2020-01", "%Y/%m"),
("2020-01-01", "%Y/%m/%d"),
("2020-01-01 00", "%Y/%m/%dT%H"),
("2020-01-01T00", "%Y/%m/%d %H"),
("2020-01-01 00:00", "%Y/%m/%dT%H:%M"),
("2020-01-01T00:00", "%Y/%m/%d %H:%M"),
("2020-01-01 00:00:00", "%Y/%m/%dT%H:%M:%S"),
("2020-01-01T00:00:00", "%Y/%m/%d %H:%M:%S"),
],
)
def test_to_datetime_iso8601_separator(self, input, format):
# https://github.com/pandas-dev/pandas/issues/12649
with pytest.raises(
ValueError,
match=(
rf"time data \"{input}\" doesn\'t match format "
rf"\"{format}\", at position 0"
),
):
to_datetime(input, format=format)
@pytest.mark.parametrize(
"input, format",
[
("2020-01", "%Y-%m"),
("2020-01-01", "%Y-%m-%d"),
("2020-01-01 00", "%Y-%m-%d %H"),
("2020-01-01T00", "%Y-%m-%dT%H"),
("2020-01-01 00:00", "%Y-%m-%d %H:%M"),
("2020-01-01T00:00", "%Y-%m-%dT%H:%M"),
("2020-01-01 00:00:00", "%Y-%m-%d %H:%M:%S"),
("2020-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S"),
("2020-01-01T00:00:00.000", "%Y-%m-%dT%H:%M:%S.%f"),
("2020-01-01T00:00:00.000000", "%Y-%m-%dT%H:%M:%S.%f"),
("2020-01-01T00:00:00.000000000", "%Y-%m-%dT%H:%M:%S.%f"),
],
)
def test_to_datetime_iso8601_valid(self, input, format):
# https://github.com/pandas-dev/pandas/issues/12649
expected = Timestamp(2020, 1, 1)
result = to_datetime(input, format=format)
assert result == expected
@pytest.mark.parametrize(
"input, format",
[
("2020-1", "%Y-%m"),
("2020-1-1", "%Y-%m-%d"),
("2020-1-1 0", "%Y-%m-%d %H"),
("2020-1-1T0", "%Y-%m-%dT%H"),
("2020-1-1 0:0", "%Y-%m-%d %H:%M"),
("2020-1-1T0:0", "%Y-%m-%dT%H:%M"),
("2020-1-1 0:0:0", "%Y-%m-%d %H:%M:%S"),
("2020-1-1T0:0:0", "%Y-%m-%dT%H:%M:%S"),
("2020-1-1T0:0:0.000", "%Y-%m-%dT%H:%M:%S.%f"),
("2020-1-1T0:0:0.000000", "%Y-%m-%dT%H:%M:%S.%f"),
("2020-1-1T0:0:0.000000000", "%Y-%m-%dT%H:%M:%S.%f"),
],
)
def test_to_datetime_iso8601_non_padded(self, input, format):
# https://github.com/pandas-dev/pandas/issues/21422
expected = Timestamp(2020, 1, 1)
result = to_datetime(input, format=format)
assert result == expected
@pytest.mark.parametrize(
"input, format",
[
("2020-01-01T00:00:00.000000000+00:00", "%Y-%m-%dT%H:%M:%S.%f%z"),
("2020-01-01T00:00:00+00:00", "%Y-%m-%dT%H:%M:%S%z"),
("2020-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%S%z"),
],
)
def test_to_datetime_iso8601_with_timezone_valid(self, input, format):
# https://github.com/pandas-dev/pandas/issues/12649
expected = Timestamp(2020, 1, 1, tzinfo=pytz.UTC)
result = to_datetime(input, format=format)
assert result == expected
def test_to_datetime_default(self, cache):
rs = to_datetime("2001", cache=cache)
xp = datetime(2001, 1, 1)
assert rs == xp
@pytest.mark.xfail(reason="fails to enforce dayfirst=True, which would raise")
def test_to_datetime_respects_dayfirst(self, cache):
# dayfirst is essentially broken
# The msg here is not important since it isn't actually raised yet.
msg = "Invalid date specified"
with pytest.raises(ValueError, match=msg):
# if dayfirst is respected, then this would parse as month=13, which
# would raise
with tm.assert_produces_warning(UserWarning, match="Provide format"):
to_datetime("01-13-2012", dayfirst=True, cache=cache)
def test_to_datetime_on_datetime64_series(self, cache):
# #2699
ser = Series(date_range("1/1/2000", periods=10))
result = to_datetime(ser, cache=cache)
assert result[0] == ser[0]
def test_to_datetime_with_space_in_series(self, cache):
# GH 6428
ser = Series(["10/18/2006", "10/18/2008", " "])
msg = (
r'^time data " " doesn\'t match format "%m/%d/%Y", '
rf"at position 2. {PARSING_ERR_MSG}$"
)
with pytest.raises(ValueError, match=msg):
to_datetime(ser, errors="raise", cache=cache)
result_coerce = to_datetime(ser, errors="coerce", cache=cache)
expected_coerce = Series([datetime(2006, 10, 18), datetime(2008, 10, 18), NaT])
tm.assert_series_equal(result_coerce, expected_coerce)
result_ignore = to_datetime(ser, errors="ignore", cache=cache)
tm.assert_series_equal(result_ignore, ser)
@td.skip_if_not_us_locale
def test_to_datetime_with_apply(self, cache):
# this is only locale tested with US/None locales
# GH 5195
# with a format and coerce a single item to_datetime fails
td = Series(["May 04", "Jun 02", "Dec 11"], index=[1, 2, 3])
expected = to_datetime(td, format="%b %y", cache=cache)
result = td.apply(to_datetime, format="%b %y", cache=cache)
tm.assert_series_equal(result, expected)
def test_to_datetime_timezone_name(self):
# https://github.com/pandas-dev/pandas/issues/49748
result = to_datetime("2020-01-01 00:00:00UTC", format="%Y-%m-%d %H:%M:%S%Z")
expected = Timestamp(2020, 1, 1).tz_localize("UTC")
assert result == expected
@td.skip_if_not_us_locale
@pytest.mark.parametrize("errors", ["raise", "coerce", "ignore"])
def test_to_datetime_with_apply_with_empty_str(self, cache, errors):
# this is only locale tested with US/None locales
# GH 5195, GH50251
# with a format and coerce a single item to_datetime fails
td = Series(["May 04", "Jun 02", ""], index=[1, 2, 3])
expected = to_datetime(td, format="%b %y", errors=errors, cache=cache)
result = td.apply(
lambda x: to_datetime(x, format="%b %y", errors="coerce", cache=cache)
)
tm.assert_series_equal(result, expected)
def test_to_datetime_empty_stt(self, cache):
# empty string
result = to_datetime("", cache=cache)
assert result is NaT
def test_to_datetime_empty_str_list(self, cache):
result = to_datetime(["", ""], cache=cache)
assert isna(result).all()
def test_to_datetime_zero(self, cache):
# ints
result = Timestamp(0)
expected = to_datetime(0, cache=cache)
assert result == expected
def test_to_datetime_strings(self, cache):
# GH 3888 (strings)
expected = to_datetime(["2012"], cache=cache)[0]
result = to_datetime("2012", cache=cache)
assert result == expected
def test_to_datetime_strings_variation(self, cache):
array = ["2012", "20120101", "20120101 12:01:01"]
expected = [to_datetime(dt_str, cache=cache) for dt_str in array]
result = [Timestamp(date_str) for date_str in array]
tm.assert_almost_equal(result, expected)
@pytest.mark.parametrize("result", [Timestamp("2012"), to_datetime("2012")])
def test_to_datetime_strings_vs_constructor(self, result):
expected = Timestamp(2012, 1, 1)
assert result == expected
def test_to_datetime_unprocessable_input(self, cache):
# GH 4928
# GH 21864
result = to_datetime([1, "1"], errors="ignore", cache=cache)
expected = Index(np.array([1, "1"], dtype="O"))
tm.assert_equal(result, expected)
msg = '^Given date string "1" not likely a datetime, at position 1$'
with pytest.raises(ValueError, match=msg):
to_datetime([1, "1"], errors="raise", cache=cache)
def test_to_datetime_unhashable_input(self, cache):
series = Series([["a"]] * 100)
result = to_datetime(series, errors="ignore", cache=cache)
tm.assert_series_equal(series, result)
def test_to_datetime_other_datetime64_units(self):
# 5/25/2012
scalar = np.int64(1337904000000000).view("M8[us]")
as_obj = scalar.astype("O")
index = DatetimeIndex([scalar])
assert index[0] == scalar.astype("O")
value = Timestamp(scalar)
assert value == as_obj
def test_to_datetime_list_of_integers(self):
rng = date_range("1/1/2000", periods=20)
rng = DatetimeIndex(rng.values)
ints = list(rng.asi8)
result = DatetimeIndex(ints)
tm.assert_index_equal(rng, result)
def test_to_datetime_overflow(self):
# gh-17637
# we are overflowing Timedelta range here
msg = "Cannot cast 139999 days 00:00:00 to unit='ns' without overflow"
with pytest.raises(OutOfBoundsTimedelta, match=msg):
date_range(start="1/1/1700", freq="B", periods=100000)
def test_string_invalid_operation(self, cache):
invalid = np.array(["87156549591102612381000001219H5"], dtype=object)
# GH #51084
with pytest.raises(ValueError, match="Unknown datetime string format"):
to_datetime(invalid, errors="raise", cache=cache)
def test_string_na_nat_conversion(self, cache):
# GH #999, #858
strings = np.array(["1/1/2000", "1/2/2000", np.nan, "1/4/2000"], dtype=object)
expected = np.empty(4, dtype="M8[ns]")
for i, val in enumerate(strings):
if isna(val):
expected[i] = iNaT
else:
expected[i] = parse(val)
result = tslib.array_to_datetime(strings)[0]
tm.assert_almost_equal(result, expected)
result2 = to_datetime(strings, cache=cache)
assert isinstance(result2, DatetimeIndex)
tm.assert_numpy_array_equal(result, result2.values)
def test_string_na_nat_conversion_malformed(self, cache):
malformed = np.array(["1/100/2000", np.nan], dtype=object)
# GH 10636, default is now 'raise'
msg = r"Unknown datetime string format"
with pytest.raises(ValueError, match=msg):
to_datetime(malformed, errors="raise", cache=cache)
result = to_datetime(malformed, errors="ignore", cache=cache)
# GH 21864
expected = Index(malformed, dtype=object)
tm.assert_index_equal(result, expected)
with pytest.raises(ValueError, match=msg):
to_datetime(malformed, errors="raise", cache=cache)
def test_string_na_nat_conversion_with_name(self, cache):
idx = ["a", "b", "c", "d", "e"]
series = Series(
["1/1/2000", np.nan, "1/3/2000", np.nan, "1/5/2000"], index=idx, name="foo"
)
dseries = Series(
[
to_datetime("1/1/2000", cache=cache),
np.nan,
to_datetime("1/3/2000", cache=cache),
np.nan,
to_datetime("1/5/2000", cache=cache),
],
index=idx,
name="foo",
)
result = to_datetime(series, cache=cache)
dresult = to_datetime(dseries, cache=cache)
expected = Series(np.empty(5, dtype="M8[ns]"), index=idx)
for i in range(5):
x = series.iloc[i]
if isna(x):
expected.iloc[i] = NaT
else:
expected.iloc[i] = to_datetime(x, cache=cache)
tm.assert_series_equal(result, expected, check_names=False)
assert result.name == "foo"
tm.assert_series_equal(dresult, expected, check_names=False)
assert dresult.name == "foo"
@pytest.mark.parametrize(
"unit",
["h", "m", "s", "ms", "us", "ns"],
)
def test_dti_constructor_numpy_timeunits(self, cache, unit):
# GH 9114
dtype = np.dtype(f"M8[{unit}]")
base = to_datetime(["2000-01-01T00:00", "2000-01-02T00:00", "NaT"], cache=cache)
values = base.values.astype(dtype)
if unit in ["h", "m"]:
# we cast to closest supported unit
unit = "s"
exp_dtype = np.dtype(f"M8[{unit}]")
expected = DatetimeIndex(base.astype(exp_dtype))
assert expected.dtype == exp_dtype
tm.assert_index_equal(DatetimeIndex(values), expected)
tm.assert_index_equal(to_datetime(values, cache=cache), expected)
def test_dayfirst(self, cache):
# GH 5917
arr = ["10/02/2014", "11/02/2014", "12/02/2014"]
expected = DatetimeIndex(
[datetime(2014, 2, 10), datetime(2014, 2, 11), datetime(2014, 2, 12)]
)
idx1 = DatetimeIndex(arr, dayfirst=True)
idx2 = DatetimeIndex(np.array(arr), dayfirst=True)
idx3 = to_datetime(arr, dayfirst=True, cache=cache)
idx4 = to_datetime(np.array(arr), dayfirst=True, cache=cache)
idx5 = DatetimeIndex(Index(arr), dayfirst=True)
idx6 = DatetimeIndex(Series(arr), dayfirst=True)
tm.assert_index_equal(expected, idx1)
tm.assert_index_equal(expected, idx2)
tm.assert_index_equal(expected, idx3)
tm.assert_index_equal(expected, idx4)
tm.assert_index_equal(expected, idx5)
tm.assert_index_equal(expected, idx6)
def test_dayfirst_warnings_valid_input(self):
# GH 12585
warning_msg = (
"Parsing dates in .* format when dayfirst=.* was specified. "
"Pass `dayfirst=.*` or specify a format to silence this warning."
)
# CASE 1: valid input
arr = ["31/12/2014", "10/03/2011"]
expected = DatetimeIndex(
["2014-12-31", "2011-03-10"], dtype="datetime64[ns]", freq=None
)
# A. dayfirst arg correct, no warning
res1 = to_datetime(arr, dayfirst=True)
tm.assert_index_equal(expected, res1)
# B. dayfirst arg incorrect, warning
with tm.assert_produces_warning(UserWarning, match=warning_msg):
res2 = to_datetime(arr, dayfirst=False)
tm.assert_index_equal(expected, res2)
def test_dayfirst_warnings_invalid_input(self):
# CASE 2: invalid input
# cannot consistently process with single format
# ValueError *always* raised
# first in DD/MM/YYYY, second in MM/DD/YYYY
arr = ["31/12/2014", "03/30/2011"]
with pytest.raises(
ValueError,
match=(
r'^time data "03/30/2011" doesn\'t match format '
rf'"%d/%m/%Y", at position 1. {PARSING_ERR_MSG}$'
),
):
to_datetime(arr, dayfirst=True)
@pytest.mark.parametrize("klass", [DatetimeIndex, DatetimeArray._from_sequence])
def test_to_datetime_dta_tz(self, klass):
# GH#27733
dti = date_range("2015-04-05", periods=3).rename("foo")
expected = dti.tz_localize("UTC")
obj = klass(dti)
expected = klass(expected)
result = to_datetime(obj, utc=True)
tm.assert_equal(result, expected)
class TestGuessDatetimeFormat:
@pytest.mark.parametrize(
"test_list",
[
[
"2011-12-30 00:00:00.000000",
"2011-12-30 00:00:00.000000",
"2011-12-30 00:00:00.000000",
],
[np.nan, np.nan, "2011-12-30 00:00:00.000000"],
["", "2011-12-30 00:00:00.000000"],
["NaT", "2011-12-30 00:00:00.000000"],
["2011-12-30 00:00:00.000000", "random_string"],
["now", "2011-12-30 00:00:00.000000"],
["today", "2011-12-30 00:00:00.000000"],
],
)
def test_guess_datetime_format_for_array(self, test_list):
expected_format = "%Y-%m-%d %H:%M:%S.%f"
test_array = np.array(test_list, dtype=object)
assert tools._guess_datetime_format_for_array(test_array) == expected_format
@td.skip_if_not_us_locale
def test_guess_datetime_format_for_array_all_nans(self):
format_for_string_of_nans = tools._guess_datetime_format_for_array(
np.array([np.nan, np.nan, np.nan], dtype="O")
)
assert format_for_string_of_nans is None
class TestToDatetimeInferFormat:
@pytest.mark.parametrize(
"test_format", ["%m-%d-%Y", "%m/%d/%Y %H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S.%f"]
)
def test_to_datetime_infer_datetime_format_consistent_format(
self, cache, test_format
):
ser = Series(date_range("20000101", periods=50, freq="h"))
s_as_dt_strings = ser.apply(lambda x: x.strftime(test_format))
with_format = to_datetime(s_as_dt_strings, format=test_format, cache=cache)
without_format = to_datetime(s_as_dt_strings, cache=cache)
# Whether the format is explicitly passed, or
# it is inferred, the results should all be the same
tm.assert_series_equal(with_format, without_format)
def test_to_datetime_inconsistent_format(self, cache):
data = ["01/01/2011 00:00:00", "01-02-2011 00:00:00", "2011-01-03T00:00:00"]
ser = Series(np.array(data))
msg = (
r'^time data "01-02-2011 00:00:00" doesn\'t match format '
rf'"%m/%d/%Y %H:%M:%S", at position 1. {PARSING_ERR_MSG}$'
)
with pytest.raises(ValueError, match=msg):
to_datetime(ser, cache=cache)
def test_to_datetime_consistent_format(self, cache):
data = ["Jan/01/2011", "Feb/01/2011", "Mar/01/2011"]
ser = Series(np.array(data))
result = to_datetime(ser, cache=cache)
expected = Series(
["2011-01-01", "2011-02-01", "2011-03-01"], dtype="datetime64[ns]"
)
tm.assert_series_equal(result, expected)
def test_to_datetime_series_with_nans(self, cache):
ser = Series(
np.array(
["01/01/2011 00:00:00", np.nan, "01/03/2011 00:00:00", np.nan],
dtype=object,
)
)
result = to_datetime(ser, cache=cache)
expected = Series(
["2011-01-01", NaT, "2011-01-03", NaT], dtype="datetime64[ns]"
)
tm.assert_series_equal(result, expected)
def test_to_datetime_series_start_with_nans(self, cache):
ser = Series(
np.array(
[
np.nan,
np.nan,
"01/01/2011 00:00:00",
"01/02/2011 00:00:00",
"01/03/2011 00:00:00",
],
dtype=object,
)
)
result = to_datetime(ser, cache=cache)
expected = Series(
[NaT, NaT, "2011-01-01", "2011-01-02", "2011-01-03"], dtype="datetime64[ns]"
)
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"tz_name, offset",
[("UTC", 0), ("UTC-3", 180), ("UTC+3", -180)],
)
def test_infer_datetime_format_tz_name(self, tz_name, offset):
# GH 33133
ser = Series([f"2019-02-02 08:07:13 {tz_name}"])
result = to_datetime(ser)
tz = timezone(timedelta(minutes=offset))
expected = Series([Timestamp("2019-02-02 08:07:13").tz_localize(tz)])
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"ts,zero_tz",
[
("2019-02-02 08:07:13", "Z"),
("2019-02-02 08:07:13", ""),
("2019-02-02 08:07:13.012345", "Z"),
("2019-02-02 08:07:13.012345", ""),
],
)
def test_infer_datetime_format_zero_tz(self, ts, zero_tz):
# GH 41047
ser = Series([ts + zero_tz])
result = to_datetime(ser)
tz = pytz.utc if zero_tz == "Z" else None
expected = Series([Timestamp(ts, tz=tz)])
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize("format", [None, "%Y-%m-%d"])
def test_to_datetime_iso8601_noleading_0s(self, cache, format):
# GH 11871
ser = Series(["2014-1-1", "2014-2-2", "2015-3-3"])
expected = Series(
[
Timestamp("2014-01-01"),
Timestamp("2014-02-02"),
Timestamp("2015-03-03"),
]
)
result = to_datetime(ser, format=format, cache=cache)
tm.assert_series_equal(result, expected)
def test_parse_dates_infer_datetime_format_warning(self):
# GH 49024
with tm.assert_produces_warning(
UserWarning,
match="The argument 'infer_datetime_format' is deprecated",
):
to_datetime(["10-10-2000"], infer_datetime_format=True)
class TestDaysInMonth:
# tests for issue #10154
@pytest.mark.parametrize(
"arg, format",
[
["2015-02-29", None],
["2015-02-29", "%Y-%m-%d"],
["2015-02-32", "%Y-%m-%d"],
["2015-04-31", "%Y-%m-%d"],
],
)
def test_day_not_in_month_coerce(self, cache, arg, format):
assert isna(to_datetime(arg, errors="coerce", format=format, cache=cache))
def test_day_not_in_month_raise(self, cache):
msg = "day is out of range for month: 2015-02-29, at position 0"
with pytest.raises(ValueError, match=msg):
to_datetime("2015-02-29", errors="raise", cache=cache)
@pytest.mark.parametrize(
"arg, format, msg",
[
(
"2015-02-29",
"%Y-%m-%d",
f"^day is out of range for month, at position 0. {PARSING_ERR_MSG}$",
),
(
"2015-29-02",
"%Y-%d-%m",
f"^day is out of range for month, at position 0. {PARSING_ERR_MSG}$",
),
(
"2015-02-32",
"%Y-%m-%d",
'^unconverted data remains when parsing with format "%Y-%m-%d": "2", '
f"at position 0. {PARSING_ERR_MSG}$",
),
(
"2015-32-02",
"%Y-%d-%m",
'^time data "2015-32-02" doesn\'t match format "%Y-%d-%m", '
f"at position 0. {PARSING_ERR_MSG}$",
),
(
"2015-04-31",
"%Y-%m-%d",
f"^day is out of range for month, at position 0. {PARSING_ERR_MSG}$",
),
(
"2015-31-04",
"%Y-%d-%m",
f"^day is out of range for month, at position 0. {PARSING_ERR_MSG}$",
),
],
)
def test_day_not_in_month_raise_value(self, cache, arg, format, msg):
# https://github.com/pandas-dev/pandas/issues/50462
with pytest.raises(ValueError, match=msg):
to_datetime(arg, errors="raise", format=format, cache=cache)
@pytest.mark.parametrize(
"expected, format",
[
["2015-02-29", None],
["2015-02-29", "%Y-%m-%d"],
["2015-02-29", "%Y-%m-%d"],
["2015-04-31", "%Y-%m-%d"],
],
)
def test_day_not_in_month_ignore(self, cache, expected, format):
result = to_datetime(expected, errors="ignore", format=format, cache=cache)
assert result == expected
class TestDatetimeParsingWrappers:
@pytest.mark.parametrize(
"date_str, expected",
[
("2011-01-01", datetime(2011, 1, 1)),
("2Q2005", datetime(2005, 4, 1)),
("2Q05", datetime(2005, 4, 1)),
("2005Q1", datetime(2005, 1, 1)),
("05Q1", datetime(2005, 1, 1)),
("2011Q3", datetime(2011, 7, 1)),
("11Q3", datetime(2011, 7, 1)),
("3Q2011", datetime(2011, 7, 1)),
("3Q11", datetime(2011, 7, 1)),
# quarterly without space
("2000Q4", datetime(2000, 10, 1)),
("00Q4", datetime(2000, 10, 1)),
("4Q2000", datetime(2000, 10, 1)),
("4Q00", datetime(2000, 10, 1)),
("2000q4", datetime(2000, 10, 1)),
("2000-Q4", datetime(2000, 10, 1)),
("00-Q4", datetime(2000, 10, 1)),
("4Q-2000", datetime(2000, 10, 1)),
("4Q-00", datetime(2000, 10, 1)),
("00q4", datetime(2000, 10, 1)),
("2005", datetime(2005, 1, 1)),
("2005-11", datetime(2005, 11, 1)),
("2005 11", datetime(2005, 11, 1)),
("11-2005", datetime(2005, 11, 1)),
("11 2005", datetime(2005, 11, 1)),
("200511", datetime(2020, 5, 11)),
("20051109", datetime(2005, 11, 9)),
("20051109 10:15", datetime(2005, 11, 9, 10, 15)),
("20051109 08H", datetime(2005, 11, 9, 8, 0)),
("2005-11-09 10:15", datetime(2005, 11, 9, 10, 15)),
("2005-11-09 08H", datetime(2005, 11, 9, 8, 0)),
("2005/11/09 10:15", datetime(2005, 11, 9, 10, 15)),
("2005/11/09 10:15:32", datetime(2005, 11, 9, 10, 15, 32)),
("2005/11/09 10:15:32 AM", datetime(2005, 11, 9, 10, 15, 32)),
("2005/11/09 10:15:32 PM", datetime(2005, 11, 9, 22, 15, 32)),
("2005/11/09 08H", datetime(2005, 11, 9, 8, 0)),
("Thu Sep 25 10:36:28 2003", datetime(2003, 9, 25, 10, 36, 28)),
("Thu Sep 25 2003", datetime(2003, 9, 25)),
("Sep 25 2003", datetime(2003, 9, 25)),
("January 1 2014", datetime(2014, 1, 1)),
# GH#10537
("2014-06", datetime(2014, 6, 1)),
("06-2014", datetime(2014, 6, 1)),
("2014-6", datetime(2014, 6, 1)),
("6-2014", datetime(2014, 6, 1)),
("20010101 12", datetime(2001, 1, 1, 12)),
("20010101 1234", datetime(2001, 1, 1, 12, 34)),
("20010101 123456", datetime(2001, 1, 1, 12, 34, 56)),
],
)
def test_parsers(self, date_str, expected, cache):
# dateutil >= 2.5.0 defaults to yearfirst=True
# https://github.com/dateutil/dateutil/issues/217
yearfirst = True
result1, _ = parsing.parse_datetime_string_with_reso(
date_str, yearfirst=yearfirst
)
result2 = to_datetime(date_str, yearfirst=yearfirst)
result3 = to_datetime([date_str], yearfirst=yearfirst)
# result5 is used below
result4 = to_datetime(
np.array([date_str], dtype=object), yearfirst=yearfirst, cache=cache
)
result6 = DatetimeIndex([date_str], yearfirst=yearfirst)
# result7 is used below
result8 = DatetimeIndex(Index([date_str]), yearfirst=yearfirst)
result9 = DatetimeIndex(Series([date_str]), yearfirst=yearfirst)
for res in [result1, result2]:
assert res == expected
for res in [result3, result4, result6, result8, result9]:
exp = DatetimeIndex([Timestamp(expected)])
tm.assert_index_equal(res, exp)
# these really need to have yearfirst, but we don't support
if not yearfirst:
result5 = Timestamp(date_str)
assert result5 == expected
result7 = date_range(date_str, freq="S", periods=1, yearfirst=yearfirst)
assert result7 == expected
def test_na_values_with_cache(
self, cache, unique_nulls_fixture, unique_nulls_fixture2
):
# GH22305
expected = Index([NaT, NaT], dtype="datetime64[ns]")
result = to_datetime([unique_nulls_fixture, unique_nulls_fixture2], cache=cache)
tm.assert_index_equal(result, expected)
def test_parsers_nat(self):
# Test that each of several string-accepting methods return pd.NaT
result1, _ = parsing.parse_datetime_string_with_reso("NaT")
result2 = to_datetime("NaT")
result3 = Timestamp("NaT")
result4 = DatetimeIndex(["NaT"])[0]
assert result1 is NaT
assert result2 is NaT
assert result3 is NaT
assert result4 is NaT
@pytest.mark.parametrize(
"date_str, dayfirst, yearfirst, expected",
[
("10-11-12", False, False, datetime(2012, 10, 11)),
("10-11-12", True, False, datetime(2012, 11, 10)),
("10-11-12", False, True, datetime(2010, 11, 12)),
("10-11-12", True, True, datetime(2010, 12, 11)),
("20/12/21", False, False, datetime(2021, 12, 20)),
("20/12/21", True, False, datetime(2021, 12, 20)),
("20/12/21", False, True, datetime(2020, 12, 21)),
("20/12/21", True, True, datetime(2020, 12, 21)),
],
)
def test_parsers_dayfirst_yearfirst(
self, cache, date_str, dayfirst, yearfirst, expected
):
# OK
# 2.5.1 10-11-12 [dayfirst=0, yearfirst=0] -> 2012-10-11 00:00:00
# 2.5.2 10-11-12 [dayfirst=0, yearfirst=1] -> 2012-10-11 00:00:00
# 2.5.3 10-11-12 [dayfirst=0, yearfirst=0] -> 2012-10-11 00:00:00
# OK
# 2.5.1 10-11-12 [dayfirst=0, yearfirst=1] -> 2010-11-12 00:00:00
# 2.5.2 10-11-12 [dayfirst=0, yearfirst=1] -> 2010-11-12 00:00:00
# 2.5.3 10-11-12 [dayfirst=0, yearfirst=1] -> 2010-11-12 00:00:00
# bug fix in 2.5.2
# 2.5.1 10-11-12 [dayfirst=1, yearfirst=1] -> 2010-11-12 00:00:00
# 2.5.2 10-11-12 [dayfirst=1, yearfirst=1] -> 2010-12-11 00:00:00
# 2.5.3 10-11-12 [dayfirst=1, yearfirst=1] -> 2010-12-11 00:00:00
# OK
# 2.5.1 10-11-12 [dayfirst=1, yearfirst=0] -> 2012-11-10 00:00:00
# 2.5.2 10-11-12 [dayfirst=1, yearfirst=0] -> 2012-11-10 00:00:00
# 2.5.3 10-11-12 [dayfirst=1, yearfirst=0] -> 2012-11-10 00:00:00
# OK
# 2.5.1 20/12/21 [dayfirst=0, yearfirst=0] -> 2021-12-20 00:00:00
# 2.5.2 20/12/21 [dayfirst=0, yearfirst=0] -> 2021-12-20 00:00:00
# 2.5.3 20/12/21 [dayfirst=0, yearfirst=0] -> 2021-12-20 00:00:00
# OK
# 2.5.1 20/12/21 [dayfirst=0, yearfirst=1] -> 2020-12-21 00:00:00
# 2.5.2 20/12/21 [dayfirst=0, yearfirst=1] -> 2020-12-21 00:00:00
# 2.5.3 20/12/21 [dayfirst=0, yearfirst=1] -> 2020-12-21 00:00:00
# revert of bug in 2.5.2
# 2.5.1 20/12/21 [dayfirst=1, yearfirst=1] -> 2020-12-21 00:00:00
# 2.5.2 20/12/21 [dayfirst=1, yearfirst=1] -> month must be in 1..12
# 2.5.3 20/12/21 [dayfirst=1, yearfirst=1] -> 2020-12-21 00:00:00
# OK
# 2.5.1 20/12/21 [dayfirst=1, yearfirst=0] -> 2021-12-20 00:00:00
# 2.5.2 20/12/21 [dayfirst=1, yearfirst=0] -> 2021-12-20 00:00:00
# 2.5.3 20/12/21 [dayfirst=1, yearfirst=0] -> 2021-12-20 00:00:00
# str : dayfirst, yearfirst, expected
# compare with dateutil result
dateutil_result = parse(date_str, dayfirst=dayfirst, yearfirst=yearfirst)
assert dateutil_result == expected
result1, _ = parsing.parse_datetime_string_with_reso(
date_str, dayfirst=dayfirst, yearfirst=yearfirst
)
# we don't support dayfirst/yearfirst here:
if not dayfirst and not yearfirst:
result2 = Timestamp(date_str)
assert result2 == expected
result3 = to_datetime(
date_str, dayfirst=dayfirst, yearfirst=yearfirst, cache=cache
)
result4 = DatetimeIndex([date_str], dayfirst=dayfirst, yearfirst=yearfirst)[0]
assert result1 == expected
assert result3 == expected
assert result4 == expected
@pytest.mark.parametrize(
"date_str, exp_def",
[["10:15", datetime(1, 1, 1, 10, 15)], ["9:05", datetime(1, 1, 1, 9, 5)]],
)
def test_parsers_timestring(self, date_str, exp_def):
# must be the same as dateutil result
exp_now = parse(date_str)
result1, _ = parsing.parse_datetime_string_with_reso(date_str)
result2 = to_datetime(date_str)
result3 = to_datetime([date_str])
result4 = Timestamp(date_str)
result5 = DatetimeIndex([date_str])[0]
# parse time string return time string based on default date
# others are not, and can't be changed because it is used in
# time series plot
assert result1 == exp_def
assert result2 == exp_now
assert result3 == exp_now
assert result4 == exp_now
assert result5 == exp_now
@pytest.mark.parametrize(
"dt_string, tz, dt_string_repr",
[
(
"2013-01-01 05:45+0545",
timezone(timedelta(minutes=345)),
"Timestamp('2013-01-01 05:45:00+0545', tz='UTC+05:45')",
),
(
"2013-01-01 05:30+0530",
timezone(timedelta(minutes=330)),
"Timestamp('2013-01-01 05:30:00+0530', tz='UTC+05:30')",
),
],
)
def test_parsers_timezone_minute_offsets_roundtrip(
self, cache, dt_string, tz, dt_string_repr
):
# GH11708
base = to_datetime("2013-01-01 00:00:00", cache=cache)
base = base.tz_localize("UTC").tz_convert(tz)
dt_time = to_datetime(dt_string, cache=cache)
assert base == dt_time
assert dt_string_repr == repr(dt_time)
@pytest.fixture(params=["D", "s", "ms", "us", "ns"])
def units(request):
"""Day and some time units.
* D
* s
* ms
* us
* ns
"""
return request.param
@pytest.fixture
def epoch_1960():
"""Timestamp at 1960-01-01."""
return Timestamp("1960-01-01")
@pytest.fixture
def units_from_epochs():
return list(range(5))
@pytest.fixture(params=["timestamp", "pydatetime", "datetime64", "str_1960"])
def epochs(epoch_1960, request):
"""Timestamp at 1960-01-01 in various forms.
* Timestamp
* datetime.datetime
* numpy.datetime64
* str
"""
assert request.param in {"timestamp", "pydatetime", "datetime64", "str_1960"}
if request.param == "timestamp":
return epoch_1960
elif request.param == "pydatetime":
return epoch_1960.to_pydatetime()
elif request.param == "datetime64":
return epoch_1960.to_datetime64()
else:
return str(epoch_1960)
@pytest.fixture
def julian_dates():
return date_range("2014-1-1", periods=10).to_julian_date().values
class TestOrigin:
def test_origin_and_unit(self):
# GH#42624
ts = to_datetime(1, unit="s", origin=1)
expected = Timestamp("1970-01-01 00:00:02")
assert ts == expected
ts = to_datetime(1, unit="s", origin=1_000_000_000)
expected = Timestamp("2001-09-09 01:46:41")
assert ts == expected
def test_julian(self, julian_dates):
# gh-11276, gh-11745
# for origin as julian
result = Series(to_datetime(julian_dates, unit="D", origin="julian"))
expected = Series(
to_datetime(julian_dates - Timestamp(0).to_julian_date(), unit="D")
)
tm.assert_series_equal(result, expected)
def test_unix(self):
result = Series(to_datetime([0, 1, 2], unit="D", origin="unix"))
expected = Series(
[Timestamp("1970-01-01"), Timestamp("1970-01-02"), Timestamp("1970-01-03")],
dtype="M8[ns]",
)
tm.assert_series_equal(result, expected)
def test_julian_round_trip(self):
result = to_datetime(2456658, origin="julian", unit="D")
assert result.to_julian_date() == 2456658
# out-of-bounds
msg = "1 is Out of Bounds for origin='julian'"
with pytest.raises(ValueError, match=msg):
to_datetime(1, origin="julian", unit="D")
def test_invalid_unit(self, units, julian_dates):
# checking for invalid combination of origin='julian' and unit != D
if units != "D":
msg = "unit must be 'D' for origin='julian'"
with pytest.raises(ValueError, match=msg):
to_datetime(julian_dates, unit=units, origin="julian")
@pytest.mark.parametrize("unit", ["ns", "D"])
def test_invalid_origin(self, unit):
# need to have a numeric specified
msg = "it must be numeric with a unit specified"
with pytest.raises(ValueError, match=msg):
to_datetime("2005-01-01", origin="1960-01-01", unit=unit)
def test_epoch(self, units, epochs, epoch_1960, units_from_epochs):
expected = Series(
[pd.Timedelta(x, unit=units) + epoch_1960 for x in units_from_epochs]
)
result = Series(to_datetime(units_from_epochs, unit=units, origin=epochs))
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"origin, exc",
[
("random_string", ValueError),
("epoch", ValueError),
("13-24-1990", ValueError),
(datetime(1, 1, 1), OutOfBoundsDatetime),
],
)
def test_invalid_origins(self, origin, exc, units, units_from_epochs):
msg = "|".join(
[
f"origin {origin} is Out of Bounds",
f"origin {origin} cannot be converted to a Timestamp",
"Cannot cast .* to unit='ns' without overflow",
]
)
with pytest.raises(exc, match=msg):
to_datetime(units_from_epochs, unit=units, origin=origin)
def test_invalid_origins_tzinfo(self):
# GH16842
with pytest.raises(ValueError, match="must be tz-naive"):
to_datetime(1, unit="D", origin=datetime(2000, 1, 1, tzinfo=pytz.utc))
def test_incorrect_value_exception(self):
# GH47495
msg = (
"Unknown datetime string format, unable to parse: yesterday, at position 1"
)
with pytest.raises(ValueError, match=msg):
to_datetime(["today", "yesterday"])
@pytest.mark.parametrize(
"format, warning",
[
(None, UserWarning),
("%Y-%m-%d %H:%M:%S", None),
("%Y-%d-%m %H:%M:%S", None),
],
)
def test_to_datetime_out_of_bounds_with_format_arg(self, format, warning):
# see gh-23830
msg = r"^Out of bounds nanosecond timestamp: 2417-10-10 00:00:00, at position 0"
with pytest.raises(OutOfBoundsDatetime, match=msg):
to_datetime("2417-10-10 00:00:00", format=format)
@pytest.mark.parametrize(
"arg, origin, expected_str",
[
[200 * 365, "unix", "2169-11-13 00:00:00"],
[200 * 365, "1870-01-01", "2069-11-13 00:00:00"],
[300 * 365, "1870-01-01", "2169-10-20 00:00:00"],
],
)
def test_processing_order(self, arg, origin, expected_str):
# make sure we handle out-of-bounds *before*
# constructing the dates
result = to_datetime(arg, unit="D", origin=origin)
expected = Timestamp(expected_str)
assert result == expected
result = to_datetime(200 * 365, unit="D", origin="1870-01-01")
expected = Timestamp("2069-11-13 00:00:00")
assert result == expected
result = to_datetime(300 * 365, unit="D", origin="1870-01-01")
expected = Timestamp("2169-10-20 00:00:00")
assert result == expected
@pytest.mark.parametrize(
"offset,utc,exp",
[
["Z", True, "2019-01-01T00:00:00.000Z"],
["Z", None, "2019-01-01T00:00:00.000Z"],
["-01:00", True, "2019-01-01T01:00:00.000Z"],
["-01:00", None, "2019-01-01T00:00:00.000-01:00"],
],
)
def test_arg_tz_ns_unit(self, offset, utc, exp):
# GH 25546
arg = "2019-01-01T00:00:00.000" + offset
result = to_datetime([arg], unit="ns", utc=utc)
expected = to_datetime([exp]).as_unit("ns")
tm.assert_index_equal(result, expected)
class TestShouldCache:
@pytest.mark.parametrize(
"listlike,do_caching",
[
([1, 2, 3, 4, 5, 6, 7, 8, 9, 0], False),
([1, 1, 1, 1, 4, 5, 6, 7, 8, 9], True),
],
)
def test_should_cache(self, listlike, do_caching):
assert (
tools.should_cache(listlike, check_count=len(listlike), unique_share=0.7)
== do_caching
)
@pytest.mark.parametrize(
"unique_share,check_count, err_message",
[
(0.5, 11, r"check_count must be in next bounds: \[0; len\(arg\)\]"),
(10, 2, r"unique_share must be in next bounds: \(0; 1\)"),
],
)
def test_should_cache_errors(self, unique_share, check_count, err_message):
arg = [5] * 10
with pytest.raises(AssertionError, match=err_message):
tools.should_cache(arg, unique_share, check_count)
@pytest.mark.parametrize(
"listlike",
[
(deque([Timestamp("2010-06-02 09:30:00")] * 51)),
([Timestamp("2010-06-02 09:30:00")] * 51),
(tuple([Timestamp("2010-06-02 09:30:00")] * 51)),
],
)
def test_no_slicing_errors_in_should_cache(self, listlike):
# GH#29403
assert tools.should_cache(listlike) is True
def test_nullable_integer_to_datetime():
# Test for #30050
ser = Series([1, 2, None, 2**61, None])
ser = ser.astype("Int64")
ser_copy = ser.copy()
res = to_datetime(ser, unit="ns")
expected = Series(
[
np.datetime64("1970-01-01 00:00:00.000000001"),
np.datetime64("1970-01-01 00:00:00.000000002"),
np.datetime64("NaT"),
np.datetime64("2043-01-25 23:56:49.213693952"),
np.datetime64("NaT"),
]
)
tm.assert_series_equal(res, expected)
# Check that ser isn't mutated
tm.assert_series_equal(ser, ser_copy)
@pytest.mark.parametrize("klass", [np.array, list])
def test_na_to_datetime(nulls_fixture, klass):
if isinstance(nulls_fixture, Decimal):
with pytest.raises(TypeError, match="not convertible to datetime"):
to_datetime(klass([nulls_fixture]))
else:
result = to_datetime(klass([nulls_fixture]))
assert result[0] is NaT
@pytest.mark.parametrize("errors", ["raise", "coerce", "ignore"])
@pytest.mark.parametrize(
"args, format",
[
(["03/24/2016", "03/25/2016", ""], "%m/%d/%Y"),
(["2016-03-24", "2016-03-25", ""], "%Y-%m-%d"),
],
ids=["non-ISO8601", "ISO8601"],
)
def test_empty_string_datetime(errors, args, format):
# GH13044, GH50251
td = Series(args)
# coerce empty string to pd.NaT
result = to_datetime(td, format=format, errors=errors)
expected = Series(["2016-03-24", "2016-03-25", NaT], dtype="datetime64[ns]")
tm.assert_series_equal(expected, result)
def test_empty_string_datetime_coerce__unit():
# GH13044
# coerce empty string to pd.NaT
result = to_datetime([1, ""], unit="s", errors="coerce")
expected = DatetimeIndex(["1970-01-01 00:00:01", "NaT"], dtype="datetime64[ns]")
tm.assert_index_equal(expected, result)
# verify that no exception is raised even when errors='raise' is set
result = to_datetime([1, ""], unit="s", errors="raise")
tm.assert_index_equal(expected, result)
@pytest.mark.parametrize("cache", [True, False])
def test_to_datetime_monotonic_increasing_index(cache):
# GH28238
cstart = start_caching_at
times = date_range(Timestamp("1980"), periods=cstart, freq="YS")
times = times.to_frame(index=False, name="DT").sample(n=cstart, random_state=1)
times.index = times.index.to_series().astype(float) / 1000
result = to_datetime(times.iloc[:, 0], cache=cache)
expected = times.iloc[:, 0]
tm.assert_series_equal(result, expected)
@pytest.mark.parametrize(
"series_length",
[40, start_caching_at, (start_caching_at + 1), (start_caching_at + 5)],
)
def test_to_datetime_cache_coerce_50_lines_outofbounds(series_length):
# GH#45319
ser = Series(
[datetime.fromisoformat("1446-04-12 00:00:00+00:00")]
+ ([datetime.fromisoformat("1991-10-20 00:00:00+00:00")] * series_length),
dtype=object,
)
result1 = to_datetime(ser, errors="coerce", utc=True)
expected1 = Series(
[NaT] + ([Timestamp("1991-10-20 00:00:00+00:00")] * series_length)
)
tm.assert_series_equal(result1, expected1)
result2 = to_datetime(ser, errors="ignore", utc=True)
expected2 = Series(
[datetime.fromisoformat("1446-04-12 00:00:00+00:00")]
+ ([datetime.fromisoformat("1991-10-20 00:00:00+00:00")] * series_length)
)
tm.assert_series_equal(result2, expected2)
with pytest.raises(OutOfBoundsDatetime, match="Out of bounds nanosecond timestamp"):
to_datetime(ser, errors="raise", utc=True)
def test_to_datetime_format_f_parse_nanos():
# GH 48767
timestamp = "15/02/2020 02:03:04.123456789"
timestamp_format = "%d/%m/%Y %H:%M:%S.%f"
result = to_datetime(timestamp, format=timestamp_format)
expected = Timestamp(
year=2020,
month=2,
day=15,
hour=2,
minute=3,
second=4,
microsecond=123456,
nanosecond=789,
)
assert result == expected
def test_to_datetime_mixed_iso8601():
# https://github.com/pandas-dev/pandas/issues/50411
result = to_datetime(["2020-01-01", "2020-01-01 05:00:00"], format="ISO8601")
expected = DatetimeIndex(["2020-01-01 00:00:00", "2020-01-01 05:00:00"])
tm.assert_index_equal(result, expected)
def test_to_datetime_mixed_other():
# https://github.com/pandas-dev/pandas/issues/50411
result = to_datetime(["01/11/2000", "12 January 2000"], format="mixed")
expected = DatetimeIndex(["2000-01-11", "2000-01-12"])
tm.assert_index_equal(result, expected)
@pytest.mark.parametrize("exact", [True, False])
@pytest.mark.parametrize("format", ["ISO8601", "mixed"])
def test_to_datetime_mixed_or_iso_exact(exact, format):
msg = "Cannot use 'exact' when 'format' is 'mixed' or 'ISO8601'"
with pytest.raises(ValueError, match=msg):
to_datetime(["2020-01-01"], exact=exact, format=format)
def test_to_datetime_mixed_not_necessarily_iso8601_raise():
# https://github.com/pandas-dev/pandas/issues/50411
with pytest.raises(
ValueError, match="Time data 01-01-2000 is not ISO8601 format, at position 1"
):
to_datetime(["2020-01-01", "01-01-2000"], format="ISO8601")
@pytest.mark.parametrize(
("errors", "expected"),
[
("coerce", DatetimeIndex(["2020-01-01 00:00:00", NaT])),
("ignore", Index(["2020-01-01", "01-01-2000"], dtype=object)),
],
)
def test_to_datetime_mixed_not_necessarily_iso8601_coerce(errors, expected):
# https://github.com/pandas-dev/pandas/issues/50411
result = to_datetime(["2020-01-01", "01-01-2000"], format="ISO8601", errors=errors)
tm.assert_index_equal(result, expected)
def test_ignoring_unknown_tz_deprecated():
# GH#18702, GH#51476
dtstr = "2014 Jan 9 05:15 FAKE"
msg = 'un-recognized timezone "FAKE". Dropping unrecognized timezones is deprecated'
with tm.assert_produces_warning(FutureWarning, match=msg):
res = Timestamp(dtstr)
assert res == Timestamp(dtstr[:-5])
with tm.assert_produces_warning(FutureWarning):
res = to_datetime(dtstr)
assert res == to_datetime(dtstr[:-5])
with tm.assert_produces_warning(FutureWarning):
res = to_datetime([dtstr])
tm.assert_index_equal(res, to_datetime([dtstr[:-5]]))
def test_from_numeric_arrow_dtype(any_numeric_ea_dtype):
# GH 52425
pytest.importorskip("pyarrow")
ser = Series([1, 2], dtype=f"{any_numeric_ea_dtype.lower()}[pyarrow]")
result = to_datetime(ser)
expected = Series([1, 2], dtype="datetime64[ns]")
tm.assert_series_equal(result, expected)
def test_to_datetime_with_empty_str_utc_false_format_mixed():
# GH 50887
vals = ["2020-01-01 00:00+00:00", ""]
result = to_datetime(vals, format="mixed")
expected = Index([Timestamp("2020-01-01 00:00+00:00"), "NaT"], dtype="M8[ns, UTC]")
tm.assert_index_equal(result, expected)
# Check that a couple of other similar paths work the same way
alt = to_datetime(vals)
tm.assert_index_equal(alt, expected)
alt2 = DatetimeIndex(vals)
tm.assert_index_equal(alt2, expected)
def test_to_datetime_with_empty_str_utc_false_offsets_and_format_mixed():
# GH 50887
msg = "parsing datetimes with mixed time zones will raise an error"
with tm.assert_produces_warning(FutureWarning, match=msg):
to_datetime(
["2020-01-01 00:00+00:00", "2020-01-01 00:00+02:00", ""], format="mixed"
)
def test_to_datetime_mixed_tzs_mixed_types():
# GH#55793, GH#55693 mismatched tzs but one is str and other is
# datetime object
ts = Timestamp("2016-01-02 03:04:05", tz="US/Pacific")
dtstr = "2023-10-30 15:06+01"
arr = [ts, dtstr]
msg = (
"Mixed timezones detected. pass utc=True in to_datetime or tz='UTC' "
"in DatetimeIndex to convert to a common timezone"
)
with pytest.raises(ValueError, match=msg):
to_datetime(arr)
with pytest.raises(ValueError, match=msg):
to_datetime(arr, format="mixed")
with pytest.raises(ValueError, match=msg):
DatetimeIndex(arr)
def test_to_datetime_mixed_types_matching_tzs():
# GH#55793
dtstr = "2023-11-01 09:22:03-07:00"
ts = Timestamp(dtstr)
arr = [ts, dtstr]
res1 = to_datetime(arr)
res2 = to_datetime(arr[::-1])[::-1]
res3 = to_datetime(arr, format="mixed")
res4 = DatetimeIndex(arr)
expected = DatetimeIndex([ts, ts])
tm.assert_index_equal(res1, expected)
tm.assert_index_equal(res2, expected)
tm.assert_index_equal(res3, expected)
tm.assert_index_equal(res4, expected)
dtstr = "2020-01-01 00:00+00:00"
ts = Timestamp(dtstr)
@pytest.mark.filterwarnings("ignore:Could not infer format:UserWarning")
@pytest.mark.parametrize(
"aware_val",
[dtstr, Timestamp(dtstr)],
ids=lambda x: type(x).__name__,
)
@pytest.mark.parametrize(
"naive_val",
[dtstr[:-6], ts.tz_localize(None), ts.date(), ts.asm8, ts.value, float(ts.value)],
ids=lambda x: type(x).__name__,
)
@pytest.mark.parametrize("naive_first", [True, False])
def test_to_datetime_mixed_awareness_mixed_types(aware_val, naive_val, naive_first):
# GH#55793, GH#55693
# Empty string parses to NaT
vals = [aware_val, naive_val, ""]
vec = vals
if naive_first:
# alas, the behavior is order-dependent, so we test both ways
vec = [naive_val, aware_val, ""]
# both_strs-> paths that were previously already deprecated with warning
# issued in _array_to_datetime_object
both_strs = isinstance(aware_val, str) and isinstance(naive_val, str)
has_numeric = isinstance(naive_val, (int, float))
depr_msg = "In a future version of pandas, parsing datetimes with mixed time zones"
first_non_null = next(x for x in vec if x != "")
# if first_non_null is a not a string, _guess_datetime_format_for_array
# doesn't guess a format so we don't go through array_strptime
if not isinstance(first_non_null, str):
# that case goes through array_strptime which has different behavior
msg = "Cannot mix tz-aware with tz-naive values"
if naive_first and isinstance(aware_val, Timestamp):
if isinstance(naive_val, Timestamp):
msg = "Tz-aware datetime.datetime cannot be converted to datetime64"
with pytest.raises(ValueError, match=msg):
to_datetime(vec)
else:
with pytest.raises(ValueError, match=msg):
to_datetime(vec)
# No warning/error with utc=True
to_datetime(vec, utc=True)
elif has_numeric and vec.index(aware_val) < vec.index(naive_val):
msg = "time data .* doesn't match format"
with pytest.raises(ValueError, match=msg):
to_datetime(vec)
with pytest.raises(ValueError, match=msg):
to_datetime(vec, utc=True)
elif both_strs and vec.index(aware_val) < vec.index(naive_val):
msg = r"time data \"2020-01-01 00:00\" doesn't match format"
with pytest.raises(ValueError, match=msg):
to_datetime(vec)
with pytest.raises(ValueError, match=msg):
to_datetime(vec, utc=True)
elif both_strs and vec.index(naive_val) < vec.index(aware_val):
msg = "unconverted data remains when parsing with format"
with pytest.raises(ValueError, match=msg):
to_datetime(vec)
with pytest.raises(ValueError, match=msg):
to_datetime(vec, utc=True)
else:
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
to_datetime(vec)
# No warning/error with utc=True
to_datetime(vec, utc=True)
if both_strs:
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
to_datetime(vec, format="mixed")
with tm.assert_produces_warning(FutureWarning, match=depr_msg):
msg = "DatetimeIndex has mixed timezones"
with pytest.raises(TypeError, match=msg):
DatetimeIndex(vec)
else:
msg = "Cannot mix tz-aware with tz-naive values"
if naive_first and isinstance(aware_val, Timestamp):
if isinstance(naive_val, Timestamp):
msg = "Tz-aware datetime.datetime cannot be converted to datetime64"
with pytest.raises(ValueError, match=msg):
to_datetime(vec, format="mixed")
with pytest.raises(ValueError, match=msg):
DatetimeIndex(vec)
else:
with pytest.raises(ValueError, match=msg):
to_datetime(vec, format="mixed")
with pytest.raises(ValueError, match=msg):
DatetimeIndex(vec)