1364 lines
45 KiB
Python
1364 lines
45 KiB
Python
import datetime
|
|
|
|
import numpy as np
|
|
import pytest
|
|
import pytz
|
|
|
|
import pandas as pd
|
|
from pandas import Timedelta, merge_asof, read_csv, to_datetime
|
|
import pandas._testing as tm
|
|
from pandas.core.reshape.merge import MergeError
|
|
|
|
|
|
class TestAsOfMerge:
|
|
def read_data(self, datapath, name, dedupe=False):
|
|
path = datapath("reshape", "merge", "data", name)
|
|
x = read_csv(path)
|
|
if dedupe:
|
|
x = x.drop_duplicates(["time", "ticker"], keep="last").reset_index(
|
|
drop=True
|
|
)
|
|
x.time = to_datetime(x.time)
|
|
return x
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def setup_method(self, datapath):
|
|
|
|
self.trades = self.read_data(datapath, "trades.csv")
|
|
self.quotes = self.read_data(datapath, "quotes.csv", dedupe=True)
|
|
self.asof = self.read_data(datapath, "asof.csv")
|
|
self.tolerance = self.read_data(datapath, "tolerance.csv")
|
|
self.allow_exact_matches = self.read_data(datapath, "allow_exact_matches.csv")
|
|
self.allow_exact_matches_and_tolerance = self.read_data(
|
|
datapath, "allow_exact_matches_and_tolerance.csv"
|
|
)
|
|
|
|
def test_examples1(self):
|
|
""" doc-string examples """
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, 3, 7]}
|
|
)
|
|
|
|
result = pd.merge_asof(left, right, on="a")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_examples2(self):
|
|
""" doc-string examples """
|
|
trades = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.038",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.048",
|
|
]
|
|
),
|
|
"ticker": ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
|
|
"price": [51.95, 51.95, 720.77, 720.92, 98.00],
|
|
"quantity": [75, 155, 100, 100, 100],
|
|
},
|
|
columns=["time", "ticker", "price", "quantity"],
|
|
)
|
|
|
|
quotes = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.030",
|
|
"20160525 13:30:00.041",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.049",
|
|
"20160525 13:30:00.072",
|
|
"20160525 13:30:00.075",
|
|
]
|
|
),
|
|
"ticker": [
|
|
"GOOG",
|
|
"MSFT",
|
|
"MSFT",
|
|
"MSFT",
|
|
"GOOG",
|
|
"AAPL",
|
|
"GOOG",
|
|
"MSFT",
|
|
],
|
|
"bid": [720.50, 51.95, 51.97, 51.99, 720.50, 97.99, 720.50, 52.01],
|
|
"ask": [720.93, 51.96, 51.98, 52.00, 720.93, 98.01, 720.88, 52.03],
|
|
},
|
|
columns=["time", "ticker", "bid", "ask"],
|
|
)
|
|
|
|
pd.merge_asof(trades, quotes, on="time", by="ticker")
|
|
|
|
pd.merge_asof(
|
|
trades, quotes, on="time", by="ticker", tolerance=Timedelta("2ms")
|
|
)
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.038",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.048",
|
|
]
|
|
),
|
|
"ticker": ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
|
|
"price": [51.95, 51.95, 720.77, 720.92, 98.00],
|
|
"quantity": [75, 155, 100, 100, 100],
|
|
"bid": [np.nan, 51.97, np.nan, np.nan, np.nan],
|
|
"ask": [np.nan, 51.98, np.nan, np.nan, np.nan],
|
|
},
|
|
columns=["time", "ticker", "price", "quantity", "bid", "ask"],
|
|
)
|
|
|
|
result = pd.merge_asof(
|
|
trades,
|
|
quotes,
|
|
on="time",
|
|
by="ticker",
|
|
tolerance=Timedelta("10ms"),
|
|
allow_exact_matches=False,
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_examples3(self):
|
|
""" doc-string examples """
|
|
# GH14887
|
|
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, 6, np.nan]}
|
|
)
|
|
|
|
result = pd.merge_asof(left, right, on="a", direction="forward")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_examples4(self):
|
|
""" doc-string examples """
|
|
# GH14887
|
|
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, 6, 7]}
|
|
)
|
|
|
|
result = pd.merge_asof(left, right, on="a", direction="nearest")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_basic(self):
|
|
|
|
expected = self.asof
|
|
trades = self.trades
|
|
quotes = self.quotes
|
|
|
|
result = merge_asof(trades, quotes, on="time", by="ticker")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_basic_categorical(self):
|
|
|
|
expected = self.asof
|
|
trades = self.trades.copy()
|
|
trades.ticker = trades.ticker.astype("category")
|
|
quotes = self.quotes.copy()
|
|
quotes.ticker = quotes.ticker.astype("category")
|
|
expected.ticker = expected.ticker.astype("category")
|
|
|
|
result = merge_asof(trades, quotes, on="time", by="ticker")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_basic_left_index(self):
|
|
|
|
# GH14253
|
|
expected = self.asof
|
|
trades = self.trades.set_index("time")
|
|
quotes = self.quotes
|
|
|
|
result = merge_asof(
|
|
trades, quotes, left_index=True, right_on="time", by="ticker"
|
|
)
|
|
# left-only index uses right"s index, oddly
|
|
expected.index = result.index
|
|
# time column appears after left"s columns
|
|
expected = expected[result.columns]
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_basic_right_index(self):
|
|
|
|
expected = self.asof
|
|
trades = self.trades
|
|
quotes = self.quotes.set_index("time")
|
|
|
|
result = merge_asof(
|
|
trades, quotes, left_on="time", right_index=True, by="ticker"
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_basic_left_index_right_index(self):
|
|
|
|
expected = self.asof.set_index("time")
|
|
trades = self.trades.set_index("time")
|
|
quotes = self.quotes.set_index("time")
|
|
|
|
result = merge_asof(
|
|
trades, quotes, left_index=True, right_index=True, by="ticker"
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_multi_index(self):
|
|
|
|
# MultiIndex is prohibited
|
|
trades = self.trades.set_index(["time", "price"])
|
|
quotes = self.quotes.set_index("time")
|
|
with pytest.raises(MergeError):
|
|
merge_asof(trades, quotes, left_index=True, right_index=True)
|
|
|
|
trades = self.trades.set_index("time")
|
|
quotes = self.quotes.set_index(["time", "bid"])
|
|
with pytest.raises(MergeError):
|
|
merge_asof(trades, quotes, left_index=True, right_index=True)
|
|
|
|
def test_on_and_index(self):
|
|
|
|
# "on" parameter and index together is prohibited
|
|
trades = self.trades.set_index("time")
|
|
quotes = self.quotes.set_index("time")
|
|
with pytest.raises(MergeError):
|
|
merge_asof(
|
|
trades, quotes, left_on="price", left_index=True, right_index=True
|
|
)
|
|
|
|
trades = self.trades.set_index("time")
|
|
quotes = self.quotes.set_index("time")
|
|
with pytest.raises(MergeError):
|
|
merge_asof(
|
|
trades, quotes, right_on="bid", left_index=True, right_index=True
|
|
)
|
|
|
|
def test_basic_left_by_right_by(self):
|
|
|
|
# GH14253
|
|
expected = self.asof
|
|
trades = self.trades
|
|
quotes = self.quotes
|
|
|
|
result = merge_asof(
|
|
trades, quotes, on="time", left_by="ticker", right_by="ticker"
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_missing_right_by(self):
|
|
|
|
expected = self.asof
|
|
trades = self.trades
|
|
quotes = self.quotes
|
|
|
|
q = quotes[quotes.ticker != "MSFT"]
|
|
result = merge_asof(trades, q, on="time", by="ticker")
|
|
expected.loc[expected.ticker == "MSFT", ["bid", "ask"]] = np.nan
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_multiby(self):
|
|
# GH13936
|
|
trades = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.046",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.050",
|
|
]
|
|
),
|
|
"ticker": ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
|
|
"exch": ["ARCA", "NSDQ", "NSDQ", "BATS", "NSDQ"],
|
|
"price": [51.95, 51.95, 720.77, 720.92, 98.00],
|
|
"quantity": [75, 155, 100, 100, 100],
|
|
},
|
|
columns=["time", "ticker", "exch", "price", "quantity"],
|
|
)
|
|
|
|
quotes = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.030",
|
|
"20160525 13:30:00.041",
|
|
"20160525 13:30:00.045",
|
|
"20160525 13:30:00.049",
|
|
]
|
|
),
|
|
"ticker": ["GOOG", "MSFT", "MSFT", "MSFT", "GOOG", "AAPL"],
|
|
"exch": ["BATS", "NSDQ", "ARCA", "ARCA", "NSDQ", "ARCA"],
|
|
"bid": [720.51, 51.95, 51.97, 51.99, 720.50, 97.99],
|
|
"ask": [720.92, 51.96, 51.98, 52.00, 720.93, 98.01],
|
|
},
|
|
columns=["time", "ticker", "exch", "bid", "ask"],
|
|
)
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.046",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.050",
|
|
]
|
|
),
|
|
"ticker": ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
|
|
"exch": ["ARCA", "NSDQ", "NSDQ", "BATS", "NSDQ"],
|
|
"price": [51.95, 51.95, 720.77, 720.92, 98.00],
|
|
"quantity": [75, 155, 100, 100, 100],
|
|
"bid": [np.nan, 51.95, 720.50, 720.51, np.nan],
|
|
"ask": [np.nan, 51.96, 720.93, 720.92, np.nan],
|
|
},
|
|
columns=["time", "ticker", "exch", "price", "quantity", "bid", "ask"],
|
|
)
|
|
|
|
result = pd.merge_asof(trades, quotes, on="time", by=["ticker", "exch"])
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_multiby_heterogeneous_types(self):
|
|
# GH13936
|
|
trades = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.046",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.050",
|
|
]
|
|
),
|
|
"ticker": [0, 0, 1, 1, 2],
|
|
"exch": ["ARCA", "NSDQ", "NSDQ", "BATS", "NSDQ"],
|
|
"price": [51.95, 51.95, 720.77, 720.92, 98.00],
|
|
"quantity": [75, 155, 100, 100, 100],
|
|
},
|
|
columns=["time", "ticker", "exch", "price", "quantity"],
|
|
)
|
|
|
|
quotes = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.030",
|
|
"20160525 13:30:00.041",
|
|
"20160525 13:30:00.045",
|
|
"20160525 13:30:00.049",
|
|
]
|
|
),
|
|
"ticker": [1, 0, 0, 0, 1, 2],
|
|
"exch": ["BATS", "NSDQ", "ARCA", "ARCA", "NSDQ", "ARCA"],
|
|
"bid": [720.51, 51.95, 51.97, 51.99, 720.50, 97.99],
|
|
"ask": [720.92, 51.96, 51.98, 52.00, 720.93, 98.01],
|
|
},
|
|
columns=["time", "ticker", "exch", "bid", "ask"],
|
|
)
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.023",
|
|
"20160525 13:30:00.046",
|
|
"20160525 13:30:00.048",
|
|
"20160525 13:30:00.050",
|
|
]
|
|
),
|
|
"ticker": [0, 0, 1, 1, 2],
|
|
"exch": ["ARCA", "NSDQ", "NSDQ", "BATS", "NSDQ"],
|
|
"price": [51.95, 51.95, 720.77, 720.92, 98.00],
|
|
"quantity": [75, 155, 100, 100, 100],
|
|
"bid": [np.nan, 51.95, 720.50, 720.51, np.nan],
|
|
"ask": [np.nan, 51.96, 720.93, 720.92, np.nan],
|
|
},
|
|
columns=["time", "ticker", "exch", "price", "quantity", "bid", "ask"],
|
|
)
|
|
|
|
result = pd.merge_asof(trades, quotes, on="time", by=["ticker", "exch"])
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_multiby_indexed(self):
|
|
# GH15676
|
|
left = pd.DataFrame(
|
|
[
|
|
[pd.to_datetime("20160602"), 1, "a"],
|
|
[pd.to_datetime("20160602"), 2, "a"],
|
|
[pd.to_datetime("20160603"), 1, "b"],
|
|
[pd.to_datetime("20160603"), 2, "b"],
|
|
],
|
|
columns=["time", "k1", "k2"],
|
|
).set_index("time")
|
|
|
|
right = pd.DataFrame(
|
|
[
|
|
[pd.to_datetime("20160502"), 1, "a", 1.0],
|
|
[pd.to_datetime("20160502"), 2, "a", 2.0],
|
|
[pd.to_datetime("20160503"), 1, "b", 3.0],
|
|
[pd.to_datetime("20160503"), 2, "b", 4.0],
|
|
],
|
|
columns=["time", "k1", "k2", "value"],
|
|
).set_index("time")
|
|
|
|
expected = pd.DataFrame(
|
|
[
|
|
[pd.to_datetime("20160602"), 1, "a", 1.0],
|
|
[pd.to_datetime("20160602"), 2, "a", 2.0],
|
|
[pd.to_datetime("20160603"), 1, "b", 3.0],
|
|
[pd.to_datetime("20160603"), 2, "b", 4.0],
|
|
],
|
|
columns=["time", "k1", "k2", "value"],
|
|
).set_index("time")
|
|
|
|
result = pd.merge_asof(
|
|
left, right, left_index=True, right_index=True, by=["k1", "k2"]
|
|
)
|
|
|
|
tm.assert_frame_equal(expected, result)
|
|
|
|
with pytest.raises(MergeError):
|
|
pd.merge_asof(
|
|
left,
|
|
right,
|
|
left_index=True,
|
|
right_index=True,
|
|
left_by=["k1", "k2"],
|
|
right_by=["k1"],
|
|
)
|
|
|
|
def test_basic2(self, datapath):
|
|
|
|
expected = self.read_data(datapath, "asof2.csv")
|
|
trades = self.read_data(datapath, "trades2.csv")
|
|
quotes = self.read_data(datapath, "quotes2.csv", dedupe=True)
|
|
|
|
result = merge_asof(trades, quotes, on="time", by="ticker")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_basic_no_by(self):
|
|
f = (
|
|
lambda x: x[x.ticker == "MSFT"]
|
|
.drop("ticker", axis=1)
|
|
.reset_index(drop=True)
|
|
)
|
|
|
|
# just use a single ticker
|
|
expected = f(self.asof)
|
|
trades = f(self.trades)
|
|
quotes = f(self.quotes)
|
|
|
|
result = merge_asof(trades, quotes, on="time")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_valid_join_keys(self):
|
|
|
|
trades = self.trades
|
|
quotes = self.quotes
|
|
|
|
with pytest.raises(MergeError):
|
|
merge_asof(trades, quotes, left_on="time", right_on="bid", by="ticker")
|
|
|
|
with pytest.raises(MergeError):
|
|
merge_asof(trades, quotes, on=["time", "ticker"], by="ticker")
|
|
|
|
with pytest.raises(MergeError):
|
|
merge_asof(trades, quotes, by="ticker")
|
|
|
|
def test_with_duplicates(self, datapath):
|
|
|
|
q = (
|
|
pd.concat([self.quotes, self.quotes])
|
|
.sort_values(["time", "ticker"])
|
|
.reset_index(drop=True)
|
|
)
|
|
result = merge_asof(self.trades, q, on="time", by="ticker")
|
|
expected = self.read_data(datapath, "asof.csv")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_with_duplicates_no_on(self):
|
|
|
|
df1 = pd.DataFrame({"key": [1, 1, 3], "left_val": [1, 2, 3]})
|
|
df2 = pd.DataFrame({"key": [1, 2, 2], "right_val": [1, 2, 3]})
|
|
result = merge_asof(df1, df2, on="key")
|
|
expected = pd.DataFrame(
|
|
{"key": [1, 1, 3], "left_val": [1, 2, 3], "right_val": [1, 1, 3]}
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_valid_allow_exact_matches(self):
|
|
|
|
trades = self.trades
|
|
quotes = self.quotes
|
|
|
|
with pytest.raises(MergeError):
|
|
merge_asof(
|
|
trades, quotes, on="time", by="ticker", allow_exact_matches="foo"
|
|
)
|
|
|
|
def test_valid_tolerance(self):
|
|
|
|
trades = self.trades
|
|
quotes = self.quotes
|
|
|
|
# dti
|
|
merge_asof(trades, quotes, on="time", by="ticker", tolerance=Timedelta("1s"))
|
|
|
|
# integer
|
|
merge_asof(
|
|
trades.reset_index(),
|
|
quotes.reset_index(),
|
|
on="index",
|
|
by="ticker",
|
|
tolerance=1,
|
|
)
|
|
|
|
# incompat
|
|
with pytest.raises(MergeError):
|
|
merge_asof(trades, quotes, on="time", by="ticker", tolerance=1)
|
|
|
|
# invalid
|
|
with pytest.raises(MergeError):
|
|
merge_asof(
|
|
trades.reset_index(),
|
|
quotes.reset_index(),
|
|
on="index",
|
|
by="ticker",
|
|
tolerance=1.0,
|
|
)
|
|
|
|
# invalid negative
|
|
with pytest.raises(MergeError):
|
|
merge_asof(
|
|
trades, quotes, on="time", by="ticker", tolerance=-Timedelta("1s")
|
|
)
|
|
|
|
with pytest.raises(MergeError):
|
|
merge_asof(
|
|
trades.reset_index(),
|
|
quotes.reset_index(),
|
|
on="index",
|
|
by="ticker",
|
|
tolerance=-1,
|
|
)
|
|
|
|
def test_non_sorted(self):
|
|
|
|
trades = self.trades.sort_values("time", ascending=False)
|
|
quotes = self.quotes.sort_values("time", ascending=False)
|
|
|
|
# we require that we are already sorted on time & quotes
|
|
assert not trades.time.is_monotonic
|
|
assert not quotes.time.is_monotonic
|
|
with pytest.raises(ValueError):
|
|
merge_asof(trades, quotes, on="time", by="ticker")
|
|
|
|
trades = self.trades.sort_values("time")
|
|
assert trades.time.is_monotonic
|
|
assert not quotes.time.is_monotonic
|
|
with pytest.raises(ValueError):
|
|
merge_asof(trades, quotes, on="time", by="ticker")
|
|
|
|
quotes = self.quotes.sort_values("time")
|
|
assert trades.time.is_monotonic
|
|
assert quotes.time.is_monotonic
|
|
|
|
# ok, though has dupes
|
|
merge_asof(trades, self.quotes, on="time", by="ticker")
|
|
|
|
@pytest.mark.parametrize(
|
|
"tolerance",
|
|
[Timedelta("1day"), datetime.timedelta(days=1)],
|
|
ids=["Timedelta", "datetime.timedelta"],
|
|
)
|
|
def test_tolerance(self, tolerance):
|
|
|
|
trades = self.trades
|
|
quotes = self.quotes
|
|
|
|
result = merge_asof(trades, quotes, on="time", by="ticker", tolerance=tolerance)
|
|
expected = self.tolerance
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_tolerance_forward(self):
|
|
# GH14887
|
|
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 2, 3, 7, 11], "right_val": [1, 2, 3, 7, 11]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, np.nan, 11]}
|
|
)
|
|
|
|
result = pd.merge_asof(left, right, on="a", direction="forward", tolerance=1)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_tolerance_nearest(self):
|
|
# GH14887
|
|
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 2, 3, 7, 11], "right_val": [1, 2, 3, 7, 11]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, np.nan, 11]}
|
|
)
|
|
|
|
result = pd.merge_asof(left, right, on="a", direction="nearest", tolerance=1)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_tolerance_tz(self):
|
|
# GH 14844
|
|
left = pd.DataFrame(
|
|
{
|
|
"date": pd.date_range(
|
|
start=pd.to_datetime("2016-01-02"),
|
|
freq="D",
|
|
periods=5,
|
|
tz=pytz.timezone("UTC"),
|
|
),
|
|
"value1": np.arange(5),
|
|
}
|
|
)
|
|
right = pd.DataFrame(
|
|
{
|
|
"date": pd.date_range(
|
|
start=pd.to_datetime("2016-01-01"),
|
|
freq="D",
|
|
periods=5,
|
|
tz=pytz.timezone("UTC"),
|
|
),
|
|
"value2": list("ABCDE"),
|
|
}
|
|
)
|
|
result = pd.merge_asof(left, right, on="date", tolerance=Timedelta("1 day"))
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"date": pd.date_range(
|
|
start=pd.to_datetime("2016-01-02"),
|
|
freq="D",
|
|
periods=5,
|
|
tz=pytz.timezone("UTC"),
|
|
),
|
|
"value1": np.arange(5),
|
|
"value2": list("BCDEE"),
|
|
}
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_tolerance_float(self):
|
|
# GH22981
|
|
left = pd.DataFrame({"a": [1.1, 3.5, 10.9], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame(
|
|
{"a": [1.0, 2.5, 3.3, 7.5, 11.5], "right_val": [1.0, 2.5, 3.3, 7.5, 11.5]}
|
|
)
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"a": [1.1, 3.5, 10.9],
|
|
"left_val": ["a", "b", "c"],
|
|
"right_val": [1, 3.3, np.nan],
|
|
}
|
|
)
|
|
|
|
result = pd.merge_asof(left, right, on="a", direction="nearest", tolerance=0.5)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_index_tolerance(self):
|
|
# GH 15135
|
|
expected = self.tolerance.set_index("time")
|
|
trades = self.trades.set_index("time")
|
|
quotes = self.quotes.set_index("time")
|
|
|
|
result = pd.merge_asof(
|
|
trades,
|
|
quotes,
|
|
left_index=True,
|
|
right_index=True,
|
|
by="ticker",
|
|
tolerance=Timedelta("1day"),
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_allow_exact_matches(self):
|
|
|
|
result = merge_asof(
|
|
self.trades, self.quotes, on="time", by="ticker", allow_exact_matches=False
|
|
)
|
|
expected = self.allow_exact_matches
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_allow_exact_matches_forward(self):
|
|
# GH14887
|
|
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 2, 3, 7, 11], "right_val": [1, 2, 3, 7, 11]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [2, 7, 11]}
|
|
)
|
|
|
|
result = pd.merge_asof(
|
|
left, right, on="a", direction="forward", allow_exact_matches=False
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_allow_exact_matches_nearest(self):
|
|
# GH14887
|
|
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 2, 3, 7, 11], "right_val": [1, 2, 3, 7, 11]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [2, 3, 11]}
|
|
)
|
|
|
|
result = pd.merge_asof(
|
|
left, right, on="a", direction="nearest", allow_exact_matches=False
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_allow_exact_matches_and_tolerance(self):
|
|
|
|
result = merge_asof(
|
|
self.trades,
|
|
self.quotes,
|
|
on="time",
|
|
by="ticker",
|
|
tolerance=Timedelta("100ms"),
|
|
allow_exact_matches=False,
|
|
)
|
|
expected = self.allow_exact_matches_and_tolerance
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_allow_exact_matches_and_tolerance2(self):
|
|
# GH 13695
|
|
df1 = pd.DataFrame(
|
|
{"time": pd.to_datetime(["2016-07-15 13:30:00.030"]), "username": ["bob"]}
|
|
)
|
|
df2 = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
["2016-07-15 13:30:00.000", "2016-07-15 13:30:00.030"]
|
|
),
|
|
"version": [1, 2],
|
|
}
|
|
)
|
|
|
|
result = pd.merge_asof(df1, df2, on="time")
|
|
expected = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(["2016-07-15 13:30:00.030"]),
|
|
"username": ["bob"],
|
|
"version": [2],
|
|
}
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
result = pd.merge_asof(df1, df2, on="time", allow_exact_matches=False)
|
|
expected = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(["2016-07-15 13:30:00.030"]),
|
|
"username": ["bob"],
|
|
"version": [1],
|
|
}
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
result = pd.merge_asof(
|
|
df1,
|
|
df2,
|
|
on="time",
|
|
allow_exact_matches=False,
|
|
tolerance=Timedelta("10ms"),
|
|
)
|
|
expected = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(["2016-07-15 13:30:00.030"]),
|
|
"username": ["bob"],
|
|
"version": [np.nan],
|
|
}
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_allow_exact_matches_and_tolerance3(self):
|
|
# GH 13709
|
|
df1 = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
["2016-07-15 13:30:00.030", "2016-07-15 13:30:00.030"]
|
|
),
|
|
"username": ["bob", "charlie"],
|
|
}
|
|
)
|
|
df2 = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
["2016-07-15 13:30:00.000", "2016-07-15 13:30:00.030"]
|
|
),
|
|
"version": [1, 2],
|
|
}
|
|
)
|
|
|
|
result = pd.merge_asof(
|
|
df1,
|
|
df2,
|
|
on="time",
|
|
allow_exact_matches=False,
|
|
tolerance=Timedelta("10ms"),
|
|
)
|
|
expected = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
["2016-07-15 13:30:00.030", "2016-07-15 13:30:00.030"]
|
|
),
|
|
"username": ["bob", "charlie"],
|
|
"version": [np.nan, np.nan],
|
|
}
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_allow_exact_matches_and_tolerance_forward(self):
|
|
# GH14887
|
|
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 3, 4, 6, 11], "right_val": [1, 3, 4, 6, 11]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [np.nan, 6, 11]}
|
|
)
|
|
|
|
result = pd.merge_asof(
|
|
left,
|
|
right,
|
|
on="a",
|
|
direction="forward",
|
|
allow_exact_matches=False,
|
|
tolerance=1,
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_allow_exact_matches_and_tolerance_nearest(self):
|
|
# GH14887
|
|
|
|
left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"a": [1, 3, 4, 6, 11], "right_val": [1, 3, 4, 7, 11]})
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [np.nan, 4, 11]}
|
|
)
|
|
|
|
result = pd.merge_asof(
|
|
left,
|
|
right,
|
|
on="a",
|
|
direction="nearest",
|
|
allow_exact_matches=False,
|
|
tolerance=1,
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_forward_by(self):
|
|
# GH14887
|
|
|
|
left = pd.DataFrame(
|
|
{
|
|
"a": [1, 5, 10, 12, 15],
|
|
"b": ["X", "X", "Y", "Z", "Y"],
|
|
"left_val": ["a", "b", "c", "d", "e"],
|
|
}
|
|
)
|
|
right = pd.DataFrame(
|
|
{
|
|
"a": [1, 6, 11, 15, 16],
|
|
"b": ["X", "Z", "Y", "Z", "Y"],
|
|
"right_val": [1, 6, 11, 15, 16],
|
|
}
|
|
)
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"a": [1, 5, 10, 12, 15],
|
|
"b": ["X", "X", "Y", "Z", "Y"],
|
|
"left_val": ["a", "b", "c", "d", "e"],
|
|
"right_val": [1, np.nan, 11, 15, 16],
|
|
}
|
|
)
|
|
|
|
result = pd.merge_asof(left, right, on="a", by="b", direction="forward")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_nearest_by(self):
|
|
# GH14887
|
|
|
|
left = pd.DataFrame(
|
|
{
|
|
"a": [1, 5, 10, 12, 15],
|
|
"b": ["X", "X", "Z", "Z", "Y"],
|
|
"left_val": ["a", "b", "c", "d", "e"],
|
|
}
|
|
)
|
|
right = pd.DataFrame(
|
|
{
|
|
"a": [1, 6, 11, 15, 16],
|
|
"b": ["X", "Z", "Z", "Z", "Y"],
|
|
"right_val": [1, 6, 11, 15, 16],
|
|
}
|
|
)
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"a": [1, 5, 10, 12, 15],
|
|
"b": ["X", "X", "Z", "Z", "Y"],
|
|
"left_val": ["a", "b", "c", "d", "e"],
|
|
"right_val": [1, 1, 11, 11, 16],
|
|
}
|
|
)
|
|
|
|
result = pd.merge_asof(left, right, on="a", by="b", direction="nearest")
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_by_int(self):
|
|
# we specialize by type, so test that this is correct
|
|
df1 = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.020",
|
|
"20160525 13:30:00.030",
|
|
"20160525 13:30:00.040",
|
|
"20160525 13:30:00.050",
|
|
"20160525 13:30:00.060",
|
|
]
|
|
),
|
|
"key": [1, 2, 1, 3, 2],
|
|
"value1": [1.1, 1.2, 1.3, 1.4, 1.5],
|
|
},
|
|
columns=["time", "key", "value1"],
|
|
)
|
|
|
|
df2 = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.015",
|
|
"20160525 13:30:00.020",
|
|
"20160525 13:30:00.025",
|
|
"20160525 13:30:00.035",
|
|
"20160525 13:30:00.040",
|
|
"20160525 13:30:00.055",
|
|
"20160525 13:30:00.060",
|
|
"20160525 13:30:00.065",
|
|
]
|
|
),
|
|
"key": [2, 1, 1, 3, 2, 1, 2, 3],
|
|
"value2": [2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8],
|
|
},
|
|
columns=["time", "key", "value2"],
|
|
)
|
|
|
|
result = pd.merge_asof(df1, df2, on="time", by="key")
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"time": pd.to_datetime(
|
|
[
|
|
"20160525 13:30:00.020",
|
|
"20160525 13:30:00.030",
|
|
"20160525 13:30:00.040",
|
|
"20160525 13:30:00.050",
|
|
"20160525 13:30:00.060",
|
|
]
|
|
),
|
|
"key": [1, 2, 1, 3, 2],
|
|
"value1": [1.1, 1.2, 1.3, 1.4, 1.5],
|
|
"value2": [2.2, 2.1, 2.3, 2.4, 2.7],
|
|
},
|
|
columns=["time", "key", "value1", "value2"],
|
|
)
|
|
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_on_float(self):
|
|
# mimics how to determine the minimum-price variation
|
|
df1 = pd.DataFrame(
|
|
{
|
|
"price": [5.01, 0.0023, 25.13, 340.05, 30.78, 1040.90, 0.0078],
|
|
"symbol": list("ABCDEFG"),
|
|
},
|
|
columns=["symbol", "price"],
|
|
)
|
|
|
|
df2 = pd.DataFrame(
|
|
{"price": [0.0, 1.0, 100.0], "mpv": [0.0001, 0.01, 0.05]},
|
|
columns=["price", "mpv"],
|
|
)
|
|
|
|
df1 = df1.sort_values("price").reset_index(drop=True)
|
|
|
|
result = pd.merge_asof(df1, df2, on="price")
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"symbol": list("BGACEDF"),
|
|
"price": [0.0023, 0.0078, 5.01, 25.13, 30.78, 340.05, 1040.90],
|
|
"mpv": [0.0001, 0.0001, 0.01, 0.01, 0.01, 0.05, 0.05],
|
|
},
|
|
columns=["symbol", "price", "mpv"],
|
|
)
|
|
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_on_specialized_type(self, any_real_dtype):
|
|
# see gh-13936
|
|
dtype = np.dtype(any_real_dtype).type
|
|
|
|
df1 = pd.DataFrame(
|
|
{"value": [5, 2, 25, 100, 78, 120, 79], "symbol": list("ABCDEFG")},
|
|
columns=["symbol", "value"],
|
|
)
|
|
df1.value = dtype(df1.value)
|
|
|
|
df2 = pd.DataFrame(
|
|
{"value": [0, 80, 120, 125], "result": list("xyzw")},
|
|
columns=["value", "result"],
|
|
)
|
|
df2.value = dtype(df2.value)
|
|
|
|
df1 = df1.sort_values("value").reset_index(drop=True)
|
|
result = pd.merge_asof(df1, df2, on="value")
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"symbol": list("BACEGDF"),
|
|
"value": [2, 5, 25, 78, 79, 100, 120],
|
|
"result": list("xxxxxyz"),
|
|
},
|
|
columns=["symbol", "value", "result"],
|
|
)
|
|
expected.value = dtype(expected.value)
|
|
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_on_specialized_type_by_int(self, any_real_dtype):
|
|
# see gh-13936
|
|
dtype = np.dtype(any_real_dtype).type
|
|
|
|
df1 = pd.DataFrame(
|
|
{
|
|
"value": [5, 2, 25, 100, 78, 120, 79],
|
|
"key": [1, 2, 3, 2, 3, 1, 2],
|
|
"symbol": list("ABCDEFG"),
|
|
},
|
|
columns=["symbol", "key", "value"],
|
|
)
|
|
df1.value = dtype(df1.value)
|
|
|
|
df2 = pd.DataFrame(
|
|
{"value": [0, 80, 120, 125], "key": [1, 2, 2, 3], "result": list("xyzw")},
|
|
columns=["value", "key", "result"],
|
|
)
|
|
df2.value = dtype(df2.value)
|
|
|
|
df1 = df1.sort_values("value").reset_index(drop=True)
|
|
result = pd.merge_asof(df1, df2, on="value", by="key")
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"symbol": list("BACEGDF"),
|
|
"key": [2, 1, 3, 3, 2, 2, 1],
|
|
"value": [2, 5, 25, 78, 79, 100, 120],
|
|
"result": [np.nan, "x", np.nan, np.nan, np.nan, "y", "x"],
|
|
},
|
|
columns=["symbol", "key", "value", "result"],
|
|
)
|
|
expected.value = dtype(expected.value)
|
|
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_on_float_by_int(self):
|
|
# type specialize both "by" and "on" parameters
|
|
df1 = pd.DataFrame(
|
|
{
|
|
"symbol": list("AAABBBCCC"),
|
|
"exch": [1, 2, 3, 1, 2, 3, 1, 2, 3],
|
|
"price": [
|
|
3.26,
|
|
3.2599,
|
|
3.2598,
|
|
12.58,
|
|
12.59,
|
|
12.5,
|
|
378.15,
|
|
378.2,
|
|
378.25,
|
|
],
|
|
},
|
|
columns=["symbol", "exch", "price"],
|
|
)
|
|
|
|
df2 = pd.DataFrame(
|
|
{
|
|
"exch": [1, 1, 1, 2, 2, 2, 3, 3, 3],
|
|
"price": [0.0, 1.0, 100.0, 0.0, 5.0, 100.0, 0.0, 5.0, 1000.0],
|
|
"mpv": [0.0001, 0.01, 0.05, 0.0001, 0.01, 0.1, 0.0001, 0.25, 1.0],
|
|
},
|
|
columns=["exch", "price", "mpv"],
|
|
)
|
|
|
|
df1 = df1.sort_values("price").reset_index(drop=True)
|
|
df2 = df2.sort_values("price").reset_index(drop=True)
|
|
|
|
result = pd.merge_asof(df1, df2, on="price", by="exch")
|
|
|
|
expected = pd.DataFrame(
|
|
{
|
|
"symbol": list("AAABBBCCC"),
|
|
"exch": [3, 2, 1, 3, 1, 2, 1, 2, 3],
|
|
"price": [
|
|
3.2598,
|
|
3.2599,
|
|
3.26,
|
|
12.5,
|
|
12.58,
|
|
12.59,
|
|
378.15,
|
|
378.2,
|
|
378.25,
|
|
],
|
|
"mpv": [0.0001, 0.0001, 0.01, 0.25, 0.01, 0.01, 0.05, 0.1, 0.25],
|
|
},
|
|
columns=["symbol", "exch", "price", "mpv"],
|
|
)
|
|
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_merge_datatype_error_raises(self):
|
|
msg = r"incompatible merge keys \[0\] .*, must be the same type"
|
|
|
|
left = pd.DataFrame({"left_val": [1, 5, 10], "a": ["a", "b", "c"]})
|
|
right = pd.DataFrame({"right_val": [1, 2, 3, 6, 7], "a": [1, 2, 3, 6, 7]})
|
|
|
|
with pytest.raises(MergeError, match=msg):
|
|
merge_asof(left, right, on="a")
|
|
|
|
def test_merge_datatype_categorical_error_raises(self):
|
|
msg = (
|
|
r"incompatible merge keys \[0\] .* both sides category, "
|
|
"but not equal ones"
|
|
)
|
|
|
|
left = pd.DataFrame(
|
|
{"left_val": [1, 5, 10], "a": pd.Categorical(["a", "b", "c"])}
|
|
)
|
|
right = pd.DataFrame(
|
|
{
|
|
"right_val": [1, 2, 3, 6, 7],
|
|
"a": pd.Categorical(["a", "X", "c", "X", "b"]),
|
|
}
|
|
)
|
|
|
|
with pytest.raises(MergeError, match=msg):
|
|
merge_asof(left, right, on="a")
|
|
|
|
def test_merge_groupby_multiple_column_with_categorical_column(self):
|
|
# GH 16454
|
|
df = pd.DataFrame({"x": [0], "y": [0], "z": pd.Categorical([0])})
|
|
result = merge_asof(df, df, on="x", by=["y", "z"])
|
|
expected = pd.DataFrame({"x": [0], "y": [0], "z": pd.Categorical([0])})
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
@pytest.mark.parametrize(
|
|
"func", [lambda x: x, lambda x: to_datetime(x)], ids=["numeric", "datetime"]
|
|
)
|
|
@pytest.mark.parametrize("side", ["left", "right"])
|
|
def test_merge_on_nans(self, func, side):
|
|
# GH 23189
|
|
msg = f"Merge keys contain null values on {side} side"
|
|
nulls = func([1.0, 5.0, np.nan])
|
|
non_nulls = func([1.0, 5.0, 10.0])
|
|
df_null = pd.DataFrame({"a": nulls, "left_val": ["a", "b", "c"]})
|
|
df = pd.DataFrame({"a": non_nulls, "right_val": [1, 6, 11]})
|
|
|
|
with pytest.raises(ValueError, match=msg):
|
|
if side == "left":
|
|
merge_asof(df_null, df, on="a")
|
|
else:
|
|
merge_asof(df, df_null, on="a")
|
|
|
|
def test_merge_by_col_tz_aware(self):
|
|
# GH 21184
|
|
left = pd.DataFrame(
|
|
{
|
|
"by_col": pd.DatetimeIndex(["2018-01-01"]).tz_localize("UTC"),
|
|
"on_col": [2],
|
|
"values": ["a"],
|
|
}
|
|
)
|
|
right = pd.DataFrame(
|
|
{
|
|
"by_col": pd.DatetimeIndex(["2018-01-01"]).tz_localize("UTC"),
|
|
"on_col": [1],
|
|
"values": ["b"],
|
|
}
|
|
)
|
|
result = pd.merge_asof(left, right, by="by_col", on="on_col")
|
|
expected = pd.DataFrame(
|
|
[[pd.Timestamp("2018-01-01", tz="UTC"), 2, "a", "b"]],
|
|
columns=["by_col", "on_col", "values_x", "values_y"],
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_by_mixed_tz_aware(self):
|
|
# GH 26649
|
|
left = pd.DataFrame(
|
|
{
|
|
"by_col1": pd.DatetimeIndex(["2018-01-01"]).tz_localize("UTC"),
|
|
"by_col2": ["HELLO"],
|
|
"on_col": [2],
|
|
"value": ["a"],
|
|
}
|
|
)
|
|
right = pd.DataFrame(
|
|
{
|
|
"by_col1": pd.DatetimeIndex(["2018-01-01"]).tz_localize("UTC"),
|
|
"by_col2": ["WORLD"],
|
|
"on_col": [1],
|
|
"value": ["b"],
|
|
}
|
|
)
|
|
result = pd.merge_asof(left, right, by=["by_col1", "by_col2"], on="on_col")
|
|
expected = pd.DataFrame(
|
|
[[pd.Timestamp("2018-01-01", tz="UTC"), "HELLO", 2, "a"]],
|
|
columns=["by_col1", "by_col2", "on_col", "value_x"],
|
|
)
|
|
expected["value_y"] = np.array([np.nan], dtype=object)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_timedelta_tolerance_nearest(self):
|
|
# GH 27642
|
|
|
|
left = pd.DataFrame(
|
|
list(zip([0, 5, 10, 15, 20, 25], [0, 1, 2, 3, 4, 5])),
|
|
columns=["time", "left"],
|
|
)
|
|
|
|
left["time"] = pd.to_timedelta(left["time"], "ms")
|
|
|
|
right = pd.DataFrame(
|
|
list(zip([0, 3, 9, 12, 15, 18], [0, 1, 2, 3, 4, 5])),
|
|
columns=["time", "right"],
|
|
)
|
|
|
|
right["time"] = pd.to_timedelta(right["time"], "ms")
|
|
|
|
expected = pd.DataFrame(
|
|
list(
|
|
zip(
|
|
[0, 5, 10, 15, 20, 25],
|
|
[0, 1, 2, 3, 4, 5],
|
|
[0, np.nan, 2, 4, np.nan, np.nan],
|
|
)
|
|
),
|
|
columns=["time", "left", "right"],
|
|
)
|
|
|
|
expected["time"] = pd.to_timedelta(expected["time"], "ms")
|
|
|
|
result = pd.merge_asof(
|
|
left, right, on="time", tolerance=Timedelta("1ms"), direction="nearest"
|
|
)
|
|
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_int_type_tolerance(self, any_int_dtype):
|
|
# GH #28870
|
|
|
|
left = pd.DataFrame({"a": [0, 10, 20], "left_val": [1, 2, 3]})
|
|
right = pd.DataFrame({"a": [5, 15, 25], "right_val": [1, 2, 3]})
|
|
left["a"] = left["a"].astype(any_int_dtype)
|
|
right["a"] = right["a"].astype(any_int_dtype)
|
|
|
|
expected = pd.DataFrame(
|
|
{"a": [0, 10, 20], "left_val": [1, 2, 3], "right_val": [np.nan, 1.0, 2.0]}
|
|
)
|
|
expected["a"] = expected["a"].astype(any_int_dtype)
|
|
|
|
result = pd.merge_asof(left, right, on="a", tolerance=10)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_merge_index_column_tz(self):
|
|
# GH 29864
|
|
index = pd.date_range("2019-10-01", freq="30min", periods=5, tz="UTC")
|
|
left = pd.DataFrame([0.9, 0.8, 0.7, 0.6], columns=["xyz"], index=index[1:])
|
|
right = pd.DataFrame({"from_date": index, "abc": [2.46] * 4 + [2.19]})
|
|
result = pd.merge_asof(
|
|
left=left, right=right, left_index=True, right_on=["from_date"]
|
|
)
|
|
expected = pd.DataFrame(
|
|
{
|
|
"xyz": [0.9, 0.8, 0.7, 0.6],
|
|
"from_date": index[1:],
|
|
"abc": [2.46] * 3 + [2.19],
|
|
},
|
|
index=pd.Index([1, 2, 3, 4]),
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
result = pd.merge_asof(
|
|
left=right, right=left, right_index=True, left_on=["from_date"]
|
|
)
|
|
expected = pd.DataFrame(
|
|
{
|
|
"from_date": index,
|
|
"abc": [2.46] * 4 + [2.19],
|
|
"xyz": [np.nan, 0.9, 0.8, 0.7, 0.6],
|
|
},
|
|
index=pd.Index([0, 1, 2, 3, 4]),
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|
|
|
|
def test_left_index_right_index_tolerance(self):
|
|
# https://github.com/pandas-dev/pandas/issues/35558
|
|
dr1 = pd.date_range(start="1/1/2020", end="1/20/2020", freq="2D") + Timedelta(
|
|
seconds=0.4
|
|
)
|
|
dr2 = pd.date_range(start="1/1/2020", end="2/1/2020")
|
|
|
|
df1 = pd.DataFrame({"val1": "foo"}, index=pd.DatetimeIndex(dr1))
|
|
df2 = pd.DataFrame({"val2": "bar"}, index=pd.DatetimeIndex(dr2))
|
|
|
|
expected = pd.DataFrame(
|
|
{"val1": "foo", "val2": "bar"}, index=pd.DatetimeIndex(dr1)
|
|
)
|
|
result = pd.merge_asof(
|
|
df1,
|
|
df2,
|
|
left_index=True,
|
|
right_index=True,
|
|
tolerance=Timedelta(seconds=0.5),
|
|
)
|
|
tm.assert_frame_equal(result, expected)
|