projektAI/venv/Lib/site-packages/pandas/tests/computation/test_eval.py

from distutils.version import LooseVersion
from functools import reduce
from itertools import product
import operator
from typing import Dict, List, Type
import warnings

import numpy as np
import pytest

from pandas.compat import is_platform_windows
from pandas.compat.numpy import np_version_under1p17
from pandas.errors import PerformanceWarning
import pandas.util._test_decorators as td

from pandas.core.dtypes.common import is_bool, is_list_like, is_scalar

import pandas as pd
from pandas import DataFrame, Series, compat, date_range
import pandas._testing as tm
from pandas.core.computation import pytables
from pandas.core.computation.check import NUMEXPR_VERSION
from pandas.core.computation.engines import ENGINES, NumExprClobberingError
import pandas.core.computation.expr as expr
from pandas.core.computation.expr import (
    BaseExprVisitor,
    PandasExprVisitor,
    PythonExprVisitor,
)
from pandas.core.computation.expressions import NUMEXPR_INSTALLED, USE_NUMEXPR
from pandas.core.computation.ops import (
    ARITH_OPS_SYMS,
    SPECIAL_CASE_ARITH_OPS_SYMS,
    _binary_math_ops,
    _binary_ops_dict,
    _unary_math_ops,
)


@pytest.fixture(
    params=(
        pytest.param(
            engine,
            marks=pytest.mark.skipif(
                engine == "numexpr" and not USE_NUMEXPR,
                reason=f"numexpr enabled->{USE_NUMEXPR}, "
                f"installed->{NUMEXPR_INSTALLED}",
            ),
        )
        for engine in ENGINES
    )
)
def engine(request):
    return request.param


@pytest.fixture(params=expr.PARSERS)
def parser(request):
    return request.param


@pytest.fixture
def ne_lt_2_6_9():
    if NUMEXPR_INSTALLED and NUMEXPR_VERSION >= LooseVersion("2.6.9"):
        pytest.skip("numexpr is >= 2.6.9")
    return "numexpr"


def _get_unary_fns_for_ne():
    if NUMEXPR_INSTALLED:
        if NUMEXPR_VERSION >= LooseVersion("2.6.9"):
            return list(_unary_math_ops)
        else:
            return [x for x in _unary_math_ops if x not in ["floor", "ceil"]]
    return []


@pytest.fixture(params=_get_unary_fns_for_ne())
def unary_fns_for_ne(request):
    return request.param


def engine_has_neg_frac(engine):
    return ENGINES[engine].has_neg_frac


def _eval_single_bin(lhs, cmp1, rhs, engine):
    c = _binary_ops_dict[cmp1]
    if engine_has_neg_frac(engine):
        try:
            return c(lhs, rhs)
        except ValueError as e:
            if str(e).startswith(
                "negative number cannot be raised to a fractional power"
            ):
                return np.nan
            raise
    return c(lhs, rhs)


def _series_and_2d_ndarray(lhs, rhs):
    return (
        isinstance(lhs, Series) and isinstance(rhs, np.ndarray) and rhs.ndim > 1
    ) or (isinstance(rhs, Series) and isinstance(lhs, np.ndarray) and lhs.ndim > 1)


def _series_and_frame(lhs, rhs):
    return (isinstance(lhs, Series) and isinstance(rhs, DataFrame)) or (
        isinstance(rhs, Series) and isinstance(lhs, DataFrame)
    )


def _bool_and_frame(lhs, rhs):
    return isinstance(lhs, bool) and isinstance(rhs, pd.core.generic.NDFrame)


def _is_py3_complex_incompat(result, expected):
    return isinstance(expected, (complex, np.complexfloating)) and np.isnan(result)


_good_arith_ops = sorted(set(ARITH_OPS_SYMS).difference(SPECIAL_CASE_ARITH_OPS_SYMS))


# TODO: using range(5) here is a kludge
@pytest.fixture(params=list(range(5)))
def lhs(request):

    nan_df1 = DataFrame(np.random.rand(10, 5))
    nan_df1[nan_df1 > 0.5] = np.nan

    opts = (
        DataFrame(np.random.randn(10, 5)),
        Series(np.random.randn(5)),
        Series([1, 2, np.nan, np.nan, 5]),
        nan_df1,
        np.random.randn(),
    )
    return opts[request.param]


rhs = lhs
midhs = lhs


@td.skip_if_no_ne
class TestEvalNumexprPandas:
    exclude_cmp: List[str] = []
    exclude_bool: List[str] = []

    engine = "numexpr"
    parser = "pandas"

    @classmethod
    def setup_class(cls):
        import numexpr as ne

        cls.ne = ne

    @property
    def current_engines(self):
        return (engine for engine in ENGINES if engine != self.engine)

    @pytest.mark.parametrize(
        "cmp1",
        ["!=", "==", "<=", ">=", "<", ">"],
        ids=["ne", "eq", "le", "ge", "lt", "gt"],
    )
    @pytest.mark.parametrize("cmp2", [">", "<"], ids=["gt", "lt"])
    @pytest.mark.parametrize("binop", expr.BOOL_OPS_SYMS)
    def test_complex_cmp_ops(self, cmp1, cmp2, binop, lhs, rhs):
        if binop in self.exclude_bool:
            pytest.skip()

        lhs_new = _eval_single_bin(lhs, cmp1, rhs, self.engine)
        rhs_new = _eval_single_bin(lhs, cmp2, rhs, self.engine)
        expected = _eval_single_bin(lhs_new, binop, rhs_new, self.engine)

        ex = f"(lhs {cmp1} rhs) {binop} (lhs {cmp2} rhs)"
        result = pd.eval(ex, engine=self.engine, parser=self.parser)
        self.check_equal(result, expected)

    @pytest.mark.parametrize("cmp_op", expr.CMP_OPS_SYMS)
    def test_simple_cmp_ops(self, cmp_op):
        if cmp_op in self.exclude_cmp:
            pytest.skip()

        bool_lhses = (
            DataFrame(tm.randbool(size=(10, 5))),
            Series(tm.randbool((5,))),
            tm.randbool(),
        )
        bool_rhses = (
            DataFrame(tm.randbool(size=(10, 5))),
            Series(tm.randbool((5,))),
            tm.randbool(),
        )
        for lhs, rhs in product(bool_lhses, bool_rhses):
            self.check_simple_cmp_op(lhs, cmp_op, rhs)

    @pytest.mark.parametrize("op", _good_arith_ops)
    def test_binary_arith_ops(self, op, lhs, rhs, request):

        if (
            op == "/"
            and isinstance(lhs, DataFrame)
            and isinstance(rhs, DataFrame)
            and not lhs.isna().any().any()
            and rhs.shape == (10, 5)
            and np_version_under1p17
            and is_platform_windows()
            and compat.PY38
        ):
            mark = pytest.mark.xfail(
                reason="GH#37328 floating point precision on Windows builds"
            )
            request.node.add_marker(mark)

        self.check_binary_arith_op(lhs, op, rhs)

    def test_modulus(self, lhs, rhs):
        self.check_modulus(lhs, "%", rhs)

    def test_floor_division(self, lhs, rhs):
        self.check_floor_division(lhs, "//", rhs)

    @td.skip_if_windows
    def test_pow(self, lhs, rhs):
        # odd failure on win32 platform, so skip
        self.check_pow(lhs, "**", rhs)

    @pytest.mark.parametrize("op", expr.CMP_OPS_SYMS)
    def test_single_invert_op(self, op, lhs):
        if op in self.exclude_cmp:
            pytest.skip()

        self.check_single_invert_op(lhs, op)

    @pytest.mark.parametrize("op", expr.CMP_OPS_SYMS)
    def test_compound_invert_op(self, op, lhs, rhs):
        if op in self.exclude_cmp:
            pytest.skip()

        self.check_compound_invert_op(lhs, op, rhs)

    @pytest.mark.parametrize("cmp1", ["<", ">"])
    @pytest.mark.parametrize("cmp2", ["<", ">"])
    def test_chained_cmp_op(self, cmp1, cmp2, lhs, midhs, rhs):
        self.check_chained_cmp_op(lhs, cmp1, midhs, cmp2, rhs)

    def check_equal(self, result, expected):
        if isinstance(result, DataFrame):
            tm.assert_frame_equal(result, expected)
        elif isinstance(result, Series):
            tm.assert_series_equal(result, expected)
        elif isinstance(result, np.ndarray):
            tm.assert_numpy_array_equal(result, expected)
        else:
            assert result == expected

    def check_chained_cmp_op(self, lhs, cmp1, mid, cmp2, rhs):
        def check_operands(left, right, cmp_op):
            return _eval_single_bin(left, cmp_op, right, self.engine)

        lhs_new = check_operands(lhs, mid, cmp1)
        rhs_new = check_operands(mid, rhs, cmp2)

        if lhs_new is not None and rhs_new is not None:
            ex1 = f"lhs {cmp1} mid {cmp2} rhs"
            ex2 = f"lhs {cmp1} mid and mid {cmp2} rhs"
            ex3 = f"(lhs {cmp1} mid) & (mid {cmp2} rhs)"
            expected = _eval_single_bin(lhs_new, "&", rhs_new, self.engine)

            for ex in (ex1, ex2, ex3):
                result = pd.eval(ex, engine=self.engine, parser=self.parser)

                tm.assert_almost_equal(result, expected)

    def check_simple_cmp_op(self, lhs, cmp1, rhs):
        ex = f"lhs {cmp1} rhs"
        msg = (
            r"only list-like( or dict-like)? objects are allowed to be "
            r"passed to (DataFrame\.)?isin\(\), you passed a "
            r"(\[|')bool(\]|')|"
            "argument of type 'bool' is not iterable"
        )
        if cmp1 in ("in", "not in") and not is_list_like(rhs):
            with pytest.raises(TypeError, match=msg):
                pd.eval(
                    ex,
                    engine=self.engine,
                    parser=self.parser,
                    local_dict={"lhs": lhs, "rhs": rhs},
                )
        else:
            expected = _eval_single_bin(lhs, cmp1, rhs, self.engine)
            result = pd.eval(ex, engine=self.engine, parser=self.parser)
            self.check_equal(result, expected)

    def check_binary_arith_op(self, lhs, arith1, rhs):
        ex = f"lhs {arith1} rhs"
        result = pd.eval(ex, engine=self.engine, parser=self.parser)
        expected = _eval_single_bin(lhs, arith1, rhs, self.engine)

        tm.assert_almost_equal(result, expected)
        ex = f"lhs {arith1} rhs {arith1} rhs"
        result = pd.eval(ex, engine=self.engine, parser=self.parser)
        nlhs = _eval_single_bin(lhs, arith1, rhs, self.engine)
        self.check_alignment(result, nlhs, rhs, arith1)

    def check_alignment(self, result, nlhs, ghs, op):
        try:
            nlhs, ghs = nlhs.align(ghs)
        except (ValueError, TypeError, AttributeError):
            # ValueError: series frame or frame series align
            # TypeError, AttributeError: series or frame with scalar align
            pass
        else:
            # direct numpy comparison
            expected = self.ne.evaluate(f"nlhs {op} ghs")
            # Update assert statement due to unreliable numerical
            # precision component (GH37328)
            # TODO: update testing code so that assert_almost_equal statement
            #  can be replaced again by the assert_numpy_array_equal statement
            tm.assert_almost_equal(result.values, expected)

    # modulus, pow, and floor division require special casing

    def check_modulus(self, lhs, arith1, rhs):
        ex = f"lhs {arith1} rhs"
        result = pd.eval(ex, engine=self.engine, parser=self.parser)
        expected = lhs % rhs

        tm.assert_almost_equal(result, expected)
        expected = self.ne.evaluate(f"expected {arith1} rhs")
        if isinstance(result, (DataFrame, Series)):
            tm.assert_almost_equal(result.values, expected)
        else:
            tm.assert_almost_equal(result, expected.item())

    def check_floor_division(self, lhs, arith1, rhs):
        ex = f"lhs {arith1} rhs"

        if self.engine == "python":
            res = pd.eval(ex, engine=self.engine, parser=self.parser)
            expected = lhs // rhs
            self.check_equal(res, expected)
        else:
            msg = (
                r"unsupported operand type\(s\) for //: 'VariableNode' and "
                "'VariableNode'"
            )
            with pytest.raises(TypeError, match=msg):
                pd.eval(
                    ex,
                    local_dict={"lhs": lhs, "rhs": rhs},
                    engine=self.engine,
                    parser=self.parser,
                )

    def get_expected_pow_result(self, lhs, rhs):
        try:
            expected = _eval_single_bin(lhs, "**", rhs, self.engine)
        except ValueError as e:
            if str(e).startswith(
                "negative number cannot be raised to a fractional power"
            ):
                if self.engine == "python":
                    pytest.skip(str(e))
                else:
                    expected = np.nan
            else:
                raise
        return expected

    def check_pow(self, lhs, arith1, rhs):
        ex = f"lhs {arith1} rhs"
        expected = self.get_expected_pow_result(lhs, rhs)
        result = pd.eval(ex, engine=self.engine, parser=self.parser)

        if (
            is_scalar(lhs)
            and is_scalar(rhs)
            and _is_py3_complex_incompat(result, expected)
        ):
            msg = "(DataFrame.columns|numpy array) are different"
            with pytest.raises(AssertionError, match=msg):
                tm.assert_numpy_array_equal(result, expected)
        else:
            tm.assert_almost_equal(result, expected)

            ex = f"(lhs {arith1} rhs) {arith1} rhs"
            result = pd.eval(ex, engine=self.engine, parser=self.parser)
            expected = self.get_expected_pow_result(
                self.get_expected_pow_result(lhs, rhs), rhs
            )
            tm.assert_almost_equal(result, expected)

    def check_single_invert_op(self, elem, cmp1):
        # simple
        try:
            elb = elem.astype(bool)
        except AttributeError:
            elb = np.array([bool(elem)])
        expected = ~elb
        result = pd.eval("~elb", engine=self.engine, parser=self.parser)
        tm.assert_almost_equal(expected, result)

        for engine in self.current_engines:
            tm.assert_almost_equal(
                result, pd.eval("~elb", engine=engine, parser=self.parser)
            )

    def check_compound_invert_op(self, lhs, cmp1, rhs):
        skip_these = ["in", "not in"]
        ex = f"~(lhs {cmp1} rhs)"

        msg = (
            r"only list-like( or dict-like)? objects are allowed to be "
            r"passed to (DataFrame\.)?isin\(\), you passed a "
            r"(\[|')float(\]|')|"
            "argument of type 'float' is not iterable"
        )
        if is_scalar(rhs) and cmp1 in skip_these:
            with pytest.raises(TypeError, match=msg):
                pd.eval(
                    ex,
                    engine=self.engine,
                    parser=self.parser,
                    local_dict={"lhs": lhs, "rhs": rhs},
                )
        else:
            # compound
            if is_scalar(lhs) and is_scalar(rhs):
                lhs, rhs = map(lambda x: np.array([x]), (lhs, rhs))
            expected = _eval_single_bin(lhs, cmp1, rhs, self.engine)
            if is_scalar(expected):
                expected = not expected
            else:
                expected = ~expected
            result = pd.eval(ex, engine=self.engine, parser=self.parser)
            tm.assert_almost_equal(expected, result)

            # make sure the other engines work the same as this one
            for engine in self.current_engines:
                ev = pd.eval(ex, engine=self.engine, parser=self.parser)
                tm.assert_almost_equal(ev, result)

    def ex(self, op, var_name="lhs"):
        return f"{op}{var_name}"

    def test_frame_invert(self):
        expr = self.ex("~")

        # ~ ##
        # frame
        # float always raises
        lhs = DataFrame(np.random.randn(5, 2))
        if self.engine == "numexpr":
            msg = "couldn't find matching opcode for 'invert_dd'"
            with pytest.raises(NotImplementedError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)
        else:
            msg = "ufunc 'invert' not supported for the input types"
            with pytest.raises(TypeError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)

        # int raises on numexpr
        lhs = DataFrame(np.random.randint(5, size=(5, 2)))
        if self.engine == "numexpr":
            msg = "couldn't find matching opcode for 'invert"
            with pytest.raises(NotImplementedError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)
        else:
            expect = ~lhs
            result = pd.eval(expr, engine=self.engine, parser=self.parser)
            tm.assert_frame_equal(expect, result)

        # bool always works
        lhs = DataFrame(np.random.rand(5, 2) > 0.5)
        expect = ~lhs
        result = pd.eval(expr, engine=self.engine, parser=self.parser)
        tm.assert_frame_equal(expect, result)

        # object raises
        lhs = DataFrame({"b": ["a", 1, 2.0], "c": np.random.rand(3) > 0.5})
        if self.engine == "numexpr":
            with pytest.raises(ValueError, match="unknown type object"):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)
        else:
            msg = "bad operand type for unary ~: 'str'"
            with pytest.raises(TypeError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)

    def test_series_invert(self):
        # ~ ####
        expr = self.ex("~")

        # series
        # float raises
        lhs = Series(np.random.randn(5))
        if self.engine == "numexpr":
            msg = "couldn't find matching opcode for 'invert_dd'"
            with pytest.raises(NotImplementedError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)
        else:
            msg = "ufunc 'invert' not supported for the input types"
            with pytest.raises(TypeError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)

        # int raises on numexpr
        lhs = Series(np.random.randint(5, size=5))
        if self.engine == "numexpr":
            msg = "couldn't find matching opcode for 'invert"
            with pytest.raises(NotImplementedError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)
        else:
            expect = ~lhs
            result = pd.eval(expr, engine=self.engine, parser=self.parser)
            tm.assert_series_equal(expect, result)

        # bool
        lhs = Series(np.random.rand(5) > 0.5)
        expect = ~lhs
        result = pd.eval(expr, engine=self.engine, parser=self.parser)
        tm.assert_series_equal(expect, result)

        # float
        # int
        # bool

        # object
        lhs = Series(["a", 1, 2.0])
        if self.engine == "numexpr":
            with pytest.raises(ValueError, match="unknown type object"):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)
        else:
            msg = "bad operand type for unary ~: 'str'"
            with pytest.raises(TypeError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)

    def test_frame_negate(self):
        expr = self.ex("-")

        # float
        lhs = DataFrame(np.random.randn(5, 2))
        expect = -lhs
        result = pd.eval(expr, engine=self.engine, parser=self.parser)
        tm.assert_frame_equal(expect, result)

        # int
        lhs = DataFrame(np.random.randint(5, size=(5, 2)))
        expect = -lhs
        result = pd.eval(expr, engine=self.engine, parser=self.parser)
        tm.assert_frame_equal(expect, result)

        # bool doesn't work with numexpr but works elsewhere
        lhs = DataFrame(np.random.rand(5, 2) > 0.5)
        if self.engine == "numexpr":
            msg = "couldn't find matching opcode for 'neg_bb'"
            with pytest.raises(NotImplementedError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)
        else:
            expect = -lhs
            result = pd.eval(expr, engine=self.engine, parser=self.parser)
            tm.assert_frame_equal(expect, result)

    def test_series_negate(self):
        expr = self.ex("-")

        # float
        lhs = Series(np.random.randn(5))
        expect = -lhs
        result = pd.eval(expr, engine=self.engine, parser=self.parser)
        tm.assert_series_equal(expect, result)

        # int
        lhs = Series(np.random.randint(5, size=5))
        expect = -lhs
        result = pd.eval(expr, engine=self.engine, parser=self.parser)
        tm.assert_series_equal(expect, result)

        # bool doesn't work with numexpr but works elsewhere
        lhs = Series(np.random.rand(5) > 0.5)
        if self.engine == "numexpr":
            msg = "couldn't find matching opcode for 'neg_bb'"
            with pytest.raises(NotImplementedError, match=msg):
                result = pd.eval(expr, engine=self.engine, parser=self.parser)
        else:
            expect = -lhs
            result = pd.eval(expr, engine=self.engine, parser=self.parser)
            tm.assert_series_equal(expect, result)

    @pytest.mark.parametrize(
        "lhs",
        [
            # Float
            DataFrame(np.random.randn(5, 2)),
            # Int
            DataFrame(np.random.randint(5, size=(5, 2))),
            # bool doesn't work with numexpr but works elsewhere
            DataFrame(np.random.rand(5, 2) > 0.5),
        ],
    )
    def test_frame_pos(self, lhs):
        expr = self.ex("+")
        expect = lhs

        result = pd.eval(expr, engine=self.engine, parser=self.parser)
        tm.assert_frame_equal(expect, result)

    @pytest.mark.parametrize(
        "lhs",
        [
            # Float
            Series(np.random.randn(5)),
            # Int
            Series(np.random.randint(5, size=5)),
            # bool doesn't work with numexpr but works elsewhere
            Series(np.random.rand(5) > 0.5),
        ],
    )
    def test_series_pos(self, lhs):
        expr = self.ex("+")
        expect = lhs

        result = pd.eval(expr, engine=self.engine, parser=self.parser)
        tm.assert_series_equal(expect, result)

    def test_scalar_unary(self):
        msg = "bad operand type for unary ~: 'float'"
        with pytest.raises(TypeError, match=msg):
            pd.eval("~1.0", engine=self.engine, parser=self.parser)

        assert pd.eval("-1.0", parser=self.parser, engine=self.engine) == -1.0
        assert pd.eval("+1.0", parser=self.parser, engine=self.engine) == +1.0
        assert pd.eval("~1", parser=self.parser, engine=self.engine) == ~1
        assert pd.eval("-1", parser=self.parser, engine=self.engine) == -1
        assert pd.eval("+1", parser=self.parser, engine=self.engine) == +1
        assert pd.eval("~True", parser=self.parser, engine=self.engine) == ~True
        assert pd.eval("~False", parser=self.parser, engine=self.engine) == ~False
        assert pd.eval("-True", parser=self.parser, engine=self.engine) == -True
        assert pd.eval("-False", parser=self.parser, engine=self.engine) == -False
        assert pd.eval("+True", parser=self.parser, engine=self.engine) == +True
        assert pd.eval("+False", parser=self.parser, engine=self.engine) == +False

    def test_unary_in_array(self):
        # GH 11235
        tm.assert_numpy_array_equal(
            pd.eval(
                "[-True, True, ~True, +True,"
                "-False, False, ~False, +False,"
                "-37, 37, ~37, +37]"
            ),
            np.array(
                [
                    -True,
                    True,
                    ~True,
                    +True,
                    -False,
                    False,
                    ~False,
                    +False,
                    -37,
                    37,
                    ~37,
                    +37,
                ],
                dtype=np.object_,
            ),
        )

    @pytest.mark.parametrize("dtype", [np.float32, np.float64])
    def test_float_comparison_bin_op(self, dtype):
        # GH 16363
        df = DataFrame({"x": np.array([0], dtype=dtype)})
        res = df.eval("x < -0.1")
        assert res.values == np.array([False])

        res = df.eval("-5 > x")
        assert res.values == np.array([False])

    def test_disallow_scalar_bool_ops(self):
        exprs = "1 or 2", "1 and 2"
        exprs += "a and b", "a or b"
        exprs += ("1 or 2 and (3 + 2) > 3",)
        exprs += ("2 * x > 2 or 1 and 2",)
        exprs += ("2 * df > 3 and 1 or a",)

        x, a, b, df = np.random.randn(3), 1, 2, DataFrame(np.random.randn(3, 2))  # noqa
        for ex in exprs:
            msg = "cannot evaluate scalar only bool ops|'BoolOp' nodes are not"
            with pytest.raises(NotImplementedError, match=msg):
                pd.eval(ex, engine=self.engine, parser=self.parser)

    def test_identical(self):
        # see gh-10546
        x = 1
        result = pd.eval("x", engine=self.engine, parser=self.parser)
        assert result == 1
        assert is_scalar(result)

        x = 1.5
        result = pd.eval("x", engine=self.engine, parser=self.parser)
        assert result == 1.5
        assert is_scalar(result)

        x = False
        result = pd.eval("x", engine=self.engine, parser=self.parser)
        assert not result
        assert is_bool(result)
        assert is_scalar(result)

        x = np.array([1])
        result = pd.eval("x", engine=self.engine, parser=self.parser)
        tm.assert_numpy_array_equal(result, np.array([1]))
        assert result.shape == (1,)

        x = np.array([1.5])
        result = pd.eval("x", engine=self.engine, parser=self.parser)
        tm.assert_numpy_array_equal(result, np.array([1.5]))
        assert result.shape == (1,)

        x = np.array([False])  # noqa
        result = pd.eval("x", engine=self.engine, parser=self.parser)
        tm.assert_numpy_array_equal(result, np.array([False]))
        assert result.shape == (1,)

    def test_line_continuation(self):
        # GH 11149
        exp = """1 + 2 * \
        5 - 1 + 2 """
        result = pd.eval(exp, engine=self.engine, parser=self.parser)
        assert result == 12

    def test_float_truncation(self):
        # GH 14241
        exp = "1000000000.006"
        result = pd.eval(exp, engine=self.engine, parser=self.parser)
        expected = np.float64(exp)
        assert result == expected

        df = DataFrame({"A": [1000000000.0009, 1000000000.0011, 1000000000.0015]})
        cutoff = 1000000000.0006
        result = df.query(f"A < {cutoff:.4f}")
        assert result.empty

        cutoff = 1000000000.0010
        result = df.query(f"A > {cutoff:.4f}")
        expected = df.loc[[1, 2], :]
        tm.assert_frame_equal(expected, result)

        exact = 1000000000.0011
        result = df.query(f"A == {exact:.4f}")
        expected = df.loc[[1], :]
        tm.assert_frame_equal(expected, result)

    def test_disallow_python_keywords(self):
        # GH 18221
        df = DataFrame([[0, 0, 0]], columns=["foo", "bar", "class"])
        msg = "Python keyword not valid identifier in numexpr query"
        with pytest.raises(SyntaxError, match=msg):
            df.query("class == 0")

        df = DataFrame()
        df.index.name = "lambda"
        with pytest.raises(SyntaxError, match=msg):
            df.query("lambda == 0")


@td.skip_if_no_ne
class TestEvalNumexprPython(TestEvalNumexprPandas):
    exclude_cmp = ["in", "not in"]
    exclude_bool = ["and", "or"]

    engine = "numexpr"
    parser = "python"

    @classmethod
    def setup_class(cls):
        super().setup_class()
        import numexpr as ne

        cls.ne = ne

    def check_chained_cmp_op(self, lhs, cmp1, mid, cmp2, rhs):
        ex1 = f"lhs {cmp1} mid {cmp2} rhs"
        msg = "'BoolOp' nodes are not implemented"
        with pytest.raises(NotImplementedError, match=msg):
            pd.eval(ex1, engine=self.engine, parser=self.parser)


class TestEvalPythonPython(TestEvalNumexprPython):
    engine = "python"
    parser = "python"

    def check_modulus(self, lhs, arith1, rhs):
        ex = f"lhs {arith1} rhs"
        result = pd.eval(ex, engine=self.engine, parser=self.parser)

        expected = lhs % rhs
        tm.assert_almost_equal(result, expected)

        expected = _eval_single_bin(expected, arith1, rhs, self.engine)
        tm.assert_almost_equal(result, expected)

    def check_alignment(self, result, nlhs, ghs, op):
        try:
            nlhs, ghs = nlhs.align(ghs)
        except (ValueError, TypeError, AttributeError):
            # ValueError: series frame or frame series align
            # TypeError, AttributeError: series or frame with scalar align
            pass
        else:
            expected = eval(f"nlhs {op} ghs")
            tm.assert_almost_equal(result, expected)


class TestEvalPythonPandas(TestEvalPythonPython):
    engine = "python"
    parser = "pandas"

    def check_chained_cmp_op(self, lhs, cmp1, mid, cmp2, rhs):
        TestEvalNumexprPandas.check_chained_cmp_op(self, lhs, cmp1, mid, cmp2, rhs)


f = lambda *args, **kwargs: np.random.randn()


# -------------------------------------
# gh-12388: Typecasting rules consistency with python


class TestTypeCasting:
    @pytest.mark.parametrize("op", ["+", "-", "*", "**", "/"])
    # maybe someday... numexpr has too many upcasting rules now
    # chain(*(np.sctypes[x] for x in ['uint', 'int', 'float']))
    @pytest.mark.parametrize("dt", [np.float32, np.float64])
    def test_binop_typecasting(self, engine, parser, op, dt):
        df = tm.makeCustomDataframe(5, 3, data_gen_f=f, dtype=dt)
        s = f"df {op} 3"
        res = pd.eval(s, engine=engine, parser=parser)
        assert df.values.dtype == dt
        assert res.values.dtype == dt
        tm.assert_frame_equal(res, eval(s))

        s = f"3 {op} df"
        res = pd.eval(s, engine=engine, parser=parser)
        assert df.values.dtype == dt
        assert res.values.dtype == dt
        tm.assert_frame_equal(res, eval(s))


# -------------------------------------
# Basic and complex alignment


def _is_datetime(x):
    return issubclass(x.dtype.type, np.datetime64)


def should_warn(*args):
    not_mono = not any(map(operator.attrgetter("is_monotonic"), args))
    only_one_dt = reduce(operator.xor, map(_is_datetime, args))
    return not_mono and only_one_dt


class TestAlignment:

    index_types = ["i", "u", "dt"]
    lhs_index_types = index_types + ["s"]  # 'p'

    def test_align_nested_unary_op(self, engine, parser):
        s = "df * ~2"
        df = tm.makeCustomDataframe(5, 3, data_gen_f=f)
        res = pd.eval(s, engine=engine, parser=parser)
        tm.assert_frame_equal(res, df * ~2)

    @pytest.mark.parametrize("lr_idx_type", lhs_index_types)
    @pytest.mark.parametrize("rr_idx_type", index_types)
    @pytest.mark.parametrize("c_idx_type", index_types)
    def test_basic_frame_alignment(
        self, engine, parser, lr_idx_type, rr_idx_type, c_idx_type
    ):
        with warnings.catch_warnings(record=True):
            warnings.simplefilter("always", RuntimeWarning)

            df = tm.makeCustomDataframe(
                10, 10, data_gen_f=f, r_idx_type=lr_idx_type, c_idx_type=c_idx_type
            )
            df2 = tm.makeCustomDataframe(
                20, 10, data_gen_f=f, r_idx_type=rr_idx_type, c_idx_type=c_idx_type
            )
            # only warns if not monotonic and not sortable
            if should_warn(df.index, df2.index):
                with tm.assert_produces_warning(RuntimeWarning):
                    res = pd.eval("df + df2", engine=engine, parser=parser)
            else:
                res = pd.eval("df + df2", engine=engine, parser=parser)
            tm.assert_frame_equal(res, df + df2)

    @pytest.mark.parametrize("r_idx_type", lhs_index_types)
    @pytest.mark.parametrize("c_idx_type", lhs_index_types)
    def test_frame_comparison(self, engine, parser, r_idx_type, c_idx_type):
        df = tm.makeCustomDataframe(
            10, 10, data_gen_f=f, r_idx_type=r_idx_type, c_idx_type=c_idx_type
        )
        res = pd.eval("df < 2", engine=engine, parser=parser)
        tm.assert_frame_equal(res, df < 2)

        df3 = DataFrame(np.random.randn(*df.shape), index=df.index, columns=df.columns)
        res = pd.eval("df < df3", engine=engine, parser=parser)
        tm.assert_frame_equal(res, df < df3)

    @pytest.mark.parametrize("r1", lhs_index_types)
    @pytest.mark.parametrize("c1", index_types)
    @pytest.mark.parametrize("r2", index_types)
    @pytest.mark.parametrize("c2", index_types)
    def test_medium_complex_frame_alignment(self, engine, parser, r1, c1, r2, c2):

        with warnings.catch_warnings(record=True):
            warnings.simplefilter("always", RuntimeWarning)

            df = tm.makeCustomDataframe(
                3, 2, data_gen_f=f, r_idx_type=r1, c_idx_type=c1
            )
            df2 = tm.makeCustomDataframe(
                4, 2, data_gen_f=f, r_idx_type=r2, c_idx_type=c2
            )
            df3 = tm.makeCustomDataframe(
                5, 2, data_gen_f=f, r_idx_type=r2, c_idx_type=c2
            )
            if should_warn(df.index, df2.index, df3.index):
                with tm.assert_produces_warning(RuntimeWarning):
                    res = pd.eval("df + df2 + df3", engine=engine, parser=parser)
            else:
                res = pd.eval("df + df2 + df3", engine=engine, parser=parser)
            tm.assert_frame_equal(res, df + df2 + df3)

    @pytest.mark.parametrize("index_name", ["index", "columns"])
    @pytest.mark.parametrize("c_idx_type", index_types)
    @pytest.mark.parametrize("r_idx_type", lhs_index_types)
    def test_basic_frame_series_alignment(
        self, engine, parser, index_name, r_idx_type, c_idx_type
    ):
        def testit(r_idx_type, c_idx_type, index_name):
            df = tm.makeCustomDataframe(
                10, 10, data_gen_f=f, r_idx_type=r_idx_type, c_idx_type=c_idx_type
            )
            index = getattr(df, index_name)
            s = Series(np.random.randn(5), index[:5])

            if should_warn(df.index, s.index):
                with tm.assert_produces_warning(RuntimeWarning):
                    res = pd.eval("df + s", engine=engine, parser=parser)
            else:
                res = pd.eval("df + s", engine=engine, parser=parser)

            if r_idx_type == "dt" or c_idx_type == "dt":
                expected = df.add(s) if engine == "numexpr" else df + s
            else:
                expected = df + s
            tm.assert_frame_equal(res, expected)

        with warnings.catch_warnings(record=True):
            warnings.simplefilter("always", RuntimeWarning)

            testit(r_idx_type, c_idx_type, index_name)

    @pytest.mark.parametrize("index_name", ["index", "columns"])
    def test_basic_series_frame_alignment(self, engine, parser, index_name):
        def testit(r_idx_type, c_idx_type, index_name):
            df = tm.makeCustomDataframe(
                10, 7, data_gen_f=f, r_idx_type=r_idx_type, c_idx_type=c_idx_type
            )
            index = getattr(df, index_name)
            s = Series(np.random.randn(5), index[:5])
            if should_warn(s.index, df.index):
                with tm.assert_produces_warning(RuntimeWarning):
                    res = pd.eval("s + df", engine=engine, parser=parser)
            else:
                res = pd.eval("s + df", engine=engine, parser=parser)

            if r_idx_type == "dt" or c_idx_type == "dt":
                expected = df.add(s) if engine == "numexpr" else s + df
            else:
                expected = s + df
            tm.assert_frame_equal(res, expected)

        # only test dt with dt, otherwise weird joins result
        args = product(["i", "u", "s"], ["i", "u", "s"])
        with warnings.catch_warnings(record=True):
            # avoid warning about comparing strings and ints
            warnings.simplefilter("ignore", RuntimeWarning)

            for r_idx_type, c_idx_type in args:
                testit(r_idx_type, c_idx_type, index_name)

        # dt with dt
        args = product(["dt"], ["dt"])
        with warnings.catch_warnings(record=True):
            # avoid warning about comparing strings and ints
            warnings.simplefilter("ignore", RuntimeWarning)

            for r_idx_type, c_idx_type in args:
                testit(r_idx_type, c_idx_type, index_name)

    @pytest.mark.parametrize("c_idx_type", index_types)
    @pytest.mark.parametrize("r_idx_type", lhs_index_types)
    @pytest.mark.parametrize("index_name", ["index", "columns"])
    @pytest.mark.parametrize("op", ["+", "*"])
    def test_series_frame_commutativity(
        self, engine, parser, index_name, op, r_idx_type, c_idx_type
    ):

        with warnings.catch_warnings(record=True):
            warnings.simplefilter("always", RuntimeWarning)

            df = tm.makeCustomDataframe(
                10, 10, data_gen_f=f, r_idx_type=r_idx_type, c_idx_type=c_idx_type
            )
            index = getattr(df, index_name)
            s = Series(np.random.randn(5), index[:5])

            lhs = f"s {op} df"
            rhs = f"df {op} s"
            if should_warn(df.index, s.index):
                with tm.assert_produces_warning(RuntimeWarning):
                    a = pd.eval(lhs, engine=engine, parser=parser)
                with tm.assert_produces_warning(RuntimeWarning):
                    b = pd.eval(rhs, engine=engine, parser=parser)
            else:
                a = pd.eval(lhs, engine=engine, parser=parser)
                b = pd.eval(rhs, engine=engine, parser=parser)

            if r_idx_type != "dt" and c_idx_type != "dt":
                if engine == "numexpr":
                    tm.assert_frame_equal(a, b)

    @pytest.mark.parametrize("r1", lhs_index_types)
    @pytest.mark.parametrize("c1", index_types)
    @pytest.mark.parametrize("r2", index_types)
    @pytest.mark.parametrize("c2", index_types)
    def test_complex_series_frame_alignment(self, engine, parser, r1, c1, r2, c2):
        import random

        n = 3
        m1 = 5
        m2 = 2 * m1

        with warnings.catch_warnings(record=True):
            warnings.simplefilter("always", RuntimeWarning)

            index_name = random.choice(["index", "columns"])
            obj_name = random.choice(["df", "df2"])

            df = tm.makeCustomDataframe(
                m1, n, data_gen_f=f, r_idx_type=r1, c_idx_type=c1
            )
            df2 = tm.makeCustomDataframe(
                m2, n, data_gen_f=f, r_idx_type=r2, c_idx_type=c2
            )
            index = getattr(locals().get(obj_name), index_name)
            ser = Series(np.random.randn(n), index[:n])

            if r2 == "dt" or c2 == "dt":
                if engine == "numexpr":
                    expected2 = df2.add(ser)
                else:
                    expected2 = df2 + ser
            else:
                expected2 = df2 + ser

            if r1 == "dt" or c1 == "dt":
                if engine == "numexpr":
                    expected = expected2.add(df)
                else:
                    expected = expected2 + df
            else:
                expected = expected2 + df

            if should_warn(df2.index, ser.index, df.index):
                with tm.assert_produces_warning(RuntimeWarning):
                    res = pd.eval("df2 + ser + df", engine=engine, parser=parser)
            else:
                res = pd.eval("df2 + ser + df", engine=engine, parser=parser)
            assert res.shape == expected.shape
            tm.assert_frame_equal(res, expected)

    def test_performance_warning_for_poor_alignment(self, engine, parser):
        df = DataFrame(np.random.randn(1000, 10))
        s = Series(np.random.randn(10000))
        if engine == "numexpr":
            seen = PerformanceWarning
        else:
            seen = False

        with tm.assert_produces_warning(seen):
            pd.eval("df + s", engine=engine, parser=parser)

        s = Series(np.random.randn(1000))
        with tm.assert_produces_warning(False):
            pd.eval("df + s", engine=engine, parser=parser)

        df = DataFrame(np.random.randn(10, 10000))
        s = Series(np.random.randn(10000))
        with tm.assert_produces_warning(False):
            pd.eval("df + s", engine=engine, parser=parser)

        df = DataFrame(np.random.randn(10, 10))
        s = Series(np.random.randn(10000))

        is_python_engine = engine == "python"

        if not is_python_engine:
            wrn = PerformanceWarning
        else:
            wrn = False

        with tm.assert_produces_warning(wrn) as w:
            pd.eval("df + s", engine=engine, parser=parser)

            if not is_python_engine:
                assert len(w) == 1
                msg = str(w[0].message)
                loged = np.log10(s.size - df.shape[1])
                expected = (
                    f"Alignment difference on axis 1 is larger "
                    f"than an order of magnitude on term 'df', "
                    f"by more than {loged:.4g}; performance may suffer"
                )
                assert msg == expected


# ------------------------------------
# Slightly more complex ops


@td.skip_if_no_ne
class TestOperationsNumExprPandas:
    exclude_arith: List[str] = []

    engine = "numexpr"
    parser = "pandas"

    @classmethod
    def setup_class(cls):
        cls.arith_ops = [
            op
            for op in expr.ARITH_OPS_SYMS + expr.CMP_OPS_SYMS
            if op not in cls.exclude_arith
        ]

    def eval(self, *args, **kwargs):
        kwargs["engine"] = self.engine
        kwargs["parser"] = self.parser
        kwargs["level"] = kwargs.pop("level", 0) + 1
        return pd.eval(*args, **kwargs)

    def test_simple_arith_ops(self):
        ops = (op for op in self.arith_ops if op != "//")

        for op in ops:
            ex = f"1 {op} 1"
            ex2 = f"x {op} 1"
            ex3 = f"1 {op} (x + 1)"

            if op in ("in", "not in"):
                msg = "argument of type 'int' is not iterable"
                with pytest.raises(TypeError, match=msg):
                    pd.eval(ex, engine=self.engine, parser=self.parser)
            else:
                expec = _eval_single_bin(1, op, 1, self.engine)
                x = self.eval(ex, engine=self.engine, parser=self.parser)
                assert x == expec

                expec = _eval_single_bin(x, op, 1, self.engine)
                y = self.eval(
                    ex2, local_dict={"x": x}, engine=self.engine, parser=self.parser
                )
                assert y == expec

                expec = _eval_single_bin(1, op, x + 1, self.engine)
                y = self.eval(
                    ex3, local_dict={"x": x}, engine=self.engine, parser=self.parser
                )
                assert y == expec

    @pytest.mark.parametrize("rhs", [True, False])
    @pytest.mark.parametrize("lhs", [True, False])
    @pytest.mark.parametrize("op", expr.BOOL_OPS_SYMS)
    def test_simple_bool_ops(self, rhs, lhs, op):
        ex = f"{lhs} {op} {rhs}"
        res = self.eval(ex)
        exp = eval(ex)
        assert res == exp

    @pytest.mark.parametrize("rhs", [True, False])
    @pytest.mark.parametrize("lhs", [True, False])
    @pytest.mark.parametrize("op", expr.BOOL_OPS_SYMS)
    def test_bool_ops_with_constants(self, rhs, lhs, op):
        ex = f"{lhs} {op} {rhs}"
        res = self.eval(ex)
        exp = eval(ex)
        assert res == exp

    def test_4d_ndarray_fails(self):
        x = np.random.randn(3, 4, 5, 6)
        y = Series(np.random.randn(10))
        msg = "N-dimensional objects, where N > 2, are not supported with eval"
        with pytest.raises(NotImplementedError, match=msg):
            self.eval("x + y", local_dict={"x": x, "y": y})

    def test_constant(self):
        x = self.eval("1")
        assert x == 1

    def test_single_variable(self):
        df = DataFrame(np.random.randn(10, 2))
        df2 = self.eval("df", local_dict={"df": df})
        tm.assert_frame_equal(df, df2)

    def test_truediv(self):
        s = np.array([1])
        ex = "s / 1"
        d = {"s": s}  # noqa

        # FutureWarning: The `truediv` parameter in pd.eval is deprecated and will be
        # removed in a future version.
        with tm.assert_produces_warning(FutureWarning):
            res = self.eval(ex, truediv=False)
        tm.assert_numpy_array_equal(res, np.array([1.0]))

        with tm.assert_produces_warning(FutureWarning):
            res = self.eval(ex, truediv=True)
        tm.assert_numpy_array_equal(res, np.array([1.0]))

        with tm.assert_produces_warning(FutureWarning):
            res = self.eval("1 / 2", truediv=True)
        expec = 0.5
        assert res == expec

        with tm.assert_produces_warning(FutureWarning):
            res = self.eval("1 / 2", truediv=False)
        expec = 0.5
        assert res == expec

        with tm.assert_produces_warning(FutureWarning):
            res = self.eval("s / 2", truediv=False)
        expec = 0.5
        assert res == expec

        with tm.assert_produces_warning(FutureWarning):
            res = self.eval("s / 2", truediv=True)
        expec = 0.5
        assert res == expec

    def test_failing_subscript_with_name_error(self):
        df = DataFrame(np.random.randn(5, 3))  # noqa
        with pytest.raises(NameError, match="name 'x' is not defined"):
            self.eval("df[x > 2] > 2")

    def test_lhs_expression_subscript(self):
        df = DataFrame(np.random.randn(5, 3))
        result = self.eval("(df + 1)[df > 2]", local_dict={"df": df})
        expected = (df + 1)[df > 2]
        tm.assert_frame_equal(result, expected)

    def test_attr_expression(self):
        df = DataFrame(np.random.randn(5, 3), columns=list("abc"))
        expr1 = "df.a < df.b"
        expec1 = df.a < df.b
        expr2 = "df.a + df.b + df.c"
        expec2 = df.a + df.b + df.c
        expr3 = "df.a + df.b + df.c[df.b < 0]"
        expec3 = df.a + df.b + df.c[df.b < 0]
        exprs = expr1, expr2, expr3
        expecs = expec1, expec2, expec3
        for e, expec in zip(exprs, expecs):
            tm.assert_series_equal(expec, self.eval(e, local_dict={"df": df}))

    def test_assignment_fails(self):
        df = DataFrame(np.random.randn(5, 3), columns=list("abc"))
        df2 = DataFrame(np.random.randn(5, 3))
        expr1 = "df = df2"
        msg = "cannot assign without a target object"
        with pytest.raises(ValueError, match=msg):
            self.eval(expr1, local_dict={"df": df, "df2": df2})

    def test_assignment_column(self):
        df = DataFrame(np.random.randn(5, 2), columns=list("ab"))
        orig_df = df.copy()

        # multiple assignees
        with pytest.raises(SyntaxError, match="invalid syntax"):
            df.eval("d c = a + b")

        # invalid assignees
        msg = "left hand side of an assignment must be a single name"
        with pytest.raises(SyntaxError, match=msg):
            df.eval("d,c = a + b")
        if compat.PY38:
            msg = "cannot assign to function call"
        else:
            msg = "can't assign to function call"
        with pytest.raises(SyntaxError, match=msg):
            df.eval('Timestamp("20131001") = a + b')

        # single assignment - existing variable
        expected = orig_df.copy()
        expected["a"] = expected["a"] + expected["b"]
        df = orig_df.copy()
        df.eval("a = a + b", inplace=True)
        tm.assert_frame_equal(df, expected)

        # single assignment - new variable
        expected = orig_df.copy()
        expected["c"] = expected["a"] + expected["b"]
        df = orig_df.copy()
        df.eval("c = a + b", inplace=True)
        tm.assert_frame_equal(df, expected)

        # with a local name overlap
        def f():
            df = orig_df.copy()
            a = 1  # noqa
            df.eval("a = 1 + b", inplace=True)
            return df

        df = f()
        expected = orig_df.copy()
        expected["a"] = 1 + expected["b"]
        tm.assert_frame_equal(df, expected)

        df = orig_df.copy()

        def f():
            a = 1  # noqa
            old_a = df.a.copy()
            df.eval("a = a + b", inplace=True)
            result = old_a + df.b
            tm.assert_series_equal(result, df.a, check_names=False)
            assert result.name is None

        f()

        # multiple assignment
        df = orig_df.copy()
        df.eval("c = a + b", inplace=True)
        msg = "can only assign a single expression"
        with pytest.raises(SyntaxError, match=msg):
            df.eval("c = a = b")

        # explicit targets
        df = orig_df.copy()
        self.eval("c = df.a + df.b", local_dict={"df": df}, target=df, inplace=True)
        expected = orig_df.copy()
        expected["c"] = expected["a"] + expected["b"]
        tm.assert_frame_equal(df, expected)

    def test_column_in(self):
        # GH 11235
        df = DataFrame({"a": [11], "b": [-32]})
        result = df.eval("a in [11, -32]")
        expected = Series([True])
        tm.assert_series_equal(result, expected)

    def assignment_not_inplace(self):
        # see gh-9297
        df = DataFrame(np.random.randn(5, 2), columns=list("ab"))

        actual = df.eval("c = a + b", inplace=False)
        assert actual is not None

        expected = df.copy()
        expected["c"] = expected["a"] + expected["b"]
        tm.assert_frame_equal(df, expected)

    def test_multi_line_expression(self):
        # GH 11149
        df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
        expected = df.copy()

        expected["c"] = expected["a"] + expected["b"]
        expected["d"] = expected["c"] + expected["b"]
        ans = df.eval(
            """
        c = a + b
        d = c + b""",
            inplace=True,
        )
        tm.assert_frame_equal(expected, df)
        assert ans is None

        expected["a"] = expected["a"] - 1
        expected["e"] = expected["a"] + 2
        ans = df.eval(
            """
        a = a - 1
        e = a + 2""",
            inplace=True,
        )
        tm.assert_frame_equal(expected, df)
        assert ans is None

        # multi-line not valid if not all assignments
        msg = "Multi-line expressions are only valid if all expressions contain"
        with pytest.raises(ValueError, match=msg):
            df.eval(
                """
            a = b + 2
            b - 2""",
                inplace=False,
            )

    def test_multi_line_expression_not_inplace(self):
        # GH 11149
        df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
        expected = df.copy()

        expected["c"] = expected["a"] + expected["b"]
        expected["d"] = expected["c"] + expected["b"]
        df = df.eval(
            """
        c = a + b
        d = c + b""",
            inplace=False,
        )
        tm.assert_frame_equal(expected, df)

        expected["a"] = expected["a"] - 1
        expected["e"] = expected["a"] + 2
        df = df.eval(
            """
        a = a - 1
        e = a + 2""",
            inplace=False,
        )
        tm.assert_frame_equal(expected, df)

    def test_multi_line_expression_local_variable(self):
        # GH 15342
        df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
        expected = df.copy()

        local_var = 7
        expected["c"] = expected["a"] * local_var
        expected["d"] = expected["c"] + local_var
        ans = df.eval(
            """
        c = a * @local_var
        d = c + @local_var
        """,
            inplace=True,
        )
        tm.assert_frame_equal(expected, df)
        assert ans is None

    def test_multi_line_expression_callable_local_variable(self):
        # 26426
        df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})

        def local_func(a, b):
            return b

        expected = df.copy()
        expected["c"] = expected["a"] * local_func(1, 7)
        expected["d"] = expected["c"] + local_func(1, 7)
        ans = df.eval(
            """
        c = a * @local_func(1, 7)
        d = c + @local_func(1, 7)
        """,
            inplace=True,
        )
        tm.assert_frame_equal(expected, df)
        assert ans is None

    def test_multi_line_expression_callable_local_variable_with_kwargs(self):
        # 26426
        df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})

        def local_func(a, b):
            return b

        expected = df.copy()
        expected["c"] = expected["a"] * local_func(b=7, a=1)
        expected["d"] = expected["c"] + local_func(b=7, a=1)
        ans = df.eval(
            """
        c = a * @local_func(b=7, a=1)
        d = c + @local_func(b=7, a=1)
        """,
            inplace=True,
        )
        tm.assert_frame_equal(expected, df)
        assert ans is None

    def test_assignment_in_query(self):
        # GH 8664
        df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
        df_orig = df.copy()
        msg = "cannot assign without a target object"
        with pytest.raises(ValueError, match=msg):
            df.query("a = 1")
        tm.assert_frame_equal(df, df_orig)

    def test_query_inplace(self):
        # see gh-11149
        df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
        expected = df.copy()
        expected = expected[expected["a"] == 2]
        df.query("a == 2", inplace=True)
        tm.assert_frame_equal(expected, df)

        df = {}
        expected = {"a": 3}

        self.eval("a = 1 + 2", target=df, inplace=True)
        tm.assert_dict_equal(df, expected)

    @pytest.mark.parametrize("invalid_target", [1, "cat", [1, 2], np.array([]), (1, 3)])
    @pytest.mark.filterwarnings("ignore::FutureWarning")
    def test_cannot_item_assign(self, invalid_target):
        msg = "Cannot assign expression output to target"
        expression = "a = 1 + 2"

        with pytest.raises(ValueError, match=msg):
            self.eval(expression, target=invalid_target, inplace=True)

        if hasattr(invalid_target, "copy"):
            with pytest.raises(ValueError, match=msg):
                self.eval(expression, target=invalid_target, inplace=False)

    @pytest.mark.parametrize("invalid_target", [1, "cat", (1, 3)])
    def test_cannot_copy_item(self, invalid_target):
        msg = "Cannot return a copy of the target"
        expression = "a = 1 + 2"

        with pytest.raises(ValueError, match=msg):
            self.eval(expression, target=invalid_target, inplace=False)

    @pytest.mark.parametrize("target", [1, "cat", [1, 2], np.array([]), (1, 3), {1: 2}])
    def test_inplace_no_assignment(self, target):
        expression = "1 + 2"

        assert self.eval(expression, target=target, inplace=False) == 3

        msg = "Cannot operate inplace if there is no assignment"
        with pytest.raises(ValueError, match=msg):
            self.eval(expression, target=target, inplace=True)

    def test_basic_period_index_boolean_expression(self):
        df = tm.makeCustomDataframe(2, 2, data_gen_f=f, c_idx_type="p", r_idx_type="i")

        e = df < 2
        r = self.eval("df < 2", local_dict={"df": df})
        x = df < 2

        tm.assert_frame_equal(r, e)
        tm.assert_frame_equal(x, e)

    def test_basic_period_index_subscript_expression(self):
        df = tm.makeCustomDataframe(2, 2, data_gen_f=f, c_idx_type="p", r_idx_type="i")
        r = self.eval("df[df < 2 + 3]", local_dict={"df": df})
        e = df[df < 2 + 3]
        tm.assert_frame_equal(r, e)

    def test_nested_period_index_subscript_expression(self):
        df = tm.makeCustomDataframe(2, 2, data_gen_f=f, c_idx_type="p", r_idx_type="i")
        r = self.eval("df[df[df < 2] < 2] + df * 2", local_dict={"df": df})
        e = df[df[df < 2] < 2] + df * 2
        tm.assert_frame_equal(r, e)

    def test_date_boolean(self):
        df = DataFrame(np.random.randn(5, 3))
        df["dates1"] = date_range("1/1/2012", periods=5)
        res = self.eval(
            "df.dates1 < 20130101",
            local_dict={"df": df},
            engine=self.engine,
            parser=self.parser,
        )
        expec = df.dates1 < "20130101"
        tm.assert_series_equal(res, expec, check_names=False)

    def test_simple_in_ops(self):
        if self.parser != "python":
            res = pd.eval("1 in [1, 2]", engine=self.engine, parser=self.parser)
            assert res

            res = pd.eval("2 in (1, 2)", engine=self.engine, parser=self.parser)
            assert res

            res = pd.eval("3 in (1, 2)", engine=self.engine, parser=self.parser)
            assert not res

            res = pd.eval("3 not in (1, 2)", engine=self.engine, parser=self.parser)
            assert res

            res = pd.eval("[3] not in (1, 2)", engine=self.engine, parser=self.parser)
            assert res

            res = pd.eval("[3] in ([3], 2)", engine=self.engine, parser=self.parser)
            assert res

            res = pd.eval("[[3]] in [[[3]], 2]", engine=self.engine, parser=self.parser)
            assert res

            res = pd.eval("(3,) in [(3,), 2]", engine=self.engine, parser=self.parser)
            assert res

            res = pd.eval(
                "(3,) not in [(3,), 2]", engine=self.engine, parser=self.parser
            )
            assert not res

            res = pd.eval(
                "[(3,)] in [[(3,)], 2]", engine=self.engine, parser=self.parser
            )
            assert res
        else:
            msg = "'In' nodes are not implemented"
            with pytest.raises(NotImplementedError, match=msg):
                pd.eval("1 in [1, 2]", engine=self.engine, parser=self.parser)
            with pytest.raises(NotImplementedError, match=msg):
                pd.eval("2 in (1, 2)", engine=self.engine, parser=self.parser)
            with pytest.raises(NotImplementedError, match=msg):
                pd.eval("3 in (1, 2)", engine=self.engine, parser=self.parser)
            with pytest.raises(NotImplementedError, match=msg):
                pd.eval(
                    "[(3,)] in (1, 2, [(3,)])", engine=self.engine, parser=self.parser
                )
            msg = "'NotIn' nodes are not implemented"
            with pytest.raises(NotImplementedError, match=msg):
                pd.eval("3 not in (1, 2)", engine=self.engine, parser=self.parser)
            with pytest.raises(NotImplementedError, match=msg):
                pd.eval(
                    "[3] not in (1, 2, [[3]])", engine=self.engine, parser=self.parser
                )


@td.skip_if_no_ne
class TestOperationsNumExprPython(TestOperationsNumExprPandas):
    exclude_arith: List[str] = ["in", "not in"]

    engine = "numexpr"
    parser = "python"

    def test_check_many_exprs(self):
        a = 1  # noqa
        expr = " * ".join("a" * 33)
        expected = 1
        res = pd.eval(expr, engine=self.engine, parser=self.parser)
        assert res == expected

    def test_fails_and(self):
        df = DataFrame(np.random.randn(5, 3))
        msg = "'BoolOp' nodes are not implemented"
        with pytest.raises(NotImplementedError, match=msg):
            pd.eval(
                "df > 2 and df > 3",
                local_dict={"df": df},
                parser=self.parser,
                engine=self.engine,
            )

    def test_fails_or(self):
        df = DataFrame(np.random.randn(5, 3))
        msg = "'BoolOp' nodes are not implemented"
        with pytest.raises(NotImplementedError, match=msg):
            pd.eval(
                "df > 2 or df > 3",
                local_dict={"df": df},
                parser=self.parser,
                engine=self.engine,
            )

    def test_fails_not(self):
        df = DataFrame(np.random.randn(5, 3))
        msg = "'Not' nodes are not implemented"
        with pytest.raises(NotImplementedError, match=msg):
            pd.eval(
                "not df > 2",
                local_dict={"df": df},
                parser=self.parser,
                engine=self.engine,
            )

    def test_fails_ampersand(self):
        df = DataFrame(np.random.randn(5, 3))  # noqa
        ex = "(df + 2)[df > 1] > 0 & (df > 0)"
        msg = "cannot evaluate scalar only bool ops"
        with pytest.raises(NotImplementedError, match=msg):
            pd.eval(ex, parser=self.parser, engine=self.engine)

    def test_fails_pipe(self):
        df = DataFrame(np.random.randn(5, 3))  # noqa
        ex = "(df + 2)[df > 1] > 0 | (df > 0)"
        msg = "cannot evaluate scalar only bool ops"
        with pytest.raises(NotImplementedError, match=msg):
            pd.eval(ex, parser=self.parser, engine=self.engine)

    @pytest.mark.parametrize("rhs", [True, False])
    @pytest.mark.parametrize("lhs", [True, False])
    @pytest.mark.parametrize("op", expr.BOOL_OPS_SYMS)
    def test_bool_ops_with_constants(self, lhs, rhs, op):
        ex = f"{lhs} {op} {rhs}"
        if op in ("and", "or"):
            msg = "'BoolOp' nodes are not implemented"
            with pytest.raises(NotImplementedError, match=msg):
                self.eval(ex)
        else:
            res = self.eval(ex)
            exp = eval(ex)
            assert res == exp

    @pytest.mark.parametrize("rhs", [True, False])
    @pytest.mark.parametrize("lhs", [True, False])
    @pytest.mark.parametrize("op", expr.BOOL_OPS_SYMS)
    def test_simple_bool_ops(self, lhs, rhs, op):
        ex = f"lhs {op} rhs"
        if op in ("and", "or"):
            msg = "'BoolOp' nodes are not implemented"
            with pytest.raises(NotImplementedError, match=msg):
                pd.eval(ex, engine=self.engine, parser=self.parser)
        else:
            res = pd.eval(ex, engine=self.engine, parser=self.parser)
            exp = eval(ex)
            assert res == exp


class TestOperationsPythonPython(TestOperationsNumExprPython):
    engine = "python"
    parser = "python"


class TestOperationsPythonPandas(TestOperationsNumExprPandas):
    exclude_arith: List[str] = []

    engine = "python"
    parser = "pandas"


@td.skip_if_no_ne
class TestMathPythonPython:
    engine = "python"
    parser = "pandas"

    def eval(self, *args, **kwargs):
        kwargs["engine"] = self.engine
        kwargs["parser"] = self.parser
        kwargs["level"] = kwargs.pop("level", 0) + 1
        return pd.eval(*args, **kwargs)

    def test_unary_functions(self, unary_fns_for_ne):
        df = DataFrame({"a": np.random.randn(10)})
        a = df.a

        fn = unary_fns_for_ne

        expr = f"{fn}(a)"
        got = self.eval(expr)
        with np.errstate(all="ignore"):
            expect = getattr(np, fn)(a)
        tm.assert_series_equal(got, expect, check_names=False)

    @pytest.mark.parametrize("fn", ["floor", "ceil"])
    def test_floor_and_ceil_functions_raise_error(self, ne_lt_2_6_9, fn):
        msg = f'"{fn}" is not a supported function'
        with pytest.raises(ValueError, match=msg):
            expr = f"{fn}(100)"
            self.eval(expr)

    @pytest.mark.parametrize("fn", _binary_math_ops)
    def test_binary_functions(self, fn):
        df = DataFrame({"a": np.random.randn(10), "b": np.random.randn(10)})
        a = df.a
        b = df.b

        expr = f"{fn}(a, b)"
        got = self.eval(expr)
        with np.errstate(all="ignore"):
            expect = getattr(np, fn)(a, b)
        tm.assert_almost_equal(got, expect, check_names=False)

    def test_df_use_case(self):
        df = DataFrame({"a": np.random.randn(10), "b": np.random.randn(10)})
        df.eval(
            "e = arctan2(sin(a), b)",
            engine=self.engine,
            parser=self.parser,
            inplace=True,
        )
        got = df.e
        expect = np.arctan2(np.sin(df.a), df.b)
        tm.assert_series_equal(got, expect, check_names=False)

    def test_df_arithmetic_subexpression(self):
        df = DataFrame({"a": np.random.randn(10), "b": np.random.randn(10)})
        df.eval("e = sin(a + b)", engine=self.engine, parser=self.parser, inplace=True)
        got = df.e
        expect = np.sin(df.a + df.b)
        tm.assert_series_equal(got, expect, check_names=False)

    def check_result_type(self, dtype, expect_dtype):
        df = DataFrame({"a": np.random.randn(10).astype(dtype)})
        assert df.a.dtype == dtype
        df.eval("b = sin(a)", engine=self.engine, parser=self.parser, inplace=True)
        got = df.b
        expect = np.sin(df.a)
        assert expect.dtype == got.dtype
        assert expect_dtype == got.dtype
        tm.assert_series_equal(got, expect, check_names=False)

    def test_result_types(self):
        self.check_result_type(np.int32, np.float64)
        self.check_result_type(np.int64, np.float64)
        self.check_result_type(np.float32, np.float32)
        self.check_result_type(np.float64, np.float64)

    @td.skip_if_windows
    def test_result_complex128(self):
        # xref https://github.com/pandas-dev/pandas/issues/12293
        #  this fails on Windows, apparently a floating point precision issue

        # Did not test complex64 because DataFrame is converting it to
        # complex128. Due to https://github.com/pandas-dev/pandas/issues/10952
        self.check_result_type(np.complex128, np.complex128)

    def test_undefined_func(self):
        df = DataFrame({"a": np.random.randn(10)})
        msg = '"mysin" is not a supported function'

        with pytest.raises(ValueError, match=msg):
            df.eval("mysin(a)", engine=self.engine, parser=self.parser)

    def test_keyword_arg(self):
        df = DataFrame({"a": np.random.randn(10)})
        msg = 'Function "sin" does not support keyword arguments'

        with pytest.raises(TypeError, match=msg):
            df.eval("sin(x=a)", engine=self.engine, parser=self.parser)


class TestMathPythonPandas(TestMathPythonPython):
    engine = "python"
    parser = "pandas"


class TestMathNumExprPandas(TestMathPythonPython):
    engine = "numexpr"
    parser = "pandas"


class TestMathNumExprPython(TestMathPythonPython):
    engine = "numexpr"
    parser = "python"


_var_s = np.random.randn(10)


class TestScope:
    def test_global_scope(self, engine, parser):
        e = "_var_s * 2"
        tm.assert_numpy_array_equal(
            _var_s * 2, pd.eval(e, engine=engine, parser=parser)
        )

    def test_no_new_locals(self, engine, parser):
        x = 1
        lcls = locals().copy()
        pd.eval("x + 1", local_dict=lcls, engine=engine, parser=parser)
        lcls2 = locals().copy()
        lcls2.pop("lcls")
        assert lcls == lcls2

    def test_no_new_globals(self, engine, parser):
        x = 1  # noqa
        gbls = globals().copy()
        pd.eval("x + 1", engine=engine, parser=parser)
        gbls2 = globals().copy()
        assert gbls == gbls2


@td.skip_if_no_ne
def test_invalid_engine():
    msg = "Invalid engine 'asdf' passed"
    with pytest.raises(KeyError, match=msg):
        pd.eval("x + y", local_dict={"x": 1, "y": 2}, engine="asdf")


@td.skip_if_no_ne
def test_invalid_parser():
    msg = "Invalid parser 'asdf' passed"
    with pytest.raises(KeyError, match=msg):
        pd.eval("x + y", local_dict={"x": 1, "y": 2}, parser="asdf")


_parsers: Dict[str, Type[BaseExprVisitor]] = {
    "python": PythonExprVisitor,
    "pytables": pytables.PyTablesExprVisitor,
    "pandas": PandasExprVisitor,
}


@pytest.mark.parametrize("engine", ENGINES)
@pytest.mark.parametrize("parser", _parsers)
def test_disallowed_nodes(engine, parser):
    VisitorClass = _parsers[parser]
    inst = VisitorClass("x + 1", engine, parser)

    for ops in VisitorClass.unsupported_nodes:

        msg = "nodes are not implemented"
        with pytest.raises(NotImplementedError, match=msg):
            getattr(inst, ops)()


def test_syntax_error_exprs(engine, parser):
    e = "s +"
    with pytest.raises(SyntaxError, match="invalid syntax"):
        pd.eval(e, engine=engine, parser=parser)


def test_name_error_exprs(engine, parser):
    e = "s + t"
    msg = "name 's' is not defined"
    with pytest.raises(NameError, match=msg):
        pd.eval(e, engine=engine, parser=parser)


@pytest.mark.parametrize("express", ["a + @b", "@a + b", "@a + @b"])
def test_invalid_local_variable_reference(engine, parser, express):
    a, b = 1, 2  # noqa

    if parser != "pandas":
        with pytest.raises(SyntaxError, match="The '@' prefix is only"):
            pd.eval(express, engine=engine, parser=parser)
    else:
        with pytest.raises(SyntaxError, match="The '@' prefix is not"):
            pd.eval(express, engine=engine, parser=parser)


def test_numexpr_builtin_raises(engine, parser):
    sin, dotted_line = 1, 2
    if engine == "numexpr":
        msg = "Variables in expression .+"
        with pytest.raises(NumExprClobberingError, match=msg):
            pd.eval("sin + dotted_line", engine=engine, parser=parser)
    else:
        res = pd.eval("sin + dotted_line", engine=engine, parser=parser)
        assert res == sin + dotted_line


def test_bad_resolver_raises(engine, parser):
    cannot_resolve = 42, 3.0
    with pytest.raises(TypeError, match="Resolver of type .+"):
        pd.eval("1 + 2", resolvers=cannot_resolve, engine=engine, parser=parser)


def test_empty_string_raises(engine, parser):
    # GH 13139
    with pytest.raises(ValueError, match="expr cannot be an empty string"):
        pd.eval("", engine=engine, parser=parser)


def test_more_than_one_expression_raises(engine, parser):
    with pytest.raises(SyntaxError, match=("only a single expression is allowed")):
        pd.eval("1 + 1; 2 + 2", engine=engine, parser=parser)


@pytest.mark.parametrize("cmp", ("and", "or"))
@pytest.mark.parametrize("lhs", (int, float))
@pytest.mark.parametrize("rhs", (int, float))
def test_bool_ops_fails_on_scalars(lhs, cmp, rhs, engine, parser):
    gen = {int: lambda: np.random.randint(10), float: np.random.randn}

    mid = gen[lhs]()  # noqa
    lhs = gen[lhs]()
    rhs = gen[rhs]()

    ex1 = f"lhs {cmp} mid {cmp} rhs"
    ex2 = f"lhs {cmp} mid and mid {cmp} rhs"
    ex3 = f"(lhs {cmp} mid) & (mid {cmp} rhs)"
    for ex in (ex1, ex2, ex3):
        msg = "cannot evaluate scalar only bool ops|'BoolOp' nodes are not"
        with pytest.raises(NotImplementedError, match=msg):
            pd.eval(ex, engine=engine, parser=parser)


@pytest.mark.parametrize(
    "other",
    [
        "'x'",
        pytest.param(
            "...", marks=pytest.mark.xfail(not compat.PY38, reason="GH-28116")
        ),
    ],
)
def test_equals_various(other):
    df = DataFrame({"A": ["a", "b", "c"]})
    result = df.eval(f"A == {other}")
    expected = Series([False, False, False], name="A")
    if USE_NUMEXPR:
        # https://github.com/pandas-dev/pandas/issues/10239
        # lose name with numexpr engine. Remove when that's fixed.
        expected.name = None
    tm.assert_series_equal(result, expected)


def test_inf(engine, parser):
    s = "inf + 1"
    expected = np.inf
    result = pd.eval(s, engine=engine, parser=parser)
    assert result == expected


def test_truediv_deprecated(engine, parser):
    # GH#29182
    match = "The `truediv` parameter in pd.eval is deprecated"

    with tm.assert_produces_warning(FutureWarning) as m:
        pd.eval("1+1", engine=engine, parser=parser, truediv=True)

    assert len(m) == 1
    assert match in str(m[0].message)

    with tm.assert_produces_warning(FutureWarning) as m:
        pd.eval("1+1", engine=engine, parser=parser, truediv=False)

    assert len(m) == 1
    assert match in str(m[0].message)


def test_negate_lt_eq_le(engine, parser):
    df = DataFrame([[0, 10], [1, 20]], columns=["cat", "count"])
    expected = df[~(df.cat > 0)]

    result = df.query("~(cat > 0)", engine=engine, parser=parser)
    tm.assert_frame_equal(result, expected)

    if parser == "python":
        msg = "'Not' nodes are not implemented"
        with pytest.raises(NotImplementedError, match=msg):
            df.query("not (cat > 0)", engine=engine, parser=parser)
    else:
        result = df.query("not (cat > 0)", engine=engine, parser=parser)
        tm.assert_frame_equal(result, expected)


class TestValidate:
    @pytest.mark.parametrize("value", [1, "True", [1, 2, 3], 5.0])
    def test_validate_bool_args(self, value):

        msg = 'For argument "inplace" expected type bool, received type'
        with pytest.raises(ValueError, match=msg):
            pd.eval("2+2", inplace=value)