#pragma once #include #include #include #include #include #include #include #ifdef __CUDACC__ #include // For CUDA_VERSION #endif #ifdef TEMPLATE_SELECTIVE_BUILD #include #else namespace at { /** * The method should_include_kernel_dtype() returns true/false * based on whether the switching code for a specific dtype should be * included based on build time constants generated from tracing model * execution. This method will be implmeneted via code-generation and * included in this file when code-gen is ready. */ inline constexpr bool should_include_kernel_dtype( const char* /*kernel_tag_str*/, at::ScalarType /*scalar_type*/ ) { return true; } } // namespace at #endif /** * In the Facebook internal build (using BUCK), this macro is enabled by * passing in -c pt.enable_record_kernel_dtype=1 when building the tracer * binary. */ #if defined ENABLE_RECORD_KERNEL_FUNCTION_DTYPE namespace at { namespace detail { TORCH_API void record_kernel_function_dtype(std::string name); } } // namespace at #define RECORD_KERNEL_FUNCTION_DTYPE(NAME, enum_type) \ at::detail::record_kernel_function_dtype( \ std::string(NAME) + "$" + toString(enum_type)); #else #define RECORD_KERNEL_FUNCTION_DTYPE(NAME, enum_type) #endif #define AT_PRIVATE_CHECK_SELECTIVE_BUILD(enum_type) \ do { \ if constexpr (!at::should_include_kernel_dtype( \ at_dispatch_name, enum_type)) { \ AT_ERROR( \ "dtype '", \ toString(enum_type), \ "' not selected for kernel tag ", \ at_dispatch_name); \ } \ } while (0) #define AT_PRIVATE_CASE_TYPE_USING_HINT(enum_type, HINT, ...) \ case enum_type: { \ AT_PRIVATE_CHECK_SELECTIVE_BUILD(enum_type); \ using HINT C10_UNUSED = c10::impl::ScalarTypeToCPPTypeT; \ return __VA_ARGS__(); \ } #define AT_DISPATCH_CASE(enum_type, ...) \ AT_PRIVATE_CASE_TYPE_USING_HINT(enum_type, scalar_t, __VA_ARGS__) #define AT_DISPATCH_CASE_QINT(enum_type, scalar_type, ...) \ case enum_type: { \ AT_PRIVATE_CHECK_SELECTIVE_BUILD(enum_type); \ using scalar_t = scalar_type; \ using underlying_t C10_UNUSED = typename scalar_t::underlying; \ const auto& SCALAR_TYPE C10_UNUSED = enum_type; \ const auto& UNDERLYING_TYPE C10_UNUSED = toUnderlying(enum_type); \ return __VA_ARGS__(); \ } #define AT_QINT_SUB_BYTE_PRIVATE_CASE_TYPE( \ enum_type, scalar_type, bitwidth, qmin, qmax, ...) \ case enum_type: { \ AT_PRIVATE_CHECK_SELECTIVE_BUILD(enum_type); \ using scalar_t = scalar_type; \ using underlying_t C10_UNUSED = typename scalar_t::underlying; \ const auto& SCALAR_TYPE C10_UNUSED = enum_type; \ const auto& UNDERLYING_TYPE C10_UNUSED = toUnderlying(enum_type); \ C10_UNUSED int bit_width = bitwidth; \ C10_UNUSED int64_t quant_min = qmin; \ C10_UNUSED int64_t quant_max = qmax; \ return __VA_ARGS__(); \ } namespace detail { inline at::ScalarType scalar_type(at::ScalarType s) { return s; } C10_DEPRECATED_MESSAGE( "passing at::DeprecatedTypeProperties to an AT_DISPATCH macro is deprecated, " "pass an at::ScalarType instead") inline at::ScalarType scalar_type(const at::DeprecatedTypeProperties& t) { return t.scalarType(); } C10_DEPRECATED_MESSAGE( "AT_DISPATCH_ALL_TYPES_AND_HALF is deprecated, " "use AT_DISPATCH_ALL_TYPES_AND(at::ScalarType::Half, ...) instead") inline void deprecated_AT_DISPATCH_ALL_TYPES_AND_HALF() {} C10_DEPRECATED_MESSAGE( "AT_DISPATCH_ALL_TYPES_AND_HALF_AND_COMPLEX is deprecated, " "use AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND(at::ScalarType::Half, ...) " "instead") inline void deprecated_AT_DISPATCH_ALL_TYPES_AND_HALF_AND_COMPLEX() {} } // namespace detail // The AT_DISPATCH_* family of macros provides the ability to // conveniently generate specializations of a kernel over all of the // dtypes we care about in PyTorch. We call it "dispatch" because // we are "dispatching" to the correct, dtype-specific kernel. // // A standard usage looks like: // // AT_DISPATCH_ALL_TYPES(self.scalar_type(), "op_name", [&] { // // Your code here, with 'scalar_t' now defined to // // be the dtype in question // }); // // There are many variations of this macro, so it's important to // understand exactly /which/ dtypes you want to get instantiated, as // well as what the "default" set is. // // The default set of dtypes that are instantiated (e.g., by // AT_DISPATCH_ALL_TYPES) are floating point types (float, double), // and integral types (int32_t, int64_t, int16_t, int8_t, uint8_t), // but NOT booleans (bool), half-precision floats (Half) or // complex number (c10::complex, c10::complex). // This "cut" is somewhat historical (the default types are the // ones that TH historically supported), but it also reflects the // fact that the non-default types are "poorly" behaved (booleans // are NOT integers mod 2, half precision operations ~essentially // don't exist on CPU, complex numbers are an experimental application). // // Here are the questions you should generally ask to decide which // dispatch you want: // // 1. Is this an integral or floating point specific operation? // (If so, you'll want one of the FLOATING or INTEGRAL macros.) // // 2. Should half be supported? (If you're on CPU, the answer is almost // definitely no. If you do want support, use one of the AND_HALF // macros) // // Much rarer situations: // // 3. Should bool be supported? (You often have to write your kernel // differently if arithmetic operations are involved.) If so, // Use AT_DISPATCH_ALL_TYPES_AND along with ScalarType::Bool // // 4. Should complex be supported? The answer is almost always no, // unless you are working on "generic" code that should work on // all dtypes. // // Parameters: // ----------- // // 1. The NAME argument is a "tag" that is used to trace and then // conditionally compile fragments of the case statements such // that the kernel functions are specialized only for the dtypes // that are needed. The NAME parameter *must* be a build time // const char* (can't be std::string, etc...) // // Please ensure that the NAME is unique for every implementation // or you run the risk of over-including code for the kernel // functions. There is no risk of missing out on any code, so // it's mostly a risk of a Type-2 error, and not a Type-1 error. // // Switch-like syntax: // ------------------- // There is also a switch-case like syntax which is useful if a kernel // needs to be specialized for particular scalar types // // AT_DISPATCH_SWITCH(self.scalar_type(), "op_name", // AT_DISPATCH_CASE_INTEGRAL_TYPES([&] { // op_integral(iter); // }) // AT_DISPATCH_CASE_FLOATING_TYPES([&] { // op_floating(iter); // }) // AT_DISPATCH_CASE(kBool, [&] { // op_bool(iter); // }) // ); // // For each AT_DISPATCH_FOO macro, there is a corresponding // AT_DISPATCH_CASE_FOO macro which can be used inside of an // AT_DISPATCH_SWITCH block. // NB: the the_type variable is not used, but we have kept it for // backwards compatibility. It's probably not used by anyone though; // but we're just being safe (and it doesn't hurt.) Note we must // use it to shut up warnings about unused store. #define AT_DISPATCH_SWITCH(TYPE, NAME, ...) \ [&] { \ const auto& the_type = TYPE; \ constexpr const char* at_dispatch_name = NAME; \ /* don't use TYPE again in case it is an expensive or side-effect op */ \ at::ScalarType _st = ::detail::scalar_type(the_type); \ RECORD_KERNEL_FUNCTION_DTYPE(at_dispatch_name, _st); \ switch (_st) { \ __VA_ARGS__ \ default: \ AT_ERROR( \ '"', \ at_dispatch_name, \ "\" not implemented for '", \ toString(_st), \ "'"); \ } \ }() #define AT_DISPATCH_CASE_FLOATING_TYPES(...) \ AT_DISPATCH_CASE(at::ScalarType::Double, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) #define AT_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_TYPES_AND_HALF(...) \ AT_DISPATCH_CASE(at::ScalarType::Double, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) #define AT_DISPATCH_FLOATING_TYPES_AND_HALF(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, NAME, AT_DISPATCH_CASE_FLOATING_TYPES_AND_HALF(__VA_ARGS__)) #define AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES(...) \ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) #define AT_DISPATCH_REDUCED_FLOATING_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, NAME, AT_DISPATCH_CASE_REDUCED_FLOATING_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_TYPES_AND(SCALARTYPE, ...) \ AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__) #define AT_DISPATCH_FLOATING_TYPES_AND(SCALARTYPE, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_TYPES_AND(SCALARTYPE, __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_TYPES_AND2(SCALARTYPE1, SCALARTYPE2, ...) \ AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) #define AT_DISPATCH_FLOATING_TYPES_AND2( \ SCALARTYPE1, SCALARTYPE2, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_TYPES_AND2( \ SCALARTYPE1, SCALARTYPE2, __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, ...) \ AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) #define AT_DISPATCH_FLOATING_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_TYPES_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, ...) \ AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) #define AT_DISPATCH_FLOATING_TYPES_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_TYPES_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, __VA_ARGS__)) #define AT_DISPATCH_CASE_COMPLEX_TYPES(...) \ AT_DISPATCH_CASE(at::ScalarType::ComplexDouble, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::ComplexFloat, __VA_ARGS__) #define AT_DISPATCH_COMPLEX_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_COMPLEX_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_COMPLEX_TYPES_AND(SCALARTYPE, ...) \ AT_DISPATCH_CASE_COMPLEX_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__) #define AT_DISPATCH_COMPLEX_TYPES_AND(SCALARTYPE, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, NAME, AT_DISPATCH_CASE_COMPLEX_TYPES_AND(SCALARTYPE, __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES(...) \ AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE_COMPLEX_TYPES(__VA_ARGS__) #define AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, NAME, AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND1(SCALARTYPE, ...) \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__) #define AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND1( \ SCALARTYPE, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND1( \ SCALARTYPE, __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND2( \ SCALARTYPE1, SCALARTYPE2, ...) \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) #define AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2( \ SCALARTYPE1, SCALARTYPE2, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND2( \ SCALARTYPE1, SCALARTYPE2, __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, ...) \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) #define AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, ...) \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) #define AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND5( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, SCALARTYPE5, ...) \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE5, __VA_ARGS__) #define AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND5( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ TYPE, \ NAME, \ ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND5( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ __VA_ARGS__)) #define AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND6( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ ...) \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE5, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE6, __VA_ARGS__) #define AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND6( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ TYPE, \ NAME, \ ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_FLOATING_AND_COMPLEX_TYPES_AND6( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ __VA_ARGS__)) #define AT_DISPATCH_CASE_INTEGRAL_TYPES(...) \ AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) #define AT_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_INTEGRAL_TYPES_AND(SCALARTYPE, ...) \ AT_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__) #define AT_DISPATCH_INTEGRAL_TYPES_AND(SCALARTYPE, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_INTEGRAL_TYPES_AND(SCALARTYPE, __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES(...) \ AT_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__) #define AT_DISPATCH_ALL_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_ALL_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_QINT_TYPES(...) \ AT_DISPATCH_CASE_QINT(at::kQInt8, at::qint8, __VA_ARGS__) \ AT_DISPATCH_CASE_QINT(at::kQUInt8, at::quint8, __VA_ARGS__) \ AT_DISPATCH_CASE_QINT(at::kQInt32, at::qint32, __VA_ARGS__) #define AT_DISPATCH_QINT_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_QINT_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_QINT_TYPES_AND(SCALARTYPE, ...) \ AT_DISPATCH_CASE_QINT_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__) #define AT_DISPATCH_QINT_TYPES_AND(SCALARTYPE, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, NAME, AT_DISPATCH_CASE_QINT_TYPES_AND(SCALARTYPE, __VA_ARGS__)) #define AT_DISPATCH_CASE_QINT_BYTE_TYPES(...) \ AT_DISPATCH_CASE_QINT(at::kQInt8, at::qint8, __VA_ARGS__) \ AT_DISPATCH_CASE_QINT(at::kQUInt8, at::quint8, __VA_ARGS__) #define AT_DISPATCH_QINT_BYTE_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_QINT_BYTE_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_QINT_AND_SUB_BYTE_TYPES(...) \ AT_QINT_SUB_BYTE_PRIVATE_CASE_TYPE( \ at::kQInt8, at::qint8, CHAR_BIT, SCHAR_MIN, SCHAR_MAX, __VA_ARGS__) \ AT_QINT_SUB_BYTE_PRIVATE_CASE_TYPE( \ at::kQUInt8, at::quint8, CHAR_BIT, 0, UCHAR_MAX, __VA_ARGS__) \ AT_QINT_SUB_BYTE_PRIVATE_CASE_TYPE( \ at::kQInt32, \ at::qint32, \ CHAR_BIT * sizeof(int), \ INT_MIN, \ INT_MAX, \ __VA_ARGS__) \ AT_QINT_SUB_BYTE_PRIVATE_CASE_TYPE( \ at::kQUInt4x2, at::quint4x2, 4, 0, 15, __VA_ARGS__) \ AT_QINT_SUB_BYTE_PRIVATE_CASE_TYPE( \ at::kQUInt2x4, at::quint2x4, 2, 0, 3, __VA_ARGS__) #define AT_DISPATCH_QINT_AND_SUB_BYTE_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, NAME, AT_DISPATCH_CASE_QINT_AND_SUB_BYTE_TYPES(__VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(...) \ AT_DISPATCH_CASE_ALL_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE_COMPLEX_TYPES(__VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, NAME, AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND(SCALARTYPE, ...) \ AT_DISPATCH_CASE_ALL_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND(SCALARTYPE, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, NAME, AT_DISPATCH_CASE_ALL_TYPES_AND(SCALARTYPE, __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND(SCALARTYPE, ...) \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND(SCALARTYPE, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND(SCALARTYPE, __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND2(SCALARTYPE1, SCALARTYPE2, ...) \ AT_DISPATCH_CASE_ALL_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND2(SCALARTYPE1, SCALARTYPE2, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND2(SCALARTYPE1, SCALARTYPE2, __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND2( \ SCALARTYPE1, SCALARTYPE2, ...) \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND2( \ SCALARTYPE1, SCALARTYPE2, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND2( \ SCALARTYPE1, SCALARTYPE2, __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, ...) \ AT_DISPATCH_CASE_ALL_TYPES(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, ...) \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND3( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, ...) \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND4( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND5( \ SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, SCALARTYPE4, SCALARTYPE5, ...) \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE5, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND5( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ TYPE, \ NAME, \ ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND5( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND6( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ ...) \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE5, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE6, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND6( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ TYPE, \ NAME, \ ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND6( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND7( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ SCALARTYPE7, \ ...) \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE5, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE6, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE7, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND7( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ SCALARTYPE7, \ TYPE, \ NAME, \ ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND7( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ SCALARTYPE7, \ __VA_ARGS__)) #define AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND8( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ SCALARTYPE7, \ SCALARTYPE8, \ ...) \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX(__VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE1, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE2, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE3, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE4, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE5, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE6, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE7, __VA_ARGS__) \ AT_DISPATCH_CASE(SCALARTYPE8, __VA_ARGS__) #define AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND8( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ SCALARTYPE7, \ SCALARTYPE8, \ TYPE, \ NAME, \ ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND_COMPLEX_AND8( \ SCALARTYPE1, \ SCALARTYPE2, \ SCALARTYPE3, \ SCALARTYPE4, \ SCALARTYPE5, \ SCALARTYPE6, \ SCALARTYPE7, \ SCALARTYPE8, \ __VA_ARGS__)) #define AT_DISPATCH_CASE_BIT_TYPES(...) \ AT_DISPATCH_CASE(at::ScalarType::Bits1x8, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Bits2x4, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Bits4x2, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Bits8, __VA_ARGS__) \ AT_DISPATCH_CASE(at::ScalarType::Bits16, __VA_ARGS__) #define AT_DISPATCH_BIT_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_BIT_TYPES(__VA_ARGS__)) #define AT_DISPATCH_INDEX_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_PRIVATE_CASE_TYPE_USING_HINT( \ at::ScalarType::Int, index_t, __VA_ARGS__) \ AT_PRIVATE_CASE_TYPE_USING_HINT( \ at::ScalarType::Long, index_t, __VA_ARGS__)) // ---------------------------------------------------------------------------- // DEPRECATED MACROS, DON'T USE THESE // ---------------------------------------------------------------------------- #define AT_DISPATCH_ALL_TYPES_AND_HALF(TYPE, NAME, ...) \ detail::deprecated_AT_DISPATCH_ALL_TYPES_AND_HALF(); \ AT_DISPATCH_SWITCH( \ TYPE, \ NAME, \ AT_DISPATCH_CASE_ALL_TYPES_AND(at::ScalarType::Half, __VA_ARGS__))