949 lines
34 KiB
Python
949 lines
34 KiB
Python
"""
|
|
This class is defined to override standard pickle functionality
|
|
|
|
The goals of it follow:
|
|
-Serialize lambdas and nested functions to compiled byte code
|
|
-Deal with main module correctly
|
|
-Deal with other non-serializable objects
|
|
|
|
It does not include an unpickler, as standard python unpickling suffices.
|
|
|
|
This module was extracted from the `cloud` package, developed by `PiCloud, Inc.
|
|
<https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
|
|
|
|
Copyright (c) 2012, Regents of the University of California.
|
|
Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
|
|
All rights reserved.
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
modification, are permitted provided that the following conditions
|
|
are met:
|
|
* Redistributions of source code must retain the above copyright
|
|
notice, this list of conditions and the following disclaimer.
|
|
* Redistributions in binary form must reproduce the above copyright
|
|
notice, this list of conditions and the following disclaimer in the
|
|
documentation and/or other materials provided with the distribution.
|
|
* Neither the name of the University of California, Berkeley nor the
|
|
names of its contributors may be used to endorse or promote
|
|
products derived from this software without specific prior written
|
|
permission.
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
|
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
"""
|
|
|
|
import builtins
|
|
import dis
|
|
import opcode
|
|
import platform
|
|
import sys
|
|
import types
|
|
import weakref
|
|
import uuid
|
|
import threading
|
|
import typing
|
|
import warnings
|
|
|
|
from .compat import pickle
|
|
from collections import OrderedDict
|
|
from typing import ClassVar, Generic, Union, Tuple, Callable
|
|
from pickle import _getattribute
|
|
from importlib._bootstrap import _find_spec
|
|
|
|
try: # pragma: no branch
|
|
import typing_extensions as _typing_extensions
|
|
from typing_extensions import Literal, Final
|
|
except ImportError:
|
|
_typing_extensions = Literal = Final = None
|
|
|
|
if sys.version_info >= (3, 8):
|
|
from types import CellType
|
|
else:
|
|
def f():
|
|
a = 1
|
|
|
|
def g():
|
|
return a
|
|
return g
|
|
CellType = type(f().__closure__[0])
|
|
|
|
|
|
# cloudpickle is meant for inter process communication: we expect all
|
|
# communicating processes to run the same Python version hence we favor
|
|
# communication speed over compatibility:
|
|
DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
|
|
|
|
# Names of modules whose resources should be treated as dynamic.
|
|
_PICKLE_BY_VALUE_MODULES = set()
|
|
|
|
# Track the provenance of reconstructed dynamic classes to make it possible to
|
|
# reconstruct instances from the matching singleton class definition when
|
|
# appropriate and preserve the usual "isinstance" semantics of Python objects.
|
|
_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary()
|
|
_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary()
|
|
_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock()
|
|
|
|
PYPY = platform.python_implementation() == "PyPy"
|
|
|
|
builtin_code_type = None
|
|
if PYPY:
|
|
# builtin-code objects only exist in pypy
|
|
builtin_code_type = type(float.__new__.__code__)
|
|
|
|
_extract_code_globals_cache = weakref.WeakKeyDictionary()
|
|
|
|
|
|
def _get_or_create_tracker_id(class_def):
|
|
with _DYNAMIC_CLASS_TRACKER_LOCK:
|
|
class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def)
|
|
if class_tracker_id is None:
|
|
class_tracker_id = uuid.uuid4().hex
|
|
_DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
|
|
_DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def
|
|
return class_tracker_id
|
|
|
|
|
|
def _lookup_class_or_track(class_tracker_id, class_def):
|
|
if class_tracker_id is not None:
|
|
with _DYNAMIC_CLASS_TRACKER_LOCK:
|
|
class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault(
|
|
class_tracker_id, class_def)
|
|
_DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
|
|
return class_def
|
|
|
|
|
|
def register_pickle_by_value(module):
|
|
"""Register a module to make it functions and classes picklable by value.
|
|
|
|
By default, functions and classes that are attributes of an importable
|
|
module are to be pickled by reference, that is relying on re-importing
|
|
the attribute from the module at load time.
|
|
|
|
If `register_pickle_by_value(module)` is called, all its functions and
|
|
classes are subsequently to be pickled by value, meaning that they can
|
|
be loaded in Python processes where the module is not importable.
|
|
|
|
This is especially useful when developing a module in a distributed
|
|
execution environment: restarting the client Python process with the new
|
|
source code is enough: there is no need to re-install the new version
|
|
of the module on all the worker nodes nor to restart the workers.
|
|
|
|
Note: this feature is considered experimental. See the cloudpickle
|
|
README.md file for more details and limitations.
|
|
"""
|
|
if not isinstance(module, types.ModuleType):
|
|
raise ValueError(
|
|
f"Input should be a module object, got {str(module)} instead"
|
|
)
|
|
# In the future, cloudpickle may need a way to access any module registered
|
|
# for pickling by value in order to introspect relative imports inside
|
|
# functions pickled by value. (see
|
|
# https://github.com/cloudpipe/cloudpickle/pull/417#issuecomment-873684633).
|
|
# This access can be ensured by checking that module is present in
|
|
# sys.modules at registering time and assuming that it will still be in
|
|
# there when accessed during pickling. Another alternative would be to
|
|
# store a weakref to the module. Even though cloudpickle does not implement
|
|
# this introspection yet, in order to avoid a possible breaking change
|
|
# later, we still enforce the presence of module inside sys.modules.
|
|
if module.__name__ not in sys.modules:
|
|
raise ValueError(
|
|
f"{module} was not imported correctly, have you used an "
|
|
f"`import` statement to access it?"
|
|
)
|
|
_PICKLE_BY_VALUE_MODULES.add(module.__name__)
|
|
|
|
|
|
def unregister_pickle_by_value(module):
|
|
"""Unregister that the input module should be pickled by value."""
|
|
if not isinstance(module, types.ModuleType):
|
|
raise ValueError(
|
|
f"Input should be a module object, got {str(module)} instead"
|
|
)
|
|
if module.__name__ not in _PICKLE_BY_VALUE_MODULES:
|
|
raise ValueError(f"{module} is not registered for pickle by value")
|
|
else:
|
|
_PICKLE_BY_VALUE_MODULES.remove(module.__name__)
|
|
|
|
|
|
def list_registry_pickle_by_value():
|
|
return _PICKLE_BY_VALUE_MODULES.copy()
|
|
|
|
|
|
def _is_registered_pickle_by_value(module):
|
|
module_name = module.__name__
|
|
if module_name in _PICKLE_BY_VALUE_MODULES:
|
|
return True
|
|
while True:
|
|
parent_name = module_name.rsplit(".", 1)[0]
|
|
if parent_name == module_name:
|
|
break
|
|
if parent_name in _PICKLE_BY_VALUE_MODULES:
|
|
return True
|
|
module_name = parent_name
|
|
return False
|
|
|
|
|
|
def _whichmodule(obj, name):
|
|
"""Find the module an object belongs to.
|
|
|
|
This function differs from ``pickle.whichmodule`` in two ways:
|
|
- it does not mangle the cases where obj's module is __main__ and obj was
|
|
not found in any module.
|
|
- Errors arising during module introspection are ignored, as those errors
|
|
are considered unwanted side effects.
|
|
"""
|
|
if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa
|
|
# Workaround bug in old Python versions: prior to Python 3.7,
|
|
# T.__module__ would always be set to "typing" even when the TypeVar T
|
|
# would be defined in a different module.
|
|
if name is not None and getattr(typing, name, None) is obj:
|
|
# Built-in TypeVar defined in typing such as AnyStr
|
|
return 'typing'
|
|
else:
|
|
# User defined or third-party TypeVar: __module__ attribute is
|
|
# irrelevant, thus trigger a exhaustive search for obj in all
|
|
# modules.
|
|
module_name = None
|
|
else:
|
|
module_name = getattr(obj, '__module__', None)
|
|
|
|
if module_name is not None:
|
|
return module_name
|
|
# Protect the iteration by using a copy of sys.modules against dynamic
|
|
# modules that trigger imports of other modules upon calls to getattr or
|
|
# other threads importing at the same time.
|
|
for module_name, module in sys.modules.copy().items():
|
|
# Some modules such as coverage can inject non-module objects inside
|
|
# sys.modules
|
|
if (
|
|
module_name == '__main__' or
|
|
module is None or
|
|
not isinstance(module, types.ModuleType)
|
|
):
|
|
continue
|
|
try:
|
|
if _getattribute(module, name)[0] is obj:
|
|
return module_name
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
|
|
def _should_pickle_by_reference(obj, name=None):
|
|
"""Test whether an function or a class should be pickled by reference
|
|
|
|
Pickling by reference means by that the object (typically a function or a
|
|
class) is an attribute of a module that is assumed to be importable in the
|
|
target Python environment. Loading will therefore rely on importing the
|
|
module and then calling `getattr` on it to access the function or class.
|
|
|
|
Pickling by reference is the only option to pickle functions and classes
|
|
in the standard library. In cloudpickle the alternative option is to
|
|
pickle by value (for instance for interactively or locally defined
|
|
functions and classes or for attributes of modules that have been
|
|
explicitly registered to be pickled by value.
|
|
"""
|
|
if isinstance(obj, types.FunctionType) or issubclass(type(obj), type):
|
|
module_and_name = _lookup_module_and_qualname(obj, name=name)
|
|
if module_and_name is None:
|
|
return False
|
|
module, name = module_and_name
|
|
return not _is_registered_pickle_by_value(module)
|
|
|
|
elif isinstance(obj, types.ModuleType):
|
|
# We assume that sys.modules is primarily used as a cache mechanism for
|
|
# the Python import machinery. Checking if a module has been added in
|
|
# is sys.modules therefore a cheap and simple heuristic to tell us
|
|
# whether we can assume that a given module could be imported by name
|
|
# in another Python process.
|
|
if _is_registered_pickle_by_value(obj):
|
|
return False
|
|
return obj.__name__ in sys.modules
|
|
else:
|
|
raise TypeError(
|
|
"cannot check importability of {} instances".format(
|
|
type(obj).__name__)
|
|
)
|
|
|
|
|
|
def _lookup_module_and_qualname(obj, name=None):
|
|
if name is None:
|
|
name = getattr(obj, '__qualname__', None)
|
|
if name is None: # pragma: no cover
|
|
# This used to be needed for Python 2.7 support but is probably not
|
|
# needed anymore. However we keep the __name__ introspection in case
|
|
# users of cloudpickle rely on this old behavior for unknown reasons.
|
|
name = getattr(obj, '__name__', None)
|
|
|
|
module_name = _whichmodule(obj, name)
|
|
|
|
if module_name is None:
|
|
# In this case, obj.__module__ is None AND obj was not found in any
|
|
# imported module. obj is thus treated as dynamic.
|
|
return None
|
|
|
|
if module_name == "__main__":
|
|
return None
|
|
|
|
# Note: if module_name is in sys.modules, the corresponding module is
|
|
# assumed importable at unpickling time. See #357
|
|
module = sys.modules.get(module_name, None)
|
|
if module is None:
|
|
# The main reason why obj's module would not be imported is that this
|
|
# module has been dynamically created, using for example
|
|
# types.ModuleType. The other possibility is that module was removed
|
|
# from sys.modules after obj was created/imported. But this case is not
|
|
# supported, as the standard pickle does not support it either.
|
|
return None
|
|
|
|
try:
|
|
obj2, parent = _getattribute(module, name)
|
|
except AttributeError:
|
|
# obj was not found inside the module it points to
|
|
return None
|
|
if obj2 is not obj:
|
|
return None
|
|
return module, name
|
|
|
|
|
|
def _extract_code_globals(co):
|
|
"""
|
|
Find all globals names read or written to by codeblock co
|
|
"""
|
|
out_names = _extract_code_globals_cache.get(co)
|
|
if out_names is None:
|
|
# We use a dict with None values instead of a set to get a
|
|
# deterministic order (assuming Python 3.6+) and avoid introducing
|
|
# non-deterministic pickle bytes as a results.
|
|
out_names = {name: None for name in _walk_global_ops(co)}
|
|
|
|
# Declaring a function inside another one using the "def ..."
|
|
# syntax generates a constant code object corresponding to the one
|
|
# of the nested function's As the nested function may itself need
|
|
# global variables, we need to introspect its code, extract its
|
|
# globals, (look for code object in it's co_consts attribute..) and
|
|
# add the result to code_globals
|
|
if co.co_consts:
|
|
for const in co.co_consts:
|
|
if isinstance(const, types.CodeType):
|
|
out_names.update(_extract_code_globals(const))
|
|
|
|
_extract_code_globals_cache[co] = out_names
|
|
|
|
return out_names
|
|
|
|
|
|
def _find_imported_submodules(code, top_level_dependencies):
|
|
"""
|
|
Find currently imported submodules used by a function.
|
|
|
|
Submodules used by a function need to be detected and referenced for the
|
|
function to work correctly at depickling time. Because submodules can be
|
|
referenced as attribute of their parent package (``package.submodule``), we
|
|
need a special introspection technique that does not rely on GLOBAL-related
|
|
opcodes to find references of them in a code object.
|
|
|
|
Example:
|
|
```
|
|
import concurrent.futures
|
|
import cloudpickle
|
|
def func():
|
|
x = concurrent.futures.ThreadPoolExecutor
|
|
if __name__ == '__main__':
|
|
cloudpickle.dumps(func)
|
|
```
|
|
The globals extracted by cloudpickle in the function's state include the
|
|
concurrent package, but not its submodule (here, concurrent.futures), which
|
|
is the module used by func. Find_imported_submodules will detect the usage
|
|
of concurrent.futures. Saving this module alongside with func will ensure
|
|
that calling func once depickled does not fail due to concurrent.futures
|
|
not being imported
|
|
"""
|
|
|
|
subimports = []
|
|
# check if any known dependency is an imported package
|
|
for x in top_level_dependencies:
|
|
if (isinstance(x, types.ModuleType) and
|
|
hasattr(x, '__package__') and x.__package__):
|
|
# check if the package has any currently loaded sub-imports
|
|
prefix = x.__name__ + '.'
|
|
# A concurrent thread could mutate sys.modules,
|
|
# make sure we iterate over a copy to avoid exceptions
|
|
for name in list(sys.modules):
|
|
# Older versions of pytest will add a "None" module to
|
|
# sys.modules.
|
|
if name is not None and name.startswith(prefix):
|
|
# check whether the function can address the sub-module
|
|
tokens = set(name[len(prefix):].split('.'))
|
|
if not tokens - set(code.co_names):
|
|
subimports.append(sys.modules[name])
|
|
return subimports
|
|
|
|
|
|
def cell_set(cell, value):
|
|
"""Set the value of a closure cell.
|
|
|
|
The point of this function is to set the cell_contents attribute of a cell
|
|
after its creation. This operation is necessary in case the cell contains a
|
|
reference to the function the cell belongs to, as when calling the
|
|
function's constructor
|
|
``f = types.FunctionType(code, globals, name, argdefs, closure)``,
|
|
closure will not be able to contain the yet-to-be-created f.
|
|
|
|
In Python3.7, cell_contents is writeable, so setting the contents of a cell
|
|
can be done simply using
|
|
>>> cell.cell_contents = value
|
|
|
|
In earlier Python3 versions, the cell_contents attribute of a cell is read
|
|
only, but this limitation can be worked around by leveraging the Python 3
|
|
``nonlocal`` keyword.
|
|
|
|
In Python2 however, this attribute is read only, and there is no
|
|
``nonlocal`` keyword. For this reason, we need to come up with more
|
|
complicated hacks to set this attribute.
|
|
|
|
The chosen approach is to create a function with a STORE_DEREF opcode,
|
|
which sets the content of a closure variable. Typically:
|
|
|
|
>>> def inner(value):
|
|
... lambda: cell # the lambda makes cell a closure
|
|
... cell = value # cell is a closure, so this triggers a STORE_DEREF
|
|
|
|
(Note that in Python2, A STORE_DEREF can never be triggered from an inner
|
|
function. The function g for example here
|
|
>>> def f(var):
|
|
... def g():
|
|
... var += 1
|
|
... return g
|
|
|
|
will not modify the closure variable ``var```inplace, but instead try to
|
|
load a local variable var and increment it. As g does not assign the local
|
|
variable ``var`` any initial value, calling f(1)() will fail at runtime.)
|
|
|
|
Our objective is to set the value of a given cell ``cell``. So we need to
|
|
somewhat reference our ``cell`` object into the ``inner`` function so that
|
|
this object (and not the smoke cell of the lambda function) gets affected
|
|
by the STORE_DEREF operation.
|
|
|
|
In inner, ``cell`` is referenced as a cell variable (an enclosing variable
|
|
that is referenced by the inner function). If we create a new function
|
|
cell_set with the exact same code as ``inner``, but with ``cell`` marked as
|
|
a free variable instead, the STORE_DEREF will be applied on its closure -
|
|
``cell``, which we can specify explicitly during construction! The new
|
|
cell_set variable thus actually sets the contents of a specified cell!
|
|
|
|
Note: we do not make use of the ``nonlocal`` keyword to set the contents of
|
|
a cell in early python3 versions to limit possible syntax errors in case
|
|
test and checker libraries decide to parse the whole file.
|
|
"""
|
|
|
|
if sys.version_info[:2] >= (3, 7): # pragma: no branch
|
|
cell.cell_contents = value
|
|
else:
|
|
_cell_set = types.FunctionType(
|
|
_cell_set_template_code, {}, '_cell_set', (), (cell,),)
|
|
_cell_set(value)
|
|
|
|
|
|
def _make_cell_set_template_code():
|
|
def _cell_set_factory(value):
|
|
lambda: cell
|
|
cell = value
|
|
|
|
co = _cell_set_factory.__code__
|
|
|
|
_cell_set_template_code = types.CodeType(
|
|
co.co_argcount,
|
|
co.co_kwonlyargcount, # Python 3 only argument
|
|
co.co_nlocals,
|
|
co.co_stacksize,
|
|
co.co_flags,
|
|
co.co_code,
|
|
co.co_consts,
|
|
co.co_names,
|
|
co.co_varnames,
|
|
co.co_filename,
|
|
co.co_name,
|
|
co.co_firstlineno,
|
|
co.co_lnotab,
|
|
co.co_cellvars, # co_freevars is initialized with co_cellvars
|
|
(), # co_cellvars is made empty
|
|
)
|
|
return _cell_set_template_code
|
|
|
|
|
|
if sys.version_info[:2] < (3, 7):
|
|
_cell_set_template_code = _make_cell_set_template_code()
|
|
|
|
# relevant opcodes
|
|
STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
|
|
DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
|
|
LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
|
|
GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)
|
|
HAVE_ARGUMENT = dis.HAVE_ARGUMENT
|
|
EXTENDED_ARG = dis.EXTENDED_ARG
|
|
|
|
|
|
_BUILTIN_TYPE_NAMES = {}
|
|
for k, v in types.__dict__.items():
|
|
if type(v) is type:
|
|
_BUILTIN_TYPE_NAMES[v] = k
|
|
|
|
|
|
def _builtin_type(name):
|
|
if name == "ClassType": # pragma: no cover
|
|
# Backward compat to load pickle files generated with cloudpickle
|
|
# < 1.3 even if loading pickle files from older versions is not
|
|
# officially supported.
|
|
return type
|
|
return getattr(types, name)
|
|
|
|
|
|
def _walk_global_ops(code):
|
|
"""
|
|
Yield referenced name for all global-referencing instructions in *code*.
|
|
"""
|
|
for instr in dis.get_instructions(code):
|
|
op = instr.opcode
|
|
if op in GLOBAL_OPS:
|
|
yield instr.argval
|
|
|
|
|
|
def _extract_class_dict(cls):
|
|
"""Retrieve a copy of the dict of a class without the inherited methods"""
|
|
clsdict = dict(cls.__dict__) # copy dict proxy to a dict
|
|
if len(cls.__bases__) == 1:
|
|
inherited_dict = cls.__bases__[0].__dict__
|
|
else:
|
|
inherited_dict = {}
|
|
for base in reversed(cls.__bases__):
|
|
inherited_dict.update(base.__dict__)
|
|
to_remove = []
|
|
for name, value in clsdict.items():
|
|
try:
|
|
base_value = inherited_dict[name]
|
|
if value is base_value:
|
|
to_remove.append(name)
|
|
except KeyError:
|
|
pass
|
|
for name in to_remove:
|
|
clsdict.pop(name)
|
|
return clsdict
|
|
|
|
|
|
if sys.version_info[:2] < (3, 7): # pragma: no branch
|
|
def _is_parametrized_type_hint(obj):
|
|
# This is very cheap but might generate false positives. So try to
|
|
# narrow it down is good as possible.
|
|
type_module = getattr(type(obj), '__module__', None)
|
|
from_typing_extensions = type_module == 'typing_extensions'
|
|
from_typing = type_module == 'typing'
|
|
|
|
# general typing Constructs
|
|
is_typing = getattr(obj, '__origin__', None) is not None
|
|
|
|
# typing_extensions.Literal
|
|
is_literal = (
|
|
(getattr(obj, '__values__', None) is not None)
|
|
and from_typing_extensions
|
|
)
|
|
|
|
# typing_extensions.Final
|
|
is_final = (
|
|
(getattr(obj, '__type__', None) is not None)
|
|
and from_typing_extensions
|
|
)
|
|
|
|
# typing.ClassVar
|
|
is_classvar = (
|
|
(getattr(obj, '__type__', None) is not None) and from_typing
|
|
)
|
|
|
|
# typing.Union/Tuple for old Python 3.5
|
|
is_union = getattr(obj, '__union_params__', None) is not None
|
|
is_tuple = getattr(obj, '__tuple_params__', None) is not None
|
|
is_callable = (
|
|
getattr(obj, '__result__', None) is not None and
|
|
getattr(obj, '__args__', None) is not None
|
|
)
|
|
return any((is_typing, is_literal, is_final, is_classvar, is_union,
|
|
is_tuple, is_callable))
|
|
|
|
def _create_parametrized_type_hint(origin, args):
|
|
return origin[args]
|
|
else:
|
|
_is_parametrized_type_hint = None
|
|
_create_parametrized_type_hint = None
|
|
|
|
|
|
def parametrized_type_hint_getinitargs(obj):
|
|
# The distorted type check sematic for typing construct becomes:
|
|
# ``type(obj) is type(TypeHint)``, which means "obj is a
|
|
# parametrized TypeHint"
|
|
if type(obj) is type(Literal): # pragma: no branch
|
|
initargs = (Literal, obj.__values__)
|
|
elif type(obj) is type(Final): # pragma: no branch
|
|
initargs = (Final, obj.__type__)
|
|
elif type(obj) is type(ClassVar):
|
|
initargs = (ClassVar, obj.__type__)
|
|
elif type(obj) is type(Generic):
|
|
initargs = (obj.__origin__, obj.__args__)
|
|
elif type(obj) is type(Union):
|
|
initargs = (Union, obj.__args__)
|
|
elif type(obj) is type(Tuple):
|
|
initargs = (Tuple, obj.__args__)
|
|
elif type(obj) is type(Callable):
|
|
(*args, result) = obj.__args__
|
|
if len(args) == 1 and args[0] is Ellipsis:
|
|
args = Ellipsis
|
|
else:
|
|
args = list(args)
|
|
initargs = (Callable, (args, result))
|
|
else: # pragma: no cover
|
|
raise pickle.PicklingError(
|
|
f"Cloudpickle Error: Unknown type {type(obj)}"
|
|
)
|
|
return initargs
|
|
|
|
|
|
# Tornado support
|
|
|
|
def is_tornado_coroutine(func):
|
|
"""
|
|
Return whether *func* is a Tornado coroutine function.
|
|
Running coroutines are not supported.
|
|
"""
|
|
if 'tornado.gen' not in sys.modules:
|
|
return False
|
|
gen = sys.modules['tornado.gen']
|
|
if not hasattr(gen, "is_coroutine_function"):
|
|
# Tornado version is too old
|
|
return False
|
|
return gen.is_coroutine_function(func)
|
|
|
|
|
|
def _rebuild_tornado_coroutine(func):
|
|
from tornado import gen
|
|
return gen.coroutine(func)
|
|
|
|
|
|
# including pickles unloading functions in this namespace
|
|
load = pickle.load
|
|
loads = pickle.loads
|
|
|
|
|
|
def subimport(name):
|
|
# We cannot do simply: `return __import__(name)`: Indeed, if ``name`` is
|
|
# the name of a submodule, __import__ will return the top-level root module
|
|
# of this submodule. For instance, __import__('os.path') returns the `os`
|
|
# module.
|
|
__import__(name)
|
|
return sys.modules[name]
|
|
|
|
|
|
def dynamic_subimport(name, vars):
|
|
mod = types.ModuleType(name)
|
|
mod.__dict__.update(vars)
|
|
mod.__dict__['__builtins__'] = builtins.__dict__
|
|
return mod
|
|
|
|
|
|
def _gen_ellipsis():
|
|
return Ellipsis
|
|
|
|
|
|
def _gen_not_implemented():
|
|
return NotImplemented
|
|
|
|
|
|
def _get_cell_contents(cell):
|
|
try:
|
|
return cell.cell_contents
|
|
except ValueError:
|
|
# sentinel used by ``_fill_function`` which will leave the cell empty
|
|
return _empty_cell_value
|
|
|
|
|
|
def instance(cls):
|
|
"""Create a new instance of a class.
|
|
|
|
Parameters
|
|
----------
|
|
cls : type
|
|
The class to create an instance of.
|
|
|
|
Returns
|
|
-------
|
|
instance : cls
|
|
A new instance of ``cls``.
|
|
"""
|
|
return cls()
|
|
|
|
|
|
@instance
|
|
class _empty_cell_value:
|
|
"""sentinel for empty closures
|
|
"""
|
|
@classmethod
|
|
def __reduce__(cls):
|
|
return cls.__name__
|
|
|
|
|
|
def _fill_function(*args):
|
|
"""Fills in the rest of function data into the skeleton function object
|
|
|
|
The skeleton itself is create by _make_skel_func().
|
|
"""
|
|
if len(args) == 2:
|
|
func = args[0]
|
|
state = args[1]
|
|
elif len(args) == 5:
|
|
# Backwards compat for cloudpickle v0.4.0, after which the `module`
|
|
# argument was introduced
|
|
func = args[0]
|
|
keys = ['globals', 'defaults', 'dict', 'closure_values']
|
|
state = dict(zip(keys, args[1:]))
|
|
elif len(args) == 6:
|
|
# Backwards compat for cloudpickle v0.4.1, after which the function
|
|
# state was passed as a dict to the _fill_function it-self.
|
|
func = args[0]
|
|
keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
|
|
state = dict(zip(keys, args[1:]))
|
|
else:
|
|
raise ValueError(f'Unexpected _fill_value arguments: {args!r}')
|
|
|
|
# - At pickling time, any dynamic global variable used by func is
|
|
# serialized by value (in state['globals']).
|
|
# - At unpickling time, func's __globals__ attribute is initialized by
|
|
# first retrieving an empty isolated namespace that will be shared
|
|
# with other functions pickled from the same original module
|
|
# by the same CloudPickler instance and then updated with the
|
|
# content of state['globals'] to populate the shared isolated
|
|
# namespace with all the global variables that are specifically
|
|
# referenced for this function.
|
|
func.__globals__.update(state['globals'])
|
|
|
|
func.__defaults__ = state['defaults']
|
|
func.__dict__ = state['dict']
|
|
if 'annotations' in state:
|
|
func.__annotations__ = state['annotations']
|
|
if 'doc' in state:
|
|
func.__doc__ = state['doc']
|
|
if 'name' in state:
|
|
func.__name__ = state['name']
|
|
if 'module' in state:
|
|
func.__module__ = state['module']
|
|
if 'qualname' in state:
|
|
func.__qualname__ = state['qualname']
|
|
if 'kwdefaults' in state:
|
|
func.__kwdefaults__ = state['kwdefaults']
|
|
# _cloudpickle_subimports is a set of submodules that must be loaded for
|
|
# the pickled function to work correctly at unpickling time. Now that these
|
|
# submodules are depickled (hence imported), they can be removed from the
|
|
# object's state (the object state only served as a reference holder to
|
|
# these submodules)
|
|
if '_cloudpickle_submodules' in state:
|
|
state.pop('_cloudpickle_submodules')
|
|
|
|
cells = func.__closure__
|
|
if cells is not None:
|
|
for cell, value in zip(cells, state['closure_values']):
|
|
if value is not _empty_cell_value:
|
|
cell_set(cell, value)
|
|
|
|
return func
|
|
|
|
|
|
def _make_function(code, globals, name, argdefs, closure):
|
|
# Setting __builtins__ in globals is needed for nogil CPython.
|
|
globals["__builtins__"] = __builtins__
|
|
return types.FunctionType(code, globals, name, argdefs, closure)
|
|
|
|
|
|
def _make_empty_cell():
|
|
if False:
|
|
# trick the compiler into creating an empty cell in our lambda
|
|
cell = None
|
|
raise AssertionError('this route should not be executed')
|
|
|
|
return (lambda: cell).__closure__[0]
|
|
|
|
|
|
def _make_cell(value=_empty_cell_value):
|
|
cell = _make_empty_cell()
|
|
if value is not _empty_cell_value:
|
|
cell_set(cell, value)
|
|
return cell
|
|
|
|
|
|
def _make_skel_func(code, cell_count, base_globals=None):
|
|
""" Creates a skeleton function object that contains just the provided
|
|
code and the correct number of cells in func_closure. All other
|
|
func attributes (e.g. func_globals) are empty.
|
|
"""
|
|
# This function is deprecated and should be removed in cloudpickle 1.7
|
|
warnings.warn(
|
|
"A pickle file created using an old (<=1.4.1) version of cloudpickle "
|
|
"is currently being loaded. This is not supported by cloudpickle and "
|
|
"will break in cloudpickle 1.7", category=UserWarning
|
|
)
|
|
# This is backward-compatibility code: for cloudpickle versions between
|
|
# 0.5.4 and 0.7, base_globals could be a string or None. base_globals
|
|
# should now always be a dictionary.
|
|
if base_globals is None or isinstance(base_globals, str):
|
|
base_globals = {}
|
|
|
|
base_globals['__builtins__'] = __builtins__
|
|
|
|
closure = (
|
|
tuple(_make_empty_cell() for _ in range(cell_count))
|
|
if cell_count >= 0 else
|
|
None
|
|
)
|
|
return types.FunctionType(code, base_globals, None, None, closure)
|
|
|
|
|
|
def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
|
|
class_tracker_id, extra):
|
|
"""Build dynamic class with an empty __dict__ to be filled once memoized
|
|
|
|
If class_tracker_id is not None, try to lookup an existing class definition
|
|
matching that id. If none is found, track a newly reconstructed class
|
|
definition under that id so that other instances stemming from the same
|
|
class id will also reuse this class definition.
|
|
|
|
The "extra" variable is meant to be a dict (or None) that can be used for
|
|
forward compatibility shall the need arise.
|
|
"""
|
|
skeleton_class = types.new_class(
|
|
name, bases, {'metaclass': type_constructor},
|
|
lambda ns: ns.update(type_kwargs)
|
|
)
|
|
return _lookup_class_or_track(class_tracker_id, skeleton_class)
|
|
|
|
|
|
def _rehydrate_skeleton_class(skeleton_class, class_dict):
|
|
"""Put attributes from `class_dict` back on `skeleton_class`.
|
|
|
|
See CloudPickler.save_dynamic_class for more info.
|
|
"""
|
|
registry = None
|
|
for attrname, attr in class_dict.items():
|
|
if attrname == "_abc_impl":
|
|
registry = attr
|
|
else:
|
|
setattr(skeleton_class, attrname, attr)
|
|
if registry is not None:
|
|
for subclass in registry:
|
|
skeleton_class.register(subclass)
|
|
|
|
return skeleton_class
|
|
|
|
|
|
def _make_skeleton_enum(bases, name, qualname, members, module,
|
|
class_tracker_id, extra):
|
|
"""Build dynamic enum with an empty __dict__ to be filled once memoized
|
|
|
|
The creation of the enum class is inspired by the code of
|
|
EnumMeta._create_.
|
|
|
|
If class_tracker_id is not None, try to lookup an existing enum definition
|
|
matching that id. If none is found, track a newly reconstructed enum
|
|
definition under that id so that other instances stemming from the same
|
|
class id will also reuse this enum definition.
|
|
|
|
The "extra" variable is meant to be a dict (or None) that can be used for
|
|
forward compatibility shall the need arise.
|
|
"""
|
|
# enums always inherit from their base Enum class at the last position in
|
|
# the list of base classes:
|
|
enum_base = bases[-1]
|
|
metacls = enum_base.__class__
|
|
classdict = metacls.__prepare__(name, bases)
|
|
|
|
for member_name, member_value in members.items():
|
|
classdict[member_name] = member_value
|
|
enum_class = metacls.__new__(metacls, name, bases, classdict)
|
|
enum_class.__module__ = module
|
|
enum_class.__qualname__ = qualname
|
|
|
|
return _lookup_class_or_track(class_tracker_id, enum_class)
|
|
|
|
|
|
def _make_typevar(name, bound, constraints, covariant, contravariant,
|
|
class_tracker_id):
|
|
tv = typing.TypeVar(
|
|
name, *constraints, bound=bound,
|
|
covariant=covariant, contravariant=contravariant
|
|
)
|
|
if class_tracker_id is not None:
|
|
return _lookup_class_or_track(class_tracker_id, tv)
|
|
else: # pragma: nocover
|
|
# Only for Python 3.5.3 compat.
|
|
return tv
|
|
|
|
|
|
def _decompose_typevar(obj):
|
|
return (
|
|
obj.__name__, obj.__bound__, obj.__constraints__,
|
|
obj.__covariant__, obj.__contravariant__,
|
|
_get_or_create_tracker_id(obj),
|
|
)
|
|
|
|
|
|
def _typevar_reduce(obj):
|
|
# TypeVar instances require the module information hence why we
|
|
# are not using the _should_pickle_by_reference directly
|
|
module_and_name = _lookup_module_and_qualname(obj, name=obj.__name__)
|
|
|
|
if module_and_name is None:
|
|
return (_make_typevar, _decompose_typevar(obj))
|
|
elif _is_registered_pickle_by_value(module_and_name[0]):
|
|
return (_make_typevar, _decompose_typevar(obj))
|
|
|
|
return (getattr, module_and_name)
|
|
|
|
|
|
def _get_bases(typ):
|
|
if '__orig_bases__' in getattr(typ, '__dict__', {}):
|
|
# For generic types (see PEP 560)
|
|
# Note that simply checking `hasattr(typ, '__orig_bases__')` is not
|
|
# correct. Subclasses of a fully-parameterized generic class does not
|
|
# have `__orig_bases__` defined, but `hasattr(typ, '__orig_bases__')`
|
|
# will return True because it's defined in the base class.
|
|
bases_attr = '__orig_bases__'
|
|
else:
|
|
# For regular class objects
|
|
bases_attr = '__bases__'
|
|
return getattr(typ, bases_attr)
|
|
|
|
|
|
def _make_dict_keys(obj, is_ordered=False):
|
|
if is_ordered:
|
|
return OrderedDict.fromkeys(obj).keys()
|
|
else:
|
|
return dict.fromkeys(obj).keys()
|
|
|
|
|
|
def _make_dict_values(obj, is_ordered=False):
|
|
if is_ordered:
|
|
return OrderedDict((i, _) for i, _ in enumerate(obj)).values()
|
|
else:
|
|
return {i: _ for i, _ in enumerate(obj)}.values()
|
|
|
|
|
|
def _make_dict_items(obj, is_ordered=False):
|
|
if is_ordered:
|
|
return OrderedDict(obj).items()
|
|
else:
|
|
return obj.items()
|