# Copyright 2017 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """State management for eager execution.""" import collections import contextlib import copy import gc import itertools import os import random import threading from absl import logging import numpy as np from tensorflow.core.framework import function_pb2 from tensorflow.core.framework import graph_debug_info_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python import pywrap_tfe from tensorflow.python import tf2 from tensorflow.python.client import pywrap_tf_session from tensorflow.python.eager import cancellation from tensorflow.python.eager import execute from tensorflow.python.eager import executor from tensorflow.python.eager import monitoring from tensorflow.python.framework import c_api_util from tensorflow.python.framework import device as pydev from tensorflow.python.framework import tfrt_utils from tensorflow.python.util import compat from tensorflow.python.util import function_utils from tensorflow.python.util import is_in_graph_mode from tensorflow.python.util import tf_contextlib from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export from tensorflow.tsl.protobuf import coordination_config_pb2 # TODO(b/307794935): Remove after a solution is found. is_oss = True # updated by copybara GRAPH_MODE = 0 EAGER_MODE = 1 default_execution_mode = EAGER_MODE if tf2.enabled() else GRAPH_MODE # Cache from (old_device_name, partial_new_device_name) -> (new_device_name, # new_device_spec). # Note that we do not protect this with a lock and instead rely on python's GIL # and the idempotent nature of writes to provide thread safety. _device_parsing_cache = {} _starting_device_spec = pydev.DeviceSpec.from_string("") _MAXINT32 = 2**31 - 1 DEVICE_PLACEMENT_EXPLICIT = pywrap_tfe.TFE_DEVICE_PLACEMENT_EXPLICIT DEVICE_PLACEMENT_WARN = pywrap_tfe.TFE_DEVICE_PLACEMENT_WARN DEVICE_PLACEMENT_SILENT = pywrap_tfe.TFE_DEVICE_PLACEMENT_SILENT DEVICE_PLACEMENT_SILENT_FOR_INT32 = ( pywrap_tfe.TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32) SYNC = 0 ASYNC = 1 _KEEP_ALIVE_SECS = 600 _python_eager_context_create_counter = monitoring.Counter( "/tensorflow/api/python/eager_context_create_counter", "Counter for number of eager contexts created in Python.") # Re-exporting through context. is_tfrt_enabled = tfrt_utils.enabled # This flag and the associated environment var are transient and will eventually # be removed, once this experiment is enabled by default. _JIT_COMPILE_REWRITE_ENABLED = os.getenv("TF_JIT_COMPILE_REWRITE") == "1" def run_eager_op_as_function_enabled(): return True # This method should only be called after the context has beein initialized. def enable_jit_compile_rewrite(): """Run jit_compile functions through rewrite pass. This runs jit_compile functions through all of the multidevice function rewrite passes. """ global _JIT_COMPILE_REWRITE_ENABLED _JIT_COMPILE_REWRITE_ENABLED = True if context_safe() is not None: context_safe().jit_compile_rewrite = True # This method should only be called after the context has been initialized. def disable_jit_compile_rewrite(): global _JIT_COMPILE_REWRITE_ENABLED _JIT_COMPILE_REWRITE_ENABLED = False if context_safe() is not None: context_safe().jit_compile_rewrite = False def jit_compile_rewrite_enabled(): if context_safe() is not None: return context_safe().jit_compile_rewrite return _JIT_COMPILE_REWRITE_ENABLED # Expose it as internally public APIs for Keras use cases in b/171080602. tf_export("__internal__.is_tfrt_enabled", v1=[])(is_tfrt_enabled) class _EagerTensorCache(object): """Simple cache which evicts items based on length in a FIFO manner.""" __slots__ = ["_data", "_max_items", "_max_tensor_size"] def __init__(self, max_items=256, max_tensor_size=10000): self._data = collections.OrderedDict() self._max_items = max_items self._max_tensor_size = max_tensor_size def put(self, key, value): if value._num_elements() > self._max_tensor_size: # pylint: disable=protected-access return self._data[key] = value if len(self._data) > self._max_items: self._data.popitem(last=False) def get(self, key): return self._data.get(key, None) def flush(self): self._data.clear() class FunctionCallOptions: """Options applied at call sites of eager functions. Eager functions are functions decorated with tf.contrib.eager.defun. """ __slots__ = ["_config_proto_serialized", "_executor_type"] def __init__(self, executor_type=None, config_proto=None): """Constructor. Args: executor_type: (optional) name of the executor to be used to execute the eager function. If None or an empty string, the default Tensorflow executor will be used. config_proto: (optional) a `config_pb2.ConfigProto` proto or a serialized string of that proto. The config used by Grappler when optimizing the function graph. Each concrete function is optimized the first time is called. Changing config_proto after the first call has no effect. If config_proto is None, an empty RewriterConfig will be used. """ self.config_proto_serialized = config_proto self.executor_type = executor_type @property def executor_type(self): return self._executor_type @executor_type.setter def executor_type(self, executor_type): self._executor_type = executor_type @property def config_proto_serialized(self): return self._config_proto_serialized @config_proto_serialized.setter def config_proto_serialized(self, config): if isinstance(config, config_pb2.ConfigProto): self._config_proto_serialized = config.SerializeToString( deterministic=True) elif isinstance(config, str): self._config_proto_serialized = config elif config is None: self._config_proto_serialized = ( config_pb2.ConfigProto().SerializeToString()) else: raise ValueError("the rewriter config must be either a " "config_pb2.ConfigProto, or a serialized string of that " "proto or None. got: {}".format(type(config))) def as_attrs(self): if self.config_proto_serialized is None: config = function_utils.get_disabled_rewriter_config() else: config = self.config_proto_serialized executor_type = self.executor_type or "" return {"executor_type": executor_type, "config_proto": config} # Map from context_id (an int) to _TensorCaches. # Dicts are thread safe in CPython. # TODO(iga): Remove this once TensorCaches are moved to C++. _tensor_caches_map = {} class _TensorCaches(threading.local): """Thread local tensor caches.""" __slots__ = ["_ones_rank_cache", "_zeros_cache"] def __init__(self): super().__init__() self._ones_rank_cache = None self._zeros_cache = None @property def ones_rank_cache(self): if not self._ones_rank_cache: self._ones_rank_cache = _EagerTensorCache() return self._ones_rank_cache @property def zeros_cache(self): if not self._zeros_cache: self._zeros_cache = _EagerTensorCache() return self._zeros_cache ContextSwitch = collections.namedtuple( "ContextSwitch", ["is_building_function", "enter_context_fn", "device_stack"]) # `_ContextSwitchStack` is a `threading.local` to match the semantics of # ``DefaultGraphStack`, which is also a `threading.local`. class _ContextSwitchStack(threading.local): """A thread-local stack of context switches.""" def __init__(self, eager): super().__init__() self.stack = [] if eager: # Initialize the stack with a pointer to enter the eager context; this # ensures that the fact that eager execution was enabled is propagated # across threads, since (1) `enable_eager_execution` modifies a # process-level flag (`default_execution_mode`) and (2) `__init__` is # called each time a threading.local object is used in a separate thread. self.push( is_building_function=False, enter_context_fn=eager_mode, device_stack=None) def push(self, is_building_function, enter_context_fn, device_stack): """Push metadata about a context switch onto the stack. A context switch can take any one of the two forms: installing a graph as the default graph, or entering the eager context. For each context switch, we record whether or not the entered context is building a function. Args: is_building_function: (bool.) Whether the context is building a function. enter_context_fn: (function.) A callable that executes the context switch. For example, `graph.as_default` or `eager_mode`. device_stack: If applicable, the device function stack for this graph. When breaking out of graphs in init_scope, the innermost nonempty device stack is used. Eager contexts put `None` here and the value is never used. """ self.stack.append( ContextSwitch(is_building_function, enter_context_fn, device_stack)) def pop(self): """Pop the stack.""" self.stack.pop() @tf_export("config.LogicalDevice") class LogicalDevice( collections.namedtuple("LogicalDevice", ["name", "device_type"])): """Abstraction for a logical device initialized by the runtime. A `tf.config.LogicalDevice` corresponds to an initialized logical device on a `tf.config.PhysicalDevice` or a remote device visible to the cluster. Tensors and operations can be placed on a specific logical device by calling `tf.device` with a specified `tf.config.LogicalDevice`. Fields: name: The fully qualified name of the device. Can be used for Op or function placement. device_type: String declaring the type of device such as "CPU" or "GPU". """ @tf_export("config.LogicalDeviceConfiguration", "config.experimental.VirtualDeviceConfiguration") class LogicalDeviceConfiguration( collections.namedtuple("LogicalDeviceConfiguration", [ "memory_limit", "experimental_priority", "experimental_device_ordinal" ])): """Configuration class for a logical devices. The class specifies the parameters to configure a `tf.config.PhysicalDevice` as it is initialized to a `tf.config.LogicalDevice` during runtime initialization. Not all fields are valid for all device types. See `tf.config.get_logical_device_configuration` and `tf.config.set_logical_device_configuration` for usage examples. Fields: memory_limit: (optional) Maximum memory (in MB) to allocate on the virtual device. Currently only supported for GPUs. experimental_priority: (optional) Priority to assign to a virtual device. Lower values have higher priorities and 0 is the default. Within a physical GPU, the GPU scheduler will prioritize ops on virtual devices with higher priority. Currently only supported for Nvidia GPUs. experimental_device_ordinal: (optional) Ordinal number to order the virtual device. LogicalDevice with lower ordinal number will receive a lower device id. Physical device id and location in the list is used to break ties. Currently only supported for Nvidia GPUs. """ def __new__(cls, memory_limit=None, experimental_priority=None, experimental_device_ordinal=None): return super().__new__(cls, memory_limit, experimental_priority, experimental_device_ordinal) @tf_export("config.PhysicalDevice") class PhysicalDevice( collections.namedtuple("PhysicalDevice", ["name", "device_type"])): """Abstraction for a locally visible physical device. TensorFlow can utilize various devices such as the CPU or multiple GPUs for computation. Before initializing a local device for use, the user can customize certain properties of the device such as it's visibility or memory configuration. Once a visible `tf.config.PhysicalDevice` is initialized one or more `tf.config.LogicalDevice` objects are created. Use `tf.config.set_visible_devices` to configure the visibility of a physical device and `tf.config.set_logical_device_configuration` to configure multiple `tf.config.LogicalDevice` objects for a `tf.config.PhysicalDevice`. This is useful when separation between models is needed or to simulate a multi-device environment. Fields: name: Unique identifier for device. device_type: String declaring the type of device such as "CPU" or "GPU". """ pass class _AtomicCounter(object): """A simple atomic counter.""" __slots__ = ["_value", "_lock"] def __init__(self): self._value = 0 self._lock = threading.Lock() def increment_and_get(self): with self._lock: self._value += 1 return self._value _context_id_counter = _AtomicCounter() class _TensorCacheDeleter(object): """Deletes tensor caches for a given context.""" __slots__ = ["_context_id"] def __init__(self, context_id): self._context_id = context_id def __del__(self): if _tensor_caches_map is None: return if self._context_id in _tensor_caches_map: del _tensor_caches_map[self._context_id] # TODO(agarwal): rename to EagerContext / EagerRuntime ? # TODO(agarwal): consider keeping the corresponding Graph here. class Context: """Environment in which eager operations execute.""" # TODO(agarwal): create and link in some documentation for `execution_mode`. # pylint: disable=redefined-outer-name def __init__(self, config=None, device_policy=None, execution_mode=None, server_def=None): """Creates a new Context. Args: config: (Optional.) A `ConfigProto` protocol buffer with configuration options for the Context. Note that a lot of these options may be currently unimplemented or irrelevant when eager execution is enabled. device_policy: (Optional.) What policy to use when trying to run an operation on a device with inputs which are not on that device. When set to None, an appropriate value will be picked automatically. The value picked may change between TensorFlow releases. Defaults to DEVICE_PLACEMENT_SILENT. Valid values: - DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is not correct. - DEVICE_PLACEMENT_WARN: copies the tensors which are not on the right device but raises a warning. - DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might hide performance problems. - DEVICE_PLACEMENT_SILENT_FOR_INT32: silently copies int32 tensors, raising errors on the other ones. execution_mode: (Optional.) Policy controlling how operations dispatched are actually executed. When set to None, an appropriate value will be picked automatically. The value picked may change between TensorFlow releases. Valid values: - SYNC: executes each operation synchronously. - ASYNC: executes each operation asynchronously. These operations may return "non-ready" handles. server_def: (Optional.) A tensorflow::ServerDef proto. Enables execution on remote devices. GrpcServers need to be started by creating an identical server_def to this, and setting the appropriate task_indexes, so that the servers can communicate. It will then be possible to execute operations on remote devices. Raises: ValueError: If execution_mode is not valid. """ # This _id is used only to index the tensor caches. # TODO(iga): Remove this when tensor caches are moved to C++. self._id = _context_id_counter.increment_and_get() self._tensor_cache_deleter = _TensorCacheDeleter(self._id) _tensor_caches_map[self._id] = _TensorCaches() self._config = config self._thread_local_data = pywrap_tfe.EagerContextThreadLocalData( self, is_eager=lambda: default_execution_mode == EAGER_MODE, device_spec=_starting_device_spec) self._context_switches = _ContextSwitchStack(self.executing_eagerly()) self._context_handle = None self._context_devices = None self._seed = None self._initialize_lock = threading.Lock() self._initialized = False if device_policy is None: device_policy = DEVICE_PLACEMENT_SILENT self._device_policy = device_policy self._mirroring_policy = None if execution_mode not in (None, SYNC, ASYNC): raise ValueError("execution_mode should be None/SYNC/ASYNC. Got %s" % execution_mode) if execution_mode is None: execution_mode = SYNC self._default_is_async = execution_mode == ASYNC self._use_tfrt = is_tfrt_enabled() self._jit_compile_rewrite = jit_compile_rewrite_enabled() self._server_def = server_def self._collective_ops_server_def = None self._collective_leader = None self._collective_scoped_allocator_enabled_ops = None self._collective_use_nccl_communication = None self._collective_device_filters = None self._coordination_service_config = None self._device_lock = threading.Lock() self._physical_devices = None self._physical_device_to_index = None self._pluggable_devices = None self._visible_device_list = [] self._memory_growth_map = None self._virtual_device_map = {} # Values set after construction self._optimizer_jit = None self._intra_op_parallelism_threads = None self._inter_op_parallelism_threads = None self._soft_device_placement = None self._log_device_placement = None self._operation_timeout_in_ms = None self._enable_mlir_graph_optimization = None self._optimizer_experimental_options = {} _python_eager_context_create_counter.get_cell().increase_by(1) self._is_global_context = False # Number of retries to give the SetServerDef step. This is useful for fault # tolerant initial connection in high-preemption settings like # ParameterServerStrategy training. self._set_server_def_retries = 0 # pylint: enable=redefined-outer-name def _set_global_seed(self, seed): """Set a global eager mode seed for random ops.""" self._seed = seed # `random.Random(seed)` needs `seed` to be hashable, while values of type # e.g. `np.int64` or `np.ndarray` are not. We use `int(...)` to convert them # to int. try: hash(seed) self._rng = random.Random(seed) except TypeError: seed = int(np.array(seed)) self._rng = random.Random(seed) # Also clear the kernel cache, to reset any existing seeds if self._context_handle is not None: pywrap_tfe.TFE_ContextClearCaches(self._context_handle) def _internal_operation_seed(self): """Returns a fake operation seed. In eager mode, user shouldn't set or depend on operation seed. Here, we generate a random seed based on global seed to make operation's randomness different and depend on the global seed. Returns: A fake operation seed based on global seed. """ return self._rng.randint(0, _MAXINT32) def _initialize_logical_devices(self): """Helper to initialize devices.""" # Store list of devices logical_devices = [] context_devices = [] device_list = pywrap_tfe.TFE_ContextListDevices(self._context_handle) try: self._num_gpus = 0 current_job, current_task = None, None server_def = self._server_def or self._collective_ops_server_def if server_def is not None: current_job, current_task = server_def.job_name, server_def.task_index for i in range(pywrap_tfe.TF_DeviceListCount(device_list)): dev_name = pywrap_tfe.TF_DeviceListName(device_list, i) context_devices.append(pydev.canonical_name(dev_name)) spec = pydev.DeviceSpec.from_string(dev_name) # If the job is localhost, we assume that the cluster has not yet been # configured and thus clear the job, replica & task. if spec.job == "localhost": spec = spec.replace(job=None, replica=None, task=None) logical_devices.append( LogicalDevice(name=spec.to_string(), device_type=spec.device_type)) dev_type = pywrap_tfe.TF_DeviceListType(device_list, i) if (dev_type == "GPU" and spec.job == current_job and spec.task == current_task): self._num_gpus += 1 finally: self._logical_devices = logical_devices self._context_devices = context_devices pywrap_tfe.TF_DeleteDeviceList(device_list) def ensure_initialized(self): """Initialize handle and devices if not already done so.""" if self._initialized: return with self._initialize_lock: if self._initialized: return assert self._context_devices is None opts = pywrap_tfe.TFE_NewContextOptions() try: config_str = self.config.SerializeToString() pywrap_tfe.TFE_ContextOptionsSetConfig(opts, config_str) if self._device_policy is not None: pywrap_tfe.TFE_ContextOptionsSetDevicePlacementPolicy( opts, self._device_policy) if self._mirroring_policy is not None: pywrap_tfe.TFE_ContextOptionsSetMirroringPolicy( opts, self._mirroring_policy) if self._default_is_async == ASYNC: pywrap_tfe.TFE_ContextOptionsSetAsync(opts, True) if self._use_tfrt is not None: pywrap_tfe.TFE_ContextOptionsSetTfrt(opts, self._use_tfrt) pywrap_tfe.TFE_ContextOptionsSetRunEagerOpAsFunction(opts, True) pywrap_tfe.TFE_ContextOptionsSetJitCompileRewrite( opts, self._jit_compile_rewrite) context_handle = pywrap_tfe.TFE_NewContext(opts) finally: pywrap_tfe.TFE_DeleteContextOptions(opts) assert not (self._server_def and self._collective_ops_server_def), ( "Cannot enable remote execution as well as collective ops at the " "moment. If this is important to you, please file an issue.") if self._server_def is not None: server_def_str = self._server_def.SerializeToString() timeout = 0 # Indicates no timeout. pywrap_tfe.TFE_ContextSetServerDefWithTimeoutAndRetries( context_handle, _KEEP_ALIVE_SECS, server_def_str, timeout, self._set_server_def_retries) elif self._collective_ops_server_def is not None: server_def_str = self._collective_ops_server_def.SerializeToString() pywrap_tfe.TFE_EnableCollectiveOps(context_handle, server_def_str) self._context_handle = context_handle self._initialize_logical_devices() self._initialized = True if self._is_global_context: pywrap_tfe.TFE_Py_SetCEagerContext(self._context_handle) def ensure_uninitialized(self): """Uninitialize handle and devices if not already done so.""" with self._initialize_lock: if not self._initialized: return self._context_devices = None self._logical_devices = None self._server_def = None self._initialized = False if self._is_global_context: pywrap_tfe.TFE_Py_SetCEagerContext(None) self._context_handle = None def mark_as_global_context(self): # If the context was already initialized, publish it. Otherwise wait with # publication until it's initialized. if self._initialized: pywrap_tfe.TFE_Py_SetCEagerContext(self._context_handle) self._is_global_context = True def _clear_caches(self): self.ones_rank_cache().flush() self.zeros_cache().flush() pywrap_tfe.TFE_ClearScalarCache() def get_server_def(self): return self._server_def def set_server_def(self, server_def, keep_alive_secs=_KEEP_ALIVE_SECS): """Allow setting a server_def on the context. When a server def is replaced, it effectively clears a bunch of caches within the context. If you attempt to use a tensor object that was pointing to a tensor on the remote device, it will raise an error. Args: server_def: A tensorflow::ServerDef proto. Enables execution on remote devices. keep_alive_secs: Num. seconds after which the remote end will hang up. As long as the client is still alive, the server state for the context will be kept alive. If the client is killed (or there is some failure), the server will clean up its context keep_alive_secs after the final RPC it receives. Raises: ValueError: if server_def is None. """ if not server_def: raise ValueError("server_def is None.") self._server_def = server_def if self._context_handle: server_def_str = server_def.SerializeToString() pywrap_tfe.TFE_ContextSetServerDef(self._context_handle, keep_alive_secs, server_def_str) self._initialize_logical_devices() # Clear all the caches in case there are remote tensors in them. self._clear_caches() # Also clear the device parsing cache since it caches the resolution of # partial device names, which may become different due to the set_server_def # call as we may have defined different devices. _device_parsing_cache.clear() def update_server_def(self, server_def, keep_alive_secs=_KEEP_ALIVE_SECS): """Update a server_def on the context. Args: server_def: A tensorflow::ServerDef proto. Enables execution on remote devices. keep_alive_secs: Num. seconds after which the remote end will hang up. As long as the client is still alive, the server state for the context will be kept alive. If the client is killed (or there is some failure), the server will clean up its context keep_alive_secs after the final RPC it receives. Raises: ValueError: if server_def is None. """ if not server_def: raise ValueError("server_def is None.") self._server_def = server_def if self._context_handle: server_def_str = server_def.SerializeToString() pywrap_tfe.TFE_ContextUpdateServerDef(self._context_handle, keep_alive_secs, server_def_str) self._initialize_logical_devices() self._clear_caches() def check_alive(self, worker_name): """Checks whether a remote worker is alive or not. Args: worker_name: a string representing the remote worker. It must be a fully specified name like "/job:worker/replica:0/task:0". Returns: a boolean indicating whether the remote worker is alive or not. Raises: ValueError: if context is not initialized. """ # TODO(yuefengz): support checking multiple workers. if self._context_handle: return pywrap_tfe.TFE_ContextCheckAlive(self._context_handle, worker_name) else: raise ValueError("Context is not initialized.") def sync_executors(self): """Sync both local executors and the ones on remote workers. In async execution mode, local function calls can return before the corresponding remote op/function execution requests are completed. Calling this method creates a synchronization barrier for remote executors. It only returns when all remote pending nodes are finished, potentially with errors if any remote executors are in error state. Raises: ValueError: if context is not initialized. """ if self._context_handle: pywrap_tfe.TFE_ContextSyncExecutors(self._context_handle) else: raise ValueError("Context is not initialized.") def clear_executor_errors(self): """Clear errors in both local executors and remote workers. After receiving errors from remote workers, additional requests on the fly could further taint the status on the remote workers due to the async nature of remote execution. Calling this method block on waiting for all pending nodes in remote executors to finish and clear their error statuses. Raises: ValueError: if context is not initialized. """ if self._context_handle: pywrap_tfe.TFE_ContextClearExecutors(self._context_handle) else: raise ValueError("Context is not initialized.") def configure_coordination_service(self, service_type, service_leader="", enable_health_check=True, cluster_register_timeout_in_ms=0, heartbeat_timeout_in_ms=0, shutdown_barrier_timeout_in_ms=0, coordinated_jobs=None, allow_new_incarnation_to_reconnect=False): """Enable distributed coordination service with specified configs.""" if self._context_handle: logging.warning("Configuring coordination service type may not be " "effective because the context is already initialized.") config = coordination_config_pb2.CoordinationServiceConfig() config.service_type = service_type if service_leader: config.service_leader = pydev.canonical_name(service_leader) config.enable_health_check = enable_health_check config.cluster_register_timeout_in_ms = cluster_register_timeout_in_ms config.heartbeat_timeout_in_ms = heartbeat_timeout_in_ms config.shutdown_barrier_timeout_in_ms = shutdown_barrier_timeout_in_ms config.allow_new_incarnation_to_reconnect = ( allow_new_incarnation_to_reconnect) if coordinated_jobs is not None: if isinstance(coordinated_jobs, list): config.coordinated_job_list.extend(coordinated_jobs) else: raise ValueError("`coordinated_jobs` must be list[CoordinatedJob] or " "None, but got: %s" % (coordinated_jobs,)) self._coordination_service_config = config @property def coordination_service(self): return self._coordination_service_config def set_config_key_value(self, key, value): ensure_initialized() pywrap_tfe.TFE_InsertConfigKeyValue(self._context_handle, key, value) # If `timeout_in_ms=0`, this will block until the key-value is set or the # worker shuts down. def get_config_key_value(self, key, timeout_in_ms=0): ensure_initialized() with c_api_util.tf_buffer() as buffer_: pywrap_tfe.TFE_GetConfigKeyValue(self._context_handle, key, timeout_in_ms, buffer_) value = pywrap_tf_session.TF_GetBuffer(buffer_).decode("utf-8") return value def delete_config_key_value(self, key): ensure_initialized() pywrap_tfe.TFE_DeleteConfigKeyValue(self._context_handle, key) def report_error_to_cluster(self, error_code, error_message): """Report error to other members in a multi-client cluster. Args: error_code: a `tf.errors` error code. error_message: a string. The error message. """ if self._context_handle: pywrap_tfe.TFE_ReportErrorToCluster(self._context_handle, error_code, error_message) else: raise ValueError("Context is not initialized.") def get_task_states(self, job_configs): """Get task states from the Coordination Service. Args: job_configs: A list of tuples of job name and task number. Returns: A list of TF_Status. """ if self._context_handle: job_names, task_nums = zip(*job_configs) return pywrap_tfe.TFE_GetTaskStates(self._context_handle, job_names, task_nums) else: raise ValueError("Context is not initialized.") def wait_at_barrier(self, barrier_id, timeout_in_ms): """Blocks until all coordinated tasks are at the barrier. The barrier may fail if it times out or if one of the tasks is unhealthy. Args: barrier_id: Unique string identifying the barrier. timeout_in_ms: Duration before the barrier times out and fails. """ ensure_initialized() pywrap_tfe.TFE_WaitAtBarrier(self._context_handle, barrier_id, timeout_in_ms) def clear_kernel_cache(self): """Clear kernel cache and reset all stateful kernels.""" if self._context_handle is not None: pywrap_tfe.TFE_ContextClearCaches(self._context_handle) def enable_collective_ops(self, server_def): """Enable distributed collective ops with an appropriate server_def. Args: server_def: A tensorflow::ServerDef proto. Enables execution on remote devices. Raises: ValueError: if server_def is None. RuntimeError: if this method is not called at program startup. """ if not server_def: raise ValueError("server_def is None.") self._collective_ops_server_def = server_def # TODO(b/129298253): Allow creating datasets/tensors before enabling # collective ops. if self._context_handle is not None: logging.warning("Enabling collective ops after program startup may cause " "error when accessing previously created tensors.") with self._initialize_lock: assert self._initialized server_def_str = self._collective_ops_server_def.SerializeToString() pywrap_tfe.TFE_EnableCollectiveOps(self._context_handle, server_def_str) self._initialize_logical_devices() self._clear_caches() def configure_collective_ops( self, collective_leader="", scoped_allocator_enabled_ops=("CollectiveReduce",), use_nccl_communication=False, device_filters=None): """Configure collective ops. Collective group leader is necessary for collective ops to run, other configurations are mainly for the purpose of performance. Args: collective_leader: a device string for collective leader, e.g. "/job:worker/replica:0/task:0"; empty string means local execution of collective ops. scoped_allocator_enabled_ops: a tuple or a list of op names for scoped allocator to run with. use_nccl_communication: whether to use nccl communication for collective ops. device_filters: a tuple or a list of device strings. If set, corresponding task can only see the devices filtered by these device filters. Raises: RuntimeError: if this method is not called at program startup. """ if self._collective_leader is not None: if (self._collective_leader != collective_leader or self._collective_scoped_allocator_enabled_ops != scoped_allocator_enabled_ops or self._collective_use_nccl_communication != use_nccl_communication or self._collective_device_filters != device_filters): raise ValueError("Collective ops are already configured.") else: return if self._context_handle is not None: raise RuntimeError("Collective ops must be configured at program startup") self._collective_leader = collective_leader self._collective_scoped_allocator_enabled_ops = scoped_allocator_enabled_ops self._collective_use_nccl_communication = use_nccl_communication self._collective_device_filters = device_filters def abort_collective_ops(self, code, message): """Abort the collective ops. This is intended to be used when a peer failure is detected, which allows the user to handle the case instead of hanging. This aborts all on-going collectives. After all subsequent collectives error immediately, and you need to reset_context() to use collectives again. Args: code: a `tf.errors` error code. message: a string. The error message. """ self.ensure_initialized() pywrap_tfe.TFE_AbortCollectiveOps(self._handle, code, message) def check_collective_ops_peer_health(self, task, timeout_in_ms): """Check collective peer health. This probes each task to see if they're still alive. Note that restarted tasks are considered a different one, and they're considered not healthy. This should only be used in multi client multi worker training. Args: task: a task string, must be in the format of /job:xxx/replica:0/task:N. timeout_in_ms: an integer, the timeout. If zero, there's no timeout. Raises: tf.errors.UnavailableError: when a peer is down. tf.errors.FailedPreconditionError: when a peer is a different one from the one this task has talked to, e.g. the peer has restarted. tf.errors.InvalidArgumentError: when the task string is invalid. """ self.ensure_initialized() pywrap_tfe.TFE_CollectiveOpsCheckPeerHealth(self._handle, task, timeout_in_ms) @property def _handle(self): if self._context_handle is None: raise AssertionError("Context must be initialized first.") return self._context_handle @property def _devices(self): if self._context_devices is None: raise AssertionError("Context must be initialized first.") return self._context_devices def __str__(self): if self._context_handle is None: return "Eager TensorFlow Context. Devices currently uninitialized." else: devices = self._devices lines = ["Eager TensorFlow Context with %d devices" % (len(devices))] for i, d in enumerate(devices): lines.append(" Device %d: %s" % (i, d)) return "\n".join(lines) @tf_contextlib.contextmanager def _mode(self, mode): """A context manager to allow setting the mode to EAGER/GRAPH.""" ctx = self._thread_local_data old_is_eager = ctx.is_eager ctx.is_eager = mode == EAGER_MODE if mode == EAGER_MODE: # Entering graph mode does not provide us with sufficient information to # record a context switch; graph-based context switches are only logged # when a graph is registered as the default graph. self.context_switches.push(False, eager_mode, None) try: yield finally: ctx.is_eager = old_is_eager if mode == EAGER_MODE: self.context_switches.pop() def executing_eagerly(self): """Returns True if current thread has eager executing enabled.""" return self._thread_local_data.is_eager def ones_rank_cache(self): """Per-device cache for scalars.""" return _tensor_caches_map[self._id].ones_rank_cache def zeros_cache(self): """Per-device cache for scalars.""" return _tensor_caches_map[self._id].zeros_cache @property def scope_name(self): """Returns scope name for the current thread.""" return self._thread_local_data.scope_name @scope_name.setter def scope_name(self, s): """Sets scope name for the current thread.""" self._thread_local_data.scope_name = s @property def device_name(self): """Returns the device name for the current thread.""" return self._thread_local_data.device_name @property def device_spec(self): """Returns the device spec for the current thread.""" return self._thread_local_data.device_spec def _set_device(self, device_name, device_spec): self._thread_local_data.device_name = device_name self._thread_local_data.device_spec = device_spec def device(self, name): """Context-manager to force placement of operations and Tensors on a device. Args: name: Name of the device or None to get default placement. Returns: Context manager that forces device placement. Raises: ValueError: If name is not a string or is an invalid device name. RuntimeError: If device scopes are not properly nested. """ if isinstance(name, LogicalDevice): name = name.name elif pydev.is_device_spec(name): name = name.to_string() return _EagerDeviceContext(self, name) def devices(self): """List of the names of devices available to execute operations.""" return self._devices def host_address_space(self): self.ensure_initialized() with c_api_util.tf_buffer() as buffer_: pywrap_tfe.TFE_HostAddressSpace(self._context_handle, buffer_) address_space = pywrap_tf_session.TF_GetBuffer(buffer_).decode("utf-8") return address_space # TODO(fishx): remove this property. @property def execution_mode(self): """Gets execution mode for current thread.""" return ASYNC if self.is_async() else SYNC @execution_mode.setter def execution_mode(self, mode): """Sets execution mode for current thread.""" if mode not in (None, SYNC, ASYNC): raise ValueError("Execution mode should be None/SYNC/ASYNC. Got %s" % mode) if mode is None: mode = SYNC enable_async = (mode == ASYNC) if self.is_async() != enable_async: # Only set the execution mode if the context has already been initialized if self._context_handle is not None: self.executor.wait() executor_new = executor.new_executor(enable_async) self._thread_local_data.executor = executor_new pywrap_tfe.TFE_ContextSetExecutorForThread(self._context_handle, executor_new.handle()) else: self._default_is_async = enable_async def is_async(self): if self._context_handle is not None: return self.executor.is_async() else: return self._default_is_async @property def executor(self): self.ensure_initialized() return executor.Executor( pywrap_tfe.TFE_ContextGetExecutorForThread(self._context_handle)) @executor.setter def executor(self, e): self.ensure_initialized() pywrap_tfe.TFE_ContextSetExecutorForThread(self._context_handle, e.handle()) @property def config(self): """Return the ConfigProto with all runtime deltas applied.""" # Ensure physical devices have been discovered and config has been imported self._initialize_physical_devices() config = config_pb2.ConfigProto() if self._config is not None: config.CopyFrom(self._config) if self._optimizer_jit is not None: config.graph_options.optimizer_options.global_jit_level = ( config_pb2.OptimizerOptions.ON_1 if self._optimizer_jit else config_pb2.OptimizerOptions.OFF) if self._intra_op_parallelism_threads is not None: config.intra_op_parallelism_threads = self._intra_op_parallelism_threads if self._inter_op_parallelism_threads is not None: config.inter_op_parallelism_threads = self._inter_op_parallelism_threads if self._soft_device_placement is not None: config.allow_soft_placement = self._soft_device_placement else: config.allow_soft_placement = self.executing_eagerly() if self._log_device_placement is not None: config.log_device_placement = self._log_device_placement if self._operation_timeout_in_ms is not None: config.operation_timeout_in_ms = self._operation_timeout_in_ms is_mlir_bridge_enabled = pywrap_tfe.TF_IsMlirBridgeEnabled() config.experimental.mlir_bridge_rollout = is_mlir_bridge_enabled if (is_mlir_bridge_enabled == config_pb2.ConfigProto.Experimental.MLIR_BRIDGE_ROLLOUT_ENABLED): config.experimental.enable_mlir_bridge = True if self._enable_mlir_graph_optimization is not None: config.experimental.enable_mlir_graph_optimization = ( self._enable_mlir_graph_optimization) def rewriter_toggle(option): toggle = self._optimizer_experimental_options.get(option, None) if toggle is None: return setattr(config.graph_options.rewrite_options, option, (rewriter_config_pb2.RewriterConfig.ON if toggle else rewriter_config_pb2.RewriterConfig.OFF)) def rewriter_bool(option): toggle = self._optimizer_experimental_options.get(option, None) if toggle is None: return setattr(config.graph_options.rewrite_options, option, toggle) rewriter_toggle("layout_optimizer") rewriter_toggle("constant_folding") rewriter_toggle("shape_optimization") rewriter_toggle("remapping") rewriter_toggle("arithmetic_optimization") rewriter_toggle("dependency_optimization") rewriter_toggle("loop_optimization") rewriter_toggle("function_optimization") rewriter_toggle("debug_stripper") rewriter_bool("disable_model_pruning") rewriter_toggle("scoped_allocator_optimization") rewriter_toggle("pin_to_host_optimization") rewriter_toggle("implementation_selector") rewriter_toggle("auto_mixed_precision") rewriter_toggle("use_plugin_optimizers") rewriter_bool("disable_meta_optimizer") rewriter_toggle("auto_mixed_precision_onednn_bfloat16") rewriter_toggle("auto_mixed_precision_mkl") nodes = self._optimizer_experimental_options.get("min_graph_nodes", None) if nodes is not None: config.graph_options.rewrite_options.min_graph_nodes = nodes # Compute device counts config.device_count["CPU"] = 0 config.device_count["GPU"] = 0 for dev in self._physical_devices: if dev not in self._visible_device_list: continue virtual_devices = self._virtual_device_map.get(dev) if virtual_devices is None: config.device_count[dev.device_type] += 1 else: config.device_count[dev.device_type] += len(virtual_devices) # Configure gpu_options gpu_options = self._compute_gpu_options() config.gpu_options.MergeFrom(gpu_options) # Configure collective ops if self._collective_leader: config.experimental.collective_group_leader = self._collective_leader if self._collective_scoped_allocator_enabled_ops: rewrite_options = config.graph_options.rewrite_options rewrite_options.scoped_allocator_optimization = ( rewriter_config_pb2.RewriterConfig.ON) del rewrite_options.scoped_allocator_opts.enable_op[:] for op in self._collective_scoped_allocator_enabled_ops: rewrite_options.scoped_allocator_opts.enable_op.append(op) if self._collective_use_nccl_communication: config.experimental.collective_nccl = True if self._collective_device_filters: del config.device_filters[:] for f in self._collective_device_filters: config.device_filters.append(f) # Configure coordination service if self._coordination_service_config: config.experimental.coordination_config.CopyFrom( self._coordination_service_config) return config def _compute_gpu_options(self): """Build the GPUOptions proto.""" visible_device_list = [] virtual_devices = [] gpu_index = -1 memory_growths = set() gpu_devices = self.list_physical_devices("GPU") pluggable_devices = self._pluggable_devices compatible_devices = gpu_devices for dev in pluggable_devices: if dev not in gpu_devices: compatible_devices.append(dev) for dev in compatible_devices: gpu_index += 1 if dev not in self._visible_device_list: continue growth = self._memory_growth_map[dev] memory_growths.add(growth) visible_device_list.append(str(gpu_index)) if self._virtual_device_map: vdevs = self._virtual_device_map.get(dev, []) device_ordinals = [] device_limits = [] priority = [] for virt_dev in vdevs: if virt_dev.experimental_device_ordinal is not None: device_ordinals.append(virt_dev.experimental_device_ordinal) device_limits.append(virt_dev.memory_limit) if virt_dev.experimental_priority is not None: priority.append(virt_dev.experimental_priority) # If priority is specified, it must be specified for all virtual # devices. if priority and len(device_limits) != len(priority): raise ValueError("priority must be specified for all virtual devices") # If device_ordinals is specified, it must be specified for all virtual # devices. if device_ordinals and len(device_limits) != len(device_ordinals): raise ValueError( "device_ordinals must be specified for all virtual devices") virtual_devices.append( config_pb2.GPUOptions.Experimental.VirtualDevices( memory_limit_mb=device_limits, priority=priority, device_ordinal=device_ordinals)) # Only compute growth if virtual devices have not been configured and we # have GPUs if not virtual_devices and memory_growths: if len(memory_growths) > 1: raise ValueError("Memory growth cannot differ between GPU devices") allow_growth = memory_growths.pop() else: allow_growth = None return config_pb2.GPUOptions( allow_growth=allow_growth, visible_device_list=",".join(visible_device_list), experimental=config_pb2.GPUOptions.Experimental( virtual_devices=virtual_devices)) @property def function_call_options(self): """Returns function call options for current thread. Note that the returned object is still referenced by the eager context. Returns: the FunctionCallOptions for current thread. """ if self._thread_local_data.function_call_options is None: config = self.config # Default to soft placement for functions unless specified if self._soft_device_placement is None: config.allow_soft_placement = True self._thread_local_data.function_call_options = FunctionCallOptions( config_proto=config) return self._thread_local_data.function_call_options @function_call_options.setter def function_call_options(self, options): """Returns function call options for current thread.""" self._thread_local_data.function_call_options = options def num_gpus(self): """The number of GPUs available to execute operations.""" self.ensure_initialized() return self._num_gpus def add_c_function(self, c_func): """Add a C API TF_Function to the context. Once added, the function (identified by its name) can be executed like any other operation. Args: c_func: A wrapped TF_Function (returned from TF_GraphToFunction_wrapper). """ self.ensure_initialized() pywrap_tfe.TFE_ContextAddFunction(self._handle, c_func) def get_c_function(self, name): """Get a C API TF_Function from the context. Args: name: Name of the function to get. Returns: A ScopedTFFunction wrapping the C API TF_Function. """ self.ensure_initialized() return c_api_util.ScopedTFFunction( pywrap_tfe.TFE_ContextGetFunction(self._handle, name), name ) def add_function_def(self, fdef): """Add a function definition to the context. Once added, the function (identified by its name) can be executed like any other operation. Args: fdef: A FunctionDef protocol buffer message. """ self.ensure_initialized() if is_oss: fdef_string = fdef.SerializeToString() pywrap_tfe.TFE_ContextAddFunctionDef( self._handle, fdef_string, len(fdef_string) ) else: pywrap_tfe.TFE_ContextAddFunctionDefNoSerialization(self._handle, fdef) def get_function_def(self, name): """Get a function definition from the context. Args: name: function signature name. Returns: The requested FunctionDef. Raises: tf.errors.NotFoundError: if name is not the name of a registered function. """ if is_oss: with c_api_util.tf_buffer() as buffer_: pywrap_tfe.TFE_ContextGetFunctionDef(self._handle, name, buffer_) proto_data = pywrap_tf_session.TF_GetBuffer(buffer_) function_def = function_pb2.FunctionDef() function_def.ParseFromString(proto_data) else: function_def = pywrap_tfe.TFE_ContextGetFunctionDefNoSerialization( self._handle, name ) return function_def def get_graph_debug_info(self, name): """Get GraphDebugInfo associated with a function from the context. Args: name: function signature name. Returns: The requested GraphDebugInfo. Raises: tf.errors.NotFoundError: if name is not the name of a registered function. """ with c_api_util.tf_buffer() as buffer_: pywrap_tfe.TFE_ContextGetGraphDebugInfo(self._handle, name, buffer_) proto_data = pywrap_tf_session.TF_GetBuffer(buffer_) graph_debug_info = graph_debug_info_pb2.GraphDebugInfo() graph_debug_info.ParseFromString(proto_data) return graph_debug_info def is_custom_device(self, device_name): """Calls TFE_IsCustomDevice. See the non-member function.""" self.ensure_initialized() return pywrap_tfe.TFE_Py_IsCustomDevice(self._handle, device_name) def register_custom_device(self, device_capsule, device_name, device_info_capsule): """Calls TFE_RegisterCustomDevice. See the non-member function.""" self.ensure_initialized() pywrap_tfe.TFE_Py_RegisterCustomDevice(self._handle, device_capsule, device_name, device_info_capsule) def pack_eager_tensors(self, tensors): """Pack multiple `EagerTensor`s of the same dtype and shape. Args: tensors: a list of EagerTensors to pack. Returns: A packed EagerTensor. """ self.ensure_initialized() return pywrap_tfe.TFE_Py_PackEagerTensors(self._handle, tensors) def list_function_names(self): """Get a list of names of registered functions. Returns: A set of names of all registered functions for the context. """ self.ensure_initialized() return set(pywrap_tfe.TFE_ContextListFunctionNames(self._handle)) def remove_function(self, name): """Remove a function from the context. Once removed, the function cannot be executed anymore. Args: name: function signature name. """ self.ensure_initialized() pywrap_tfe.TFE_ContextRemoveFunction(self._handle, name) def has_function(self, name): """Check if a function `name` is registered.""" self.ensure_initialized() return bool(pywrap_tfe.TFE_ContextHasFunction(self._handle, name)) @property def function_scope_id(self): """Returns an id that is unique to each scope holding functions.""" return id(self._context_handle) def call_function(self, name, tensor_inputs, num_outputs): """Calls the function associated with the given name.""" attrs = tuple( itertools.chain( *self.function_call_options.as_attrs().items() ) ) cancellation_context = cancellation.context() if cancellation_context is None: outputs = execute.execute( name.decode("utf-8"), num_outputs=num_outputs, inputs=tensor_inputs, attrs=attrs, ctx=self, ) else: outputs = execute.execute_with_cancellation( name.decode("utf-8"), num_outputs=num_outputs, inputs=tensor_inputs, attrs=attrs, ctx=self, cancellation_manager=cancellation_context, ) # Empty list means no function outputs so return None outputs = outputs or None return outputs def add_op_callback(self, callback): """Add a post-op callback to the context. A post-op callback is invoked immediately after an eager operation or function has finished execution or after a op has been added to a graph, providing access to the op's type, name input and output tensors. Multiple op callbacks can be added, in which case the callbacks will be invoked in the order in which they are added. Args: callback: a callable of the signature `f(op_type, inputs, attrs, outputs, op_name=None, graph=None)`. See doc strings in `op_callbacks.py` for details on the function signature and its semantics. """ if callback not in self._thread_local_data.op_callbacks: self._thread_local_data.op_callbacks.append(callback) def remove_op_callback(self, callback): """Remove an already-registered op callback. Args: callback: The op callback to be removed. Raises: KeyError: If `callback` is not already registered. """ if callback not in self._thread_local_data.op_callbacks: raise KeyError("The specified op callback has not been registered, " "and hence cannot be removed.") del self._thread_local_data.op_callbacks[ self._thread_local_data.op_callbacks.index(callback)] @property def op_callbacks(self): return self._thread_local_data.op_callbacks @property def invoking_op_callbacks(self): return self._thread_local_data.invoking_op_callbacks @invoking_op_callbacks.setter def invoking_op_callbacks(self, value): self._thread_local_data.invoking_op_callbacks = value def _initialize_physical_devices(self, reinitialize=False): """Gets local devices visible to the system. Args: reinitialize: If True, reinitializes self._physical_devices so that dynamic registered devices will also be visible to the python front-end. """ # We lazy initialize self._physical_devices since we do not want to do this # the constructor since the backend may not be initialized yet. with self._device_lock: if not reinitialize and self._physical_devices is not None: return devs = pywrap_tfe.TF_ListPhysicalDevices() self._physical_devices = [ PhysicalDevice(name=d.decode(), device_type=d.decode().split(":")[1]) for d in devs ] self._physical_device_to_index = { p: i for i, p in enumerate(self._physical_devices) } # We maintain a separate list just so we can check whether the device in # _physical_devices is a PluggableDevice. pluggable_devs = pywrap_tfe.TF_ListPluggablePhysicalDevices() self._pluggable_devices = [ PhysicalDevice(name=d.decode(), device_type=d.decode().split(":")[1]) for d in pluggable_devs ] self._visible_device_list = list(self._physical_devices) self._memory_growth_map = { d: None for d in self._physical_devices if d.device_type == "GPU" or d in self._pluggable_devices } # Import device settings that may have been passed into the constructor self._import_config() def reinitialize_physical_devices(self): """Gets local devices visible to the system.""" # Reinitialize the physical device list after registering # the pluggable device. self._initialize_physical_devices(True) def list_physical_devices(self, device_type=None): """List local devices visible to the system. This API allows a client to query the devices before they have been initialized by the eager runtime. Additionally a user can filter by device type, to get only CPUs or GPUs. Args: device_type: Optional device type to limit results to Returns: List of PhysicalDevice objects. """ self._initialize_physical_devices() if device_type is None: return list(self._physical_devices) return [d for d in self._physical_devices if d.device_type == device_type] def get_device_details(self, device): # pylint: disable=redefined-outer-name """Returns details about a physical devices. Args: device: A `tf.config.PhysicalDevice` returned by `tf.config.list_physical_devices` or `tf.config.get_visible_devices`. Returns: A dict with string keys. """ if not isinstance(device, PhysicalDevice): raise ValueError("device must be a tf.config.PhysicalDevice, but got: " "%s" % (device,)) if (self._physical_device_to_index is None or device not in self._physical_device_to_index): raise ValueError("The PhysicalDevice must be one obtained from " "calling `tf.config.list_physical_devices`, but got: " "%s" % (device,)) index = self._physical_device_to_index[device] details = pywrap_tfe.TF_GetDeviceDetails(index) # Change compute_capability from a string to a tuple if "compute_capability" in details: try: major, minor = details["compute_capability"].split(".") details["compute_capability"] = (int(major), int(minor)) except ValueError: raise RuntimeError("Device returned compute capability an in invalid " "format: %s" % details["compute_capability"]) return details def _import_config(self): """Import config if passed in during construction. If Context was created with a ConfigProto such as when calling tf.compat.v1.enable_eager_execution(), then we need to pull out the various pieces we might be replacing and import then into our internal class representation. """ if self._config is None: return num_cpus = self._config.device_count.get("CPU", 1) if num_cpus != 1: cpus = [d for d in self._physical_devices if d.device_type == "CPU"] if num_cpus == 0: self.set_visible_devices([], "CPU") elif num_cpus > 1: self.set_logical_device_configuration( cpus[0], [LogicalDeviceConfiguration() for _ in range(num_cpus)]) # Parse GPU options gpus = [d for d in self._physical_devices if d.device_type == "GPU"] # If there are no GPUs detected, simply ignore all the GPU options passed in # rather than doing any validation checks. if not gpus: return gpu_count = self._config.device_count.get("GPU", None) visible_gpus = [] # TODO(gjn): Handle importing existing virtual GPU configuration visible_indices = self._config.gpu_options.visible_device_list if visible_indices: for index in visible_indices.split(","): if int(index) >= len(gpus): raise ValueError("Invalid visible device index: %s" % index) visible_gpus.append(gpus[int(index)]) else: visible_gpus = gpus if gpu_count is not None: visible_gpus = visible_gpus[:gpu_count] self.set_visible_devices(visible_gpus, "GPU") def list_logical_devices(self, device_type=None): """Return logical devices.""" self.ensure_initialized() if device_type is None: return list(self._logical_devices) return [d for d in self._logical_devices if d.device_type == device_type] def get_visible_devices(self, device_type=None): """Get the list of visible devices.""" self._initialize_physical_devices() if device_type is None: return list(self._visible_device_list) return [ d for d in self._visible_device_list if d.device_type == device_type ] def set_visible_devices(self, devices, device_type=None): """Set the list of visible devices.""" self._initialize_physical_devices() if not isinstance(devices, list): devices = [devices] for d in devices: if d not in self._physical_devices: raise ValueError("Unrecognized device: %s" % repr(d)) if device_type is not None and d.device_type != device_type: raise ValueError("Unrecognized device: %s" % repr(d)) visible_device_list = [] if device_type is not None: visible_device_list = [ d for d in self._visible_device_list if d.device_type != device_type ] visible_device_list += devices if self._visible_device_list == visible_device_list: return if self._context_handle is not None: raise RuntimeError( "Visible devices cannot be modified after being initialized") self._visible_device_list = visible_device_list def get_memory_info(self, dev): """Returns a dict of memory info for the device.""" self._initialize_physical_devices() self.ensure_initialized() return pywrap_tfe.TFE_GetMemoryInfo(self._context_handle, dev) def reset_memory_stats(self, dev): """Resets the tracked memory stats for the device.""" self._initialize_physical_devices() self.ensure_initialized() pywrap_tfe.TFE_ResetMemoryStats(self._context_handle, dev) def get_memory_growth(self, dev): """Get if memory growth is enabled for a PhysicalDevice.""" self._initialize_physical_devices() if dev not in self._physical_devices: raise ValueError("Unrecognized device: %s" % repr(dev)) return self._memory_growth_map[dev] def set_memory_growth(self, dev, enable): """Set if memory growth should be enabled for a PhysicalDevice.""" self._initialize_physical_devices() if dev not in self._physical_devices: raise ValueError("Unrecognized device: %s" % repr(dev)) if dev in self._virtual_device_map: raise ValueError( "Cannot set memory growth on device when virtual devices configured") if dev.device_type != "GPU" and dev not in self._pluggable_devices: raise ValueError( "Cannot set memory growth on non-GPU and non-Pluggable devices") if self._memory_growth_map.get(dev) == enable: return if self._context_handle is not None: raise RuntimeError( "Physical devices cannot be modified after being initialized") self._memory_growth_map[dev] = enable def get_logical_device_configuration(self, dev): """Get the virtual device configuration for a PhysicalDevice.""" self._initialize_physical_devices() if dev not in self._physical_devices: raise ValueError("Unrecognized device: %s" % repr(dev)) return self._virtual_device_map.get(dev) def set_logical_device_configuration(self, dev, virtual_devices): """Set the virtual device configuration for a PhysicalDevice.""" self._initialize_physical_devices() if dev not in self._physical_devices: raise ValueError("Unrecognized device: %s" % repr(dev)) if dev.device_type == "CPU": for vdev in virtual_devices: if vdev.memory_limit is not None: raise ValueError("Setting memory limit on CPU virtual devices is " "currently not supported") if vdev.experimental_priority is not None: raise ValueError("Setting experimental_priority on CPU virtual " " devices is currently not supported") if vdev.experimental_device_ordinal is not None: raise ValueError("Setting experimental_device_ordinal on CPU virtual " " devices is currently not supported") elif dev.device_type == "GPU": for vdev in virtual_devices: if vdev.memory_limit is None: raise ValueError( "Setting memory limit is required for GPU virtual devices") else: raise ValueError("Virtual devices are not supported for %s" % dev.device_type) if self._virtual_device_map.get(dev) == virtual_devices: return if self._context_handle is not None: raise RuntimeError( "Virtual devices cannot be modified after being initialized") self._virtual_device_map[dev] = virtual_devices def set_logical_cpu_devices(self, num_cpus, prefix=""): """Set virtual CPU devices in context. If virtual CPU devices are already configured at context initialization by tf.config.set_logical_device_configuration(), this method should not be called. Args: num_cpus: Number of virtual CPUs. prefix: Device name prefix. Raises: RuntimeError: If virtual CPUs are already configured at context initialization. """ server_def = self._server_def or self._collective_ops_server_def local_prefix = ["/device"] if server_def is not None: local_prefix.append("/job:%s/replica:0/task:%d" % (server_def.job_name, server_def.task_index)) logical_local_devices = [d for d in self.list_logical_devices("CPU") if d.name.startswith(tuple(local_prefix))] self.ensure_initialized() # Error out if there are already multiple logical CPU in the context. if len(logical_local_devices) > 1: raise RuntimeError("Virtual CPUs already set, cannot modify again.") pywrap_tfe.TFE_SetLogicalCpuDevices(self._context_handle, num_cpus, prefix) self._initialize_logical_devices() def get_compiler_ir( self, device_name, platform_name, function_name, flat_args, captured_inputs, stage="hlo", ): """Get the compiler IR bytes. Args: device_name: The name of the device with the form as "/job:localhost/replica:0/task:0/device:CPU:0", "/device:TPU:0" etc. When this is used, actual device is needed for getting the compiler IR. platform_name: The name of the platform, e.g. "TPU". When this is used, first we find a device whose name contains the platform, if it is found we get the compiler IR by device. Otherwise the compiler IR is obtained as if using that device. The former logic of falling back to device is necessary, as there are cases of TF variables that need to access devices, but the upper layer may generally choose platform for getting compiler IR in a device-agnostic way. function_name: The name of the function to get the compiler IR. flat_args: The flat argument inputs. captured_inputs: The inputs that are captured. stage: The exported stage for the given function. Returns: The compiler IR bytes. """ return pywrap_tfe.TF_GetCompilerIr( self._context_handle, function_name, stage, device_name, flat_args, captured_inputs, platform_name, ) @deprecated( None, "XLA:CPU and XLA:GPU devices are deprecated", warn_once=True) def enable_xla_devices(self): """Enables XLA:CPU and XLA:GPU devices registration.""" pywrap_tfe.TF_EnableXlaDevices() @property def enable_mlir_bridge(self): return pywrap_tfe.TF_IsMlirBridgeEnabled() @property def enable_mlir_graph_optimization(self): return self._enable_mlir_graph_optimization @enable_mlir_bridge.setter def enable_mlir_bridge(self, enabled): pywrap_tfe.TF_EnableMlirBridge(enabled) self._thread_local_data.function_call_options = None @enable_mlir_graph_optimization.setter def enable_mlir_graph_optimization(self, enabled): self._enable_mlir_graph_optimization = enabled self._thread_local_data.function_call_options = None @property def optimizer_jit(self): level = self.config.graph_options.optimizer_options.global_jit_level return (level == config_pb2.OptimizerOptions.ON_1 or level == config_pb2.OptimizerOptions.ON_2) @optimizer_jit.setter def optimizer_jit(self, enabled): self._optimizer_jit = enabled self._thread_local_data.function_call_options = None def get_optimizer_experimental_options(self): """Get experimental options for the optimizer. Returns: Dictionary of current option values """ rewrite_options = self.config.graph_options.rewrite_options options = {} def rewriter_toggle(option): attr = getattr(rewrite_options, option) if attr != 0: options[option] = (attr == rewriter_config_pb2.RewriterConfig.ON) def rewriter_bool(option): options[option] = getattr(rewrite_options, option) rewriter_toggle("layout_optimizer") rewriter_toggle("constant_folding") rewriter_toggle("shape_optimization") rewriter_toggle("remapping") rewriter_toggle("arithmetic_optimization") rewriter_toggle("dependency_optimization") rewriter_toggle("loop_optimization") rewriter_toggle("function_optimization") rewriter_toggle("debug_stripper") rewriter_bool("disable_model_pruning") rewriter_toggle("scoped_allocator_optimization") rewriter_toggle("pin_to_host_optimization") rewriter_toggle("implementation_selector") rewriter_toggle("auto_mixed_precision") rewriter_toggle("use_plugin_optimizers") rewriter_bool("disable_meta_optimizer") rewriter_toggle("auto_mixed_precision_onednn_bfloat16") rewriter_toggle("auto_mixed_precision_mkl") if rewrite_options.min_graph_nodes != 0: options["min_graph_nodes"] = rewrite_options.min_graph_nodes return options def set_optimizer_experimental_options(self, options): """Set experimental options for the optimizer. Args: options: Dictionary of options to modify """ self._optimizer_experimental_options.update(options) self._thread_local_data.function_call_options = None @property def intra_op_parallelism_threads(self): return self.config.intra_op_parallelism_threads @intra_op_parallelism_threads.setter def intra_op_parallelism_threads(self, num_threads): if self._intra_op_parallelism_threads == num_threads: return if self._context_handle is not None: raise RuntimeError( "Intra op parallelism cannot be modified after initialization.") self._intra_op_parallelism_threads = num_threads @property def inter_op_parallelism_threads(self): return self.config.inter_op_parallelism_threads @inter_op_parallelism_threads.setter def inter_op_parallelism_threads(self, num_threads): if self._inter_op_parallelism_threads == num_threads: return if self._context_handle is not None: raise RuntimeError( "Inter op parallelism cannot be modified after initialization.") self._inter_op_parallelism_threads = num_threads @property def soft_device_placement(self): return self.config.allow_soft_placement @soft_device_placement.setter def soft_device_placement(self, enable): if self._context_handle is not None: pywrap_tfe.TFE_ContextSetSoftDevicePlacement(self._handle, enable) self._soft_device_placement = enable self._thread_local_data.function_call_options = None @property def log_device_placement(self): return self.config.log_device_placement @log_device_placement.setter def log_device_placement(self, enable): if self._context_handle is not None: pywrap_tfe.TFE_ContextSetLogDevicePlacement(self._handle, enable) self._log_device_placement = enable self._thread_local_data.function_call_options = None @property def jit_compile_rewrite(self): return self._jit_compile_rewrite @jit_compile_rewrite.setter def jit_compile_rewrite(self, enable): if self._context_handle is not None: pywrap_tfe.TFE_ContextSetJitCompileRewrite(self._handle, enable) self._jit_compile_rewrite = enable @property def device_policy(self): # Only get the policy from the context if it has already been initialized if self._context_handle is not None: return pywrap_tfe.TFE_ContextGetDevicePlacementPolicy(self._handle) return self._device_policy @device_policy.setter def device_policy(self, policy): if policy is None: policy = DEVICE_PLACEMENT_SILENT if self._device_policy != policy: self._device_policy = policy # Only set the policy if the context has already been initialized if self._context_handle is not None: pywrap_tfe.TFE_ContextSetThreadLocalDevicePlacementPolicy( self._handle, self._device_policy) @property def use_tfrt(self): return self._use_tfrt @use_tfrt.setter def use_tfrt(self, tfrt): """Sets whether to use TFRT.""" if not isinstance(tfrt, bool): raise ValueError("Expecting a boolean but got %s" % type(tfrt)) if self._use_tfrt != tfrt: if self._initialized: raise ValueError("use_tfrt should be set before being initialized.") self._use_tfrt = tfrt @property def operation_timeout_in_ms(self): return self.config.operation_timeout_in_ms @operation_timeout_in_ms.setter def operation_timeout_in_ms(self, timeout_in_ms): if self._operation_timeout_in_ms == timeout_in_ms: return if self._context_handle is not None: raise RuntimeError( "Operation timeout cannot be modified after initialization.") self._operation_timeout_in_ms = timeout_in_ms def enable_run_metadata(self): """Enables tracing of op execution via RunMetadata. To retrieve the accumulated metadata call context.export_run_metadata() and to stop tracing call context.disable_run_metadata(). """ self.ensure_initialized() pywrap_tfe.TFE_ContextEnableRunMetadata(self._handle) def disable_run_metadata(self): """Disables tracing of op execution via RunMetadata.""" if not self._context_handle: return pywrap_tfe.TFE_ContextDisableRunMetadata(self._context_handle) def enable_graph_collection(self): """Enables graph collection of executed functions. To retrieve the accumulated graphs call context.export_run_metadata() and to stop collecting graphs call context.disable_graph_collection(). """ self.ensure_initialized() pywrap_tfe.TFE_ContextEnableGraphCollection(self._handle) def disable_graph_collection(self): """Disables graph collection of executed functions.""" if not self._context_handle: return pywrap_tfe.TFE_ContextDisableGraphCollection(self._context_handle) def export_run_metadata(self): """Returns a RunMetadata proto with accumulated information. The returned protocol buffer contains information since the most recent call to either enable_run_metadata or export_run_metadata. Returns: A RunMetadata protocol buffer. Or None if not enabled. """ if not self._context_handle: return None with c_api_util.tf_buffer() as buffer_: pywrap_tfe.TFE_ContextExportRunMetadata(self._context_handle, buffer_) proto_data = pywrap_tf_session.TF_GetBuffer(buffer_) run_metadata = config_pb2.RunMetadata() run_metadata.ParseFromString(compat.as_bytes(proto_data)) return run_metadata def set_server_def_retries(self, retries): """Set the number of retries to use when calling SetServerDef. In cases where many servers run in high-preemption environments, jobs could be preempted during startup and initial connection via SetServerDef. Retries allow for more robust connection in these environments. Args: retries: int specifying the number of connection retries before failing. Retries follow an exponential backoff waiting period with min value 1ms, max value 10s, and exponent 1.3. """ self._set_server_def_retries = retries @property def context_switches(self): """Returns a stack of context switches.""" return self._context_switches class _EagerDeviceContext(object): """Context-manager forcing placement of ops and Tensors on a device.""" __slots__ = ["_device_name", "_ctx", "_stack"] def __init__(self, ctx, device_name): self._device_name = device_name self._ctx = ctx self._stack = [] # TODO(b/189233748): Consolidate the device string parsing logic with # tensorflow/core/util/device_name_utils.cc. def __enter__(self): ctx = self._ctx old_device_name = ctx.device_name old_device_spec = ctx.device_spec new_device_name = self._device_name cache_key = (old_device_name, new_device_name) try: new_device_name, new_device_spec = _device_parsing_cache[cache_key] except TypeError: # Error while trying to compute the cache key. raise ValueError("Expecting a string device name. Got %s(%s)" % (type(new_device_name), new_device_name)) except KeyError: # Handle a cache miss. if new_device_name is not None: if not isinstance(new_device_name, str): raise ValueError("Expecting a string device name. Got %s(%s)" % (type(new_device_name), new_device_name)) device_spec = pydev.DeviceSpec.from_string(new_device_name) if old_device_name: new_device_spec = copy.copy(old_device_spec) else: ctx.ensure_initialized() new_device_spec = pydev.DeviceSpec.from_string( ctx._context_devices[0]) # pylint: disable=protected-access new_device_spec = new_device_spec.make_merged_spec(device_spec) else: new_device_spec = pydev.DeviceSpec.from_string("") new_device_name = new_device_spec.to_string() _device_parsing_cache[cache_key] = (new_device_name, new_device_spec) ctx._set_device(new_device_name, new_device_spec) # pylint: disable=protected-access self._stack.append((old_device_name, old_device_spec, new_device_spec)) def __exit__(self, *ex_info): ctx = self._ctx old_device_name, old_device_spec, new_device_spec = self._stack[-1] if ctx.device_spec is not new_device_spec: raise RuntimeError("Exiting device scope without proper scope nesting") del self._stack[-1] ctx._set_device(old_device_name, old_device_spec) # pylint: disable=protected-access # Do not change directly. _context = None _context_lock = threading.Lock() def _set_context_locked(ctx): global _context pywrap_tfe.TFE_Py_SetEagerContext(ctx) ctx.mark_as_global_context() _context = ctx def _set_context(ctx): with _context_lock: _set_context_locked(ctx) def _create_context(): with _context_lock: if _context is None: ctx = Context() _set_context_locked(ctx) def _reset_context(): """Clears and re-initializes the singleton context. Should only be used for testing. """ global _context global _device_parsing_cache # Garbage collect and clear scalar cache to avoid Tensor from current context # polluting next context. gc.collect() pywrap_tfe.TFE_ClearScalarCache() with _context_lock: if _context is not None: _context._clear_caches() _context = None _create_context() _device_parsing_cache = {} def _reset_jit_compiler_flags(): """Clears and re-initializes the TF JIT compiler flags. Should only be used for testing. """ pywrap_tfe.TF_ResetJitCompilerFlags() def context() -> Context: """Returns a singleton context object.""" if _context is None: _create_context() return _context def context_safe(): """Returns current context (or None if one hasn't been initialized).""" return _context def ensure_initialized(): """Initialize the context.""" context().ensure_initialized() def initialize_logical_devices(): """Initialize the virtual devices.""" context()._initialize_logical_devices() # pylint: disable=protected-access def set_global_seed(seed): """Sets the eager mode seed.""" context()._set_global_seed(seed) # pylint: disable=protected-access def global_seed(): """Returns the eager mode seed.""" return context()._seed # pylint: disable=protected-access def internal_operation_seed(): """Returns the operation seed generated based on global seed.""" return context()._internal_operation_seed() # pylint: disable=protected-access @tf_export("executing_eagerly", v1=[]) def executing_eagerly(): """Checks whether the current thread has eager execution enabled. Eager execution is enabled by default and this API returns `True` in most of cases. However, this API might return `False` in the following use cases. * Executing inside `tf.function`, unless under `tf.init_scope` or `tf.config.run_functions_eagerly(True)` is previously called. * Executing inside a transformation function for `tf.dataset`. * `tf.compat.v1.disable_eager_execution()` is called. General case: >>> print(tf.executing_eagerly()) True Inside `tf.function`: >>> @tf.function ... def fn(): ... with tf.init_scope(): ... print(tf.executing_eagerly()) ... print(tf.executing_eagerly()) >>> fn() True False Inside `tf.function` after `tf.config.run_functions_eagerly(True)` is called: >>> tf.config.run_functions_eagerly(True) >>> @tf.function ... def fn(): ... with tf.init_scope(): ... print(tf.executing_eagerly()) ... print(tf.executing_eagerly()) >>> fn() True True >>> tf.config.run_functions_eagerly(False) Inside a transformation function for `tf.dataset`: >>> def data_fn(x): ... print(tf.executing_eagerly()) ... return x >>> dataset = tf.data.Dataset.range(100) >>> dataset = dataset.map(data_fn) False Returns: `True` if the current thread has eager execution enabled. """ ctx = context_safe() if ctx is None: return default_execution_mode == EAGER_MODE return ctx.executing_eagerly() @tf_export(v1=["executing_eagerly"]) def executing_eagerly_v1(): """Checks whether the current thread has eager execution enabled. Eager execution is typically enabled via `tf.compat.v1.enable_eager_execution`, but may also be enabled within the context of a Python function via tf.contrib.eager.py_func. When eager execution is enabled, returns `True` in most cases. However, this API might return `False` in the following use cases. * Executing inside `tf.function`, unless under `tf.init_scope` or `tf.config.run_functions_eagerly(True)` is previously called. * Executing inside a transformation function for `tf.dataset`. * `tf.compat.v1.disable_eager_execution()` is called. >>> tf.compat.v1.enable_eager_execution() General case: >>> print(tf.executing_eagerly()) True Inside `tf.function`: >>> @tf.function ... def fn(): ... with tf.init_scope(): ... print(tf.executing_eagerly()) ... print(tf.executing_eagerly()) >>> fn() True False Inside `tf.function` after `tf.config.run_functions_eagerly(True)` is called: >>> tf.config.run_functions_eagerly(True) >>> @tf.function ... def fn(): ... with tf.init_scope(): ... print(tf.executing_eagerly()) ... print(tf.executing_eagerly()) >>> fn() True True >>> tf.config.run_functions_eagerly(False) Inside a transformation function for `tf.dataset`: >>> def data_fn(x): ... print(tf.executing_eagerly()) ... return x >>> dataset = tf.data.Dataset.range(100) >>> dataset = dataset.map(data_fn) False Returns: `True` if the current thread has eager execution enabled. """ return executing_eagerly() def in_eager_mode(): """Use executing_eagerly() instead. This function will be removed.""" return executing_eagerly() def anonymous_name(): """Returns the anonymous shared name. In eager mode we create anonymous resources to avoid spurious sharing issues. The runtime generates a unique name on our behalf when the reserved anonymous shared name is used as a shared name. Returns: The anonymous shared name. """ # The magic value is defined as # `tensorflow::ResourceHandle::ANONYMOUS_NAME` in C++. return "cd2c89b7-88b7-44c8-ad83-06c2a9158347" def graph_mode(): """Context-manager to disable eager execution for the current thread.""" return context()._mode(GRAPH_MODE) # pylint: disable=protected-access # Used by b/167638505 for keras backend API and Lambda layer. @tf_export("__internal__.eager_context.eager_mode", v1=[]) def eager_mode(): """Context-manager to enable eager execution for the current thread.""" return context()._mode(EAGER_MODE) # pylint: disable=protected-access def scope_name(): """Name of the current scope.""" return context().scope_name def device(name): """Context-manager to force placement of operations and Tensors on a device. Example: ```python with tf.device('gpu:0'): with tf.device('cpu:0'): shape = tf.constant([], dtype=tf.int32) x = tf.random.truncated_normal(shape, tf.float32) ``` will ensure that the `shape` Tensor is on CPU but the `truncated_normal` operation runs on GPU 0. Args: name: Name of the device (see context().devices()), or None to perform automatic placement. Returns: Context manager for setting the device. """ ensure_initialized() return context().device(name) # Expose some properties of Context as internally public APIs (b/160348781). @tf_export("__internal__.eager_context.get_config", v1=[]) def get_config(): """Get the ConfigProto of Context. Returns: The ConfigProto of Context. """ return context().config @tf_export("__internal__.eager_context.get_device_name", v1=[]) def get_device_name(): """Get the device name for the current thread. Returns: The device name for the current thread. """ return context().device_name @tf_export("__internal__.eager_context.set_soft_device_placement", v1=[]) def set_soft_device_placement(enabled): """Set if soft device placements should be allowed. Args: enabled: Whether to enable soft device placement. """ context().soft_device_placement = enabled @tf_export("__internal__.eager_context.get_executor", v1=[]) def get_executor(): """Get the Executor of the current thread. Returns: The Executor of the current thread. """ return context().executor @tf_export("debugging.get_log_device_placement") def get_log_device_placement(): """Get if device placements are logged. Returns: If device placements are logged. """ return context().log_device_placement @tf_export("debugging.set_log_device_placement") def set_log_device_placement(enabled): """Turns logging for device placement decisions on or off. Operations execute on a particular device, producing and consuming tensors on that device. This may change the performance of the operation or require TensorFlow to copy data to or from an accelerator, so knowing where operations execute is useful for debugging performance issues. For more advanced profiling, use the [TensorFlow profiler](https://www.tensorflow.org/guide/profiler). Device placement for operations is typically controlled by a `tf.device` scope, but there are exceptions, for example operations on a `tf.Variable` which follow the initial placement of the variable. Turning off soft device placement (with `tf.config.set_soft_device_placement`) provides more explicit control. >>> tf.debugging.set_log_device_placement(True) >>> tf.ones([]) >>> # [...] op Fill in device /job:localhost/replica:0/task:0/device:GPU:0 >>> with tf.device("CPU"): ... tf.ones([]) >>> # [...] op Fill in device /job:localhost/replica:0/task:0/device:CPU:0 >>> tf.debugging.set_log_device_placement(False) Turning on `tf.debugging.set_log_device_placement` also logs the placement of ops inside `tf.function` when the function is called. Args: enabled: Whether to enabled device placement logging. """ context().log_device_placement = enabled @tf_contextlib.contextmanager def device_policy(policy): """Context manager for setting device placement policy for current thread.""" ctx = context() old_policy = ctx.device_policy try: ctx.device_policy = policy yield finally: ctx.device_policy = old_policy def set_execution_mode(mode): """Sets execution mode for the current thread.""" context().execution_mode = mode # TODO(fishx): remove this method. @tf_contextlib.contextmanager def execution_mode(mode): """Context manager for setting execution mode for current thread.""" if mode is None: yield else: ctx = context() executor_new = executor.new_executor(mode == ASYNC) executor_old = ctx.executor try: executor_old.wait() ctx.executor = executor_new yield finally: ctx.executor = executor_old executor_new.wait() @tf_contextlib.contextmanager def executor_scope(e): """Context manager for changing executor for current thread. Args: e: A Executor to execute eager ops under this scope. Setting it to None will switch back to use the default executor for the context. Yields: Context manager for setting the executor for current thread. """ ctx = context() executor_old = ctx.executor try: ctx.executor = e yield finally: ctx.executor = executor_old @tf_export("experimental.function_executor_type") @tf_contextlib.contextmanager def function_executor_type(executor_type): """Context manager for setting the executor of eager defined functions. Eager defined functions are functions decorated by tf.contrib.eager.defun. Args: executor_type: a string for the name of the executor to be used to execute functions defined by tf.contrib.eager.defun. Yields: Context manager for setting the executor of eager defined functions. """ current_options = context().function_call_options old_options = copy.copy(current_options) try: current_options.executor_type = executor_type yield finally: context().function_call_options = old_options def is_async(): """Returns true if current thread is in async mode.""" return context().is_async() def num_gpus(): """Get the number of available GPU devices. Returns: The number of available GPU devices. """ return context().num_gpus() def enable_run_metadata(): """Enables tracing of op execution via RunMetadata. To retrieve the accumulated metadata call context.export_run_metadata() and to stop tracing call context.disable_run_metadata(). """ context().enable_run_metadata() def disable_run_metadata(): """Disables tracing of op execution via RunMetadata.""" context().disable_run_metadata() def enable_graph_collection(): """Enables graph collection of executed functions. To retrieve the accumulated graphs call context.export_run_metadata() and to stop collecting graphs call context.disable_graph_collection(). """ context().enable_graph_collection() def disable_graph_collection(): """Disables graph collection of executed functions.""" context().disable_graph_collection() def export_run_metadata(): """Returns a RunMetadata proto with accumulated information. The returned protocol buffer contains information since the most recent call to either enable_run_metadata or export_run_metadata. Returns: A RunMetadata protocol buffer. """ return context().export_run_metadata() @contextlib.contextmanager def collect_graphs(optimized=True): """Collects a flat list of pre- or post-optimization graphs. The collected graphs include device placements, which can be useful for testing. Usage: ``` @def_function.function def f(x): return x + constant_op.constant(1.) with context.collect_graphs() as graphs: with ops.device("CPU:0"): f(constant_op.constant(1.)) graph, = graphs # `graph` contains a single GraphDef for inspection ``` Args: optimized: whether to collect optimized graphs or non-optimized graphs Yields: A list of GraphDefs, populated when the context manager exits. """ ctx = context() ctx.enable_graph_collection() try: graphs = [] yield graphs metadata = ctx.export_run_metadata() finally: ctx.disable_graph_collection() for graph in metadata.function_graphs: if optimized: graphs.append(graph.post_optimization_graph) else: graphs.append(graph.pre_optimization_graph) def get_server_def(): return context().get_server_def() def set_server_def(server_def): context().set_server_def(server_def) def set_server_def_retries(retries): """Set the number of retries to use when calling SetServerDef. In cases where many servers run in high-preemption environments, jobs could be preempted during startup and initial connection via SetServerDef. Retries allow for more robust connection in these environments. Args: retries: int specifying the number of connection retries before failing. Retries follow an exponential backoff waiting period with min value 1ms, max value 10s, and exponent 1.3. """ context().set_server_def_retries(retries) def update_server_def(server_def): context().update_server_def(server_def) def check_alive(worker_name): return context().check_alive(worker_name) @tf_export("experimental.async_scope") @tf_contextlib.contextmanager def async_scope(): """Context manager for grouping async operations. Ops/function calls inside the scope can return before finishing the actual execution. When exiting the async scope, a synchronization barrier will be automatically added to ensure the completion of all async op and function execution, potentially raising exceptions if async execution results in an error state. Users may write the following code to asynchronously invoke `train_step_fn` and log the `loss` metric for every `num_steps` steps in a training loop. `train_step_fn` internally consumes data using `iterator.get_next()`, and may throw OutOfRangeError when running out of data. In the case: ``` try: with tf.experimental.async_scope(): for _ in range(num_steps): # Step function updates the metric `loss` internally train_step_fn() except tf.errors.OutOfRangeError: tf.experimental.async_clear_error() logging.info('loss = %s', loss.numpy()) ``` Yields: Context manager for grouping async operations. """ # TODO(haoyuzhang): replace env var once we have a config method to turn on # and off async streaming RPC remote_async_env_var = "TF_ENABLE_EAGER_CLIENT_STREAMING_ENQUEUE" old_policy = os.environ.get(remote_async_env_var) try: os.environ[remote_async_env_var] = str(True) yield # Note: sync local and remote executors iff the async block does not raise # an exception. Triggering sync after an exception may lead to derived # runtime errors and unexpected exception types. context().sync_executors() finally: if old_policy is None: del os.environ[remote_async_env_var] else: os.environ[remote_async_env_var] = old_policy def async_wait(): """Sync all async operations and raise any errors during execution. In async execution mode, an op/function call can return before finishing the actual execution. Calling this method creates a synchronization barrier for all async op and function execution. It only returns when all pending nodes are finished, potentially raising exceptions if async execution results in an error state. It is a no-op if the context is not initialized. """ disable_async_executor_env_var = "TF_PS_DISABLE_ASYNC_EXECUTOR_GLOBALLY" if os.environ.get(disable_async_executor_env_var) == str(True): return if context()._context_handle is not None: # pylint: disable=protected-access context().sync_executors() @tf_export("experimental.async_clear_error") def async_clear_error(): """Clear pending operations and error statuses in async execution. In async execution mode, an error in op/function execution can lead to errors in subsequent ops/functions that are scheduled but not yet executed. Calling this method clears all pending operations and reset the async execution state. Example: ``` while True: try: # Step function updates the metric `loss` internally train_step_fn() except tf.errors.OutOfRangeError: tf.experimental.async_clear_error() break logging.info('loss = %s', loss.numpy()) ``` """ context().clear_executor_errors() def add_c_function(c_func): """Add a C API TF_Function to the context.""" context().add_c_function(c_func) def get_c_function(name): """Get a C API TF_Function from the context.""" return context().get_c_function(name) def remove_function(name): """Remove a function from the context.""" context().remove_function(name) def get_function_def(name): return context().get_function_def(name) def is_custom_device(device_name): """Calls TFE_IsCustomDevice. Enables using C extensions specifying a custom device from Python. See the experimental eager C API in tensorflow/c/eager/c_api_experimental.h for details. Args: device_name: A string indicating the name to check whether it is a registered custom device. Returns: A boolean. """ return context().is_custom_device(device_name) def register_custom_device(device_capsule, device_name, device_info_capsule): """Calls TFE_RegisterCustomDevice to register a custom device with Python. Enables using C extensions specifying a custom device from Python. See the experimental eager C API in tensorflow/c/eager/c_api_experimental.h for details. Note that custom devices are not currently supported inside `tf.function`s. Args: device_capsule: A PyCapsule with the name set to 'TFE_CustomDevice' containing a pointer to a TFE_CustomDevice struct. The capsule retains ownership of the memory. device_name: A string indicating the name to register the custom device under, e.g. '/job:localhost/replica:0/task:0/device:CUSTOM:0'. It may subsequently be passed to `with tf.device(...):`. device_info_capsule: A PyCapsule with the name set to 'TFE_CustomDevice_DeviceInfo' containing a pointer to a device-specific struct with the initial state of the custom device (the void* device_info argument to TFE_RegisterCustomDevice). This method takes ownership of the memory and clears the capsule destructor. """ context().register_custom_device(device_capsule, device_name, device_info_capsule) # Not every user creates a Context via context.context() # (for example, enable_eager_execution in python/framework/ops.py), # but they do all import this file. Note that IS_IN_GRAPH_MODE and # in_graph_mode are both parameterless functions. def _tmp_in_graph_mode(): if context_safe() is None: # Context not yet initialized. Assume graph mode following the # default implementation in `is_in_graph_mode`. return True return not executing_eagerly() is_in_graph_mode.IS_IN_GRAPH_MODE = _tmp_in_graph_mode