# Copyright 2015 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Variable class.""" import abc import enum import functools import itertools import os from tensorflow.core.framework import variable_pb2 from tensorflow.python import pywrap_tensorflow # pylint: disable=unused-import from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor as tensor_lib from tensorflow.python.framework import tensor_conversion_registry from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import array_ops_stack from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import tensor_getitem_override from tensorflow.python.trackable import base as trackable from tensorflow.python.util import object_identity from tensorflow.python.util import tf_should_use from tensorflow.python.util import traceback_utils from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.deprecation import deprecated_args from tensorflow.python.util.tf_export import tf_export def default_variable_creator_v2(next_creator=None, **kwds): from tensorflow.python.ops import resource_variable_ops # pylint: disable=g-import-not-at-top return resource_variable_ops.default_variable_creator_v2( next_creator=next_creator, **kwds) def _make_getter(captured_getter, captured_previous): """To avoid capturing loop variables.""" def getter(**kwargs): return captured_getter(captured_previous, **kwargs) return getter @tf_export("VariableSynchronization") class VariableSynchronization(enum.Enum): """Indicates when a distributed variable will be synced. * `AUTO`: Indicates that the synchronization will be determined by the current `DistributionStrategy` (eg. With `MirroredStrategy` this would be `ON_WRITE`). * `NONE`: Indicates that there will only be one copy of the variable, so there is no need to sync. * `ON_WRITE`: Indicates that the variable will be updated across devices every time it is written. * `ON_READ`: Indicates that the variable will be aggregated across devices when it is read (eg. when checkpointing or when evaluating an op that uses the variable). Example: >>> temp_grad=[tf.Variable([0.], trainable=False, ... synchronization=tf.VariableSynchronization.ON_READ, ... aggregation=tf.VariableAggregation.MEAN ... )] """ AUTO = 0 NONE = 1 ON_WRITE = 2 ON_READ = 3 # LINT.IfChange @tf_export("VariableAggregation", v1=[]) class VariableAggregationV2(enum.Enum): """Indicates how a distributed variable will be aggregated. `tf.distribute.Strategy` distributes a model by making multiple copies (called "replicas") acting on different elements of the input batch in a data parallel model. When performing some variable-update operation, for example `var.assign_add(x)`, in a model, we need to resolve how to combine the different values for `x` computed in the different replicas. * `NONE`: This is the default, giving an error if you use a variable-update operation with multiple replicas. * `SUM`: Add the updates across replicas. * `MEAN`: Take the arithmetic mean ("average") of the updates across replicas. * `ONLY_FIRST_REPLICA`: This is for when every replica is performing the same update, but we only want to perform the update once. Used, e.g., for the global step counter. For example: >>> strategy = tf.distribute.MirroredStrategy(["GPU:0", "GPU:1"]) >>> with strategy.scope(): ... v = tf.Variable(5.0, aggregation=tf.VariableAggregation.MEAN) >>> @tf.function ... def update_fn(): ... return v.assign_add(1.0) >>> strategy.run(update_fn) PerReplica:{ 0: , 1: } """ NONE = 0 SUM = 1 MEAN = 2 ONLY_FIRST_REPLICA = 3 def __hash__(self): return hash(self.value) def __eq__(self, other): if self is other: return True elif isinstance(other, VariableAggregation): return int(self.value) == int(other.value) else: return False @tf_export(v1=["VariableAggregation"]) class VariableAggregation(enum.Enum): NONE = 0 SUM = 1 MEAN = 2 ONLY_FIRST_REPLICA = 3 ONLY_FIRST_TOWER = 3 # DEPRECATED def __hash__(self): return hash(self.value) # LINT.ThenChange(//tensorflow/core/framework/variable.proto) # # Note that we are currently relying on the integer values of the Python enums # matching the integer values of the proto enums. VariableAggregation.__doc__ = ( VariableAggregationV2.__doc__ + "* `ONLY_FIRST_TOWER`: Deprecated alias for `ONLY_FIRST_REPLICA`.\n ") def validate_synchronization_aggregation_trainable(synchronization, aggregation, trainable, name): """Given user-provided variable properties, sets defaults and validates.""" if aggregation is None: aggregation = VariableAggregation.NONE else: if not isinstance(aggregation, (VariableAggregation, VariableAggregationV2)): try: aggregation = VariableAggregationV2(aggregation) except ValueError: raise ValueError( "Invalid variable aggregation mode: {} for variable: {}".format( aggregation, name)) if synchronization is None: synchronization = VariableSynchronization.AUTO else: try: synchronization = VariableSynchronization(synchronization) except ValueError: raise ValueError( "Invalid variable synchronization mode: {} for variable: {}".format( synchronization, name)) if trainable is None: trainable = synchronization != VariableSynchronization.ON_READ return synchronization, aggregation, trainable class VariableMetaclass(abc.ABCMeta): """Metaclass to allow construction of tf.Variable to be overridden.""" @traceback_utils.filter_traceback def __call__(cls, *args, **kwargs): if hasattr(cls, "_variable_call") and callable(cls._variable_call): variable_call = cls._variable_call(*args, **kwargs) if variable_call is not None: return variable_call return super(VariableMetaclass, cls).__call__(*args, **kwargs) @tf_export("Variable", v1=[]) # TODO(mdan): This should subclass core.Tensor, and not all its subclasses? class Variable(trackable.Trackable, metaclass=VariableMetaclass): """See the [variable guide](https://tensorflow.org/guide/variable). A variable maintains shared, persistent state manipulated by a program. The `Variable()` constructor requires an initial value for the variable, which can be a `Tensor` of any type and shape. This initial value defines the type and shape of the variable. After construction, the type and shape of the variable are fixed. The value can be changed using one of the assign methods. >>> v = tf.Variable(1.) >>> v.assign(2.) >>> v.assign_add(0.5) The `shape` argument to `Variable`'s constructor allows you to construct a variable with a less defined shape than its `initial_value`: >>> v = tf.Variable(1., shape=tf.TensorShape(None)) >>> v.assign([[1.]]) dtype=float32, numpy=array([[1.]], ...)> Just like any `Tensor`, variables created with `Variable()` can be used as inputs to operations. Additionally, all the operators overloaded for the `Tensor` class are carried over to variables. >>> w = tf.Variable([[1.], [2.]]) >>> x = tf.constant([[3., 4.]]) >>> tf.matmul(w, x) >>> tf.sigmoid(w + x) When building a machine learning model it is often convenient to distinguish between variables holding trainable model parameters and other variables such as a `step` variable used to count training steps. To make this easier, the variable constructor supports a `trainable=` parameter. `tf.GradientTape` watches trainable variables by default: >>> with tf.GradientTape(persistent=True) as tape: ... trainable = tf.Variable(1.) ... non_trainable = tf.Variable(2., trainable=False) ... x1 = trainable * 2. ... x2 = non_trainable * 3. >>> tape.gradient(x1, trainable) >>> assert tape.gradient(x2, non_trainable) is None # Unwatched Variables are automatically tracked when assigned to attributes of types inheriting from `tf.Module`. >>> m = tf.Module() >>> m.v = tf.Variable([1.]) >>> m.trainable_variables (,) This tracking then allows saving variable values to [training checkpoints](https://www.tensorflow.org/guide/checkpoint), or to [SavedModels](https://www.tensorflow.org/guide/saved_model) which include serialized TensorFlow graphs. Variables are often captured and manipulated by `tf.function`s. This works the same way the un-decorated function would have: >>> v = tf.Variable(0.) >>> read_and_decrement = tf.function(lambda: v.assign_sub(0.1)) >>> read_and_decrement() >>> read_and_decrement() Variables created inside a `tf.function` must be owned outside the function and be created only once: >>> class M(tf.Module): ... @tf.function ... def __call__(self, x): ... if not hasattr(self, "v"): # Or set self.v to None in __init__ ... self.v = tf.Variable(x) ... return self.v * x >>> m = M() >>> m(2.) >>> m(3.) >>> m.v See the `tf.function` documentation for details. """ @deprecated_args( None, "A variable's value can be manually cached by calling " "tf.Variable.read_value() under a tf.device scope. The caching_device " "argument does not work properly.", "caching_device") def __init__(self, initial_value=None, trainable=None, validate_shape=True, caching_device=None, name=None, variable_def=None, dtype=None, import_scope=None, constraint=None, synchronization=VariableSynchronization.AUTO, aggregation=VariableAggregation.NONE, shape=None, experimental_enable_variable_lifting=True, ): """Creates a new variable with value `initial_value`. Args: initial_value: A `Tensor`, or Python object convertible to a `Tensor`, which is the initial value for the Variable. The initial value must have a shape specified unless `validate_shape` is set to False. Can also be a callable with no argument that returns the initial value when called. In that case, `dtype` must be specified. (Note that initializer functions from init_ops.py must first be bound to a shape before being used here.) trainable: If `True`, GradientTapes automatically watch uses of this variable. Defaults to `True`, unless `synchronization` is set to `ON_READ`, in which case it defaults to `False`. validate_shape: If `False`, allows the variable to be initialized with a value of unknown shape. If `True`, the default, the shape of `initial_value` must be known. caching_device: Note: This argument is only valid when using a v1-style `Session`. Optional device string describing where the Variable should be cached for reading. Defaults to the Variable's device. If not `None`, caches on another device. Typical use is to cache on the device where the Ops using the Variable reside, to deduplicate copying through `Switch` and other conditional statements. name: Optional name for the variable. Defaults to `'Variable'` and gets uniquified automatically. variable_def: `VariableDef` protocol buffer. If not `None`, recreates the Variable object with its contents, referencing the variable's nodes in the graph, which must already exist. The graph is not changed. `variable_def` and the other arguments are mutually exclusive. dtype: If set, initial_value will be converted to the given type. If `None`, either the datatype will be kept (if `initial_value` is a Tensor), or `convert_to_tensor` will decide. import_scope: Optional `string`. Name scope to add to the `Variable.` Only used when initializing from protocol buffer. constraint: An optional projection function to be applied to the variable after being updated by an `Optimizer` (e.g. used to implement norm constraints or value constraints for layer weights). The function must take as input the unprojected Tensor representing the value of the variable and return the Tensor for the projected value (which must have the same shape). Constraints are not safe to use when doing asynchronous distributed training. synchronization: Indicates when a distributed variable will be aggregated. Accepted values are constants defined in the class `tf.VariableSynchronization`. By default the synchronization is set to `AUTO` and the current `DistributionStrategy` chooses when to synchronize. aggregation: Indicates how a distributed variable will be aggregated. Accepted values are constants defined in the class `tf.VariableAggregation`. shape: (optional) The shape of this variable. If None, the shape of `initial_value` will be used. When setting this argument to `tf.TensorShape(None)` (representing an unspecified shape), the variable can be assigned with values of different shapes. experimental_enable_variable_lifting: Whether to lift the variable out if it's in a `tf.function`. Default is `True`. When this argument is `True`, variable creation will follow the behavior and restrictions described [here](https://www.tensorflow.org/guide/function#creating_tfvariables). If this argument is `False`, that description doesn't apply, and you can freely create and use the variable in the `tf.function`, as if it's a "mutable `tf.Tensor`". You can't return the variable though. Raises: ValueError: If both `variable_def` and initial_value are specified. ValueError: If the initial value is not specified, or does not have a shape and `validate_shape` is `True`. """ raise NotImplementedError def __repr__(self): raise NotImplementedError def value(self): """Returns the last snapshot of this variable. You usually do not need to call this method as all ops that need the value of the variable call it automatically through a `convert_to_tensor()` call. Returns a `Tensor` which holds the value of the variable. You can not assign a new value to this tensor as it is not a reference to the variable. To avoid copies, if the consumer of the returned value is on the same device as the variable, this actually returns the live value of the variable, not a copy. Updates to the variable are seen by the consumer. If the consumer is on a different device it will get a copy of the variable. Returns: A `Tensor` containing the value of the variable. """ raise NotImplementedError def read_value(self): """Returns the value of this variable, read in the current context. Can be different from value() if it's on another device, with control dependencies, etc. Returns: A `Tensor` containing the value of the variable. """ raise NotImplementedError def set_shape(self, shape): """Overrides the shape for this variable. Args: shape: the `TensorShape` representing the overridden shape. """ raise NotImplementedError @property def trainable(self): raise NotImplementedError @property def synchronization(self): raise NotImplementedError @property def aggregation(self): raise NotImplementedError def eval(self, session=None): """In a session, computes and returns the value of this variable. This is not a graph construction method, it does not add ops to the graph. This convenience method requires a session where the graph containing this variable has been launched. If no session is passed, the default session is used. See `tf.compat.v1.Session` for more information on launching a graph and on sessions. ```python v = tf.Variable([1, 2]) init = tf.compat.v1.global_variables_initializer() with tf.compat.v1.Session() as sess: sess.run(init) # Usage passing the session explicitly. print(v.eval(sess)) # Usage with the default session. The 'with' block # above makes 'sess' the default session. print(v.eval()) ``` Args: session: The session to use to evaluate this variable. If none, the default session is used. Returns: A numpy `ndarray` with a copy of the value of this variable. """ raise NotImplementedError @deprecated( None, "Use Variable.read_value. Variables in 2.X are initialized " "automatically both in eager and graph (inside tf.defun) contexts.") def initialized_value(self): """Returns the value of the initialized variable. You should use this instead of the variable itself to initialize another variable with a value that depends on the value of this variable. ```python # Initialize 'v' with a random tensor. v = tf.Variable(tf.random.truncated_normal([10, 40])) # Use `initialized_value` to guarantee that `v` has been # initialized before its value is used to initialize `w`. # The random values are picked only once. w = tf.Variable(v.initialized_value() * 2.0) ``` Returns: A `Tensor` holding the value of this variable after its initializer has run. """ raise NotImplementedError @property def initial_value(self): """Returns the Tensor used as the initial value for the variable. Note that this is different from `initialized_value()` which runs the op that initializes the variable before returning its value. This method returns the tensor that is used by the op that initializes the variable. Returns: A `Tensor`. """ raise NotImplementedError @property def constraint(self): """Returns the constraint function associated with this variable. Returns: The constraint function that was passed to the variable constructor. Can be `None` if no constraint was passed. """ raise NotImplementedError def assign(self, value, use_locking=False, name=None, read_value=True): """Assigns a new value to the variable. This is essentially a shortcut for `assign(self, value)`. Args: value: A `Tensor`. The new value for this variable. use_locking: If `True`, use locking during the assignment. name: The name of the operation to be created read_value: if True, will return something which evaluates to the new value of the variable; if False will return the assign op. Returns: The updated variable. If `read_value` is false, instead returns None in Eager mode and the assign op in graph mode. """ raise NotImplementedError def assign_add(self, delta, use_locking=False, name=None, read_value=True): """Adds a value to this variable. This is essentially a shortcut for `assign_add(self, delta)`. Args: delta: A `Tensor`. The value to add to this variable. use_locking: If `True`, use locking during the operation. name: The name of the operation to be created read_value: if True, will return something which evaluates to the new value of the variable; if False will return the assign op. Returns: The updated variable. If `read_value` is false, instead returns None in Eager mode and the assign op in graph mode. """ raise NotImplementedError def assign_sub(self, delta, use_locking=False, name=None, read_value=True): """Subtracts a value from this variable. This is essentially a shortcut for `assign_sub(self, delta)`. Args: delta: A `Tensor`. The value to subtract from this variable. use_locking: If `True`, use locking during the operation. name: The name of the operation to be created read_value: if True, will return something which evaluates to the new value of the variable; if False will return the assign op. Returns: The updated variable. If `read_value` is false, instead returns None in Eager mode and the assign op in graph mode. """ raise NotImplementedError def scatter_sub(self, sparse_delta, use_locking=False, name=None): """Subtracts `tf.IndexedSlices` from this variable. Args: sparse_delta: `tf.IndexedSlices` to be subtracted from this variable. use_locking: If `True`, use locking during the operation. name: the name of the operation. Returns: The updated variable. Raises: TypeError: if `sparse_delta` is not an `IndexedSlices`. """ raise NotImplementedError def scatter_add(self, sparse_delta, use_locking=False, name=None): """Adds `tf.IndexedSlices` to this variable. Args: sparse_delta: `tf.IndexedSlices` to be added to this variable. use_locking: If `True`, use locking during the operation. name: the name of the operation. Returns: The updated variable. Raises: TypeError: if `sparse_delta` is not an `IndexedSlices`. """ raise NotImplementedError def scatter_max(self, sparse_delta, use_locking=False, name=None): """Updates this variable with the max of `tf.IndexedSlices` and itself. Args: sparse_delta: `tf.IndexedSlices` to use as an argument of max with this variable. use_locking: If `True`, use locking during the operation. name: the name of the operation. Returns: The updated variable. Raises: TypeError: if `sparse_delta` is not an `IndexedSlices`. """ raise NotImplementedError def scatter_min(self, sparse_delta, use_locking=False, name=None): """Updates this variable with the min of `tf.IndexedSlices` and itself. Args: sparse_delta: `tf.IndexedSlices` to use as an argument of min with this variable. use_locking: If `True`, use locking during the operation. name: the name of the operation. Returns: The updated variable. Raises: TypeError: if `sparse_delta` is not an `IndexedSlices`. """ raise NotImplementedError def scatter_mul(self, sparse_delta, use_locking=False, name=None): """Multiply this variable by `tf.IndexedSlices`. Args: sparse_delta: `tf.IndexedSlices` to multiply this variable by. use_locking: If `True`, use locking during the operation. name: the name of the operation. Returns: The updated variable. Raises: TypeError: if `sparse_delta` is not an `IndexedSlices`. """ raise NotImplementedError def scatter_div(self, sparse_delta, use_locking=False, name=None): """Divide this variable by `tf.IndexedSlices`. Args: sparse_delta: `tf.IndexedSlices` to divide this variable by. use_locking: If `True`, use locking during the operation. name: the name of the operation. Returns: The updated variable. Raises: TypeError: if `sparse_delta` is not an `IndexedSlices`. """ raise NotImplementedError def scatter_update(self, sparse_delta, use_locking=False, name=None): """Assigns `tf.IndexedSlices` to this variable. Args: sparse_delta: `tf.IndexedSlices` to be assigned to this variable. use_locking: If `True`, use locking during the operation. name: the name of the operation. Returns: The updated variable. Raises: TypeError: if `sparse_delta` is not an `IndexedSlices`. """ raise NotImplementedError def batch_scatter_update(self, sparse_delta, use_locking=False, name=None): """Assigns `tf.IndexedSlices` to this variable batch-wise. Analogous to `batch_gather`. This assumes that this variable and the sparse_delta IndexedSlices have a series of leading dimensions that are the same for all of them, and the updates are performed on the last dimension of indices. In other words, the dimensions should be the following: `num_prefix_dims = sparse_delta.indices.ndims - 1` `batch_dim = num_prefix_dims + 1` `sparse_delta.updates.shape = sparse_delta.indices.shape + var.shape[ batch_dim:]` where `sparse_delta.updates.shape[:num_prefix_dims]` `== sparse_delta.indices.shape[:num_prefix_dims]` `== var.shape[:num_prefix_dims]` And the operation performed can be expressed as: `var[i_1, ..., i_n, sparse_delta.indices[i_1, ..., i_n, j]] = sparse_delta.updates[ i_1, ..., i_n, j]` When sparse_delta.indices is a 1D tensor, this operation is equivalent to `scatter_update`. To avoid this operation one can looping over the first `ndims` of the variable and using `scatter_update` on the subtensors that result of slicing the first dimension. This is a valid option for `ndims = 1`, but less efficient than this implementation. Args: sparse_delta: `tf.IndexedSlices` to be assigned to this variable. use_locking: If `True`, use locking during the operation. name: the name of the operation. Returns: The updated variable. Raises: TypeError: if `sparse_delta` is not an `IndexedSlices`. """ raise NotImplementedError def scatter_nd_sub(self, indices, updates, name=None): """Applies sparse subtraction to individual values or slices in a Variable. Assuming the variable has rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into self. It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th dimension of self. `updates` is `Tensor` of rank `Q-1+P-K` with shape: ``` [d_0, ..., d_{Q-2}, self.shape[K], ..., self.shape[P-1]]. ``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that update would look like this: ```python v = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8]) indices = tf.constant([[4], [3], [1] ,[7]]) updates = tf.constant([9, 10, 11, 12]) v.scatter_nd_sub(indices, updates) print(v) ``` After the update `v` would look like this: [1, -9, 3, -6, -4, 6, 7, -4] See `tf.scatter_nd` for more details about how to make updates to slices. Args: indices: The indices to be used in the operation. updates: The values to be used in the operation. name: the name of the operation. Returns: The updated variable. """ raise NotImplementedError def scatter_nd_add(self, indices, updates, name=None): """Applies sparse addition to individual values or slices in a Variable. The Variable has rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into self. It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th dimension of self. `updates` is `Tensor` of rank `Q-1+P-K` with shape: ``` [d_0, ..., d_{Q-2}, self.shape[K], ..., self.shape[P-1]]. ``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that update would look like this: ```python v = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8]) indices = tf.constant([[4], [3], [1] ,[7]]) updates = tf.constant([9, 10, 11, 12]) v.scatter_nd_add(indices, updates) print(v) ``` The resulting update to v would look like this: [1, 13, 3, 14, 14, 6, 7, 20] See `tf.scatter_nd` for more details about how to make updates to slices. Args: indices: The indices to be used in the operation. updates: The values to be used in the operation. name: the name of the operation. Returns: The updated variable. """ raise NotImplementedError def scatter_nd_update(self, indices, updates, name=None): """Applies sparse assignment to individual values or slices in a Variable. The Variable has rank `P` and `indices` is a `Tensor` of rank `Q`. `indices` must be integer tensor, containing indices into self. It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`. The innermost dimension of `indices` (with length `K`) corresponds to indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th dimension of self. `updates` is `Tensor` of rank `Q-1+P-K` with shape: ``` [d_0, ..., d_{Q-2}, self.shape[K], ..., self.shape[P-1]]. ``` For example, say we want to add 4 scattered elements to a rank-1 tensor to 8 elements. In Python, that update would look like this: ```python v = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8]) indices = tf.constant([[4], [3], [1] ,[7]]) updates = tf.constant([9, 10, 11, 12]) v.scatter_nd_update(indices, updates) print(v) ``` The resulting update to v would look like this: [1, 11, 3, 10, 9, 6, 7, 12] See `tf.scatter_nd` for more details about how to make updates to slices. Args: indices: The indices to be used in the operation. updates: The values to be used in the operation. name: the name of the operation. Returns: The updated variable. """ raise NotImplementedError def sparse_read(self, indices, name=None): r"""Gather slices from params axis axis according to indices. This function supports a subset of tf.gather, see tf.gather for details on usage. Args: indices: The index `Tensor`. Must be one of the following types: `int32`, `int64`. Must be in range `[0, params.shape[axis])`. name: A name for the operation (optional). Returns: A `Tensor`. Has the same type as `params`. """ raise AttributeError def gather_nd(self, indices, name=None): r"""Gather slices from `params` into a Tensor with shape specified by `indices`. See tf.gather_nd for details. Args: indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. Index tensor. name: A name for the operation (optional). Returns: A `Tensor`. Has the same type as `params`. """ raise AttributeError @deprecated(None, "Prefer Dataset.range instead.") def count_up_to(self, limit): """Increments this variable until it reaches `limit`. When that Op is run it tries to increment the variable by `1`. If incrementing the variable would bring it above `limit` then the Op raises the exception `OutOfRangeError`. If no error is raised, the Op outputs the value of the variable before the increment. This is essentially a shortcut for `count_up_to(self, limit)`. Args: limit: value at which incrementing the variable raises an error. Returns: A `Tensor` that will hold the variable value before the increment. If no other Op modifies this variable, the values produced will all be distinct. """ raise NotImplementedError @deprecated(None, "Prefer Variable.assign which has equivalent behavior in 2.X.") def load(self, value, session=None): """Load new value into this variable. Writes new value to variable's memory. Doesn't add ops to the graph. This convenience method requires a session where the graph containing this variable has been launched. If no session is passed, the default session is used. See `tf.compat.v1.Session` for more information on launching a graph and on sessions. ```python v = tf.Variable([1, 2]) init = tf.compat.v1.global_variables_initializer() with tf.compat.v1.Session() as sess: sess.run(init) # Usage passing the session explicitly. v.load([2, 3], sess) print(v.eval(sess)) # prints [2 3] # Usage with the default session. The 'with' block # above makes 'sess' the default session. v.load([3, 4], sess) print(v.eval()) # prints [3 4] ``` Args: value: New variable value session: The session to use to evaluate this variable. If none, the default session is used. Raises: ValueError: Session is not passed and no default session """ if context.executing_eagerly(): self.assign(value) else: session = session or ops.get_default_session() if session is None: raise ValueError( "Either session argument should be provided or default session " "should be established") session.run(self.initializer, {self.initializer.inputs[1]: value}) # Conversion to tensor. @staticmethod def _TensorConversionFunction(v, dtype=None, name=None, as_ref=False): # pylint: disable=invalid-name """Utility function for converting a Variable to a Tensor.""" _ = name if dtype and not dtype.is_compatible_with(v.dtype): raise ValueError( f"Incompatible type conversion requested to type '{dtype.name}' for " f"variable of type '{v.dtype.name}' (Variable: {v}).") if as_ref: return v._ref() # pylint: disable=protected-access else: return v.value() @classmethod def _OverloadAllOperators(cls): # pylint: disable=invalid-name """Register overloads for all operators.""" for operator in tensor_lib.Tensor.OVERLOADABLE_OPERATORS: cls._OverloadOperator(operator) # For slicing, bind getitem differently than a tensor (use _slice_helper_var # instead) # pylint: disable=protected-access setattr(cls, "__getitem__", tensor_getitem_override._slice_helper_var) @classmethod def _OverloadOperator(cls, operator): # pylint: disable=invalid-name """Defer an operator overload to `tensor_lib.Tensor`. We pull the operator out of tensor_lib.Tensor dynamically to avoid ordering issues. Args: operator: string. The operator name. """ # We can't use the overload mechanism on __eq__ & __ne__ since __eq__ is # called when adding a variable to sets. As a result we call a.value() which # causes infinite recursion when operating within a GradientTape # TODO(gjn): Consider removing this if operator == "__eq__" or operator == "__ne__": return tensor_oper = getattr(tensor_lib.Tensor, operator) def _run_op(a, *args, **kwargs): # pylint: disable=protected-access return tensor_oper(a.value(), *args, **kwargs) functools.update_wrapper(_run_op, tensor_oper) setattr(cls, operator, _run_op) def __hash__(self): if ( tensor_lib.Tensor._USE_EQUALITY and ops.executing_eagerly_outside_functions() ): # pylint: disable=protected-access raise TypeError( "Variable is unhashable. " f"Instead, use variable.ref() as the key. (Variable: {self})" ) else: return id(self) # TODO(gjn): duplicate of math_ops.tensor_equals, consider removing def __eq__(self, other): """Compares two variables element-wise for equality.""" if ( tensor_lib.Tensor._USE_EQUALITY and ops.executing_eagerly_outside_functions() ): # pylint: disable=protected-access return gen_math_ops.equal(self, other, incompatible_shape_error=False) else: # In legacy graph mode, tensor equality is object equality return self is other # TODO(gjn): duplicate of math_ops.tensor_not_equals, consider removing def __ne__(self, other): """Compares two variables element-wise for equality.""" if ( tensor_lib.Tensor._USE_EQUALITY and ops.executing_eagerly_outside_functions() ): # pylint: disable=protected-access return gen_math_ops.not_equal(self, other, incompatible_shape_error=False) else: # In legacy graph mode, tensor equality is object equality return self is not other def __iter__(self): """When executing eagerly, iterates over the value of the variable.""" return iter(self.read_value()) # NOTE(mrry): This enables the Variable's overloaded "right" binary # operators to run when the left operand is an ndarray, because it # accords the Variable class higher priority than an ndarray, or a # numpy matrix. # TODO(mrry): Convert this to using numpy's __numpy_ufunc__ # mechanism, which allows more control over how Variables interact # with ndarrays. __array_priority__ = 100 @property def name(self): """The name of this variable.""" raise NotImplementedError @property def _shared_name(self): """The shared name of the variable. Unlike name(), shared_name doesn't have ":0" suffix. It is user-specified name with name scope prefix. Returns: variable name. """ return self.name[:self.name.index(":")] @property def initializer(self): """The initializer operation for this variable.""" raise NotImplementedError @property def device(self): """The device of this variable.""" raise NotImplementedError @property def dtype(self): """The `DType` of this variable.""" raise NotImplementedError @property def op(self): """The `Operation` of this variable.""" raise NotImplementedError @property def graph(self): """The `Graph` of this variable.""" raise NotImplementedError @property def shape(self): """The `TensorShape` of this variable. Returns: A `TensorShape`. """ raise NotImplementedError def get_shape(self) -> tensor_shape.TensorShape: """Alias of `Variable.shape`.""" return self.shape def _gather_saveables_for_checkpoint(self): """For implementing `Trackable`. This object is saveable on its own.""" return {trackable.VARIABLE_VALUE_KEY: self} def to_proto(self, export_scope=None): """Converts a `Variable` to a `VariableDef` protocol buffer. Args: export_scope: Optional `string`. Name scope to remove. Returns: A `VariableDef` protocol buffer, or `None` if the `Variable` is not in the specified name scope. """ raise NotImplementedError @staticmethod def from_proto(variable_def, import_scope=None): """Returns a `Variable` object created from `variable_def`.""" raise NotImplementedError def _set_save_slice_info(self, save_slice_info): """Sets the slice info for this `Variable`. Args: save_slice_info: A `Variable.SaveSliceInfo` object. """ self._save_slice_info = save_slice_info def _get_save_slice_info(self): return self._save_slice_info @deprecated(None, "Use ref() instead.") def experimental_ref(self): return self.ref() def ref(self): # tf.Tensor also has the same ref() API. If you update the # documentation here, please update tf.Tensor.ref() as well. """Returns a hashable reference object to this Variable. The primary use case for this API is to put variables in a set/dictionary. We can't put variables in a set/dictionary as `variable.__hash__()` is no longer available starting Tensorflow 2.0. The following will raise an exception starting 2.0 >>> x = tf.Variable(5) >>> y = tf.Variable(10) >>> z = tf.Variable(10) >>> variable_set = {x, y, z} Traceback (most recent call last): ... TypeError: Variable is unhashable. Instead, use tensor.ref() as the key. >>> variable_dict = {x: 'five', y: 'ten'} Traceback (most recent call last): ... TypeError: Variable is unhashable. Instead, use tensor.ref() as the key. Instead, we can use `variable.ref()`. >>> variable_set = {x.ref(), y.ref(), z.ref()} >>> x.ref() in variable_set True >>> variable_dict = {x.ref(): 'five', y.ref(): 'ten', z.ref(): 'ten'} >>> variable_dict[y.ref()] 'ten' Also, the reference object provides `.deref()` function that returns the original Variable. >>> x = tf.Variable(5) >>> x.ref().deref() """ return object_identity.Reference(self) @classmethod def _variable_call( cls, initial_value=None, trainable=None, validate_shape=True, caching_device=None, name=None, variable_def=None, dtype=None, import_scope=None, constraint=None, synchronization=VariableSynchronization.AUTO, aggregation=VariableAggregation.NONE, shape=None, experimental_enable_variable_lifting=None, **kwargs, ): """Variable class getter. Useful to force the signature.""" if cls is not Variable: return None previous_getter = lambda **kws: default_variable_creator_v2(None, **kws) for _, getter in ops.get_default_graph()._variable_creator_stack: # pylint: disable=protected-access previous_getter = _make_getter(getter, previous_getter) # Reset `aggregation` that is explicitly set as `None` to the enum NONE. if aggregation is None: aggregation = VariableAggregation.NONE return previous_getter( initial_value=initial_value, trainable=trainable, validate_shape=validate_shape, caching_device=caching_device, name=name, variable_def=variable_def, dtype=dtype, import_scope=import_scope, constraint=constraint, synchronization=synchronization, aggregation=aggregation, shape=shape, experimental_enable_variable_lifting=experimental_enable_variable_lifting, **kwargs ) class SaveSliceInfo: """Information on how to save this Variable as a slice. Provides internal support for saving variables as slices of a larger variable. This API is not public and is subject to change. Available properties: * full_name * full_shape * var_offset * var_shape """ def __init__(self, full_name=None, full_shape=None, var_offset=None, var_shape=None, save_slice_info_def=None, import_scope=None): """Create a `SaveSliceInfo`. Args: full_name: Name of the full variable of which this `Variable` is a slice. full_shape: Shape of the full variable, as a list of int. var_offset: Offset of this `Variable` into the full variable, as a list of int. var_shape: Shape of this `Variable`, as a list of int. save_slice_info_def: `SaveSliceInfoDef` protocol buffer. If not `None`, recreates the SaveSliceInfo object its contents. `save_slice_info_def` and other arguments are mutually exclusive. import_scope: Optional `string`. Name scope to add. Only used when initializing from protocol buffer. """ if save_slice_info_def: assert isinstance(save_slice_info_def, variable_pb2.SaveSliceInfoDef) self.full_name = ops.prepend_name_scope( save_slice_info_def.full_name, import_scope=import_scope) self.full_shape = list(save_slice_info_def.full_shape) self.var_offset = list(save_slice_info_def.var_offset) self.var_shape = list(save_slice_info_def.var_shape) else: self.full_name = full_name self.full_shape = full_shape self.var_offset = var_offset self.var_shape = var_shape @property def spec(self): """Computes the spec string used for saving.""" full_shape_str = " ".join("%d" % d for d in self.full_shape) + " " sl_spec = ":".join( "%d,%d" % (o, s) for o, s in zip(self.var_offset, self.var_shape)) return full_shape_str + sl_spec def to_proto(self, export_scope=None): """Returns a SaveSliceInfoDef() proto. Args: export_scope: Optional `string`. Name scope to remove. Returns: A `SaveSliceInfoDef` protocol buffer, or None if the `Variable` is not in the specified name scope. """ if (export_scope is None or self.full_name.startswith(export_scope)): save_slice_info_def = variable_pb2.SaveSliceInfoDef() save_slice_info_def.full_name = ops.strip_name_scope( self.full_name, export_scope) for i in self.full_shape: save_slice_info_def.full_shape.append(i) for i in self.var_offset: save_slice_info_def.var_offset.append(i) for i in self.var_shape: save_slice_info_def.var_shape.append(i) return save_slice_info_def else: return None Variable._OverloadAllOperators() # pylint: disable=protected-access def _try_guard_against_uninitialized_dependencies(name, initial_value): """Attempt to guard against dependencies on uninitialized variables. Replace references to variables in `initial_value` with references to the variable's initialized values. The initialized values are essentially conditional TensorFlow graphs that return a variable's value if it is initialized or its `initial_value` if it hasn't been initialized. This replacement is done on a best effort basis: - If the `initial_value` graph contains cycles, we don't do any replacements for that graph. - If the variables that `initial_value` depends on are not present in the `GLOBAL_VARIABLES` or `LOCAL_VARIABLES` we don't replace them. In these cases, it is up to the caller to ensure that the `initial_value` graph uses initialized variables or that they guard access to variables using their `initialized_value` method. Args: name: Variable name. initial_value: `Tensor`. The initial value. Returns: A `Tensor` suitable to initialize a variable. Raises: TypeError: If `initial_value` is not a `Tensor`. """ if not isinstance(initial_value, tensor_lib.Tensor): raise TypeError("initial_value needs to be a Tensor: %s" % initial_value) # Don't modify initial_value if it contains any cyclic dependencies. if _has_cycle(initial_value.op, state={}): return initial_value return _safe_initial_value_from_tensor(name, initial_value, op_cache={}) _UNKNOWN, _STARTED, _FINISHED = range(3) def _has_cycle(op, state): """Detect cycles in the dependencies of `initial_value`.""" op_state = state.get(op.name, _UNKNOWN) if op_state == _STARTED: return True elif op_state == _FINISHED: return False state[op.name] = _STARTED for i in itertools.chain((i.op for i in op.inputs), op.control_inputs): if _has_cycle(i, state): return True state[op.name] = _FINISHED return False def _safe_initial_value_from_tensor(name, tensor, op_cache): """Replace dependencies on variables with their initialized values. Args: name: Variable name. tensor: A `Tensor`. The tensor to replace. op_cache: A dict mapping operation names to `Operation`s. Used to memoize the results so as to avoid creating redundant operations. Returns: A `Tensor` compatible with `tensor`. Any inputs that lead to variable values will be replaced with a corresponding graph that uses the variable's initialized values. This is done on a best-effort basis. If no modifications need to be made then `tensor` will be returned unchanged. """ op = tensor.op new_op = op_cache.get(op.name) if new_op is None: new_op = _safe_initial_value_from_op(name, op, op_cache) op_cache[op.name] = new_op return new_op.outputs[tensor.value_index] def _safe_initial_value_from_op(name, op, op_cache): """Replace dependencies on variables with their initialized values. Args: name: Variable name. op: An `Operation`. The operation to replace. op_cache: A dict mapping operation names to `Operation`s. Used to memoize the results so as to avoid creating redundant operations. Returns: An `Operation` compatible with `op`. Any inputs that lead to variable values will be replaced with a corresponding graph that uses the variable's initialized values. This is done on a best-effort basis. If no modifications need to be made then `op` will be returned unchanged. """ op_type = op.node_def.op if op_type in ("IsVariableInitialized", "VarIsInitializedOp", "ReadVariableOp", "If"): return op # Attempt to find the initialized_value of any variable reference / handles. # TODO(b/70206927): Fix handling of ResourceVariables. if op_type in ("Variable", "VariableV2", "VarHandleOp"): initialized_value = _find_initialized_value_for_variable(op) return op if initialized_value is None else initialized_value.op # Recursively build initializer expressions for inputs. modified = False new_op_inputs = [] for op_input in op.inputs: new_op_input = _safe_initial_value_from_tensor(name, op_input, op_cache) new_op_inputs.append(new_op_input) modified = modified or (new_op_input != op_input) # If at least one input was modified, replace the op. if modified: new_op_type = op_type if new_op_type == "RefSwitch": new_op_type = "Switch" new_op_name = op.node_def.name + "_" + name new_op_name = new_op_name.replace(":", "_") return op.graph.create_op( new_op_type, new_op_inputs, op._output_types, # pylint: disable=protected-access name=new_op_name, attrs=op.node_def.attr) return op def _find_initialized_value_for_variable(variable_op): """Find the initialized value for a variable op. To do so, lookup the variable op in the variables collection. Args: variable_op: A variable `Operation`. Returns: A `Tensor` representing the initialized value for the variable or `None` if the initialized value could not be found. """ try: var_names = [variable_op.node_def.name, variable_op.node_def.name + ":0"] for collection_name in (ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.LOCAL_VARIABLES): for var in variable_op.graph.get_collection(collection_name): if var.name in var_names: return var.initialized_value() except AttributeError: # Return None when an incomplete user-defined variable type was put in # the collection. return None return None class PartitionedVariable: """A container for partitioned `Variable` objects. @compatibility(eager) `tf.PartitionedVariable` is not compatible with eager execution. Use `tf.Variable` instead which is compatible with both eager execution and graph construction. See [the TensorFlow Eager Execution guide](https://www.tensorflow.org/guide/eager#variables_and_optimizers) for details on how variables work in eager execution. @end_compatibility """ def __init__(self, name, shape, dtype, variable_list, partitions): """Creates a new partitioned variable wrapper. Variables passed via the variable_list must contain a save_slice_info field. Concatenation and iteration is in lexicographic order according to the var_offset property of the save_slice_info. Args: name: String. Overall name of the variables. shape: List of integers. Overall shape of the variables. dtype: Type of the variables. variable_list: List of `Variable` that comprise this partitioned variable. partitions: List of integers. Number of partitions for each dimension. Raises: TypeError: If `variable_list` is not a list of `Variable` objects, or `partitions` is not a list. ValueError: If `variable_list` is empty, or the `Variable` shape information does not match `shape`, or `partitions` has invalid values. """ if not isinstance(variable_list, (list, tuple)): raise TypeError("variable_list is not a list or tuple: %s" % variable_list) if not isinstance(partitions, (list, tuple)): raise TypeError("partitions is not a list or tuple: %s" % partitions) if not all(p >= 1 for p in partitions): raise ValueError("partition values must be positive: %s" % partitions) if not variable_list: raise ValueError("variable_list may not be empty") # pylint: disable=protected-access for v in variable_list: # Sort the variable_list lexicographically according to var offset value. if not all(v._get_save_slice_info() is not None for v in variable_list): raise ValueError( "All variables must have a save_slice_info available: %s" % [v.name for v in variable_list]) if len(shape) != len(partitions): raise ValueError("len(shape) != len(partitions): %s vs. %s" % (shape, partitions)) if v._get_save_slice_info().full_shape != shape: raise ValueError("All variables' full shapes must match shape: %s; " "but full shapes were: %s" % (shape, str([v._get_save_slice_info().full_shape]))) self._variable_list = sorted( variable_list, key=lambda v: v._get_save_slice_info().var_offset) # pylint: enable=protected-access self._name = name self._shape = shape self._dtype = dtype self._partitions = partitions self._as_tensor = None def __iter__(self): """Return an iterable for accessing the underlying partition Variables.""" return iter(self._variable_list) def __len__(self): num_partition_axes = len(self._partition_axes()) if num_partition_axes > 1: raise ValueError("Cannot get a length for %d > 1 partition axes" % num_partition_axes) return len(self._variable_list) def _partition_axes(self): if all(p == 1 for p in self._partitions): return [0] else: return [i for i, p in enumerate(self._partitions) if p > 1] def _concat(self): """Returns the overall concatenated value as a `Tensor`. This is different from using the partitioned variable directly as a tensor (through tensor conversion and `as_tensor`) in that it creates a new set of operations that keeps the control dependencies from its scope. Returns: `Tensor` containing the concatenated value. """ if len(self._variable_list) == 1: with ops.name_scope(None): return array_ops.identity(self._variable_list[0], name=self._name) partition_axes = self._partition_axes() if len(partition_axes) > 1: raise NotImplementedError( "Cannot concatenate along more than one dimension: %s. " "Multi-axis partition concat is not supported" % str(partition_axes)) partition_ix = partition_axes[0] with ops.name_scope(self._name + "/ConcatPartitions/"): concatenated = array_ops.concat(self._variable_list, partition_ix) with ops.name_scope(None): return array_ops.identity(concatenated, name=self._name) def as_tensor(self): """Returns the overall concatenated value as a `Tensor`. The returned tensor will not inherit the control dependencies from the scope where the value is used, which is similar to getting the value of `Variable`. Returns: `Tensor` containing the concatenated value. """ with ops.control_dependencies(None): return self._concat() @staticmethod def _TensorConversionFunction(v, dtype=None, name=None, as_ref=False): # pylint: disable=invalid-name _ = name if dtype is not None and not dtype.is_compatible_with(v.dtype): raise ValueError( "Incompatible type conversion requested to type '%s' for variable " "of type '%s'" % (dtype.name, v.dtype.name)) if as_ref: raise NotImplementedError( "PartitionedVariable doesn't support being used as a reference.") else: return v.as_tensor() @property def name(self): return self._name @property def dtype(self): return self._dtype @property def shape(self): return self.get_shape() @property def _distribute_strategy(self): """The `tf.distribute.Strategy` that this variable was created under.""" # NOTE(yuefengz): Today, no partitioned variables in a distribute strategy. return None def get_shape(self) -> tensor_shape.TensorShape: return self._shape def _get_variable_list(self): return self._variable_list def _get_partitions(self): return self._partitions def _apply_assign_fn(self, assign_fn, value): partition_axes = self._partition_axes() if len(partition_axes) > 1: raise NotImplementedError( "Cannot do assign action along more than one dimension: %s. " "Multi-axis partition assign action is not supported " % str(partition_axes)) if isinstance(value, list): assert len(value) == len(self._variable_list) value_list = value elif isinstance(value, PartitionedVariable): value_list = list(value) else: partition_ix = partition_axes[0] size_splits_list = [ tensor_shape.dimension_value(var.shape[partition_ix]) for var in self._variable_list ] value_list = array_ops.split(value, size_splits_list, axis=partition_ix) op_list = [ assign_fn(var, value_list[idx]) for idx, var in enumerate(self._variable_list) ] return op_list def assign(self, value, use_locking=False, name=None, read_value=True): assign_fn = lambda var, r_value: var.assign( r_value, use_locking=use_locking, name=name, read_value=read_value) assign_list = self._apply_assign_fn(assign_fn, value) if read_value: return assign_list return [assign.op for assign in assign_list] def assign_add(self, value, use_locking=False, name=None, read_value=True): assign_fn = lambda var, r_value: var.assign_add( r_value, use_locking=use_locking, name=name, read_value=read_value) assign_list = self._apply_assign_fn(assign_fn, value) if read_value: return assign_list return [assign.op for assign in assign_list] def assign_sub(self, value, use_locking=False, name=None, read_value=True): assign_fn = lambda var, r_value: var.assign_sub( r_value, use_locking=use_locking, name=name, read_value=read_value) assign_list = self._apply_assign_fn(assign_fn, value) if read_value: return assign_list return [assign.op for assign in assign_list] @tf_export(v1=["global_variables"]) def global_variables(scope=None): """Returns global variables. Global variables are variables that are shared across machines in a distributed environment. The `Variable()` constructor or `get_variable()` automatically adds new variables to the graph collection `GraphKeys.GLOBAL_VARIABLES`. This convenience function returns the contents of that collection. An alternative to global variables are local variables. See `tf.compat.v1.local_variables` @compatibility(TF2) Not compatible with eager execution and `tf.function`. In particular, Graph collections are deprecated in TF2. Instead please create a [tf.Module](https://www.tensorflow.org/guide/intro_to_modules) container for all your model state, including variables. You can then list all the variables in your `tf.Module` through the `variables` attribute. @end_compatibility Args: scope: (Optional.) A string. If supplied, the resulting list is filtered to include only items whose `name` attribute matches `scope` using `re.match`. Items without a `name` attribute are never returned if a scope is supplied. The choice of `re.match` means that a `scope` without special tokens filters by prefix. Returns: A list of `Variable` objects. """ return ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, scope) @tf_export(v1=["all_variables"]) @deprecated("2017-03-02", "Please use tf.global_variables instead.") def all_variables(): """Use `tf.compat.v1.global_variables` instead.""" return global_variables() def _all_saveable_objects(scope=None): """Returns all variables and `SaveableObject`s that must be checkpointed. Args: scope: (Optional.) A string. If supplied, the resulting list is filtered to include only items whose `name` attribute matches `scope` using `re.match`. Items without a `name` attribute are never returned if a scope is supplied. The choice of `re.match` means that a `scope` without special tokens filters by prefix. Returns: A list of `Variable` and `SaveableObject` to be checkpointed """ # TODO(andreasst): make this function public once things are settled. return (ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES, scope) + ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS, scope)) @tf_export(v1=["local_variables"]) def local_variables(scope=None): """Returns local variables. Local variables - per process variables, usually not saved/restored to checkpoint and used for temporary or intermediate values. For example, they can be used as counters for metrics computation or number of epochs this machine has read data. The `tf.contrib.framework.local_variable()` function automatically adds the new variable to `GraphKeys.LOCAL_VARIABLES`. This convenience function returns the contents of that collection. An alternative to local variables are global variables. See `tf.compat.v1.global_variables` Args: scope: (Optional.) A string. If supplied, the resulting list is filtered to include only items whose `name` attribute matches `scope` using `re.match`. Items without a `name` attribute are never returned if a scope is supplied. The choice of `re.match` means that a `scope` without special tokens filters by prefix. Returns: A list of local `Variable` objects. """ return ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES, scope) @tf_export(v1=["model_variables"]) def model_variables(scope=None): """Returns all variables in the MODEL_VARIABLES collection. Args: scope: (Optional.) A string. If supplied, the resulting list is filtered to include only items whose `name` attribute matches `scope` using `re.match`. Items without a `name` attribute are never returned if a scope is supplied. The choice of `re.match` means that a `scope` without special tokens filters by prefix. Returns: A list of local Variable objects. """ return ops.get_collection(ops.GraphKeys.MODEL_VARIABLES, scope) @tf_export(v1=["trainable_variables"]) def trainable_variables(scope=None): """Returns all variables created with `trainable=True`. When passed `trainable=True`, the `Variable()` constructor automatically adds new variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES`. This convenience function returns the contents of that collection. @compatibility(TF2) Not compatible with eager execution and `tf.function`. In particular, Graph collections are deprecated in TF2. Instead please create a `tf.Module` container for all your model state, including variables. You can then list all the trainable variables in your `tf.Module` through the `trainable_variables` attribute. @end_compatibility Args: scope: (Optional.) A string. If supplied, the resulting list is filtered to include only items whose `name` attribute matches `scope` using `re.match`. Items without a `name` attribute are never returned if a scope is supplied. The choice of `re.match` means that a `scope` without special tokens filters by prefix. Returns: A list of Variable objects. """ return ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES, scope) @tf_export(v1=["moving_average_variables"]) def moving_average_variables(scope=None): """Returns all variables that maintain their moving averages. If an `ExponentialMovingAverage` object is created and the `apply()` method is called on a list of variables, these variables will be added to the `GraphKeys.MOVING_AVERAGE_VARIABLES` collection. This convenience function returns the contents of that collection. Args: scope: (Optional.) A string. If supplied, the resulting list is filtered to include only items whose `name` attribute matches `scope` using `re.match`. Items without a `name` attribute are never returned if a scope is supplied. The choice of `re.match` means that a `scope` without special tokens filters by prefix. Returns: A list of Variable objects. """ return ops.get_collection(ops.GraphKeys.MOVING_AVERAGE_VARIABLES, scope) @tf_export(v1=["initializers.variables", "variables_initializer"]) def variables_initializer(var_list, name="init"): """Returns an Op that initializes a list of variables. After you launch the graph in a session, you can run the returned Op to initialize all the variables in `var_list`. This Op runs all the initializers of the variables in `var_list` in parallel. Calling `initialize_variables()` is equivalent to passing the list of initializers to `Group()`. If `var_list` is empty, however, the function still returns an Op that can be run. That Op just has no effect. @compatibility(TF2) In TF2, variables are initialized immediately when they are created. There is no longer a need to run variable initializers before using them. @end_compatibility Args: var_list: List of `Variable` objects to initialize. name: Optional name for the returned operation. Returns: An Op that run the initializers of all the specified variables. """ if var_list and not context.executing_eagerly(): return control_flow_ops.group(*[v.initializer for v in var_list], name=name) return control_flow_ops.no_op(name=name) @tf_export(v1=["initialize_variables"]) @tf_should_use.should_use_result @deprecated("2017-03-02", "Use `tf.variables_initializer` instead.") def initialize_variables(var_list, name="init"): """See `tf.compat.v1.variables_initializer`.""" return variables_initializer(var_list, name=name) @tf_export(v1=["initializers.global_variables", "global_variables_initializer"]) def global_variables_initializer(): """Returns an Op that initializes global variables. This is just a shortcut for `variables_initializer(global_variables())` @compatibility(TF2) In TF2, variables are initialized immediately when they are created. There is no longer a need to run variable initializers before using them. @end_compatibility Returns: An Op that initializes global variables in the graph. """ if context.executing_eagerly(): return control_flow_ops.no_op(name="global_variables_initializer") return variables_initializer(global_variables()) @tf_export(v1=["initialize_all_variables"]) @tf_should_use.should_use_result @deprecated("2017-03-02", "Use `tf.global_variables_initializer` instead.") def initialize_all_variables(): """See `tf.compat.v1.global_variables_initializer`.""" return global_variables_initializer() @tf_export(v1=["initializers.local_variables", "local_variables_initializer"]) def local_variables_initializer(): """Returns an Op that initializes all local variables. This is just a shortcut for `variables_initializer(local_variables())` @compatibility(TF2) In TF2, variables are initialized immediately when they are created. There is no longer a need to run variable initializers before using them. @end_compatibility Returns: An Op that initializes all local variables in the graph. """ if context.executing_eagerly(): return control_flow_ops.no_op(name="local_variables_initializer") return variables_initializer(local_variables()) @tf_export(v1=["initialize_local_variables"]) @tf_should_use.should_use_result @deprecated("2017-03-02", "Use `tf.local_variables_initializer` instead.") def initialize_local_variables(): """See `tf.compat.v1.local_variables_initializer`.""" return local_variables_initializer() @tf_export(v1=["assert_variables_initialized"]) @tf_should_use.should_use_result def assert_variables_initialized(var_list=None): """Returns an Op to check if variables are initialized. NOTE: This function is obsolete and will be removed in 6 months. Please change your implementation to use `report_uninitialized_variables()`. When run, the returned Op will raise the exception `FailedPreconditionError` if any of the variables has not yet been initialized. Note: This function is implemented by trying to fetch the values of the variables. If one of the variables is not initialized a message may be logged by the C++ runtime. This is expected. Args: var_list: List of `Variable` objects to check. Defaults to the value of `global_variables().` Returns: An Op, or None if there are no variables. """ if var_list is None: var_list = global_variables() + local_variables() # Backwards compatibility for old-style variables. TODO(touts): remove. if not var_list: var_list = [] for op in ops.get_default_graph().get_operations(): if op.type in ["Variable", "VariableV2", "AutoReloadVariable"]: var_list.append(op.outputs[0]) if not var_list: return None else: ranks = [] for var in var_list: with ops.colocate_with(var.op): ranks.append(array_ops.rank_internal(var, optimize=False)) if len(ranks) == 1: return ranks[0] else: return array_ops_stack.stack(ranks) @tf_export(v1=["report_uninitialized_variables"]) @tf_should_use.should_use_result def report_uninitialized_variables(var_list=None, name="report_uninitialized_variables"): """Adds ops to list the names of uninitialized variables. When run, it returns a 1-D tensor containing the names of uninitialized variables if there are any, or an empty array if there are none. Args: var_list: List of `Variable` objects to check. Defaults to the value of `global_variables() + local_variables()` name: Optional name of the `Operation`. Returns: A 1-D tensor containing names of the uninitialized variables, or an empty 1-D tensor if there are no variables or no uninitialized variables. """ if var_list is None: var_list = global_variables() + local_variables() # Backwards compatibility for old-style variables. TODO(touts): remove. if not var_list: var_list = [] for op in ops.get_default_graph().get_operations(): if op.type in ["Variable", "VariableV2", "AutoReloadVariable"]: var_list.append(op.outputs[0]) with ops.name_scope(name): # Run all operations on CPU if var_list: init_vars = [state_ops.is_variable_initialized(v) for v in var_list] local_device = os.environ.get( "TF_DEVICE_FOR_UNINITIALIZED_VARIABLE_REPORTING", "/cpu:0") with ops.device(local_device): if not var_list: # Return an empty tensor so we only need to check for returned tensor # size being 0 as an indication of model ready. return array_ops.constant([], dtype=dtypes.string) else: # Get a 1-D boolean tensor listing whether each variable is initialized. variables_mask = math_ops.logical_not(array_ops_stack.stack(init_vars)) # Get a 1-D string tensor containing all the variable names. variable_names_tensor = array_ops.constant( [s.op.name for s in var_list]) # Return a 1-D tensor containing all the names of # uninitialized variables. return array_ops.boolean_mask(variable_names_tensor, variables_mask) tensor_conversion_registry.register_tensor_conversion_function( PartitionedVariable, PartitionedVariable._TensorConversionFunction) # pylint: disable=protected-access