8373 lines
385 KiB
Python
8373 lines
385 KiB
Python
"""Python wrappers around TensorFlow ops.
|
|
|
|
This file is MACHINE GENERATED! Do not edit.
|
|
"""
|
|
|
|
import collections
|
|
|
|
from tensorflow.python import pywrap_tfe as pywrap_tfe
|
|
from tensorflow.python.eager import context as _context
|
|
from tensorflow.python.eager import core as _core
|
|
from tensorflow.python.eager import execute as _execute
|
|
from tensorflow.python.framework import dtypes as _dtypes
|
|
from tensorflow.security.fuzzing.py import annotation_types as _atypes
|
|
|
|
from tensorflow.python.framework import op_def_registry as _op_def_registry
|
|
from tensorflow.python.framework import ops as _ops
|
|
from tensorflow.python.framework import op_def_library as _op_def_library
|
|
from tensorflow.python.util.deprecation import deprecated_endpoints
|
|
from tensorflow.python.util import dispatch as _dispatch
|
|
from tensorflow.python.util.tf_export import tf_export
|
|
|
|
from typing import TypeVar, List, Any
|
|
from typing_extensions import Annotated
|
|
|
|
def anonymous_iterator(output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""A container for an iterator resource.
|
|
|
|
Args:
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "AnonymousIterator", name, "output_types", output_types,
|
|
"output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return anonymous_iterator_eager_fallback(
|
|
output_types=output_types, output_shapes=output_shapes, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"AnonymousIterator", output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"AnonymousIterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
AnonymousIterator = tf_export("raw_ops.AnonymousIterator")(_ops.to_raw_op(anonymous_iterator))
|
|
|
|
|
|
def anonymous_iterator_eager_fallback(output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_inputs_flat = []
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"AnonymousIterator", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"AnonymousIterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
_AnonymousIteratorV2Output = collections.namedtuple(
|
|
"AnonymousIteratorV2",
|
|
["handle", "deleter"])
|
|
|
|
|
|
def anonymous_iterator_v2(output_types, output_shapes, name=None):
|
|
r"""A container for an iterator resource.
|
|
|
|
Args:
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A tuple of `Tensor` objects (handle, deleter).
|
|
|
|
handle: A `Tensor` of type `resource`.
|
|
deleter: A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "AnonymousIteratorV2", name, "output_types", output_types,
|
|
"output_shapes", output_shapes)
|
|
_result = _AnonymousIteratorV2Output._make(_result)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return anonymous_iterator_v2_eager_fallback(
|
|
output_types=output_types, output_shapes=output_shapes, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_iterator_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_iterator_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"AnonymousIteratorV2", output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"AnonymousIteratorV2", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousIteratorV2Output._make(_result)
|
|
return _result
|
|
|
|
AnonymousIteratorV2 = tf_export("raw_ops.AnonymousIteratorV2")(_ops.to_raw_op(anonymous_iterator_v2))
|
|
|
|
|
|
def anonymous_iterator_v2_eager_fallback(output_types, output_shapes, name, ctx):
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_iterator_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_iterator_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_inputs_flat = []
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"AnonymousIteratorV2", 2, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"AnonymousIteratorV2", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousIteratorV2Output._make(_result)
|
|
return _result
|
|
|
|
|
|
def anonymous_iterator_v3(output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""A container for an iterator resource.
|
|
|
|
Args:
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "AnonymousIteratorV3", name, "output_types", output_types,
|
|
"output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return anonymous_iterator_v3_eager_fallback(
|
|
output_types=output_types, output_shapes=output_shapes, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_iterator_v3' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_iterator_v3' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"AnonymousIteratorV3", output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"AnonymousIteratorV3", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
AnonymousIteratorV3 = tf_export("raw_ops.AnonymousIteratorV3")(_ops.to_raw_op(anonymous_iterator_v3))
|
|
|
|
|
|
def anonymous_iterator_v3_eager_fallback(output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_iterator_v3' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_iterator_v3' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_inputs_flat = []
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"AnonymousIteratorV3", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"AnonymousIteratorV3", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
_AnonymousMemoryCacheOutput = collections.namedtuple(
|
|
"AnonymousMemoryCache",
|
|
["handle", "deleter"])
|
|
|
|
|
|
def anonymous_memory_cache(name=None):
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A tuple of `Tensor` objects (handle, deleter).
|
|
|
|
handle: A `Tensor` of type `resource`.
|
|
deleter: A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "AnonymousMemoryCache", name)
|
|
_result = _AnonymousMemoryCacheOutput._make(_result)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return anonymous_memory_cache_eager_fallback(
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"AnonymousMemoryCache", name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"AnonymousMemoryCache", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousMemoryCacheOutput._make(_result)
|
|
return _result
|
|
|
|
AnonymousMemoryCache = tf_export("raw_ops.AnonymousMemoryCache")(_ops.to_raw_op(anonymous_memory_cache))
|
|
|
|
|
|
def anonymous_memory_cache_eager_fallback(name, ctx):
|
|
_inputs_flat = []
|
|
_attrs = None
|
|
_result = _execute.execute(b"AnonymousMemoryCache", 2, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"AnonymousMemoryCache", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousMemoryCacheOutput._make(_result)
|
|
return _result
|
|
|
|
_AnonymousMultiDeviceIteratorOutput = collections.namedtuple(
|
|
"AnonymousMultiDeviceIterator",
|
|
["handle", "deleter"])
|
|
|
|
|
|
def anonymous_multi_device_iterator(devices, output_types, output_shapes, name=None):
|
|
r"""A container for a multi device iterator resource.
|
|
|
|
Args:
|
|
devices: A list of `strings` that has length `>= 1`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A tuple of `Tensor` objects (handle, deleter).
|
|
|
|
handle: A `Tensor` of type `resource`.
|
|
deleter: A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "AnonymousMultiDeviceIterator", name, "devices", devices,
|
|
"output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _AnonymousMultiDeviceIteratorOutput._make(_result)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return anonymous_multi_device_iterator_eager_fallback(
|
|
devices=devices, output_types=output_types,
|
|
output_shapes=output_shapes, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(devices, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'devices' argument to "
|
|
"'anonymous_multi_device_iterator' Op, not %r." % devices)
|
|
devices = [_execute.make_str(_s, "devices") for _s in devices]
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_multi_device_iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_multi_device_iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"AnonymousMultiDeviceIterator", devices=devices,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("devices", _op.get_attr("devices"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"AnonymousMultiDeviceIterator", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousMultiDeviceIteratorOutput._make(_result)
|
|
return _result
|
|
|
|
AnonymousMultiDeviceIterator = tf_export("raw_ops.AnonymousMultiDeviceIterator")(_ops.to_raw_op(anonymous_multi_device_iterator))
|
|
|
|
|
|
def anonymous_multi_device_iterator_eager_fallback(devices, output_types, output_shapes, name, ctx):
|
|
if not isinstance(devices, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'devices' argument to "
|
|
"'anonymous_multi_device_iterator' Op, not %r." % devices)
|
|
devices = [_execute.make_str(_s, "devices") for _s in devices]
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_multi_device_iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_multi_device_iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_inputs_flat = []
|
|
_attrs = ("devices", devices, "output_types", output_types, "output_shapes",
|
|
output_shapes)
|
|
_result = _execute.execute(b"AnonymousMultiDeviceIterator", 2,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"AnonymousMultiDeviceIterator", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousMultiDeviceIteratorOutput._make(_result)
|
|
return _result
|
|
|
|
|
|
def anonymous_multi_device_iterator_v3(devices, output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""A container for a multi device iterator resource.
|
|
|
|
Args:
|
|
devices: A list of `strings` that has length `>= 1`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "AnonymousMultiDeviceIteratorV3", name, "devices", devices,
|
|
"output_types", output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return anonymous_multi_device_iterator_v3_eager_fallback(
|
|
devices=devices, output_types=output_types,
|
|
output_shapes=output_shapes, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(devices, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'devices' argument to "
|
|
"'anonymous_multi_device_iterator_v3' Op, not %r." % devices)
|
|
devices = [_execute.make_str(_s, "devices") for _s in devices]
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_multi_device_iterator_v3' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_multi_device_iterator_v3' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"AnonymousMultiDeviceIteratorV3", devices=devices,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("devices", _op.get_attr("devices"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"AnonymousMultiDeviceIteratorV3", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
AnonymousMultiDeviceIteratorV3 = tf_export("raw_ops.AnonymousMultiDeviceIteratorV3")(_ops.to_raw_op(anonymous_multi_device_iterator_v3))
|
|
|
|
|
|
def anonymous_multi_device_iterator_v3_eager_fallback(devices, output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
if not isinstance(devices, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'devices' argument to "
|
|
"'anonymous_multi_device_iterator_v3' Op, not %r." % devices)
|
|
devices = [_execute.make_str(_s, "devices") for _s in devices]
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'anonymous_multi_device_iterator_v3' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'anonymous_multi_device_iterator_v3' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_inputs_flat = []
|
|
_attrs = ("devices", devices, "output_types", output_types, "output_shapes",
|
|
output_shapes)
|
|
_result = _execute.execute(b"AnonymousMultiDeviceIteratorV3", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"AnonymousMultiDeviceIteratorV3", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
_AnonymousRandomSeedGeneratorOutput = collections.namedtuple(
|
|
"AnonymousRandomSeedGenerator",
|
|
["handle", "deleter"])
|
|
|
|
|
|
def anonymous_random_seed_generator(seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], name=None):
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
seed: A `Tensor` of type `int64`.
|
|
seed2: A `Tensor` of type `int64`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A tuple of `Tensor` objects (handle, deleter).
|
|
|
|
handle: A `Tensor` of type `resource`.
|
|
deleter: A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "AnonymousRandomSeedGenerator", name, seed, seed2)
|
|
_result = _AnonymousRandomSeedGeneratorOutput._make(_result)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return anonymous_random_seed_generator_eager_fallback(
|
|
seed, seed2, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"AnonymousRandomSeedGenerator", seed=seed, seed2=seed2, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"AnonymousRandomSeedGenerator", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousRandomSeedGeneratorOutput._make(_result)
|
|
return _result
|
|
|
|
AnonymousRandomSeedGenerator = tf_export("raw_ops.AnonymousRandomSeedGenerator")(_ops.to_raw_op(anonymous_random_seed_generator))
|
|
|
|
|
|
def anonymous_random_seed_generator_eager_fallback(seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], name, ctx):
|
|
seed = _ops.convert_to_tensor(seed, _dtypes.int64)
|
|
seed2 = _ops.convert_to_tensor(seed2, _dtypes.int64)
|
|
_inputs_flat = [seed, seed2]
|
|
_attrs = None
|
|
_result = _execute.execute(b"AnonymousRandomSeedGenerator", 2,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"AnonymousRandomSeedGenerator", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousRandomSeedGeneratorOutput._make(_result)
|
|
return _result
|
|
|
|
_AnonymousSeedGeneratorOutput = collections.namedtuple(
|
|
"AnonymousSeedGenerator",
|
|
["handle", "deleter"])
|
|
|
|
|
|
def anonymous_seed_generator(seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], reshuffle: Annotated[Any, _atypes.Bool], name=None):
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
seed: A `Tensor` of type `int64`.
|
|
seed2: A `Tensor` of type `int64`.
|
|
reshuffle: A `Tensor` of type `bool`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A tuple of `Tensor` objects (handle, deleter).
|
|
|
|
handle: A `Tensor` of type `resource`.
|
|
deleter: A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "AnonymousSeedGenerator", name, seed, seed2, reshuffle)
|
|
_result = _AnonymousSeedGeneratorOutput._make(_result)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return anonymous_seed_generator_eager_fallback(
|
|
seed, seed2, reshuffle, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"AnonymousSeedGenerator", seed=seed, seed2=seed2, reshuffle=reshuffle,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"AnonymousSeedGenerator", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousSeedGeneratorOutput._make(_result)
|
|
return _result
|
|
|
|
AnonymousSeedGenerator = tf_export("raw_ops.AnonymousSeedGenerator")(_ops.to_raw_op(anonymous_seed_generator))
|
|
|
|
|
|
def anonymous_seed_generator_eager_fallback(seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], reshuffle: Annotated[Any, _atypes.Bool], name, ctx):
|
|
seed = _ops.convert_to_tensor(seed, _dtypes.int64)
|
|
seed2 = _ops.convert_to_tensor(seed2, _dtypes.int64)
|
|
reshuffle = _ops.convert_to_tensor(reshuffle, _dtypes.bool)
|
|
_inputs_flat = [seed, seed2, reshuffle]
|
|
_attrs = None
|
|
_result = _execute.execute(b"AnonymousSeedGenerator", 2,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"AnonymousSeedGenerator", _inputs_flat, _attrs, _result)
|
|
_result = _AnonymousSeedGeneratorOutput._make(_result)
|
|
return _result
|
|
|
|
|
|
def batch_dataset(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that batches `batch_size` elements from `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
batch_size: A `Tensor` of type `int64`.
|
|
A scalar representing the number of elements to accumulate in a
|
|
batch.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "BatchDataset", name, input_dataset, batch_size, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return batch_dataset_eager_fallback(
|
|
input_dataset, batch_size, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'batch_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'batch_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"BatchDataset", input_dataset=input_dataset, batch_size=batch_size,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"BatchDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
BatchDataset = tf_export("raw_ops.BatchDataset")(_ops.to_raw_op(batch_dataset))
|
|
|
|
|
|
def batch_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'batch_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'batch_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
batch_size = _ops.convert_to_tensor(batch_size, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, batch_size]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"BatchDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"BatchDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def batch_dataset_v2(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], drop_remainder: Annotated[Any, _atypes.Bool], output_types, output_shapes, parallel_copy:bool=False, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that batches `batch_size` elements from `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
batch_size: A `Tensor` of type `int64`.
|
|
A scalar representing the number of elements to accumulate in a batch.
|
|
drop_remainder: A `Tensor` of type `bool`.
|
|
A scalar representing whether the last batch should be dropped in case its size
|
|
is smaller than desired.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
parallel_copy: An optional `bool`. Defaults to `False`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "BatchDatasetV2", name, input_dataset, batch_size,
|
|
drop_remainder, "parallel_copy", parallel_copy, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return batch_dataset_v2_eager_fallback(
|
|
input_dataset, batch_size, drop_remainder,
|
|
parallel_copy=parallel_copy, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'batch_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'batch_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if parallel_copy is None:
|
|
parallel_copy = False
|
|
parallel_copy = _execute.make_bool(parallel_copy, "parallel_copy")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"BatchDatasetV2", input_dataset=input_dataset, batch_size=batch_size,
|
|
drop_remainder=drop_remainder,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
parallel_copy=parallel_copy, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("parallel_copy", _op._get_attr_bool("parallel_copy"),
|
|
"output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"BatchDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
BatchDatasetV2 = tf_export("raw_ops.BatchDatasetV2")(_ops.to_raw_op(batch_dataset_v2))
|
|
|
|
|
|
def batch_dataset_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], drop_remainder: Annotated[Any, _atypes.Bool], output_types, output_shapes, parallel_copy: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'batch_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'batch_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if parallel_copy is None:
|
|
parallel_copy = False
|
|
parallel_copy = _execute.make_bool(parallel_copy, "parallel_copy")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
batch_size = _ops.convert_to_tensor(batch_size, _dtypes.int64)
|
|
drop_remainder = _ops.convert_to_tensor(drop_remainder, _dtypes.bool)
|
|
_inputs_flat = [input_dataset, batch_size, drop_remainder]
|
|
_attrs = ("parallel_copy", parallel_copy, "output_types", output_types,
|
|
"output_shapes", output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"BatchDatasetV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"BatchDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def cache_dataset(input_dataset: Annotated[Any, _atypes.Variant], filename: Annotated[Any, _atypes.String], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that caches elements from `input_dataset`.
|
|
|
|
A CacheDataset will iterate over the input_dataset, and store tensors. If the
|
|
cache already exists, the cache will be used. If the cache is inappropriate
|
|
(e.g. cannot be opened, contains tensors of the wrong shape / size), an error
|
|
will the returned when used.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
filename: A `Tensor` of type `string`.
|
|
A path on the filesystem where we should cache the dataset. Note: this
|
|
will be a directory.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "CacheDataset", name, input_dataset, filename, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return cache_dataset_eager_fallback(
|
|
input_dataset, filename, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'cache_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'cache_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"CacheDataset", input_dataset=input_dataset, filename=filename,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"CacheDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
CacheDataset = tf_export("raw_ops.CacheDataset")(_ops.to_raw_op(cache_dataset))
|
|
|
|
|
|
def cache_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], filename: Annotated[Any, _atypes.String], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'cache_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'cache_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
filename = _ops.convert_to_tensor(filename, _dtypes.string)
|
|
_inputs_flat = [input_dataset, filename]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"CacheDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"CacheDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def cache_dataset_v2(input_dataset: Annotated[Any, _atypes.Variant], filename: Annotated[Any, _atypes.String], cache: Annotated[Any, _atypes.Resource], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
filename: A `Tensor` of type `string`.
|
|
cache: A `Tensor` of type `resource`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "CacheDatasetV2", name, input_dataset, filename, cache,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return cache_dataset_v2_eager_fallback(
|
|
input_dataset, filename, cache, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'cache_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'cache_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"CacheDatasetV2", input_dataset=input_dataset, filename=filename,
|
|
cache=cache, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"CacheDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
CacheDatasetV2 = tf_export("raw_ops.CacheDatasetV2")(_ops.to_raw_op(cache_dataset_v2))
|
|
|
|
|
|
def cache_dataset_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], filename: Annotated[Any, _atypes.String], cache: Annotated[Any, _atypes.Resource], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'cache_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'cache_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
filename = _ops.convert_to_tensor(filename, _dtypes.string)
|
|
cache = _ops.convert_to_tensor(cache, _dtypes.resource)
|
|
_inputs_flat = [input_dataset, filename, cache]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"CacheDatasetV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"CacheDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def concatenate_dataset(input_dataset: Annotated[Any, _atypes.Variant], another_dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that concatenates `input_dataset` with `another_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
another_dataset: A `Tensor` of type `variant`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ConcatenateDataset", name, input_dataset, another_dataset,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return concatenate_dataset_eager_fallback(
|
|
input_dataset, another_dataset, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'concatenate_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'concatenate_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ConcatenateDataset", input_dataset=input_dataset,
|
|
another_dataset=another_dataset,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ConcatenateDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ConcatenateDataset = tf_export("raw_ops.ConcatenateDataset")(_ops.to_raw_op(concatenate_dataset))
|
|
|
|
|
|
def concatenate_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], another_dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'concatenate_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'concatenate_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
another_dataset = _ops.convert_to_tensor(another_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset, another_dataset]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"ConcatenateDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ConcatenateDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def dataset_cardinality(input_dataset: Annotated[Any, _atypes.Variant], cardinality_options:str="", name=None) -> Annotated[Any, _atypes.Int64]:
|
|
r"""Returns the cardinality of `input_dataset`.
|
|
|
|
Returns the cardinality of `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the dataset to return cardinality for.
|
|
cardinality_options: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `int64`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DatasetCardinality", name, input_dataset,
|
|
"cardinality_options", cardinality_options)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return dataset_cardinality_eager_fallback(
|
|
input_dataset, cardinality_options=cardinality_options, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if cardinality_options is None:
|
|
cardinality_options = ""
|
|
cardinality_options = _execute.make_str(cardinality_options, "cardinality_options")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DatasetCardinality", input_dataset=input_dataset,
|
|
cardinality_options=cardinality_options,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("cardinality_options", _op.get_attr("cardinality_options"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"DatasetCardinality", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
DatasetCardinality = tf_export("raw_ops.DatasetCardinality")(_ops.to_raw_op(dataset_cardinality))
|
|
|
|
|
|
def dataset_cardinality_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], cardinality_options: str, name, ctx) -> Annotated[Any, _atypes.Int64]:
|
|
if cardinality_options is None:
|
|
cardinality_options = ""
|
|
cardinality_options = _execute.make_str(cardinality_options, "cardinality_options")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = ("cardinality_options", cardinality_options)
|
|
_result = _execute.execute(b"DatasetCardinality", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"DatasetCardinality", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def dataset_fingerprint(input_dataset: Annotated[Any, _atypes.Variant], name=None) -> Annotated[Any, _atypes.UInt64]:
|
|
r"""Returns the fingerprint of `input_dataset`.
|
|
|
|
Returns the fingerprint of `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the dataset to return fingerprint for.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `uint64`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DatasetFingerprint", name, input_dataset)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return dataset_fingerprint_eager_fallback(
|
|
input_dataset, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DatasetFingerprint", input_dataset=input_dataset, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"DatasetFingerprint", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
DatasetFingerprint = tf_export("raw_ops.DatasetFingerprint")(_ops.to_raw_op(dataset_fingerprint))
|
|
|
|
|
|
def dataset_fingerprint_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], name, ctx) -> Annotated[Any, _atypes.UInt64]:
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = None
|
|
_result = _execute.execute(b"DatasetFingerprint", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"DatasetFingerprint", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def dataset_to_graph(input_dataset: Annotated[Any, _atypes.Variant], stateful_whitelist=[], allow_stateful:bool=False, strip_device_assignment:bool=False, name=None) -> Annotated[Any, _atypes.String]:
|
|
r"""Returns a serialized GraphDef representing `input_dataset`.
|
|
|
|
Returns a graph representation for `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the dataset to return the graph representation for.
|
|
stateful_whitelist: An optional list of `strings`. Defaults to `[]`.
|
|
allow_stateful: An optional `bool`. Defaults to `False`.
|
|
strip_device_assignment: An optional `bool`. Defaults to `False`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `string`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DatasetToGraph", name, input_dataset, "stateful_whitelist",
|
|
stateful_whitelist, "allow_stateful", allow_stateful,
|
|
"strip_device_assignment", strip_device_assignment)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return dataset_to_graph_eager_fallback(
|
|
input_dataset, stateful_whitelist=stateful_whitelist,
|
|
allow_stateful=allow_stateful,
|
|
strip_device_assignment=strip_device_assignment, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if stateful_whitelist is None:
|
|
stateful_whitelist = []
|
|
if not isinstance(stateful_whitelist, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'stateful_whitelist' argument to "
|
|
"'dataset_to_graph' Op, not %r." % stateful_whitelist)
|
|
stateful_whitelist = [_execute.make_str(_s, "stateful_whitelist") for _s in stateful_whitelist]
|
|
if allow_stateful is None:
|
|
allow_stateful = False
|
|
allow_stateful = _execute.make_bool(allow_stateful, "allow_stateful")
|
|
if strip_device_assignment is None:
|
|
strip_device_assignment = False
|
|
strip_device_assignment = _execute.make_bool(strip_device_assignment, "strip_device_assignment")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DatasetToGraph", input_dataset=input_dataset,
|
|
stateful_whitelist=stateful_whitelist,
|
|
allow_stateful=allow_stateful,
|
|
strip_device_assignment=strip_device_assignment,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("stateful_whitelist", _op.get_attr("stateful_whitelist"),
|
|
"allow_stateful", _op._get_attr_bool("allow_stateful"),
|
|
"strip_device_assignment",
|
|
_op._get_attr_bool("strip_device_assignment"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"DatasetToGraph", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
DatasetToGraph = tf_export("raw_ops.DatasetToGraph")(_ops.to_raw_op(dataset_to_graph))
|
|
|
|
|
|
def dataset_to_graph_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], stateful_whitelist, allow_stateful: bool, strip_device_assignment: bool, name, ctx) -> Annotated[Any, _atypes.String]:
|
|
if stateful_whitelist is None:
|
|
stateful_whitelist = []
|
|
if not isinstance(stateful_whitelist, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'stateful_whitelist' argument to "
|
|
"'dataset_to_graph' Op, not %r." % stateful_whitelist)
|
|
stateful_whitelist = [_execute.make_str(_s, "stateful_whitelist") for _s in stateful_whitelist]
|
|
if allow_stateful is None:
|
|
allow_stateful = False
|
|
allow_stateful = _execute.make_bool(allow_stateful, "allow_stateful")
|
|
if strip_device_assignment is None:
|
|
strip_device_assignment = False
|
|
strip_device_assignment = _execute.make_bool(strip_device_assignment, "strip_device_assignment")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = ("stateful_whitelist", stateful_whitelist, "allow_stateful",
|
|
allow_stateful, "strip_device_assignment", strip_device_assignment)
|
|
_result = _execute.execute(b"DatasetToGraph", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"DatasetToGraph", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def dataset_to_graph_v2(input_dataset: Annotated[Any, _atypes.Variant], external_state_policy:int=0, strip_device_assignment:bool=False, name=None) -> Annotated[Any, _atypes.String]:
|
|
r"""Returns a serialized GraphDef representing `input_dataset`.
|
|
|
|
Returns a graph representation for `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the dataset to return the graph representation for.
|
|
external_state_policy: An optional `int`. Defaults to `0`.
|
|
strip_device_assignment: An optional `bool`. Defaults to `False`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `string`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DatasetToGraphV2", name, input_dataset,
|
|
"external_state_policy", external_state_policy,
|
|
"strip_device_assignment", strip_device_assignment)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return dataset_to_graph_v2_eager_fallback(
|
|
input_dataset, external_state_policy=external_state_policy,
|
|
strip_device_assignment=strip_device_assignment, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if external_state_policy is None:
|
|
external_state_policy = 0
|
|
external_state_policy = _execute.make_int(external_state_policy, "external_state_policy")
|
|
if strip_device_assignment is None:
|
|
strip_device_assignment = False
|
|
strip_device_assignment = _execute.make_bool(strip_device_assignment, "strip_device_assignment")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DatasetToGraphV2", input_dataset=input_dataset,
|
|
external_state_policy=external_state_policy,
|
|
strip_device_assignment=strip_device_assignment,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("external_state_policy",
|
|
_op._get_attr_int("external_state_policy"),
|
|
"strip_device_assignment",
|
|
_op._get_attr_bool("strip_device_assignment"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"DatasetToGraphV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
DatasetToGraphV2 = tf_export("raw_ops.DatasetToGraphV2")(_ops.to_raw_op(dataset_to_graph_v2))
|
|
|
|
|
|
def dataset_to_graph_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], external_state_policy: int, strip_device_assignment: bool, name, ctx) -> Annotated[Any, _atypes.String]:
|
|
if external_state_policy is None:
|
|
external_state_policy = 0
|
|
external_state_policy = _execute.make_int(external_state_policy, "external_state_policy")
|
|
if strip_device_assignment is None:
|
|
strip_device_assignment = False
|
|
strip_device_assignment = _execute.make_bool(strip_device_assignment, "strip_device_assignment")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = ("external_state_policy", external_state_policy,
|
|
"strip_device_assignment", strip_device_assignment)
|
|
_result = _execute.execute(b"DatasetToGraphV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"DatasetToGraphV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def dataset_to_single_element(dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, metadata:str="", name=None):
|
|
r"""Outputs the single element from the given dataset.
|
|
|
|
Args:
|
|
dataset: A `Tensor` of type `variant`.
|
|
A handle to a dataset that contains a single element.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A list of `Tensor` objects of type `output_types`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DatasetToSingleElement", name, dataset, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return dataset_to_single_element_eager_fallback(
|
|
dataset, output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'dataset_to_single_element' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'dataset_to_single_element' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DatasetToSingleElement", dataset=dataset, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if not _result:
|
|
return _op
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"DatasetToSingleElement", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
DatasetToSingleElement = tf_export("raw_ops.DatasetToSingleElement")(_ops.to_raw_op(dataset_to_single_element))
|
|
|
|
|
|
def dataset_to_single_element_eager_fallback(dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, metadata: str, name, ctx):
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'dataset_to_single_element' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'dataset_to_single_element' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
dataset = _ops.convert_to_tensor(dataset, _dtypes.variant)
|
|
_inputs_flat = [dataset]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"DatasetToSingleElement", len(output_types),
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"DatasetToSingleElement", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
|
|
def delete_iterator(handle: Annotated[Any, _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name=None):
|
|
r"""A container for an iterator resource.
|
|
|
|
Args:
|
|
handle: A `Tensor` of type `resource`. A handle to the iterator to delete.
|
|
deleter: A `Tensor` of type `variant`. A variant deleter.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
The created Operation.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DeleteIterator", name, handle, deleter)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return delete_iterator_eager_fallback(
|
|
handle, deleter, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DeleteIterator", handle=handle, deleter=deleter, name=name)
|
|
return _op
|
|
DeleteIterator = tf_export("raw_ops.DeleteIterator")(_ops.to_raw_op(delete_iterator))
|
|
|
|
|
|
def delete_iterator_eager_fallback(handle: Annotated[Any, _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name, ctx):
|
|
handle = _ops.convert_to_tensor(handle, _dtypes.resource)
|
|
deleter = _ops.convert_to_tensor(deleter, _dtypes.variant)
|
|
_inputs_flat = [handle, deleter]
|
|
_attrs = None
|
|
_result = _execute.execute(b"DeleteIterator", 0, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
_result = None
|
|
return _result
|
|
|
|
|
|
def delete_memory_cache(handle: Annotated[Any, _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name=None):
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
handle: A `Tensor` of type `resource`.
|
|
deleter: A `Tensor` of type `variant`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
The created Operation.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DeleteMemoryCache", name, handle, deleter)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return delete_memory_cache_eager_fallback(
|
|
handle, deleter, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DeleteMemoryCache", handle=handle, deleter=deleter, name=name)
|
|
return _op
|
|
DeleteMemoryCache = tf_export("raw_ops.DeleteMemoryCache")(_ops.to_raw_op(delete_memory_cache))
|
|
|
|
|
|
def delete_memory_cache_eager_fallback(handle: Annotated[Any, _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name, ctx):
|
|
handle = _ops.convert_to_tensor(handle, _dtypes.resource)
|
|
deleter = _ops.convert_to_tensor(deleter, _dtypes.variant)
|
|
_inputs_flat = [handle, deleter]
|
|
_attrs = None
|
|
_result = _execute.execute(b"DeleteMemoryCache", 0, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
_result = None
|
|
return _result
|
|
|
|
|
|
def delete_multi_device_iterator(multi_device_iterator: Annotated[Any, _atypes.Resource], iterators: Annotated[List[Any], _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name=None):
|
|
r"""A container for an iterator resource.
|
|
|
|
Args:
|
|
multi_device_iterator: A `Tensor` of type `resource`.
|
|
A handle to the multi device iterator to delete.
|
|
iterators: A list of `Tensor` objects with type `resource`.
|
|
A list of iterator handles (unused). This is added so that automatic control dependencies get added during function tracing that ensure this op runs after all the dependent iterators are deleted.
|
|
deleter: A `Tensor` of type `variant`. A variant deleter.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
The created Operation.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DeleteMultiDeviceIterator", name, multi_device_iterator,
|
|
iterators, deleter)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return delete_multi_device_iterator_eager_fallback(
|
|
multi_device_iterator, iterators, deleter, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(iterators, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'iterators' argument to "
|
|
"'delete_multi_device_iterator' Op, not %r." % iterators)
|
|
_attr_N = len(iterators)
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DeleteMultiDeviceIterator", multi_device_iterator=multi_device_iterator,
|
|
iterators=iterators, deleter=deleter,
|
|
name=name)
|
|
return _op
|
|
DeleteMultiDeviceIterator = tf_export("raw_ops.DeleteMultiDeviceIterator")(_ops.to_raw_op(delete_multi_device_iterator))
|
|
|
|
|
|
def delete_multi_device_iterator_eager_fallback(multi_device_iterator: Annotated[Any, _atypes.Resource], iterators: Annotated[List[Any], _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name, ctx):
|
|
if not isinstance(iterators, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'iterators' argument to "
|
|
"'delete_multi_device_iterator' Op, not %r." % iterators)
|
|
_attr_N = len(iterators)
|
|
multi_device_iterator = _ops.convert_to_tensor(multi_device_iterator, _dtypes.resource)
|
|
iterators = _ops.convert_n_to_tensor(iterators, _dtypes.resource)
|
|
deleter = _ops.convert_to_tensor(deleter, _dtypes.variant)
|
|
_inputs_flat = [multi_device_iterator] + list(iterators) + [deleter]
|
|
_attrs = ("N", _attr_N)
|
|
_result = _execute.execute(b"DeleteMultiDeviceIterator", 0,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
_result = None
|
|
return _result
|
|
|
|
|
|
def delete_random_seed_generator(handle: Annotated[Any, _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name=None):
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
handle: A `Tensor` of type `resource`.
|
|
deleter: A `Tensor` of type `variant`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
The created Operation.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DeleteRandomSeedGenerator", name, handle, deleter)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return delete_random_seed_generator_eager_fallback(
|
|
handle, deleter, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DeleteRandomSeedGenerator", handle=handle, deleter=deleter,
|
|
name=name)
|
|
return _op
|
|
DeleteRandomSeedGenerator = tf_export("raw_ops.DeleteRandomSeedGenerator")(_ops.to_raw_op(delete_random_seed_generator))
|
|
|
|
|
|
def delete_random_seed_generator_eager_fallback(handle: Annotated[Any, _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name, ctx):
|
|
handle = _ops.convert_to_tensor(handle, _dtypes.resource)
|
|
deleter = _ops.convert_to_tensor(deleter, _dtypes.variant)
|
|
_inputs_flat = [handle, deleter]
|
|
_attrs = None
|
|
_result = _execute.execute(b"DeleteRandomSeedGenerator", 0,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
_result = None
|
|
return _result
|
|
|
|
|
|
def delete_seed_generator(handle: Annotated[Any, _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name=None):
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
handle: A `Tensor` of type `resource`.
|
|
deleter: A `Tensor` of type `variant`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
The created Operation.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DeleteSeedGenerator", name, handle, deleter)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return delete_seed_generator_eager_fallback(
|
|
handle, deleter, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DeleteSeedGenerator", handle=handle, deleter=deleter, name=name)
|
|
return _op
|
|
DeleteSeedGenerator = tf_export("raw_ops.DeleteSeedGenerator")(_ops.to_raw_op(delete_seed_generator))
|
|
|
|
|
|
def delete_seed_generator_eager_fallback(handle: Annotated[Any, _atypes.Resource], deleter: Annotated[Any, _atypes.Variant], name, ctx):
|
|
handle = _ops.convert_to_tensor(handle, _dtypes.resource)
|
|
deleter = _ops.convert_to_tensor(deleter, _dtypes.variant)
|
|
_inputs_flat = [handle, deleter]
|
|
_attrs = None
|
|
_result = _execute.execute(b"DeleteSeedGenerator", 0, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
_result = None
|
|
return _result
|
|
|
|
|
|
def deserialize_iterator(resource_handle: Annotated[Any, _atypes.Resource], serialized: Annotated[Any, _atypes.Variant], name=None):
|
|
r"""Converts the given variant tensor to an iterator and stores it in the given resource.
|
|
|
|
Args:
|
|
resource_handle: A `Tensor` of type `resource`.
|
|
A handle to an iterator resource.
|
|
serialized: A `Tensor` of type `variant`.
|
|
A variant tensor storing the state of the iterator contained in the
|
|
resource.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
The created Operation.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DeserializeIterator", name, resource_handle, serialized)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return deserialize_iterator_eager_fallback(
|
|
resource_handle, serialized, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DeserializeIterator", resource_handle=resource_handle,
|
|
serialized=serialized, name=name)
|
|
return _op
|
|
DeserializeIterator = tf_export("raw_ops.DeserializeIterator")(_ops.to_raw_op(deserialize_iterator))
|
|
|
|
|
|
def deserialize_iterator_eager_fallback(resource_handle: Annotated[Any, _atypes.Resource], serialized: Annotated[Any, _atypes.Variant], name, ctx):
|
|
resource_handle = _ops.convert_to_tensor(resource_handle, _dtypes.resource)
|
|
serialized = _ops.convert_to_tensor(serialized, _dtypes.variant)
|
|
_inputs_flat = [resource_handle, serialized]
|
|
_attrs = None
|
|
_result = _execute.execute(b"DeserializeIterator", 0, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
_result = None
|
|
return _result
|
|
|
|
|
|
def dummy_memory_cache(name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DummyMemoryCache", name)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return dummy_memory_cache_eager_fallback(
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DummyMemoryCache", name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"DummyMemoryCache", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
DummyMemoryCache = tf_export("raw_ops.DummyMemoryCache")(_ops.to_raw_op(dummy_memory_cache))
|
|
|
|
|
|
def dummy_memory_cache_eager_fallback(name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
_inputs_flat = []
|
|
_attrs = None
|
|
_result = _execute.execute(b"DummyMemoryCache", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"DummyMemoryCache", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def dummy_seed_generator(name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "DummySeedGenerator", name)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return dummy_seed_generator_eager_fallback(
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"DummySeedGenerator", name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"DummySeedGenerator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
DummySeedGenerator = tf_export("raw_ops.DummySeedGenerator")(_ops.to_raw_op(dummy_seed_generator))
|
|
|
|
|
|
def dummy_seed_generator_eager_fallback(name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
_inputs_flat = []
|
|
_attrs = None
|
|
_result = _execute.execute(b"DummySeedGenerator", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"DummySeedGenerator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def filter_by_last_component_dataset(input_dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset containing elements of first component of `input_dataset` having true in the last component.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "FilterByLastComponentDataset", name, input_dataset,
|
|
"output_types", output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return filter_by_last_component_dataset_eager_fallback(
|
|
input_dataset, output_types=output_types,
|
|
output_shapes=output_shapes, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'filter_by_last_component_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'filter_by_last_component_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"FilterByLastComponentDataset", input_dataset=input_dataset,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"FilterByLastComponentDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
FilterByLastComponentDataset = tf_export("raw_ops.FilterByLastComponentDataset")(_ops.to_raw_op(filter_by_last_component_dataset))
|
|
|
|
|
|
def filter_by_last_component_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'filter_by_last_component_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'filter_by_last_component_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"FilterByLastComponentDataset", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"FilterByLastComponentDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def filter_dataset(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, predicate, output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset containing elements of `input_dataset` matching `predicate`.
|
|
|
|
The `predicate` function must return a scalar boolean and accept the
|
|
following arguments:
|
|
|
|
* One tensor for each component of an element of `input_dataset`.
|
|
* One tensor for each value in `other_arguments`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
A list of tensors, typically values that were captured when
|
|
building a closure for `predicate`.
|
|
predicate: A function decorated with @Defun.
|
|
A function returning a scalar boolean.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "FilterDataset", name, input_dataset, other_arguments,
|
|
"predicate", predicate, "output_types", output_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return filter_dataset_eager_fallback(
|
|
input_dataset, other_arguments, predicate=predicate,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'filter_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'filter_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"FilterDataset", input_dataset=input_dataset,
|
|
other_arguments=other_arguments, predicate=predicate,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("predicate", _op.get_attr("predicate"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"FilterDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
FilterDataset = tf_export("raw_ops.FilterDataset")(_ops.to_raw_op(filter_dataset))
|
|
|
|
|
|
def filter_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, predicate, output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'filter_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'filter_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset] + list(other_arguments)
|
|
_attrs = ("predicate", predicate, "Targuments", _attr_Targuments,
|
|
"output_types", output_types, "output_shapes", output_shapes, "metadata",
|
|
metadata)
|
|
_result = _execute.execute(b"FilterDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"FilterDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def finalize_dataset(input_dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, has_captured_ref:bool=False, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset by applying `tf.data.Options` to `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the input dataset.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
has_captured_ref: An optional `bool`. Defaults to `False`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "FinalizeDataset", name, input_dataset, "has_captured_ref",
|
|
has_captured_ref, "output_types", output_types, "output_shapes",
|
|
output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return finalize_dataset_eager_fallback(
|
|
input_dataset, has_captured_ref=has_captured_ref,
|
|
output_types=output_types, output_shapes=output_shapes, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'finalize_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'finalize_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if has_captured_ref is None:
|
|
has_captured_ref = False
|
|
has_captured_ref = _execute.make_bool(has_captured_ref, "has_captured_ref")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"FinalizeDataset", input_dataset=input_dataset,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
has_captured_ref=has_captured_ref, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("has_captured_ref", _op._get_attr_bool("has_captured_ref"),
|
|
"output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"FinalizeDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
FinalizeDataset = tf_export("raw_ops.FinalizeDataset")(_ops.to_raw_op(finalize_dataset))
|
|
|
|
|
|
def finalize_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, has_captured_ref: bool, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'finalize_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'finalize_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if has_captured_ref is None:
|
|
has_captured_ref = False
|
|
has_captured_ref = _execute.make_bool(has_captured_ref, "has_captured_ref")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = ("has_captured_ref", has_captured_ref, "output_types",
|
|
output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"FinalizeDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"FinalizeDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def fixed_length_record_dataset(filenames: Annotated[Any, _atypes.String], header_bytes: Annotated[Any, _atypes.Int64], record_bytes: Annotated[Any, _atypes.Int64], footer_bytes: Annotated[Any, _atypes.Int64], buffer_size: Annotated[Any, _atypes.Int64], metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that emits the records from one or more binary files.
|
|
|
|
Args:
|
|
filenames: A `Tensor` of type `string`.
|
|
A scalar or a vector containing the name(s) of the file(s) to be
|
|
read.
|
|
header_bytes: A `Tensor` of type `int64`.
|
|
A scalar representing the number of bytes to skip at the
|
|
beginning of a file.
|
|
record_bytes: A `Tensor` of type `int64`.
|
|
A scalar representing the number of bytes in each record.
|
|
footer_bytes: A `Tensor` of type `int64`.
|
|
A scalar representing the number of bytes to skip at the end
|
|
of a file.
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
A scalar representing the number of bytes to buffer. Must be > 0.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "FixedLengthRecordDataset", name, filenames, header_bytes,
|
|
record_bytes, footer_bytes, buffer_size, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return fixed_length_record_dataset_eager_fallback(
|
|
filenames, header_bytes, record_bytes, footer_bytes, buffer_size,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"FixedLengthRecordDataset", filenames=filenames,
|
|
header_bytes=header_bytes,
|
|
record_bytes=record_bytes,
|
|
footer_bytes=footer_bytes,
|
|
buffer_size=buffer_size,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("metadata", _op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"FixedLengthRecordDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
FixedLengthRecordDataset = tf_export("raw_ops.FixedLengthRecordDataset")(_ops.to_raw_op(fixed_length_record_dataset))
|
|
|
|
|
|
def fixed_length_record_dataset_eager_fallback(filenames: Annotated[Any, _atypes.String], header_bytes: Annotated[Any, _atypes.Int64], record_bytes: Annotated[Any, _atypes.Int64], footer_bytes: Annotated[Any, _atypes.Int64], buffer_size: Annotated[Any, _atypes.Int64], metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
filenames = _ops.convert_to_tensor(filenames, _dtypes.string)
|
|
header_bytes = _ops.convert_to_tensor(header_bytes, _dtypes.int64)
|
|
record_bytes = _ops.convert_to_tensor(record_bytes, _dtypes.int64)
|
|
footer_bytes = _ops.convert_to_tensor(footer_bytes, _dtypes.int64)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
_inputs_flat = [filenames, header_bytes, record_bytes, footer_bytes, buffer_size]
|
|
_attrs = ("metadata", metadata)
|
|
_result = _execute.execute(b"FixedLengthRecordDataset", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"FixedLengthRecordDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def fixed_length_record_dataset_v2(filenames: Annotated[Any, _atypes.String], header_bytes: Annotated[Any, _atypes.Int64], record_bytes: Annotated[Any, _atypes.Int64], footer_bytes: Annotated[Any, _atypes.Int64], buffer_size: Annotated[Any, _atypes.Int64], compression_type: Annotated[Any, _atypes.String], metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
filenames: A `Tensor` of type `string`.
|
|
header_bytes: A `Tensor` of type `int64`.
|
|
record_bytes: A `Tensor` of type `int64`.
|
|
footer_bytes: A `Tensor` of type `int64`.
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
compression_type: A `Tensor` of type `string`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "FixedLengthRecordDatasetV2", name, filenames, header_bytes,
|
|
record_bytes, footer_bytes, buffer_size, compression_type, "metadata",
|
|
metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return fixed_length_record_dataset_v2_eager_fallback(
|
|
filenames, header_bytes, record_bytes, footer_bytes, buffer_size,
|
|
compression_type, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"FixedLengthRecordDatasetV2", filenames=filenames,
|
|
header_bytes=header_bytes,
|
|
record_bytes=record_bytes,
|
|
footer_bytes=footer_bytes,
|
|
buffer_size=buffer_size,
|
|
compression_type=compression_type,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("metadata", _op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"FixedLengthRecordDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
FixedLengthRecordDatasetV2 = tf_export("raw_ops.FixedLengthRecordDatasetV2")(_ops.to_raw_op(fixed_length_record_dataset_v2))
|
|
|
|
|
|
def fixed_length_record_dataset_v2_eager_fallback(filenames: Annotated[Any, _atypes.String], header_bytes: Annotated[Any, _atypes.Int64], record_bytes: Annotated[Any, _atypes.Int64], footer_bytes: Annotated[Any, _atypes.Int64], buffer_size: Annotated[Any, _atypes.Int64], compression_type: Annotated[Any, _atypes.String], metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
filenames = _ops.convert_to_tensor(filenames, _dtypes.string)
|
|
header_bytes = _ops.convert_to_tensor(header_bytes, _dtypes.int64)
|
|
record_bytes = _ops.convert_to_tensor(record_bytes, _dtypes.int64)
|
|
footer_bytes = _ops.convert_to_tensor(footer_bytes, _dtypes.int64)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
compression_type = _ops.convert_to_tensor(compression_type, _dtypes.string)
|
|
_inputs_flat = [filenames, header_bytes, record_bytes, footer_bytes, buffer_size, compression_type]
|
|
_attrs = ("metadata", metadata)
|
|
_result = _execute.execute(b"FixedLengthRecordDatasetV2", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"FixedLengthRecordDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def flat_map_dataset(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, f, output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that applies `f` to the outputs of `input_dataset`.
|
|
|
|
Unlike MapDataset, the `f` in FlatMapDataset is expected to return a
|
|
Dataset variant, and FlatMapDataset will flatten successive results
|
|
into a single Dataset.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
f: A function decorated with @Defun.
|
|
A function mapping elements of `input_dataset`, concatenated with
|
|
`other_arguments`, to a Dataset variant that contains elements matching
|
|
`output_types` and `output_shapes`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "FlatMapDataset", name, input_dataset, other_arguments, "f", f,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return flat_map_dataset_eager_fallback(
|
|
input_dataset, other_arguments, f=f, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'flat_map_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'flat_map_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"FlatMapDataset", input_dataset=input_dataset,
|
|
other_arguments=other_arguments, f=f,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"FlatMapDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
FlatMapDataset = tf_export("raw_ops.FlatMapDataset")(_ops.to_raw_op(flat_map_dataset))
|
|
|
|
|
|
def flat_map_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, f, output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'flat_map_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'flat_map_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset] + list(other_arguments)
|
|
_attrs = ("f", f, "Targuments", _attr_Targuments, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"FlatMapDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"FlatMapDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def generator_dataset(init_func_other_args, next_func_other_args, finalize_func_other_args, init_func, next_func, finalize_func, output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that invokes a function to generate elements.
|
|
|
|
Args:
|
|
init_func_other_args: A list of `Tensor` objects.
|
|
next_func_other_args: A list of `Tensor` objects.
|
|
finalize_func_other_args: A list of `Tensor` objects.
|
|
init_func: A function decorated with @Defun.
|
|
next_func: A function decorated with @Defun.
|
|
finalize_func: A function decorated with @Defun.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "GeneratorDataset", name, init_func_other_args,
|
|
next_func_other_args, finalize_func_other_args, "init_func",
|
|
init_func, "next_func", next_func, "finalize_func", finalize_func,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return generator_dataset_eager_fallback(
|
|
init_func_other_args, next_func_other_args,
|
|
finalize_func_other_args, init_func=init_func, next_func=next_func,
|
|
finalize_func=finalize_func, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'generator_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'generator_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"GeneratorDataset", init_func_other_args=init_func_other_args,
|
|
next_func_other_args=next_func_other_args,
|
|
finalize_func_other_args=finalize_func_other_args,
|
|
init_func=init_func, next_func=next_func,
|
|
finalize_func=finalize_func,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("init_func", _op.get_attr("init_func"), "next_func",
|
|
_op.get_attr("next_func"), "finalize_func",
|
|
_op.get_attr("finalize_func"), "Tinit_func_args",
|
|
_op.get_attr("Tinit_func_args"), "Tnext_func_args",
|
|
_op.get_attr("Tnext_func_args"), "Tfinalize_func_args",
|
|
_op.get_attr("Tfinalize_func_args"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"GeneratorDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
GeneratorDataset = tf_export("raw_ops.GeneratorDataset")(_ops.to_raw_op(generator_dataset))
|
|
|
|
|
|
def generator_dataset_eager_fallback(init_func_other_args, next_func_other_args, finalize_func_other_args, init_func, next_func, finalize_func, output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'generator_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'generator_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Tinit_func_args, init_func_other_args = _execute.convert_to_mixed_eager_tensors(init_func_other_args, ctx)
|
|
_attr_Tnext_func_args, next_func_other_args = _execute.convert_to_mixed_eager_tensors(next_func_other_args, ctx)
|
|
_attr_Tfinalize_func_args, finalize_func_other_args = _execute.convert_to_mixed_eager_tensors(finalize_func_other_args, ctx)
|
|
_inputs_flat = list(init_func_other_args) + list(next_func_other_args) + list(finalize_func_other_args)
|
|
_attrs = ("init_func", init_func, "next_func", next_func, "finalize_func",
|
|
finalize_func, "Tinit_func_args", _attr_Tinit_func_args, "Tnext_func_args",
|
|
_attr_Tnext_func_args, "Tfinalize_func_args", _attr_Tfinalize_func_args,
|
|
"output_types", output_types, "output_shapes", output_shapes, "metadata",
|
|
metadata)
|
|
_result = _execute.execute(b"GeneratorDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"GeneratorDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def get_options(input_dataset: Annotated[Any, _atypes.Variant], name=None) -> Annotated[Any, _atypes.String]:
|
|
r"""Returns the `tf.data.Options` attached to `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the input dataset.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `string`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "GetOptions", name, input_dataset)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return get_options_eager_fallback(
|
|
input_dataset, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"GetOptions", input_dataset=input_dataset, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"GetOptions", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
GetOptions = tf_export("raw_ops.GetOptions")(_ops.to_raw_op(get_options))
|
|
|
|
|
|
def get_options_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], name, ctx) -> Annotated[Any, _atypes.String]:
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = None
|
|
_result = _execute.execute(b"GetOptions", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"GetOptions", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def interleave_dataset(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, cycle_length: Annotated[Any, _atypes.Int64], block_length: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that applies `f` to the outputs of `input_dataset`.
|
|
|
|
Unlike MapDataset, the `f` in InterleaveDataset is expected to return
|
|
a Dataset variant, and InterleaveDataset will flatten successive
|
|
results into a single Dataset. Unlike FlatMapDataset,
|
|
InterleaveDataset will interleave sequences of up to `block_length`
|
|
consecutive elements from `cycle_length` input elements.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
cycle_length: A `Tensor` of type `int64`.
|
|
block_length: A `Tensor` of type `int64`.
|
|
f: A function decorated with @Defun.
|
|
A function mapping elements of `input_dataset`, concatenated with
|
|
`other_arguments`, to a Dataset variant that contains elements matching
|
|
`output_types` and `output_shapes`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "InterleaveDataset", name, input_dataset, other_arguments,
|
|
cycle_length, block_length, "f", f, "output_types", output_types,
|
|
"output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return interleave_dataset_eager_fallback(
|
|
input_dataset, other_arguments, cycle_length, block_length, f=f,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'interleave_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'interleave_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"InterleaveDataset", input_dataset=input_dataset,
|
|
other_arguments=other_arguments,
|
|
cycle_length=cycle_length,
|
|
block_length=block_length, f=f,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"InterleaveDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
InterleaveDataset = tf_export("raw_ops.InterleaveDataset")(_ops.to_raw_op(interleave_dataset))
|
|
|
|
|
|
def interleave_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, cycle_length: Annotated[Any, _atypes.Int64], block_length: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'interleave_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'interleave_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
cycle_length = _ops.convert_to_tensor(cycle_length, _dtypes.int64)
|
|
block_length = _ops.convert_to_tensor(block_length, _dtypes.int64)
|
|
_inputs_flat = [input_dataset] + list(other_arguments) + [cycle_length, block_length]
|
|
_attrs = ("f", f, "Targuments", _attr_Targuments, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"InterleaveDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"InterleaveDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def iterator(shared_name: str, container: str, output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""A container for an iterator resource.
|
|
|
|
Args:
|
|
shared_name: A `string`.
|
|
container: A `string`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "Iterator", name, "shared_name", shared_name, "container",
|
|
container, "output_types", output_types, "output_shapes",
|
|
output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return iterator_eager_fallback(
|
|
shared_name=shared_name, container=container,
|
|
output_types=output_types, output_shapes=output_shapes, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
shared_name = _execute.make_str(shared_name, "shared_name")
|
|
container = _execute.make_str(container, "container")
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"Iterator", shared_name=shared_name, container=container,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("shared_name", _op.get_attr("shared_name"), "container",
|
|
_op.get_attr("container"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"Iterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
Iterator = tf_export("raw_ops.Iterator")(_ops.to_raw_op(iterator))
|
|
|
|
|
|
def iterator_eager_fallback(shared_name: str, container: str, output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
shared_name = _execute.make_str(shared_name, "shared_name")
|
|
container = _execute.make_str(container, "container")
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_inputs_flat = []
|
|
_attrs = ("shared_name", shared_name, "container", container,
|
|
"output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"Iterator", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"Iterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def iterator_from_string_handle(string_handle: Annotated[Any, _atypes.String], output_types=[], output_shapes=[], name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""Converts the given string representing a handle to an iterator to a resource.
|
|
|
|
Args:
|
|
string_handle: A `Tensor` of type `string`.
|
|
A string representation of the given handle.
|
|
output_types: An optional list of `tf.DTypes`. Defaults to `[]`.
|
|
If specified, defines the type of each tuple component in an
|
|
element produced by the resulting iterator.
|
|
output_shapes: An optional list of shapes (each a `tf.TensorShape` or list of `ints`). Defaults to `[]`.
|
|
If specified, defines the shape of each tuple component in an
|
|
element produced by the resulting iterator.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "IteratorFromStringHandle", name, string_handle, "output_types",
|
|
output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return iterator_from_string_handle_eager_fallback(
|
|
string_handle, output_types=output_types,
|
|
output_shapes=output_shapes, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if output_types is None:
|
|
output_types = []
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_from_string_handle' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if output_shapes is None:
|
|
output_shapes = []
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_from_string_handle' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"IteratorFromStringHandle", string_handle=string_handle,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"IteratorFromStringHandle", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
IteratorFromStringHandle = tf_export("raw_ops.IteratorFromStringHandle")(_ops.to_raw_op(iterator_from_string_handle))
|
|
|
|
|
|
def iterator_from_string_handle_eager_fallback(string_handle: Annotated[Any, _atypes.String], output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
if output_types is None:
|
|
output_types = []
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_from_string_handle' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if output_shapes is None:
|
|
output_shapes = []
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_from_string_handle' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
string_handle = _ops.convert_to_tensor(string_handle, _dtypes.string)
|
|
_inputs_flat = [string_handle]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"IteratorFromStringHandle", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"IteratorFromStringHandle", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def iterator_from_string_handle_v2(string_handle: Annotated[Any, _atypes.String], output_types=[], output_shapes=[], name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
string_handle: A `Tensor` of type `string`.
|
|
output_types: An optional list of `tf.DTypes`. Defaults to `[]`.
|
|
output_shapes: An optional list of shapes (each a `tf.TensorShape` or list of `ints`). Defaults to `[]`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "IteratorFromStringHandleV2", name, string_handle,
|
|
"output_types", output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return iterator_from_string_handle_v2_eager_fallback(
|
|
string_handle, output_types=output_types,
|
|
output_shapes=output_shapes, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if output_types is None:
|
|
output_types = []
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_from_string_handle_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if output_shapes is None:
|
|
output_shapes = []
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_from_string_handle_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"IteratorFromStringHandleV2", string_handle=string_handle,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"IteratorFromStringHandleV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
IteratorFromStringHandleV2 = tf_export("raw_ops.IteratorFromStringHandleV2")(_ops.to_raw_op(iterator_from_string_handle_v2))
|
|
|
|
|
|
def iterator_from_string_handle_v2_eager_fallback(string_handle: Annotated[Any, _atypes.String], output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
if output_types is None:
|
|
output_types = []
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_from_string_handle_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if output_shapes is None:
|
|
output_shapes = []
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_from_string_handle_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
string_handle = _ops.convert_to_tensor(string_handle, _dtypes.string)
|
|
_inputs_flat = [string_handle]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"IteratorFromStringHandleV2", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"IteratorFromStringHandleV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def iterator_get_next(iterator: Annotated[Any, _atypes.Resource], output_types, output_shapes, name=None):
|
|
r"""Gets the next output from the given iterator .
|
|
|
|
Args:
|
|
iterator: A `Tensor` of type `resource`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A list of `Tensor` objects of type `output_types`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "IteratorGetNext", name, iterator, "output_types", output_types,
|
|
"output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return iterator_get_next_eager_fallback(
|
|
iterator, output_types=output_types, output_shapes=output_shapes,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_get_next' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_get_next' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"IteratorGetNext", iterator=iterator, output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if not _result:
|
|
return _op
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"IteratorGetNext", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
IteratorGetNext = tf_export("raw_ops.IteratorGetNext")(_ops.to_raw_op(iterator_get_next))
|
|
|
|
|
|
def iterator_get_next_eager_fallback(iterator: Annotated[Any, _atypes.Resource], output_types, output_shapes, name, ctx):
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_get_next' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_get_next' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
iterator = _ops.convert_to_tensor(iterator, _dtypes.resource)
|
|
_inputs_flat = [iterator]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"IteratorGetNext", len(output_types),
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"IteratorGetNext", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
|
|
def iterator_get_next_as_optional(iterator: Annotated[Any, _atypes.Resource], output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Gets the next output from the given iterator as an Optional variant.
|
|
|
|
Args:
|
|
iterator: A `Tensor` of type `resource`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "IteratorGetNextAsOptional", name, iterator, "output_types",
|
|
output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return iterator_get_next_as_optional_eager_fallback(
|
|
iterator, output_types=output_types, output_shapes=output_shapes,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_get_next_as_optional' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_get_next_as_optional' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"IteratorGetNextAsOptional", iterator=iterator,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"IteratorGetNextAsOptional", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
IteratorGetNextAsOptional = tf_export("raw_ops.IteratorGetNextAsOptional")(_ops.to_raw_op(iterator_get_next_as_optional))
|
|
|
|
|
|
def iterator_get_next_as_optional_eager_fallback(iterator: Annotated[Any, _atypes.Resource], output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_get_next_as_optional' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_get_next_as_optional' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
iterator = _ops.convert_to_tensor(iterator, _dtypes.resource)
|
|
_inputs_flat = [iterator]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"IteratorGetNextAsOptional", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"IteratorGetNextAsOptional", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def iterator_get_next_sync(iterator: Annotated[Any, _atypes.Resource], output_types, output_shapes, name=None):
|
|
r"""Gets the next output from the given iterator.
|
|
|
|
This operation is a synchronous version IteratorGetNext. It should only be used
|
|
in situations where the iterator does not block the calling thread, or where
|
|
the calling thread is not a member of the thread pool used to execute parallel
|
|
operations (e.g. in eager mode).
|
|
|
|
Args:
|
|
iterator: A `Tensor` of type `resource`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A list of `Tensor` objects of type `output_types`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "IteratorGetNextSync", name, iterator, "output_types",
|
|
output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return iterator_get_next_sync_eager_fallback(
|
|
iterator, output_types=output_types, output_shapes=output_shapes,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_get_next_sync' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_get_next_sync' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"IteratorGetNextSync", iterator=iterator, output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if not _result:
|
|
return _op
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"IteratorGetNextSync", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
IteratorGetNextSync = tf_export("raw_ops.IteratorGetNextSync")(_ops.to_raw_op(iterator_get_next_sync))
|
|
|
|
|
|
def iterator_get_next_sync_eager_fallback(iterator: Annotated[Any, _atypes.Resource], output_types, output_shapes, name, ctx):
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_get_next_sync' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_get_next_sync' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
iterator = _ops.convert_to_tensor(iterator, _dtypes.resource)
|
|
_inputs_flat = [iterator]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"IteratorGetNextSync", len(output_types),
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"IteratorGetNextSync", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
|
|
def iterator_to_string_handle(resource_handle: Annotated[Any, _atypes.Resource], name=None) -> Annotated[Any, _atypes.String]:
|
|
r"""Converts the given `resource_handle` representing an iterator to a string.
|
|
|
|
Args:
|
|
resource_handle: A `Tensor` of type `resource`.
|
|
A handle to an iterator resource.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `string`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "IteratorToStringHandle", name, resource_handle)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return iterator_to_string_handle_eager_fallback(
|
|
resource_handle, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"IteratorToStringHandle", resource_handle=resource_handle, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"IteratorToStringHandle", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
IteratorToStringHandle = tf_export("raw_ops.IteratorToStringHandle")(_ops.to_raw_op(iterator_to_string_handle))
|
|
|
|
|
|
def iterator_to_string_handle_eager_fallback(resource_handle: Annotated[Any, _atypes.Resource], name, ctx) -> Annotated[Any, _atypes.String]:
|
|
resource_handle = _ops.convert_to_tensor(resource_handle, _dtypes.resource)
|
|
_inputs_flat = [resource_handle]
|
|
_attrs = None
|
|
_result = _execute.execute(b"IteratorToStringHandle", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"IteratorToStringHandle", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def iterator_v2(shared_name: str, container: str, output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
shared_name: A `string`.
|
|
container: A `string`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "IteratorV2", name, "shared_name", shared_name, "container",
|
|
container, "output_types", output_types, "output_shapes",
|
|
output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return iterator_v2_eager_fallback(
|
|
shared_name=shared_name, container=container,
|
|
output_types=output_types, output_shapes=output_shapes, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
shared_name = _execute.make_str(shared_name, "shared_name")
|
|
container = _execute.make_str(container, "container")
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"IteratorV2", shared_name=shared_name, container=container,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("shared_name", _op.get_attr("shared_name"), "container",
|
|
_op.get_attr("container"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"IteratorV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
IteratorV2 = tf_export("raw_ops.IteratorV2")(_ops.to_raw_op(iterator_v2))
|
|
|
|
|
|
def iterator_v2_eager_fallback(shared_name: str, container: str, output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
shared_name = _execute.make_str(shared_name, "shared_name")
|
|
container = _execute.make_str(container, "container")
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'iterator_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'iterator_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_inputs_flat = []
|
|
_attrs = ("shared_name", shared_name, "container", container,
|
|
"output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"IteratorV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"IteratorV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def make_iterator(dataset: Annotated[Any, _atypes.Variant], iterator: Annotated[Any, _atypes.Resource], name=None):
|
|
r"""Makes a new iterator from the given `dataset` and stores it in `iterator`.
|
|
|
|
This operation may be executed multiple times. Each execution will reset the
|
|
iterator in `iterator` to the first element of `dataset`.
|
|
|
|
Args:
|
|
dataset: A `Tensor` of type `variant`.
|
|
iterator: A `Tensor` of type `resource`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
The created Operation.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "MakeIterator", name, dataset, iterator)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return make_iterator_eager_fallback(
|
|
dataset, iterator, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"MakeIterator", dataset=dataset, iterator=iterator, name=name)
|
|
return _op
|
|
MakeIterator = tf_export("raw_ops.MakeIterator")(_ops.to_raw_op(make_iterator))
|
|
|
|
|
|
def make_iterator_eager_fallback(dataset: Annotated[Any, _atypes.Variant], iterator: Annotated[Any, _atypes.Resource], name, ctx):
|
|
dataset = _ops.convert_to_tensor(dataset, _dtypes.variant)
|
|
iterator = _ops.convert_to_tensor(iterator, _dtypes.resource)
|
|
_inputs_flat = [dataset, iterator]
|
|
_attrs = None
|
|
_result = _execute.execute(b"MakeIterator", 0, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
_result = None
|
|
return _result
|
|
|
|
|
|
def map_dataset(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, f, output_types, output_shapes, use_inter_op_parallelism:bool=True, preserve_cardinality:bool=False, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that applies `f` to the outputs of `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
f: A function decorated with @Defun.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
use_inter_op_parallelism: An optional `bool`. Defaults to `True`.
|
|
preserve_cardinality: An optional `bool`. Defaults to `False`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "MapDataset", name, input_dataset, other_arguments, "f", f,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"use_inter_op_parallelism", use_inter_op_parallelism,
|
|
"preserve_cardinality", preserve_cardinality, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return map_dataset_eager_fallback(
|
|
input_dataset, other_arguments, f=f, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
use_inter_op_parallelism=use_inter_op_parallelism,
|
|
preserve_cardinality=preserve_cardinality, metadata=metadata,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'map_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'map_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if use_inter_op_parallelism is None:
|
|
use_inter_op_parallelism = True
|
|
use_inter_op_parallelism = _execute.make_bool(use_inter_op_parallelism, "use_inter_op_parallelism")
|
|
if preserve_cardinality is None:
|
|
preserve_cardinality = False
|
|
preserve_cardinality = _execute.make_bool(preserve_cardinality, "preserve_cardinality")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"MapDataset", input_dataset=input_dataset,
|
|
other_arguments=other_arguments, f=f,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
use_inter_op_parallelism=use_inter_op_parallelism,
|
|
preserve_cardinality=preserve_cardinality,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "use_inter_op_parallelism",
|
|
_op._get_attr_bool("use_inter_op_parallelism"),
|
|
"preserve_cardinality",
|
|
_op._get_attr_bool("preserve_cardinality"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"MapDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
MapDataset = tf_export("raw_ops.MapDataset")(_ops.to_raw_op(map_dataset))
|
|
|
|
|
|
def map_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, f, output_types, output_shapes, use_inter_op_parallelism: bool, preserve_cardinality: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'map_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'map_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if use_inter_op_parallelism is None:
|
|
use_inter_op_parallelism = True
|
|
use_inter_op_parallelism = _execute.make_bool(use_inter_op_parallelism, "use_inter_op_parallelism")
|
|
if preserve_cardinality is None:
|
|
preserve_cardinality = False
|
|
preserve_cardinality = _execute.make_bool(preserve_cardinality, "preserve_cardinality")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset] + list(other_arguments)
|
|
_attrs = ("f", f, "Targuments", _attr_Targuments, "output_types",
|
|
output_types, "output_shapes", output_shapes, "use_inter_op_parallelism",
|
|
use_inter_op_parallelism, "preserve_cardinality", preserve_cardinality,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"MapDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"MapDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def map_defun(arguments, captured_inputs, output_types, output_shapes, f, max_intra_op_parallelism:int=1, name=None):
|
|
r""" Maps a function on the list of tensors unpacked from arguments on dimension 0.
|
|
The function given by `f` is assumed to be stateless, and is executed
|
|
concurrently on all the slices; up to batch_size (i.e. the size of the 0th
|
|
dimension of each argument) functions will be scheduled at once.
|
|
|
|
The `max_intra_op_parallelism` attr, which defaults to 1, can be used to
|
|
limit the intra op parallelism. To limit inter-op parallelism, a user can
|
|
set a private threadpool on the dataset using `tf.data.Options`'s
|
|
`ThreadingOptions`.
|
|
|
|
Note that this op is not exposed to users directly, but is invoked in tf.data
|
|
rewrites.
|
|
|
|
Args:
|
|
arguments: A list of `Tensor` objects.
|
|
A list of tensors whose types are `Targuments`, corresponding to the inputs
|
|
the function should be mapped over.
|
|
captured_inputs: A list of `Tensor` objects.
|
|
A list of tensors whose types are `Tcaptured`, corresponding to the captured
|
|
inputs of the defun.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
A list of types.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
A list of shapes.
|
|
f: A function decorated with @Defun.
|
|
max_intra_op_parallelism: An optional `int`. Defaults to `1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A list of `Tensor` objects of type `output_types`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "MapDefun", name, arguments, captured_inputs, "output_types",
|
|
output_types, "output_shapes", output_shapes, "f", f,
|
|
"max_intra_op_parallelism", max_intra_op_parallelism)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return map_defun_eager_fallback(
|
|
arguments, captured_inputs, output_types=output_types,
|
|
output_shapes=output_shapes, f=f,
|
|
max_intra_op_parallelism=max_intra_op_parallelism, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'map_defun' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'map_defun' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if max_intra_op_parallelism is None:
|
|
max_intra_op_parallelism = 1
|
|
max_intra_op_parallelism = _execute.make_int(max_intra_op_parallelism, "max_intra_op_parallelism")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"MapDefun", arguments=arguments, captured_inputs=captured_inputs,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
f=f, max_intra_op_parallelism=max_intra_op_parallelism,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("Targuments", _op.get_attr("Targuments"), "Tcaptured",
|
|
_op.get_attr("Tcaptured"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "f", _op.get_attr("f"),
|
|
"max_intra_op_parallelism",
|
|
_op._get_attr_int("max_intra_op_parallelism"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"MapDefun", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
MapDefun = tf_export("raw_ops.MapDefun")(_ops.to_raw_op(map_defun))
|
|
|
|
|
|
def map_defun_eager_fallback(arguments, captured_inputs, output_types, output_shapes, f, max_intra_op_parallelism: int, name, ctx):
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'map_defun' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'map_defun' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if max_intra_op_parallelism is None:
|
|
max_intra_op_parallelism = 1
|
|
max_intra_op_parallelism = _execute.make_int(max_intra_op_parallelism, "max_intra_op_parallelism")
|
|
_attr_Targuments, arguments = _execute.convert_to_mixed_eager_tensors(arguments, ctx)
|
|
_attr_Tcaptured, captured_inputs = _execute.convert_to_mixed_eager_tensors(captured_inputs, ctx)
|
|
_inputs_flat = list(arguments) + list(captured_inputs)
|
|
_attrs = ("Targuments", _attr_Targuments, "Tcaptured", _attr_Tcaptured,
|
|
"output_types", output_types, "output_shapes", output_shapes, "f", f,
|
|
"max_intra_op_parallelism", max_intra_op_parallelism)
|
|
_result = _execute.execute(b"MapDefun", len(output_types),
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"MapDefun", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
|
|
def model_dataset(input_dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, algorithm:int=0, cpu_budget:int=0, ram_budget:int=0, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Identity transformation that models performance.
|
|
|
|
Identity transformation that models performance.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the input dataset.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
algorithm: An optional `int`. Defaults to `0`.
|
|
cpu_budget: An optional `int`. Defaults to `0`.
|
|
ram_budget: An optional `int`. Defaults to `0`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ModelDataset", name, input_dataset, "algorithm", algorithm,
|
|
"cpu_budget", cpu_budget, "ram_budget", ram_budget, "output_types",
|
|
output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return model_dataset_eager_fallback(
|
|
input_dataset, algorithm=algorithm, cpu_budget=cpu_budget,
|
|
ram_budget=ram_budget, output_types=output_types,
|
|
output_shapes=output_shapes, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'model_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'model_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if algorithm is None:
|
|
algorithm = 0
|
|
algorithm = _execute.make_int(algorithm, "algorithm")
|
|
if cpu_budget is None:
|
|
cpu_budget = 0
|
|
cpu_budget = _execute.make_int(cpu_budget, "cpu_budget")
|
|
if ram_budget is None:
|
|
ram_budget = 0
|
|
ram_budget = _execute.make_int(ram_budget, "ram_budget")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ModelDataset", input_dataset=input_dataset,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, algorithm=algorithm,
|
|
cpu_budget=cpu_budget, ram_budget=ram_budget,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("algorithm", _op._get_attr_int("algorithm"), "cpu_budget",
|
|
_op._get_attr_int("cpu_budget"), "ram_budget",
|
|
_op._get_attr_int("ram_budget"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ModelDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ModelDataset = tf_export("raw_ops.ModelDataset")(_ops.to_raw_op(model_dataset))
|
|
|
|
|
|
def model_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], output_types, output_shapes, algorithm: int, cpu_budget: int, ram_budget: int, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'model_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'model_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if algorithm is None:
|
|
algorithm = 0
|
|
algorithm = _execute.make_int(algorithm, "algorithm")
|
|
if cpu_budget is None:
|
|
cpu_budget = 0
|
|
cpu_budget = _execute.make_int(cpu_budget, "cpu_budget")
|
|
if ram_budget is None:
|
|
ram_budget = 0
|
|
ram_budget = _execute.make_int(ram_budget, "ram_budget")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = ("algorithm", algorithm, "cpu_budget", cpu_budget, "ram_budget",
|
|
ram_budget, "output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"ModelDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ModelDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def multi_device_iterator(devices, shared_name: str, container: str, output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""Creates a MultiDeviceIterator resource.
|
|
|
|
Args:
|
|
devices: A list of `strings` that has length `>= 1`.
|
|
A list of devices the iterator works across.
|
|
shared_name: A `string`.
|
|
If non-empty, this resource will be shared under the given name
|
|
across multiple sessions.
|
|
container: A `string`.
|
|
If non-empty, this resource is placed in the given container.
|
|
Otherwise, a default container is used.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
The type list for the return values.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
The list of shapes being produced.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "MultiDeviceIterator", name, "devices", devices, "shared_name",
|
|
shared_name, "container", container, "output_types", output_types,
|
|
"output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return multi_device_iterator_eager_fallback(
|
|
devices=devices, shared_name=shared_name, container=container,
|
|
output_types=output_types, output_shapes=output_shapes, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(devices, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'devices' argument to "
|
|
"'multi_device_iterator' Op, not %r." % devices)
|
|
devices = [_execute.make_str(_s, "devices") for _s in devices]
|
|
shared_name = _execute.make_str(shared_name, "shared_name")
|
|
container = _execute.make_str(container, "container")
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'multi_device_iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'multi_device_iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"MultiDeviceIterator", devices=devices, shared_name=shared_name,
|
|
container=container, output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("devices", _op.get_attr("devices"), "shared_name",
|
|
_op.get_attr("shared_name"), "container",
|
|
_op.get_attr("container"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"MultiDeviceIterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
MultiDeviceIterator = tf_export("raw_ops.MultiDeviceIterator")(_ops.to_raw_op(multi_device_iterator))
|
|
|
|
|
|
def multi_device_iterator_eager_fallback(devices, shared_name: str, container: str, output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
if not isinstance(devices, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'devices' argument to "
|
|
"'multi_device_iterator' Op, not %r." % devices)
|
|
devices = [_execute.make_str(_s, "devices") for _s in devices]
|
|
shared_name = _execute.make_str(shared_name, "shared_name")
|
|
container = _execute.make_str(container, "container")
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'multi_device_iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'multi_device_iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_inputs_flat = []
|
|
_attrs = ("devices", devices, "shared_name", shared_name, "container",
|
|
container, "output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"MultiDeviceIterator", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"MultiDeviceIterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def multi_device_iterator_from_string_handle(string_handle: Annotated[Any, _atypes.String], output_types=[], output_shapes=[], name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""Generates a MultiDeviceIterator resource from its provided string handle.
|
|
|
|
Args:
|
|
string_handle: A `Tensor` of type `string`.
|
|
String representing the resource.
|
|
output_types: An optional list of `tf.DTypes`. Defaults to `[]`.
|
|
The type list for the return values.
|
|
output_shapes: An optional list of shapes (each a `tf.TensorShape` or list of `ints`). Defaults to `[]`.
|
|
The list of shapes being produced.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "MultiDeviceIteratorFromStringHandle", name, string_handle,
|
|
"output_types", output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return multi_device_iterator_from_string_handle_eager_fallback(
|
|
string_handle, output_types=output_types,
|
|
output_shapes=output_shapes, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if output_types is None:
|
|
output_types = []
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'multi_device_iterator_from_string_handle' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if output_shapes is None:
|
|
output_shapes = []
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'multi_device_iterator_from_string_handle' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"MultiDeviceIteratorFromStringHandle", string_handle=string_handle,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"MultiDeviceIteratorFromStringHandle", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
MultiDeviceIteratorFromStringHandle = tf_export("raw_ops.MultiDeviceIteratorFromStringHandle")(_ops.to_raw_op(multi_device_iterator_from_string_handle))
|
|
|
|
|
|
def multi_device_iterator_from_string_handle_eager_fallback(string_handle: Annotated[Any, _atypes.String], output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
if output_types is None:
|
|
output_types = []
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'multi_device_iterator_from_string_handle' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if output_shapes is None:
|
|
output_shapes = []
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'multi_device_iterator_from_string_handle' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
string_handle = _ops.convert_to_tensor(string_handle, _dtypes.string)
|
|
_inputs_flat = [string_handle]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"MultiDeviceIteratorFromStringHandle", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"MultiDeviceIteratorFromStringHandle", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def multi_device_iterator_get_next_from_shard(multi_device_iterator: Annotated[Any, _atypes.Resource], shard_num: Annotated[Any, _atypes.Int32], incarnation_id: Annotated[Any, _atypes.Int64], output_types, output_shapes, name=None):
|
|
r"""Gets next element for the provided shard number.
|
|
|
|
Args:
|
|
multi_device_iterator: A `Tensor` of type `resource`.
|
|
A MultiDeviceIterator resource.
|
|
shard_num: A `Tensor` of type `int32`.
|
|
Integer representing which shard to fetch data for.
|
|
incarnation_id: A `Tensor` of type `int64`.
|
|
Which incarnation of the MultiDeviceIterator is running.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
The type list for the return values.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
The list of shapes being produced.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A list of `Tensor` objects of type `output_types`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "MultiDeviceIteratorGetNextFromShard", name,
|
|
multi_device_iterator, shard_num, incarnation_id, "output_types",
|
|
output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return multi_device_iterator_get_next_from_shard_eager_fallback(
|
|
multi_device_iterator, shard_num, incarnation_id,
|
|
output_types=output_types, output_shapes=output_shapes, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'multi_device_iterator_get_next_from_shard' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'multi_device_iterator_get_next_from_shard' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"MultiDeviceIteratorGetNextFromShard", multi_device_iterator=multi_device_iterator,
|
|
shard_num=shard_num,
|
|
incarnation_id=incarnation_id,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if not _result:
|
|
return _op
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"MultiDeviceIteratorGetNextFromShard", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
MultiDeviceIteratorGetNextFromShard = tf_export("raw_ops.MultiDeviceIteratorGetNextFromShard")(_ops.to_raw_op(multi_device_iterator_get_next_from_shard))
|
|
|
|
|
|
def multi_device_iterator_get_next_from_shard_eager_fallback(multi_device_iterator: Annotated[Any, _atypes.Resource], shard_num: Annotated[Any, _atypes.Int32], incarnation_id: Annotated[Any, _atypes.Int64], output_types, output_shapes, name, ctx):
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'multi_device_iterator_get_next_from_shard' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'multi_device_iterator_get_next_from_shard' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
multi_device_iterator = _ops.convert_to_tensor(multi_device_iterator, _dtypes.resource)
|
|
shard_num = _ops.convert_to_tensor(shard_num, _dtypes.int32)
|
|
incarnation_id = _ops.convert_to_tensor(incarnation_id, _dtypes.int64)
|
|
_inputs_flat = [multi_device_iterator, shard_num, incarnation_id]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"MultiDeviceIteratorGetNextFromShard",
|
|
len(output_types), inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"MultiDeviceIteratorGetNextFromShard", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
|
|
def multi_device_iterator_init(dataset: Annotated[Any, _atypes.Variant], multi_device_iterator: Annotated[Any, _atypes.Resource], max_buffer_size: Annotated[Any, _atypes.Int64], name=None) -> Annotated[Any, _atypes.Int64]:
|
|
r"""Initializes the multi device iterator with the given dataset.
|
|
|
|
Args:
|
|
dataset: A `Tensor` of type `variant`. Dataset to be iterated upon.
|
|
multi_device_iterator: A `Tensor` of type `resource`.
|
|
A MultiDeviceIteratorResource.
|
|
max_buffer_size: A `Tensor` of type `int64`.
|
|
The maximum size of the host side per device buffer to keep.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `int64`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "MultiDeviceIteratorInit", name, dataset, multi_device_iterator,
|
|
max_buffer_size)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return multi_device_iterator_init_eager_fallback(
|
|
dataset, multi_device_iterator, max_buffer_size, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"MultiDeviceIteratorInit", dataset=dataset,
|
|
multi_device_iterator=multi_device_iterator,
|
|
max_buffer_size=max_buffer_size, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"MultiDeviceIteratorInit", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
MultiDeviceIteratorInit = tf_export("raw_ops.MultiDeviceIteratorInit")(_ops.to_raw_op(multi_device_iterator_init))
|
|
|
|
|
|
def multi_device_iterator_init_eager_fallback(dataset: Annotated[Any, _atypes.Variant], multi_device_iterator: Annotated[Any, _atypes.Resource], max_buffer_size: Annotated[Any, _atypes.Int64], name, ctx) -> Annotated[Any, _atypes.Int64]:
|
|
dataset = _ops.convert_to_tensor(dataset, _dtypes.variant)
|
|
multi_device_iterator = _ops.convert_to_tensor(multi_device_iterator, _dtypes.resource)
|
|
max_buffer_size = _ops.convert_to_tensor(max_buffer_size, _dtypes.int64)
|
|
_inputs_flat = [dataset, multi_device_iterator, max_buffer_size]
|
|
_attrs = None
|
|
_result = _execute.execute(b"MultiDeviceIteratorInit", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"MultiDeviceIteratorInit", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def multi_device_iterator_to_string_handle(multi_device_iterator: Annotated[Any, _atypes.Resource], name=None) -> Annotated[Any, _atypes.String]:
|
|
r"""Produces a string handle for the given MultiDeviceIterator.
|
|
|
|
Args:
|
|
multi_device_iterator: A `Tensor` of type `resource`.
|
|
A MultiDeviceIterator resource.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `string`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "MultiDeviceIteratorToStringHandle", name,
|
|
multi_device_iterator)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return multi_device_iterator_to_string_handle_eager_fallback(
|
|
multi_device_iterator, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"MultiDeviceIteratorToStringHandle", multi_device_iterator=multi_device_iterator,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"MultiDeviceIteratorToStringHandle", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
MultiDeviceIteratorToStringHandle = tf_export("raw_ops.MultiDeviceIteratorToStringHandle")(_ops.to_raw_op(multi_device_iterator_to_string_handle))
|
|
|
|
|
|
def multi_device_iterator_to_string_handle_eager_fallback(multi_device_iterator: Annotated[Any, _atypes.Resource], name, ctx) -> Annotated[Any, _atypes.String]:
|
|
multi_device_iterator = _ops.convert_to_tensor(multi_device_iterator, _dtypes.resource)
|
|
_inputs_flat = [multi_device_iterator]
|
|
_attrs = None
|
|
_result = _execute.execute(b"MultiDeviceIteratorToStringHandle", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"MultiDeviceIteratorToStringHandle", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def one_shot_iterator(dataset_factory, output_types, output_shapes, container:str="", shared_name:str="", name=None) -> Annotated[Any, _atypes.Resource]:
|
|
r"""Makes a "one-shot" iterator that can be iterated only once.
|
|
|
|
A one-shot iterator bundles the logic for defining the dataset and
|
|
the state of the iterator in a single op, which allows simple input
|
|
pipelines to be defined without an additional initialization
|
|
("MakeIterator") step.
|
|
|
|
One-shot iterators have the following limitations:
|
|
|
|
* They do not support parameterization: all logic for creating the underlying
|
|
dataset must be bundled in the `dataset_factory` function.
|
|
* They are not resettable. Once a one-shot iterator reaches the end of its
|
|
underlying dataset, subsequent "IteratorGetNext" operations on that
|
|
iterator will always produce an `OutOfRange` error.
|
|
|
|
For greater flexibility, use "Iterator" and "MakeIterator" to define
|
|
an iterator using an arbitrary subgraph, which may capture tensors
|
|
(including fed values) as parameters, and which may be reset multiple
|
|
times by rerunning "MakeIterator".
|
|
|
|
Args:
|
|
dataset_factory: A function decorated with @Defun.
|
|
A function of type `() -> DT_VARIANT`, where the returned
|
|
DT_VARIANT is a dataset.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
container: An optional `string`. Defaults to `""`.
|
|
shared_name: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `resource`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "OneShotIterator", name, "dataset_factory", dataset_factory,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"container", container, "shared_name", shared_name)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return one_shot_iterator_eager_fallback(
|
|
dataset_factory=dataset_factory, output_types=output_types,
|
|
output_shapes=output_shapes, container=container,
|
|
shared_name=shared_name, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'one_shot_iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'one_shot_iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if container is None:
|
|
container = ""
|
|
container = _execute.make_str(container, "container")
|
|
if shared_name is None:
|
|
shared_name = ""
|
|
shared_name = _execute.make_str(shared_name, "shared_name")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"OneShotIterator", dataset_factory=dataset_factory,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, container=container,
|
|
shared_name=shared_name, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("dataset_factory", _op.get_attr("dataset_factory"),
|
|
"output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "container",
|
|
_op.get_attr("container"), "shared_name",
|
|
_op.get_attr("shared_name"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"OneShotIterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
OneShotIterator = tf_export("raw_ops.OneShotIterator")(_ops.to_raw_op(one_shot_iterator))
|
|
|
|
|
|
def one_shot_iterator_eager_fallback(dataset_factory, output_types, output_shapes, container: str, shared_name: str, name, ctx) -> Annotated[Any, _atypes.Resource]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'one_shot_iterator' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'one_shot_iterator' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if container is None:
|
|
container = ""
|
|
container = _execute.make_str(container, "container")
|
|
if shared_name is None:
|
|
shared_name = ""
|
|
shared_name = _execute.make_str(shared_name, "shared_name")
|
|
_inputs_flat = []
|
|
_attrs = ("dataset_factory", dataset_factory, "output_types", output_types,
|
|
"output_shapes", output_shapes, "container", container, "shared_name",
|
|
shared_name)
|
|
_result = _execute.execute(b"OneShotIterator", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"OneShotIterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def optimize_dataset(input_dataset: Annotated[Any, _atypes.Variant], optimizations: Annotated[Any, _atypes.String], output_types, output_shapes, optimization_configs=[], name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset by applying optimizations to `input_dataset`.
|
|
|
|
Creates a dataset by applying optimizations to `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the input dataset.
|
|
optimizations: A `Tensor` of type `string`.
|
|
A `tf.string` vector `tf.Tensor` identifying optimizations to use.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
optimization_configs: An optional list of `strings`. Defaults to `[]`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "OptimizeDataset", name, input_dataset, optimizations,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"optimization_configs", optimization_configs)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return optimize_dataset_eager_fallback(
|
|
input_dataset, optimizations, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
optimization_configs=optimization_configs, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'optimize_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'optimize_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if optimization_configs is None:
|
|
optimization_configs = []
|
|
if not isinstance(optimization_configs, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'optimization_configs' argument to "
|
|
"'optimize_dataset' Op, not %r." % optimization_configs)
|
|
optimization_configs = [_execute.make_str(_s, "optimization_configs") for _s in optimization_configs]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"OptimizeDataset", input_dataset=input_dataset,
|
|
optimizations=optimizations,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
optimization_configs=optimization_configs,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "optimization_configs",
|
|
_op.get_attr("optimization_configs"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"OptimizeDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
OptimizeDataset = tf_export("raw_ops.OptimizeDataset")(_ops.to_raw_op(optimize_dataset))
|
|
|
|
|
|
def optimize_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], optimizations: Annotated[Any, _atypes.String], output_types, output_shapes, optimization_configs, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'optimize_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'optimize_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if optimization_configs is None:
|
|
optimization_configs = []
|
|
if not isinstance(optimization_configs, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'optimization_configs' argument to "
|
|
"'optimize_dataset' Op, not %r." % optimization_configs)
|
|
optimization_configs = [_execute.make_str(_s, "optimization_configs") for _s in optimization_configs]
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
optimizations = _ops.convert_to_tensor(optimizations, _dtypes.string)
|
|
_inputs_flat = [input_dataset, optimizations]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"optimization_configs", optimization_configs)
|
|
_result = _execute.execute(b"OptimizeDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"OptimizeDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def optimize_dataset_v2(input_dataset: Annotated[Any, _atypes.Variant], optimizations_enabled: Annotated[Any, _atypes.String], optimizations_disabled: Annotated[Any, _atypes.String], optimizations_default: Annotated[Any, _atypes.String], output_types, output_shapes, optimization_configs=[], name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset by applying related optimizations to `input_dataset`.
|
|
|
|
Creates a dataset by applying related optimizations to `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the input dataset.
|
|
optimizations_enabled: A `Tensor` of type `string`.
|
|
A `tf.string` vector `tf.Tensor` identifying user enabled optimizations.
|
|
optimizations_disabled: A `Tensor` of type `string`.
|
|
A `tf.string` vector `tf.Tensor` identifying user disabled optimizations.
|
|
optimizations_default: A `Tensor` of type `string`.
|
|
A `tf.string` vector `tf.Tensor` identifying optimizations by default.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
optimization_configs: An optional list of `strings`. Defaults to `[]`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "OptimizeDatasetV2", name, input_dataset, optimizations_enabled,
|
|
optimizations_disabled, optimizations_default, "output_types",
|
|
output_types, "output_shapes", output_shapes, "optimization_configs",
|
|
optimization_configs)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return optimize_dataset_v2_eager_fallback(
|
|
input_dataset, optimizations_enabled, optimizations_disabled,
|
|
optimizations_default, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
optimization_configs=optimization_configs, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'optimize_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'optimize_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if optimization_configs is None:
|
|
optimization_configs = []
|
|
if not isinstance(optimization_configs, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'optimization_configs' argument to "
|
|
"'optimize_dataset_v2' Op, not %r." % optimization_configs)
|
|
optimization_configs = [_execute.make_str(_s, "optimization_configs") for _s in optimization_configs]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"OptimizeDatasetV2", input_dataset=input_dataset,
|
|
optimizations_enabled=optimizations_enabled,
|
|
optimizations_disabled=optimizations_disabled,
|
|
optimizations_default=optimizations_default,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
optimization_configs=optimization_configs,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "optimization_configs",
|
|
_op.get_attr("optimization_configs"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"OptimizeDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
OptimizeDatasetV2 = tf_export("raw_ops.OptimizeDatasetV2")(_ops.to_raw_op(optimize_dataset_v2))
|
|
|
|
|
|
def optimize_dataset_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], optimizations_enabled: Annotated[Any, _atypes.String], optimizations_disabled: Annotated[Any, _atypes.String], optimizations_default: Annotated[Any, _atypes.String], output_types, output_shapes, optimization_configs, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'optimize_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'optimize_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if optimization_configs is None:
|
|
optimization_configs = []
|
|
if not isinstance(optimization_configs, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'optimization_configs' argument to "
|
|
"'optimize_dataset_v2' Op, not %r." % optimization_configs)
|
|
optimization_configs = [_execute.make_str(_s, "optimization_configs") for _s in optimization_configs]
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
optimizations_enabled = _ops.convert_to_tensor(optimizations_enabled, _dtypes.string)
|
|
optimizations_disabled = _ops.convert_to_tensor(optimizations_disabled, _dtypes.string)
|
|
optimizations_default = _ops.convert_to_tensor(optimizations_default, _dtypes.string)
|
|
_inputs_flat = [input_dataset, optimizations_enabled, optimizations_disabled, optimizations_default]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"optimization_configs", optimization_configs)
|
|
_result = _execute.execute(b"OptimizeDatasetV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"OptimizeDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def options_dataset(input_dataset: Annotated[Any, _atypes.Variant], serialized_options: str, output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset by attaching tf.data.Options to `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the input dataset.
|
|
serialized_options: A `string`.
|
|
A `tf.string` scalar `tf.Tensor` of serialized `tf.data.Options` protocol buffer.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "OptionsDataset", name, input_dataset, "serialized_options",
|
|
serialized_options, "output_types", output_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return options_dataset_eager_fallback(
|
|
input_dataset, serialized_options=serialized_options,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
serialized_options = _execute.make_str(serialized_options, "serialized_options")
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'options_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'options_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"OptionsDataset", input_dataset=input_dataset,
|
|
serialized_options=serialized_options,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("serialized_options", _op.get_attr("serialized_options"),
|
|
"output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"OptionsDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
OptionsDataset = tf_export("raw_ops.OptionsDataset")(_ops.to_raw_op(options_dataset))
|
|
|
|
|
|
def options_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], serialized_options: str, output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
serialized_options = _execute.make_str(serialized_options, "serialized_options")
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'options_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'options_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset]
|
|
_attrs = ("serialized_options", serialized_options, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"OptionsDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"OptionsDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def padded_batch_dataset(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], padded_shapes: Annotated[List[Any], _atypes.Int64], padding_values, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that batches and pads `batch_size` elements from the input.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
batch_size: A `Tensor` of type `int64`.
|
|
A scalar representing the number of elements to accumulate in a
|
|
batch.
|
|
padded_shapes: A list of at least 1 `Tensor` objects with type `int64`.
|
|
A list of int64 tensors representing the desired padded shapes
|
|
of the corresponding output components. These shapes may be partially
|
|
specified, using `-1` to indicate that a particular dimension should be
|
|
padded to the maximum size of all batch elements.
|
|
padding_values: A list of `Tensor` objects.
|
|
A list of scalars containing the padding value to use for
|
|
each of the outputs.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "PaddedBatchDataset", name, input_dataset, batch_size,
|
|
padded_shapes, padding_values, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return padded_batch_dataset_eager_fallback(
|
|
input_dataset, batch_size, padded_shapes, padding_values,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(padded_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'padded_shapes' argument to "
|
|
"'padded_batch_dataset' Op, not %r." % padded_shapes)
|
|
_attr_N = len(padded_shapes)
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'padded_batch_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"PaddedBatchDataset", input_dataset=input_dataset,
|
|
batch_size=batch_size,
|
|
padded_shapes=padded_shapes,
|
|
padding_values=padding_values,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("Toutput_types", _op.get_attr("Toutput_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "N", _op._get_attr_int("N"),
|
|
"metadata", _op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"PaddedBatchDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
PaddedBatchDataset = tf_export("raw_ops.PaddedBatchDataset")(_ops.to_raw_op(padded_batch_dataset))
|
|
|
|
|
|
def padded_batch_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], padded_shapes: Annotated[List[Any], _atypes.Int64], padding_values, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(padded_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'padded_shapes' argument to "
|
|
"'padded_batch_dataset' Op, not %r." % padded_shapes)
|
|
_attr_N = len(padded_shapes)
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'padded_batch_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Toutput_types, padding_values = _execute.convert_to_mixed_eager_tensors(padding_values, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
batch_size = _ops.convert_to_tensor(batch_size, _dtypes.int64)
|
|
padded_shapes = _ops.convert_n_to_tensor(padded_shapes, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, batch_size] + list(padded_shapes) + list(padding_values)
|
|
_attrs = ("Toutput_types", _attr_Toutput_types, "output_shapes",
|
|
output_shapes, "N", _attr_N, "metadata", metadata)
|
|
_result = _execute.execute(b"PaddedBatchDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"PaddedBatchDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def padded_batch_dataset_v2(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], padded_shapes: Annotated[List[Any], _atypes.Int64], padding_values, drop_remainder: Annotated[Any, _atypes.Bool], output_shapes, parallel_copy:bool=False, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that batches and pads `batch_size` elements from the input.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
batch_size: A `Tensor` of type `int64`.
|
|
A scalar representing the number of elements to accumulate in a
|
|
batch.
|
|
padded_shapes: A list of at least 1 `Tensor` objects with type `int64`.
|
|
A list of int64 tensors representing the desired padded shapes
|
|
of the corresponding output components. These shapes may be partially
|
|
specified, using `-1` to indicate that a particular dimension should be
|
|
padded to the maximum size of all batch elements.
|
|
padding_values: A list of `Tensor` objects.
|
|
A list of scalars containing the padding value to use for
|
|
each of the outputs.
|
|
drop_remainder: A `Tensor` of type `bool`.
|
|
A scalar representing whether the last batch should be dropped in case its size
|
|
is smaller than desired.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
parallel_copy: An optional `bool`. Defaults to `False`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "PaddedBatchDatasetV2", name, input_dataset, batch_size,
|
|
padded_shapes, padding_values, drop_remainder, "parallel_copy",
|
|
parallel_copy, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return padded_batch_dataset_v2_eager_fallback(
|
|
input_dataset, batch_size, padded_shapes, padding_values,
|
|
drop_remainder, parallel_copy=parallel_copy,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(padded_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'padded_shapes' argument to "
|
|
"'padded_batch_dataset_v2' Op, not %r." % padded_shapes)
|
|
_attr_N = len(padded_shapes)
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'padded_batch_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if parallel_copy is None:
|
|
parallel_copy = False
|
|
parallel_copy = _execute.make_bool(parallel_copy, "parallel_copy")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"PaddedBatchDatasetV2", input_dataset=input_dataset,
|
|
batch_size=batch_size,
|
|
padded_shapes=padded_shapes,
|
|
padding_values=padding_values,
|
|
drop_remainder=drop_remainder,
|
|
output_shapes=output_shapes,
|
|
parallel_copy=parallel_copy,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("parallel_copy", _op._get_attr_bool("parallel_copy"),
|
|
"Toutput_types", _op.get_attr("Toutput_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "N", _op._get_attr_int("N"),
|
|
"metadata", _op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"PaddedBatchDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
PaddedBatchDatasetV2 = tf_export("raw_ops.PaddedBatchDatasetV2")(_ops.to_raw_op(padded_batch_dataset_v2))
|
|
|
|
|
|
def padded_batch_dataset_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], padded_shapes: Annotated[List[Any], _atypes.Int64], padding_values, drop_remainder: Annotated[Any, _atypes.Bool], output_shapes, parallel_copy: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(padded_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'padded_shapes' argument to "
|
|
"'padded_batch_dataset_v2' Op, not %r." % padded_shapes)
|
|
_attr_N = len(padded_shapes)
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'padded_batch_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if parallel_copy is None:
|
|
parallel_copy = False
|
|
parallel_copy = _execute.make_bool(parallel_copy, "parallel_copy")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Toutput_types, padding_values = _execute.convert_to_mixed_eager_tensors(padding_values, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
batch_size = _ops.convert_to_tensor(batch_size, _dtypes.int64)
|
|
padded_shapes = _ops.convert_n_to_tensor(padded_shapes, _dtypes.int64)
|
|
drop_remainder = _ops.convert_to_tensor(drop_remainder, _dtypes.bool)
|
|
_inputs_flat = [input_dataset, batch_size] + list(padded_shapes) + list(padding_values) + [drop_remainder]
|
|
_attrs = ("parallel_copy", parallel_copy, "Toutput_types",
|
|
_attr_Toutput_types, "output_shapes", output_shapes, "N", _attr_N,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"PaddedBatchDatasetV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"PaddedBatchDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def parallel_batch_dataset(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], num_parallel_calls: Annotated[Any, _atypes.Int64], drop_remainder: Annotated[Any, _atypes.Bool], output_types, output_shapes, parallel_copy:bool=False, deterministic:str="default", metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
batch_size: A `Tensor` of type `int64`.
|
|
num_parallel_calls: A `Tensor` of type `int64`.
|
|
drop_remainder: A `Tensor` of type `bool`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
parallel_copy: An optional `bool`. Defaults to `False`.
|
|
deterministic: An optional `string`. Defaults to `"default"`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ParallelBatchDataset", name, input_dataset, batch_size,
|
|
num_parallel_calls, drop_remainder, "parallel_copy", parallel_copy,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"deterministic", deterministic, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return parallel_batch_dataset_eager_fallback(
|
|
input_dataset, batch_size, num_parallel_calls, drop_remainder,
|
|
parallel_copy=parallel_copy, output_types=output_types,
|
|
output_shapes=output_shapes, deterministic=deterministic,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_batch_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_batch_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if parallel_copy is None:
|
|
parallel_copy = False
|
|
parallel_copy = _execute.make_bool(parallel_copy, "parallel_copy")
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ParallelBatchDataset", input_dataset=input_dataset,
|
|
batch_size=batch_size,
|
|
num_parallel_calls=num_parallel_calls,
|
|
drop_remainder=drop_remainder,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
parallel_copy=parallel_copy,
|
|
deterministic=deterministic,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("parallel_copy", _op._get_attr_bool("parallel_copy"),
|
|
"output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "deterministic",
|
|
_op.get_attr("deterministic"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ParallelBatchDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ParallelBatchDataset = tf_export("raw_ops.ParallelBatchDataset")(_ops.to_raw_op(parallel_batch_dataset))
|
|
|
|
|
|
def parallel_batch_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], batch_size: Annotated[Any, _atypes.Int64], num_parallel_calls: Annotated[Any, _atypes.Int64], drop_remainder: Annotated[Any, _atypes.Bool], output_types, output_shapes, parallel_copy: bool, deterministic: str, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_batch_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_batch_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if parallel_copy is None:
|
|
parallel_copy = False
|
|
parallel_copy = _execute.make_bool(parallel_copy, "parallel_copy")
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
batch_size = _ops.convert_to_tensor(batch_size, _dtypes.int64)
|
|
num_parallel_calls = _ops.convert_to_tensor(num_parallel_calls, _dtypes.int64)
|
|
drop_remainder = _ops.convert_to_tensor(drop_remainder, _dtypes.bool)
|
|
_inputs_flat = [input_dataset, batch_size, num_parallel_calls, drop_remainder]
|
|
_attrs = ("parallel_copy", parallel_copy, "output_types", output_types,
|
|
"output_shapes", output_shapes, "deterministic", deterministic, "metadata",
|
|
metadata)
|
|
_result = _execute.execute(b"ParallelBatchDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ParallelBatchDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def parallel_filter_dataset(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, num_parallel_calls: Annotated[Any, _atypes.Int64], predicate, output_types, output_shapes, deterministic:str="default", metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset containing elements of `input_dataset` matching `predicate`.
|
|
|
|
The `predicate` function must return a scalar boolean and accept the
|
|
following arguments:
|
|
|
|
* One tensor for each component of an element of `input_dataset`.
|
|
* One tensor for each value in `other_arguments`.
|
|
|
|
Unlike a "FilterDataset", which applies `predicate` sequentially, this dataset
|
|
invokes up to `num_parallel_calls` copies of `predicate` in parallel.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
A list of tensors, typically values that were captured when
|
|
building a closure for `predicate`.
|
|
num_parallel_calls: A `Tensor` of type `int64`.
|
|
The number of concurrent invocations of `predicate` that process
|
|
elements from `input_dataset` in parallel.
|
|
predicate: A function decorated with @Defun.
|
|
A function returning a scalar boolean.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
deterministic: An optional `string`. Defaults to `"default"`.
|
|
A string indicating the op-level determinism to use. Deterministic controls
|
|
whether the interleave is allowed to return elements out of order if the next
|
|
element to be returned isn't available, but a later element is. Options are
|
|
"true", "false", and "default". "default" indicates that determinism should be
|
|
decided by the `experimental_deterministic` parameter of `tf.data.Options`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ParallelFilterDataset", name, input_dataset, other_arguments,
|
|
num_parallel_calls, "predicate", predicate, "deterministic",
|
|
deterministic, "output_types", output_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return parallel_filter_dataset_eager_fallback(
|
|
input_dataset, other_arguments, num_parallel_calls,
|
|
predicate=predicate, deterministic=deterministic,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_filter_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_filter_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ParallelFilterDataset", input_dataset=input_dataset,
|
|
other_arguments=other_arguments,
|
|
num_parallel_calls=num_parallel_calls,
|
|
predicate=predicate,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
deterministic=deterministic,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("predicate", _op.get_attr("predicate"), "deterministic",
|
|
_op.get_attr("deterministic"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ParallelFilterDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ParallelFilterDataset = tf_export("raw_ops.ParallelFilterDataset")(_ops.to_raw_op(parallel_filter_dataset))
|
|
|
|
|
|
def parallel_filter_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, num_parallel_calls: Annotated[Any, _atypes.Int64], predicate, output_types, output_shapes, deterministic: str, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_filter_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_filter_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
num_parallel_calls = _ops.convert_to_tensor(num_parallel_calls, _dtypes.int64)
|
|
_inputs_flat = [input_dataset] + list(other_arguments) + [num_parallel_calls]
|
|
_attrs = ("predicate", predicate, "deterministic", deterministic,
|
|
"Targuments", _attr_Targuments, "output_types", output_types,
|
|
"output_shapes", output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"ParallelFilterDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ParallelFilterDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def parallel_interleave_dataset_v2(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, cycle_length: Annotated[Any, _atypes.Int64], block_length: Annotated[Any, _atypes.Int64], num_parallel_calls: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, sloppy:bool=False, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that applies `f` to the outputs of `input_dataset`.
|
|
|
|
The resulting dataset is similar to the `InterleaveDataset`, except that the
|
|
dataset will fetch records from the interleaved datasets in parallel.
|
|
|
|
The `tf.data` Python API creates instances of this op from
|
|
`Dataset.interleave()` when the `num_parallel_calls` parameter of that method
|
|
is set to any value other than `None`.
|
|
|
|
By default, the output of this dataset will be deterministic, which may result
|
|
in the dataset blocking if the next data item to be returned isn't available.
|
|
In order to avoid head-of-line blocking, one can set the
|
|
`experimental_deterministic` parameter of `tf.data.Options` to `False`,
|
|
which can improve performance at the expense of non-determinism.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
Dataset that produces a stream of arguments for the function `f`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
Additional arguments to pass to `f` beyond those produced by `input_dataset`.
|
|
Evaluated once when the dataset is instantiated.
|
|
cycle_length: A `Tensor` of type `int64`.
|
|
Number of datasets (each created by applying `f` to the elements of
|
|
`input_dataset`) among which the `ParallelInterleaveDatasetV2` will cycle in a
|
|
round-robin fashion.
|
|
block_length: A `Tensor` of type `int64`.
|
|
Number of elements at a time to produce from each interleaved invocation of a
|
|
dataset returned by `f`.
|
|
num_parallel_calls: A `Tensor` of type `int64`.
|
|
Determines the number of threads that should be used for fetching data from
|
|
input datasets in parallel. The Python API `tf.data.experimental.AUTOTUNE`
|
|
constant can be used to indicate that the level of parallelism should be autotuned.
|
|
f: A function decorated with @Defun.
|
|
A function mapping elements of `input_dataset`, concatenated with
|
|
`other_arguments`, to a Dataset variant that contains elements matching
|
|
`output_types` and `output_shapes`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
sloppy: An optional `bool`. Defaults to `False`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ParallelInterleaveDatasetV2", name, input_dataset,
|
|
other_arguments, cycle_length, block_length, num_parallel_calls, "f",
|
|
f, "output_types", output_types, "output_shapes", output_shapes,
|
|
"sloppy", sloppy, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return parallel_interleave_dataset_v2_eager_fallback(
|
|
input_dataset, other_arguments, cycle_length, block_length,
|
|
num_parallel_calls, f=f, output_types=output_types,
|
|
output_shapes=output_shapes, sloppy=sloppy, metadata=metadata,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_interleave_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_interleave_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if sloppy is None:
|
|
sloppy = False
|
|
sloppy = _execute.make_bool(sloppy, "sloppy")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ParallelInterleaveDatasetV2", input_dataset=input_dataset,
|
|
other_arguments=other_arguments,
|
|
cycle_length=cycle_length,
|
|
block_length=block_length,
|
|
num_parallel_calls=num_parallel_calls,
|
|
f=f, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
sloppy=sloppy, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "sloppy",
|
|
_op._get_attr_bool("sloppy"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ParallelInterleaveDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ParallelInterleaveDatasetV2 = tf_export("raw_ops.ParallelInterleaveDatasetV2")(_ops.to_raw_op(parallel_interleave_dataset_v2))
|
|
|
|
|
|
def parallel_interleave_dataset_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, cycle_length: Annotated[Any, _atypes.Int64], block_length: Annotated[Any, _atypes.Int64], num_parallel_calls: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, sloppy: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_interleave_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_interleave_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if sloppy is None:
|
|
sloppy = False
|
|
sloppy = _execute.make_bool(sloppy, "sloppy")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
cycle_length = _ops.convert_to_tensor(cycle_length, _dtypes.int64)
|
|
block_length = _ops.convert_to_tensor(block_length, _dtypes.int64)
|
|
num_parallel_calls = _ops.convert_to_tensor(num_parallel_calls, _dtypes.int64)
|
|
_inputs_flat = [input_dataset] + list(other_arguments) + [cycle_length, block_length, num_parallel_calls]
|
|
_attrs = ("f", f, "Targuments", _attr_Targuments, "output_types",
|
|
output_types, "output_shapes", output_shapes, "sloppy", sloppy, "metadata",
|
|
metadata)
|
|
_result = _execute.execute(b"ParallelInterleaveDatasetV2", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ParallelInterleaveDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def parallel_interleave_dataset_v3(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, cycle_length: Annotated[Any, _atypes.Int64], block_length: Annotated[Any, _atypes.Int64], num_parallel_calls: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, deterministic:str="default", metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that applies `f` to the outputs of `input_dataset`.
|
|
|
|
The resulting dataset is similar to the `InterleaveDataset`, except that the
|
|
dataset will fetch records from the interleaved datasets in parallel.
|
|
|
|
The `tf.data` Python API creates instances of this op from
|
|
`Dataset.interleave()` when the `num_parallel_calls` parameter of that method
|
|
is set to any value other than `None`.
|
|
|
|
By default, the output of this dataset will be deterministic, which may result
|
|
in the dataset blocking if the next data item to be returned isn't available.
|
|
In order to avoid head-of-line blocking, one can either set the `deterministic`
|
|
attribute to "false", or leave it as "default" and set the
|
|
`experimental_deterministic` parameter of `tf.data.Options` to `False`.
|
|
This can improve performance at the expense of non-determinism.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
Dataset that produces a stream of arguments for the function `f`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
Additional arguments to pass to `f` beyond those produced by `input_dataset`.
|
|
Evaluated once when the dataset is instantiated.
|
|
cycle_length: A `Tensor` of type `int64`.
|
|
Number of datasets (each created by applying `f` to the elements of
|
|
`input_dataset`) among which the `ParallelInterleaveDatasetV2` will cycle in a
|
|
round-robin fashion.
|
|
block_length: A `Tensor` of type `int64`.
|
|
Number of elements at a time to produce from each interleaved invocation of a
|
|
dataset returned by `f`.
|
|
num_parallel_calls: A `Tensor` of type `int64`.
|
|
Determines the number of threads that should be used for fetching data from
|
|
input datasets in parallel. The Python API `tf.data.experimental.AUTOTUNE`
|
|
constant can be used to indicate that the level of parallelism should be autotuned.
|
|
f: A function decorated with @Defun.
|
|
A function mapping elements of `input_dataset`, concatenated with
|
|
`other_arguments`, to a Dataset variant that contains elements matching
|
|
`output_types` and `output_shapes`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
deterministic: An optional `string`. Defaults to `"default"`.
|
|
A string indicating the op-level determinism to use. Deterministic controls
|
|
whether the interleave is allowed to return elements out of order if the next
|
|
element to be returned isn't available, but a later element is. Options are
|
|
"true", "false", and "default". "default" indicates that determinism should be
|
|
decided by the `experimental_deterministic` parameter of `tf.data.Options`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ParallelInterleaveDatasetV3", name, input_dataset,
|
|
other_arguments, cycle_length, block_length, num_parallel_calls, "f",
|
|
f, "deterministic", deterministic, "output_types", output_types,
|
|
"output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return parallel_interleave_dataset_v3_eager_fallback(
|
|
input_dataset, other_arguments, cycle_length, block_length,
|
|
num_parallel_calls, f=f, deterministic=deterministic,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_interleave_dataset_v3' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_interleave_dataset_v3' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ParallelInterleaveDatasetV3", input_dataset=input_dataset,
|
|
other_arguments=other_arguments,
|
|
cycle_length=cycle_length,
|
|
block_length=block_length,
|
|
num_parallel_calls=num_parallel_calls,
|
|
f=f, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
deterministic=deterministic,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "deterministic",
|
|
_op.get_attr("deterministic"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ParallelInterleaveDatasetV3", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ParallelInterleaveDatasetV3 = tf_export("raw_ops.ParallelInterleaveDatasetV3")(_ops.to_raw_op(parallel_interleave_dataset_v3))
|
|
|
|
|
|
def parallel_interleave_dataset_v3_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, cycle_length: Annotated[Any, _atypes.Int64], block_length: Annotated[Any, _atypes.Int64], num_parallel_calls: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, deterministic: str, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_interleave_dataset_v3' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_interleave_dataset_v3' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
cycle_length = _ops.convert_to_tensor(cycle_length, _dtypes.int64)
|
|
block_length = _ops.convert_to_tensor(block_length, _dtypes.int64)
|
|
num_parallel_calls = _ops.convert_to_tensor(num_parallel_calls, _dtypes.int64)
|
|
_inputs_flat = [input_dataset] + list(other_arguments) + [cycle_length, block_length, num_parallel_calls]
|
|
_attrs = ("f", f, "deterministic", deterministic, "Targuments",
|
|
_attr_Targuments, "output_types", output_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"ParallelInterleaveDatasetV3", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ParallelInterleaveDatasetV3", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def parallel_interleave_dataset_v4(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, cycle_length: Annotated[Any, _atypes.Int64], block_length: Annotated[Any, _atypes.Int64], buffer_output_elements: Annotated[Any, _atypes.Int64], prefetch_input_elements: Annotated[Any, _atypes.Int64], num_parallel_calls: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, deterministic:str="default", metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that applies `f` to the outputs of `input_dataset`.
|
|
|
|
The resulting dataset is similar to the `InterleaveDataset`, except that the
|
|
dataset will fetch records from the interleaved datasets in parallel.
|
|
|
|
The `tf.data` Python API creates instances of this op from
|
|
`Dataset.interleave()` when the `num_parallel_calls` parameter of that method
|
|
is set to any value other than `None`.
|
|
|
|
By default, the output of this dataset will be deterministic, which may result
|
|
in the dataset blocking if the next data item to be returned isn't available.
|
|
In order to avoid head-of-line blocking, one can either set the `deterministic`
|
|
attribute to "false", or leave it as "default" and set the
|
|
`experimental_deterministic` parameter of `tf.data.Options` to `False`.
|
|
This can improve performance at the expense of non-determinism.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
Dataset that produces a stream of arguments for the function `f`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
Additional arguments to pass to `f` beyond those produced by `input_dataset`.
|
|
Evaluated once when the dataset is instantiated.
|
|
cycle_length: A `Tensor` of type `int64`.
|
|
Number of datasets (each created by applying `f` to the elements of
|
|
`input_dataset`) among which the `ParallelInterleaveDatasetV2` will cycle in a
|
|
round-robin fashion.
|
|
block_length: A `Tensor` of type `int64`.
|
|
Number of elements at a time to produce from each interleaved invocation of a
|
|
dataset returned by `f`.
|
|
buffer_output_elements: A `Tensor` of type `int64`.
|
|
The number of elements each iterator being interleaved should buffer (similar
|
|
to the `.prefetch()` transformation for each interleaved iterator).
|
|
prefetch_input_elements: A `Tensor` of type `int64`.
|
|
Determines the number of iterators to prefetch, allowing buffers to warm up and
|
|
data to be pre-fetched without blocking the main thread.
|
|
num_parallel_calls: A `Tensor` of type `int64`.
|
|
Determines the number of threads that should be used for fetching data from
|
|
input datasets in parallel. The Python API `tf.data.experimental.AUTOTUNE`
|
|
constant can be used to indicate that the level of parallelism should be autotuned.
|
|
f: A function decorated with @Defun.
|
|
A function mapping elements of `input_dataset`, concatenated with
|
|
`other_arguments`, to a Dataset variant that contains elements matching
|
|
`output_types` and `output_shapes`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
deterministic: An optional `string`. Defaults to `"default"`.
|
|
A string indicating the op-level determinism to use. Deterministic controls
|
|
whether the interleave is allowed to return elements out of order if the next
|
|
element to be returned isn't available, but a later element is. Options are
|
|
"true", "false", and "default". "default" indicates that determinism should be
|
|
decided by the `experimental_deterministic` parameter of `tf.data.Options`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ParallelInterleaveDatasetV4", name, input_dataset,
|
|
other_arguments, cycle_length, block_length, buffer_output_elements,
|
|
prefetch_input_elements, num_parallel_calls, "f", f, "deterministic",
|
|
deterministic, "output_types", output_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return parallel_interleave_dataset_v4_eager_fallback(
|
|
input_dataset, other_arguments, cycle_length, block_length,
|
|
buffer_output_elements, prefetch_input_elements, num_parallel_calls,
|
|
f=f, deterministic=deterministic, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_interleave_dataset_v4' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_interleave_dataset_v4' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ParallelInterleaveDatasetV4", input_dataset=input_dataset,
|
|
other_arguments=other_arguments,
|
|
cycle_length=cycle_length,
|
|
block_length=block_length,
|
|
buffer_output_elements=buffer_output_elements,
|
|
prefetch_input_elements=prefetch_input_elements,
|
|
num_parallel_calls=num_parallel_calls,
|
|
f=f, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
deterministic=deterministic,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "deterministic",
|
|
_op.get_attr("deterministic"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ParallelInterleaveDatasetV4", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ParallelInterleaveDatasetV4 = tf_export("raw_ops.ParallelInterleaveDatasetV4")(_ops.to_raw_op(parallel_interleave_dataset_v4))
|
|
|
|
|
|
def parallel_interleave_dataset_v4_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, cycle_length: Annotated[Any, _atypes.Int64], block_length: Annotated[Any, _atypes.Int64], buffer_output_elements: Annotated[Any, _atypes.Int64], prefetch_input_elements: Annotated[Any, _atypes.Int64], num_parallel_calls: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, deterministic: str, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_interleave_dataset_v4' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_interleave_dataset_v4' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
cycle_length = _ops.convert_to_tensor(cycle_length, _dtypes.int64)
|
|
block_length = _ops.convert_to_tensor(block_length, _dtypes.int64)
|
|
buffer_output_elements = _ops.convert_to_tensor(buffer_output_elements, _dtypes.int64)
|
|
prefetch_input_elements = _ops.convert_to_tensor(prefetch_input_elements, _dtypes.int64)
|
|
num_parallel_calls = _ops.convert_to_tensor(num_parallel_calls, _dtypes.int64)
|
|
_inputs_flat = [input_dataset] + list(other_arguments) + [cycle_length, block_length, buffer_output_elements, prefetch_input_elements, num_parallel_calls]
|
|
_attrs = ("f", f, "deterministic", deterministic, "Targuments",
|
|
_attr_Targuments, "output_types", output_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"ParallelInterleaveDatasetV4", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ParallelInterleaveDatasetV4", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def parallel_map_dataset(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, num_parallel_calls: Annotated[Any, _atypes.Int32], f, output_types, output_shapes, use_inter_op_parallelism:bool=True, sloppy:bool=False, preserve_cardinality:bool=False, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that applies `f` to the outputs of `input_dataset`.
|
|
|
|
Unlike a "MapDataset", which applies `f` sequentially, this dataset invokes up
|
|
to `num_parallel_calls` copies of `f` in parallel.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
num_parallel_calls: A `Tensor` of type `int32`.
|
|
The number of concurrent invocations of `f` that process
|
|
elements from `input_dataset` in parallel.
|
|
f: A function decorated with @Defun.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
use_inter_op_parallelism: An optional `bool`. Defaults to `True`.
|
|
sloppy: An optional `bool`. Defaults to `False`.
|
|
preserve_cardinality: An optional `bool`. Defaults to `False`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ParallelMapDataset", name, input_dataset, other_arguments,
|
|
num_parallel_calls, "f", f, "output_types", output_types,
|
|
"output_shapes", output_shapes, "use_inter_op_parallelism",
|
|
use_inter_op_parallelism, "sloppy", sloppy, "preserve_cardinality",
|
|
preserve_cardinality, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return parallel_map_dataset_eager_fallback(
|
|
input_dataset, other_arguments, num_parallel_calls, f=f,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
use_inter_op_parallelism=use_inter_op_parallelism, sloppy=sloppy,
|
|
preserve_cardinality=preserve_cardinality, metadata=metadata,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_map_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_map_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if use_inter_op_parallelism is None:
|
|
use_inter_op_parallelism = True
|
|
use_inter_op_parallelism = _execute.make_bool(use_inter_op_parallelism, "use_inter_op_parallelism")
|
|
if sloppy is None:
|
|
sloppy = False
|
|
sloppy = _execute.make_bool(sloppy, "sloppy")
|
|
if preserve_cardinality is None:
|
|
preserve_cardinality = False
|
|
preserve_cardinality = _execute.make_bool(preserve_cardinality, "preserve_cardinality")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ParallelMapDataset", input_dataset=input_dataset,
|
|
other_arguments=other_arguments,
|
|
num_parallel_calls=num_parallel_calls, f=f,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
use_inter_op_parallelism=use_inter_op_parallelism,
|
|
sloppy=sloppy,
|
|
preserve_cardinality=preserve_cardinality,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "use_inter_op_parallelism",
|
|
_op._get_attr_bool("use_inter_op_parallelism"), "sloppy",
|
|
_op._get_attr_bool("sloppy"), "preserve_cardinality",
|
|
_op._get_attr_bool("preserve_cardinality"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ParallelMapDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ParallelMapDataset = tf_export("raw_ops.ParallelMapDataset")(_ops.to_raw_op(parallel_map_dataset))
|
|
|
|
|
|
def parallel_map_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, num_parallel_calls: Annotated[Any, _atypes.Int32], f, output_types, output_shapes, use_inter_op_parallelism: bool, sloppy: bool, preserve_cardinality: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_map_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_map_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if use_inter_op_parallelism is None:
|
|
use_inter_op_parallelism = True
|
|
use_inter_op_parallelism = _execute.make_bool(use_inter_op_parallelism, "use_inter_op_parallelism")
|
|
if sloppy is None:
|
|
sloppy = False
|
|
sloppy = _execute.make_bool(sloppy, "sloppy")
|
|
if preserve_cardinality is None:
|
|
preserve_cardinality = False
|
|
preserve_cardinality = _execute.make_bool(preserve_cardinality, "preserve_cardinality")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
num_parallel_calls = _ops.convert_to_tensor(num_parallel_calls, _dtypes.int32)
|
|
_inputs_flat = [input_dataset] + list(other_arguments) + [num_parallel_calls]
|
|
_attrs = ("f", f, "Targuments", _attr_Targuments, "output_types",
|
|
output_types, "output_shapes", output_shapes, "use_inter_op_parallelism",
|
|
use_inter_op_parallelism, "sloppy", sloppy, "preserve_cardinality",
|
|
preserve_cardinality, "metadata", metadata)
|
|
_result = _execute.execute(b"ParallelMapDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ParallelMapDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def parallel_map_dataset_v2(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, num_parallel_calls: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, use_inter_op_parallelism:bool=True, deterministic:str="default", preserve_cardinality:bool=False, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that applies `f` to the outputs of `input_dataset`.
|
|
|
|
Unlike a "MapDataset", which applies `f` sequentially, this dataset invokes up
|
|
to `num_parallel_calls` copies of `f` in parallel.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
other_arguments: A list of `Tensor` objects.
|
|
num_parallel_calls: A `Tensor` of type `int64`.
|
|
The number of concurrent invocations of `f` that process
|
|
elements from `input_dataset` in parallel.
|
|
f: A function decorated with @Defun.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
use_inter_op_parallelism: An optional `bool`. Defaults to `True`.
|
|
deterministic: An optional `string`. Defaults to `"default"`.
|
|
preserve_cardinality: An optional `bool`. Defaults to `False`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ParallelMapDatasetV2", name, input_dataset, other_arguments,
|
|
num_parallel_calls, "f", f, "output_types", output_types,
|
|
"output_shapes", output_shapes, "use_inter_op_parallelism",
|
|
use_inter_op_parallelism, "deterministic", deterministic,
|
|
"preserve_cardinality", preserve_cardinality, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return parallel_map_dataset_v2_eager_fallback(
|
|
input_dataset, other_arguments, num_parallel_calls, f=f,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
use_inter_op_parallelism=use_inter_op_parallelism,
|
|
deterministic=deterministic,
|
|
preserve_cardinality=preserve_cardinality, metadata=metadata,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_map_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_map_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if use_inter_op_parallelism is None:
|
|
use_inter_op_parallelism = True
|
|
use_inter_op_parallelism = _execute.make_bool(use_inter_op_parallelism, "use_inter_op_parallelism")
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if preserve_cardinality is None:
|
|
preserve_cardinality = False
|
|
preserve_cardinality = _execute.make_bool(preserve_cardinality, "preserve_cardinality")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ParallelMapDatasetV2", input_dataset=input_dataset,
|
|
other_arguments=other_arguments,
|
|
num_parallel_calls=num_parallel_calls, f=f,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
use_inter_op_parallelism=use_inter_op_parallelism,
|
|
deterministic=deterministic,
|
|
preserve_cardinality=preserve_cardinality,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "Targuments",
|
|
_op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "use_inter_op_parallelism",
|
|
_op._get_attr_bool("use_inter_op_parallelism"), "deterministic",
|
|
_op.get_attr("deterministic"), "preserve_cardinality",
|
|
_op._get_attr_bool("preserve_cardinality"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ParallelMapDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ParallelMapDatasetV2 = tf_export("raw_ops.ParallelMapDatasetV2")(_ops.to_raw_op(parallel_map_dataset_v2))
|
|
|
|
|
|
def parallel_map_dataset_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], other_arguments, num_parallel_calls: Annotated[Any, _atypes.Int64], f, output_types, output_shapes, use_inter_op_parallelism: bool, deterministic: str, preserve_cardinality: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'parallel_map_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'parallel_map_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if use_inter_op_parallelism is None:
|
|
use_inter_op_parallelism = True
|
|
use_inter_op_parallelism = _execute.make_bool(use_inter_op_parallelism, "use_inter_op_parallelism")
|
|
if deterministic is None:
|
|
deterministic = "default"
|
|
deterministic = _execute.make_str(deterministic, "deterministic")
|
|
if preserve_cardinality is None:
|
|
preserve_cardinality = False
|
|
preserve_cardinality = _execute.make_bool(preserve_cardinality, "preserve_cardinality")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
num_parallel_calls = _ops.convert_to_tensor(num_parallel_calls, _dtypes.int64)
|
|
_inputs_flat = [input_dataset] + list(other_arguments) + [num_parallel_calls]
|
|
_attrs = ("f", f, "Targuments", _attr_Targuments, "output_types",
|
|
output_types, "output_shapes", output_shapes, "use_inter_op_parallelism",
|
|
use_inter_op_parallelism, "deterministic", deterministic,
|
|
"preserve_cardinality", preserve_cardinality, "metadata", metadata)
|
|
_result = _execute.execute(b"ParallelMapDatasetV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ParallelMapDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def prefetch_dataset(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], output_types, output_shapes, slack_period:int=0, legacy_autotune:bool=True, buffer_size_min:int=0, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that asynchronously prefetches elements from `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
The maximum number of elements to buffer in an iterator over
|
|
this dataset.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
slack_period: An optional `int`. Defaults to `0`.
|
|
legacy_autotune: An optional `bool`. Defaults to `True`.
|
|
buffer_size_min: An optional `int`. Defaults to `0`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "PrefetchDataset", name, input_dataset, buffer_size,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"slack_period", slack_period, "legacy_autotune", legacy_autotune,
|
|
"buffer_size_min", buffer_size_min, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return prefetch_dataset_eager_fallback(
|
|
input_dataset, buffer_size, output_types=output_types,
|
|
output_shapes=output_shapes, slack_period=slack_period,
|
|
legacy_autotune=legacy_autotune, buffer_size_min=buffer_size_min,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'prefetch_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'prefetch_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if slack_period is None:
|
|
slack_period = 0
|
|
slack_period = _execute.make_int(slack_period, "slack_period")
|
|
if legacy_autotune is None:
|
|
legacy_autotune = True
|
|
legacy_autotune = _execute.make_bool(legacy_autotune, "legacy_autotune")
|
|
if buffer_size_min is None:
|
|
buffer_size_min = 0
|
|
buffer_size_min = _execute.make_int(buffer_size_min, "buffer_size_min")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"PrefetchDataset", input_dataset=input_dataset,
|
|
buffer_size=buffer_size, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
slack_period=slack_period,
|
|
legacy_autotune=legacy_autotune,
|
|
buffer_size_min=buffer_size_min, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "slack_period",
|
|
_op._get_attr_int("slack_period"), "legacy_autotune",
|
|
_op._get_attr_bool("legacy_autotune"), "buffer_size_min",
|
|
_op._get_attr_int("buffer_size_min"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"PrefetchDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
PrefetchDataset = tf_export("raw_ops.PrefetchDataset")(_ops.to_raw_op(prefetch_dataset))
|
|
|
|
|
|
def prefetch_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], output_types, output_shapes, slack_period: int, legacy_autotune: bool, buffer_size_min: int, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'prefetch_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'prefetch_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if slack_period is None:
|
|
slack_period = 0
|
|
slack_period = _execute.make_int(slack_period, "slack_period")
|
|
if legacy_autotune is None:
|
|
legacy_autotune = True
|
|
legacy_autotune = _execute.make_bool(legacy_autotune, "legacy_autotune")
|
|
if buffer_size_min is None:
|
|
buffer_size_min = 0
|
|
buffer_size_min = _execute.make_int(buffer_size_min, "buffer_size_min")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, buffer_size]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"slack_period", slack_period, "legacy_autotune", legacy_autotune,
|
|
"buffer_size_min", buffer_size_min, "metadata", metadata)
|
|
_result = _execute.execute(b"PrefetchDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"PrefetchDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def range_dataset(start: Annotated[Any, _atypes.Int64], stop: Annotated[Any, _atypes.Int64], step: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata:str="", replicate_on_split:bool=False, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset with a range of values. Corresponds to python's xrange.
|
|
|
|
Args:
|
|
start: A `Tensor` of type `int64`.
|
|
corresponds to start in python's xrange().
|
|
stop: A `Tensor` of type `int64`.
|
|
corresponds to stop in python's xrange().
|
|
step: A `Tensor` of type `int64`.
|
|
corresponds to step in python's xrange().
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
replicate_on_split: An optional `bool`. Defaults to `False`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "RangeDataset", name, start, stop, step, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata,
|
|
"replicate_on_split", replicate_on_split)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return range_dataset_eager_fallback(
|
|
start, stop, step, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
replicate_on_split=replicate_on_split, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'range_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'range_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
if replicate_on_split is None:
|
|
replicate_on_split = False
|
|
replicate_on_split = _execute.make_bool(replicate_on_split, "replicate_on_split")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"RangeDataset", start=start, stop=stop, step=step,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
replicate_on_split=replicate_on_split, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"), "replicate_on_split",
|
|
_op._get_attr_bool("replicate_on_split"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"RangeDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
RangeDataset = tf_export("raw_ops.RangeDataset")(_ops.to_raw_op(range_dataset))
|
|
|
|
|
|
def range_dataset_eager_fallback(start: Annotated[Any, _atypes.Int64], stop: Annotated[Any, _atypes.Int64], step: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata: str, replicate_on_split: bool, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'range_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'range_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
if replicate_on_split is None:
|
|
replicate_on_split = False
|
|
replicate_on_split = _execute.make_bool(replicate_on_split, "replicate_on_split")
|
|
start = _ops.convert_to_tensor(start, _dtypes.int64)
|
|
stop = _ops.convert_to_tensor(stop, _dtypes.int64)
|
|
step = _ops.convert_to_tensor(step, _dtypes.int64)
|
|
_inputs_flat = [start, stop, step]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata, "replicate_on_split", replicate_on_split)
|
|
_result = _execute.execute(b"RangeDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"RangeDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def reduce_dataset(input_dataset: Annotated[Any, _atypes.Variant], initial_state, other_arguments, f, output_types, output_shapes, use_inter_op_parallelism:bool=True, metadata:str="", name=None):
|
|
r"""Reduces the input dataset to a singleton using a reduce function.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
A variant tensor representing the input dataset.
|
|
initial_state: A list of `Tensor` objects.
|
|
A nested structure of tensors, representing the initial state of the
|
|
transformation.
|
|
other_arguments: A list of `Tensor` objects.
|
|
f: A function decorated with @Defun.
|
|
A function that maps `(old_state, input_element)` to `new_state`. It must take
|
|
two arguments and return a nested structures of tensors. The structure of
|
|
`new_state` must match the structure of `initial_state`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
use_inter_op_parallelism: An optional `bool`. Defaults to `True`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A list of `Tensor` objects of type `output_types`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ReduceDataset", name, input_dataset, initial_state,
|
|
other_arguments, "f", f, "output_types", output_types,
|
|
"output_shapes", output_shapes, "use_inter_op_parallelism",
|
|
use_inter_op_parallelism, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return reduce_dataset_eager_fallback(
|
|
input_dataset, initial_state, other_arguments, f=f,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
use_inter_op_parallelism=use_inter_op_parallelism,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'reduce_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'reduce_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if use_inter_op_parallelism is None:
|
|
use_inter_op_parallelism = True
|
|
use_inter_op_parallelism = _execute.make_bool(use_inter_op_parallelism, "use_inter_op_parallelism")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ReduceDataset", input_dataset=input_dataset,
|
|
initial_state=initial_state,
|
|
other_arguments=other_arguments, f=f,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
use_inter_op_parallelism=use_inter_op_parallelism,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if not _result:
|
|
return _op
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("f", _op.get_attr("f"), "Tstate", _op.get_attr("Tstate"),
|
|
"Targuments", _op.get_attr("Targuments"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "use_inter_op_parallelism",
|
|
_op._get_attr_bool("use_inter_op_parallelism"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ReduceDataset", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
ReduceDataset = tf_export("raw_ops.ReduceDataset")(_ops.to_raw_op(reduce_dataset))
|
|
|
|
|
|
def reduce_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], initial_state, other_arguments, f, output_types, output_shapes, use_inter_op_parallelism: bool, metadata: str, name, ctx):
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'reduce_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'reduce_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if use_inter_op_parallelism is None:
|
|
use_inter_op_parallelism = True
|
|
use_inter_op_parallelism = _execute.make_bool(use_inter_op_parallelism, "use_inter_op_parallelism")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Tstate, initial_state = _execute.convert_to_mixed_eager_tensors(initial_state, ctx)
|
|
_attr_Targuments, other_arguments = _execute.convert_to_mixed_eager_tensors(other_arguments, ctx)
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
_inputs_flat = [input_dataset] + list(initial_state) + list(other_arguments)
|
|
_attrs = ("f", f, "Tstate", _attr_Tstate, "Targuments", _attr_Targuments,
|
|
"output_types", output_types, "output_shapes", output_shapes,
|
|
"use_inter_op_parallelism", use_inter_op_parallelism, "metadata", metadata)
|
|
_result = _execute.execute(b"ReduceDataset", len(output_types),
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ReduceDataset", _inputs_flat, _attrs, _result)
|
|
return _result
|
|
|
|
|
|
def repeat_dataset(input_dataset: Annotated[Any, _atypes.Variant], count: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that emits the outputs of `input_dataset` `count` times.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
count: A `Tensor` of type `int64`.
|
|
A scalar representing the number of times that `input_dataset` should
|
|
be repeated. A value of `-1` indicates that it should be repeated infinitely.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "RepeatDataset", name, input_dataset, count, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return repeat_dataset_eager_fallback(
|
|
input_dataset, count, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'repeat_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'repeat_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"RepeatDataset", input_dataset=input_dataset, count=count,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"RepeatDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
RepeatDataset = tf_export("raw_ops.RepeatDataset")(_ops.to_raw_op(repeat_dataset))
|
|
|
|
|
|
def repeat_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], count: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'repeat_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'repeat_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
count = _ops.convert_to_tensor(count, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, count]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"RepeatDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"RepeatDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def rewrite_dataset(input_dataset: Annotated[Any, _atypes.Variant], rewrite_name: Annotated[Any, _atypes.String], output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
rewrite_name: A `Tensor` of type `string`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "RewriteDataset", name, input_dataset, rewrite_name,
|
|
"output_types", output_types, "output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return rewrite_dataset_eager_fallback(
|
|
input_dataset, rewrite_name, output_types=output_types,
|
|
output_shapes=output_shapes, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'rewrite_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'rewrite_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"RewriteDataset", input_dataset=input_dataset,
|
|
rewrite_name=rewrite_name,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"RewriteDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
RewriteDataset = tf_export("raw_ops.RewriteDataset")(_ops.to_raw_op(rewrite_dataset))
|
|
|
|
|
|
def rewrite_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], rewrite_name: Annotated[Any, _atypes.String], output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'rewrite_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'rewrite_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
rewrite_name = _ops.convert_to_tensor(rewrite_name, _dtypes.string)
|
|
_inputs_flat = [input_dataset, rewrite_name]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes)
|
|
_result = _execute.execute(b"RewriteDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"RewriteDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def serialize_iterator(resource_handle: Annotated[Any, _atypes.Resource], external_state_policy:int=0, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Converts the given `resource_handle` representing an iterator to a variant tensor.
|
|
|
|
Args:
|
|
resource_handle: A `Tensor` of type `resource`.
|
|
A handle to an iterator resource.
|
|
external_state_policy: An optional `int`. Defaults to `0`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "SerializeIterator", name, resource_handle,
|
|
"external_state_policy", external_state_policy)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return serialize_iterator_eager_fallback(
|
|
resource_handle, external_state_policy=external_state_policy,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if external_state_policy is None:
|
|
external_state_policy = 0
|
|
external_state_policy = _execute.make_int(external_state_policy, "external_state_policy")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"SerializeIterator", resource_handle=resource_handle,
|
|
external_state_policy=external_state_policy,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("external_state_policy",
|
|
_op._get_attr_int("external_state_policy"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"SerializeIterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
SerializeIterator = tf_export("raw_ops.SerializeIterator")(_ops.to_raw_op(serialize_iterator))
|
|
|
|
|
|
def serialize_iterator_eager_fallback(resource_handle: Annotated[Any, _atypes.Resource], external_state_policy: int, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if external_state_policy is None:
|
|
external_state_policy = 0
|
|
external_state_policy = _execute.make_int(external_state_policy, "external_state_policy")
|
|
resource_handle = _ops.convert_to_tensor(resource_handle, _dtypes.resource)
|
|
_inputs_flat = [resource_handle]
|
|
_attrs = ("external_state_policy", external_state_policy)
|
|
_result = _execute.execute(b"SerializeIterator", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"SerializeIterator", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def shard_dataset(input_dataset: Annotated[Any, _atypes.Variant], num_shards: Annotated[Any, _atypes.Int64], index: Annotated[Any, _atypes.Int64], output_types, output_shapes, require_non_empty:bool=False, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a `Dataset` that includes only 1/`num_shards` of this dataset.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
num_shards: A `Tensor` of type `int64`.
|
|
An integer representing the number of shards operating in parallel.
|
|
index: A `Tensor` of type `int64`.
|
|
An integer representing the current worker index.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
require_non_empty: An optional `bool`. Defaults to `False`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ShardDataset", name, input_dataset, num_shards, index,
|
|
"require_non_empty", require_non_empty, "output_types", output_types,
|
|
"output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return shard_dataset_eager_fallback(
|
|
input_dataset, num_shards, index,
|
|
require_non_empty=require_non_empty, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shard_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shard_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if require_non_empty is None:
|
|
require_non_empty = False
|
|
require_non_empty = _execute.make_bool(require_non_empty, "require_non_empty")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ShardDataset", input_dataset=input_dataset, num_shards=num_shards,
|
|
index=index, output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
require_non_empty=require_non_empty,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("require_non_empty", _op._get_attr_bool("require_non_empty"),
|
|
"output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ShardDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ShardDataset = tf_export("raw_ops.ShardDataset")(_ops.to_raw_op(shard_dataset))
|
|
|
|
|
|
def shard_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], num_shards: Annotated[Any, _atypes.Int64], index: Annotated[Any, _atypes.Int64], output_types, output_shapes, require_non_empty: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shard_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shard_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if require_non_empty is None:
|
|
require_non_empty = False
|
|
require_non_empty = _execute.make_bool(require_non_empty, "require_non_empty")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
num_shards = _ops.convert_to_tensor(num_shards, _dtypes.int64)
|
|
index = _ops.convert_to_tensor(index, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, num_shards, index]
|
|
_attrs = ("require_non_empty", require_non_empty, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"ShardDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ShardDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def shuffle_and_repeat_dataset(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], count: Annotated[Any, _atypes.Int64], output_types, output_shapes, reshuffle_each_iteration:bool=True, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that shuffles and repeats elements from `input_dataset`
|
|
|
|
pseudorandomly.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
The number of output elements to buffer in an iterator over
|
|
this dataset. Compare with the `min_after_dequeue` attr when creating a
|
|
`RandomShuffleQueue`.
|
|
seed: A `Tensor` of type `int64`.
|
|
A scalar seed for the random number generator. If either `seed` or
|
|
`seed2` is set to be non-zero, the random number generator is seeded
|
|
by the given seed. Otherwise, a random seed is used.
|
|
seed2: A `Tensor` of type `int64`.
|
|
A second scalar seed to avoid seed collision.
|
|
count: A `Tensor` of type `int64`.
|
|
A scalar representing the number of times the underlying dataset
|
|
should be repeated. The default is `-1`, which results in infinite repetition.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
reshuffle_each_iteration: An optional `bool`. Defaults to `True`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ShuffleAndRepeatDataset", name, input_dataset, buffer_size,
|
|
seed, seed2, count, "output_types", output_types, "output_shapes",
|
|
output_shapes, "reshuffle_each_iteration", reshuffle_each_iteration,
|
|
"metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return shuffle_and_repeat_dataset_eager_fallback(
|
|
input_dataset, buffer_size, seed, seed2, count,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
reshuffle_each_iteration=reshuffle_each_iteration,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_and_repeat_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_and_repeat_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if reshuffle_each_iteration is None:
|
|
reshuffle_each_iteration = True
|
|
reshuffle_each_iteration = _execute.make_bool(reshuffle_each_iteration, "reshuffle_each_iteration")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ShuffleAndRepeatDataset", input_dataset=input_dataset,
|
|
buffer_size=buffer_size, seed=seed,
|
|
seed2=seed2, count=count,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
reshuffle_each_iteration=reshuffle_each_iteration,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "reshuffle_each_iteration",
|
|
_op._get_attr_bool("reshuffle_each_iteration"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ShuffleAndRepeatDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ShuffleAndRepeatDataset = tf_export("raw_ops.ShuffleAndRepeatDataset")(_ops.to_raw_op(shuffle_and_repeat_dataset))
|
|
|
|
|
|
def shuffle_and_repeat_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], count: Annotated[Any, _atypes.Int64], output_types, output_shapes, reshuffle_each_iteration: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_and_repeat_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_and_repeat_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if reshuffle_each_iteration is None:
|
|
reshuffle_each_iteration = True
|
|
reshuffle_each_iteration = _execute.make_bool(reshuffle_each_iteration, "reshuffle_each_iteration")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
seed = _ops.convert_to_tensor(seed, _dtypes.int64)
|
|
seed2 = _ops.convert_to_tensor(seed2, _dtypes.int64)
|
|
count = _ops.convert_to_tensor(count, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, buffer_size, seed, seed2, count]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"reshuffle_each_iteration", reshuffle_each_iteration, "metadata", metadata)
|
|
_result = _execute.execute(b"ShuffleAndRepeatDataset", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ShuffleAndRepeatDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def shuffle_and_repeat_dataset_v2(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], count: Annotated[Any, _atypes.Int64], seed_generator: Annotated[Any, _atypes.Resource], output_types, output_shapes, reshuffle_each_iteration:bool=True, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
seed: A `Tensor` of type `int64`.
|
|
seed2: A `Tensor` of type `int64`.
|
|
count: A `Tensor` of type `int64`.
|
|
seed_generator: A `Tensor` of type `resource`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
reshuffle_each_iteration: An optional `bool`. Defaults to `True`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ShuffleAndRepeatDatasetV2", name, input_dataset, buffer_size,
|
|
seed, seed2, count, seed_generator, "reshuffle_each_iteration",
|
|
reshuffle_each_iteration, "output_types", output_types,
|
|
"output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return shuffle_and_repeat_dataset_v2_eager_fallback(
|
|
input_dataset, buffer_size, seed, seed2, count, seed_generator,
|
|
reshuffle_each_iteration=reshuffle_each_iteration,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_and_repeat_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_and_repeat_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if reshuffle_each_iteration is None:
|
|
reshuffle_each_iteration = True
|
|
reshuffle_each_iteration = _execute.make_bool(reshuffle_each_iteration, "reshuffle_each_iteration")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ShuffleAndRepeatDatasetV2", input_dataset=input_dataset,
|
|
buffer_size=buffer_size, seed=seed,
|
|
seed2=seed2, count=count,
|
|
seed_generator=seed_generator,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
reshuffle_each_iteration=reshuffle_each_iteration,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("reshuffle_each_iteration",
|
|
_op._get_attr_bool("reshuffle_each_iteration"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ShuffleAndRepeatDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ShuffleAndRepeatDatasetV2 = tf_export("raw_ops.ShuffleAndRepeatDatasetV2")(_ops.to_raw_op(shuffle_and_repeat_dataset_v2))
|
|
|
|
|
|
def shuffle_and_repeat_dataset_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], count: Annotated[Any, _atypes.Int64], seed_generator: Annotated[Any, _atypes.Resource], output_types, output_shapes, reshuffle_each_iteration: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_and_repeat_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_and_repeat_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if reshuffle_each_iteration is None:
|
|
reshuffle_each_iteration = True
|
|
reshuffle_each_iteration = _execute.make_bool(reshuffle_each_iteration, "reshuffle_each_iteration")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
seed = _ops.convert_to_tensor(seed, _dtypes.int64)
|
|
seed2 = _ops.convert_to_tensor(seed2, _dtypes.int64)
|
|
count = _ops.convert_to_tensor(count, _dtypes.int64)
|
|
seed_generator = _ops.convert_to_tensor(seed_generator, _dtypes.resource)
|
|
_inputs_flat = [input_dataset, buffer_size, seed, seed2, count, seed_generator]
|
|
_attrs = ("reshuffle_each_iteration", reshuffle_each_iteration,
|
|
"output_types", output_types, "output_shapes", output_shapes, "metadata",
|
|
metadata)
|
|
_result = _execute.execute(b"ShuffleAndRepeatDatasetV2", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ShuffleAndRepeatDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def shuffle_dataset(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], output_types, output_shapes, reshuffle_each_iteration:bool=True, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that shuffles elements from `input_dataset` pseudorandomly.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
The number of output elements to buffer in an iterator over
|
|
this dataset. Compare with the `min_after_dequeue` attr when creating a
|
|
`RandomShuffleQueue`.
|
|
seed: A `Tensor` of type `int64`.
|
|
A scalar seed for the random number generator. If either `seed` or
|
|
`seed2` is set to be non-zero, the random number generator is seeded
|
|
by the given seed. Otherwise, a random seed is used.
|
|
seed2: A `Tensor` of type `int64`.
|
|
A second scalar seed to avoid seed collision.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
reshuffle_each_iteration: An optional `bool`. Defaults to `True`.
|
|
If true, each iterator over this dataset will be given
|
|
a different pseudorandomly generated seed, based on a sequence seeded by the
|
|
`seed` and `seed2` inputs. If false, each iterator will be given the same
|
|
seed, and repeated iteration over this dataset will yield the exact same
|
|
sequence of results.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ShuffleDataset", name, input_dataset, buffer_size, seed, seed2,
|
|
"reshuffle_each_iteration", reshuffle_each_iteration, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return shuffle_dataset_eager_fallback(
|
|
input_dataset, buffer_size, seed, seed2,
|
|
reshuffle_each_iteration=reshuffle_each_iteration,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if reshuffle_each_iteration is None:
|
|
reshuffle_each_iteration = True
|
|
reshuffle_each_iteration = _execute.make_bool(reshuffle_each_iteration, "reshuffle_each_iteration")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ShuffleDataset", input_dataset=input_dataset,
|
|
buffer_size=buffer_size, seed=seed, seed2=seed2,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
reshuffle_each_iteration=reshuffle_each_iteration,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("reshuffle_each_iteration",
|
|
_op._get_attr_bool("reshuffle_each_iteration"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ShuffleDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ShuffleDataset = tf_export("raw_ops.ShuffleDataset")(_ops.to_raw_op(shuffle_dataset))
|
|
|
|
|
|
def shuffle_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], output_types, output_shapes, reshuffle_each_iteration: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if reshuffle_each_iteration is None:
|
|
reshuffle_each_iteration = True
|
|
reshuffle_each_iteration = _execute.make_bool(reshuffle_each_iteration, "reshuffle_each_iteration")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
seed = _ops.convert_to_tensor(seed, _dtypes.int64)
|
|
seed2 = _ops.convert_to_tensor(seed2, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, buffer_size, seed, seed2]
|
|
_attrs = ("reshuffle_each_iteration", reshuffle_each_iteration,
|
|
"output_types", output_types, "output_shapes", output_shapes, "metadata",
|
|
metadata)
|
|
_result = _execute.execute(b"ShuffleDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ShuffleDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def shuffle_dataset_v2(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed_generator: Annotated[Any, _atypes.Resource], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
seed_generator: A `Tensor` of type `resource`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ShuffleDatasetV2", name, input_dataset, buffer_size,
|
|
seed_generator, "output_types", output_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return shuffle_dataset_v2_eager_fallback(
|
|
input_dataset, buffer_size, seed_generator,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ShuffleDatasetV2", input_dataset=input_dataset,
|
|
buffer_size=buffer_size,
|
|
seed_generator=seed_generator,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ShuffleDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ShuffleDatasetV2 = tf_export("raw_ops.ShuffleDatasetV2")(_ops.to_raw_op(shuffle_dataset_v2))
|
|
|
|
|
|
def shuffle_dataset_v2_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed_generator: Annotated[Any, _atypes.Resource], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_dataset_v2' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_dataset_v2' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
seed_generator = _ops.convert_to_tensor(seed_generator, _dtypes.resource)
|
|
_inputs_flat = [input_dataset, buffer_size, seed_generator]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"ShuffleDatasetV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ShuffleDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def shuffle_dataset_v3(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], seed_generator: Annotated[Any, _atypes.Resource], output_types, output_shapes, reshuffle_each_iteration:bool=True, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
seed: A `Tensor` of type `int64`.
|
|
seed2: A `Tensor` of type `int64`.
|
|
seed_generator: A `Tensor` of type `resource`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
reshuffle_each_iteration: An optional `bool`. Defaults to `True`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ShuffleDatasetV3", name, input_dataset, buffer_size, seed,
|
|
seed2, seed_generator, "reshuffle_each_iteration",
|
|
reshuffle_each_iteration, "output_types", output_types,
|
|
"output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return shuffle_dataset_v3_eager_fallback(
|
|
input_dataset, buffer_size, seed, seed2, seed_generator,
|
|
reshuffle_each_iteration=reshuffle_each_iteration,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_dataset_v3' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_dataset_v3' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if reshuffle_each_iteration is None:
|
|
reshuffle_each_iteration = True
|
|
reshuffle_each_iteration = _execute.make_bool(reshuffle_each_iteration, "reshuffle_each_iteration")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ShuffleDatasetV3", input_dataset=input_dataset,
|
|
buffer_size=buffer_size, seed=seed, seed2=seed2,
|
|
seed_generator=seed_generator,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes,
|
|
reshuffle_each_iteration=reshuffle_each_iteration,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("reshuffle_each_iteration",
|
|
_op._get_attr_bool("reshuffle_each_iteration"), "output_types",
|
|
_op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ShuffleDatasetV3", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ShuffleDatasetV3 = tf_export("raw_ops.ShuffleDatasetV3")(_ops.to_raw_op(shuffle_dataset_v3))
|
|
|
|
|
|
def shuffle_dataset_v3_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], buffer_size: Annotated[Any, _atypes.Int64], seed: Annotated[Any, _atypes.Int64], seed2: Annotated[Any, _atypes.Int64], seed_generator: Annotated[Any, _atypes.Resource], output_types, output_shapes, reshuffle_each_iteration: bool, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'shuffle_dataset_v3' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'shuffle_dataset_v3' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if reshuffle_each_iteration is None:
|
|
reshuffle_each_iteration = True
|
|
reshuffle_each_iteration = _execute.make_bool(reshuffle_each_iteration, "reshuffle_each_iteration")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
seed = _ops.convert_to_tensor(seed, _dtypes.int64)
|
|
seed2 = _ops.convert_to_tensor(seed2, _dtypes.int64)
|
|
seed_generator = _ops.convert_to_tensor(seed_generator, _dtypes.resource)
|
|
_inputs_flat = [input_dataset, buffer_size, seed, seed2, seed_generator]
|
|
_attrs = ("reshuffle_each_iteration", reshuffle_each_iteration,
|
|
"output_types", output_types, "output_shapes", output_shapes, "metadata",
|
|
metadata)
|
|
_result = _execute.execute(b"ShuffleDatasetV3", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ShuffleDatasetV3", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def skip_dataset(input_dataset: Annotated[Any, _atypes.Variant], count: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that skips `count` elements from the `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
count: A `Tensor` of type `int64`.
|
|
A scalar representing the number of elements from the `input_dataset`
|
|
that should be skipped. If count is -1, skips everything.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "SkipDataset", name, input_dataset, count, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return skip_dataset_eager_fallback(
|
|
input_dataset, count, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'skip_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'skip_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"SkipDataset", input_dataset=input_dataset, count=count,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"SkipDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
SkipDataset = tf_export("raw_ops.SkipDataset")(_ops.to_raw_op(skip_dataset))
|
|
|
|
|
|
def skip_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], count: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'skip_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'skip_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
count = _ops.convert_to_tensor(count, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, count]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"SkipDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"SkipDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
TV_SparseTensorSliceDataset_Tvalues = TypeVar("TV_SparseTensorSliceDataset_Tvalues", _atypes.BFloat16, _atypes.Bool, _atypes.Complex128, _atypes.Complex64, _atypes.Float16, _atypes.Float32, _atypes.Float64, _atypes.Float8e4m3fn, _atypes.Float8e5m2, _atypes.Half, _atypes.Int16, _atypes.Int32, _atypes.Int4, _atypes.Int64, _atypes.Int8, _atypes.QInt16, _atypes.QInt32, _atypes.QInt8, _atypes.QUInt16, _atypes.QUInt8, _atypes.Resource, _atypes.String, _atypes.UInt16, _atypes.UInt32, _atypes.UInt4, _atypes.UInt64, _atypes.UInt8, _atypes.Variant)
|
|
|
|
def sparse_tensor_slice_dataset(indices: Annotated[Any, _atypes.Int64], values: Annotated[Any, TV_SparseTensorSliceDataset_Tvalues], dense_shape: Annotated[Any, _atypes.Int64], name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that splits a SparseTensor into elements row-wise.
|
|
|
|
Args:
|
|
indices: A `Tensor` of type `int64`.
|
|
values: A `Tensor`.
|
|
dense_shape: A `Tensor` of type `int64`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "SparseTensorSliceDataset", name, indices, values, dense_shape)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return sparse_tensor_slice_dataset_eager_fallback(
|
|
indices, values, dense_shape, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"SparseTensorSliceDataset", indices=indices, values=values,
|
|
dense_shape=dense_shape, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("Tvalues", _op._get_attr_type("Tvalues"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"SparseTensorSliceDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
SparseTensorSliceDataset = tf_export("raw_ops.SparseTensorSliceDataset")(_ops.to_raw_op(sparse_tensor_slice_dataset))
|
|
|
|
|
|
def sparse_tensor_slice_dataset_eager_fallback(indices: Annotated[Any, _atypes.Int64], values: Annotated[Any, TV_SparseTensorSliceDataset_Tvalues], dense_shape: Annotated[Any, _atypes.Int64], name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
_attr_Tvalues, (values,) = _execute.args_to_matching_eager([values], ctx, [])
|
|
indices = _ops.convert_to_tensor(indices, _dtypes.int64)
|
|
dense_shape = _ops.convert_to_tensor(dense_shape, _dtypes.int64)
|
|
_inputs_flat = [indices, values, dense_shape]
|
|
_attrs = ("Tvalues", _attr_Tvalues)
|
|
_result = _execute.execute(b"SparseTensorSliceDataset", 1,
|
|
inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
|
|
name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"SparseTensorSliceDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def tf_record_dataset(filenames: Annotated[Any, _atypes.String], compression_type: Annotated[Any, _atypes.String], buffer_size: Annotated[Any, _atypes.Int64], metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that emits the records from one or more TFRecord files.
|
|
|
|
Args:
|
|
filenames: A `Tensor` of type `string`.
|
|
A scalar or vector containing the name(s) of the file(s) to be
|
|
read.
|
|
compression_type: A `Tensor` of type `string`.
|
|
A scalar containing either (i) the empty string (no
|
|
compression), (ii) "ZLIB", or (iii) "GZIP".
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
A scalar representing the number of bytes to buffer. A value of
|
|
0 means no buffering will be performed.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "TFRecordDataset", name, filenames, compression_type,
|
|
buffer_size, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return tf_record_dataset_eager_fallback(
|
|
filenames, compression_type, buffer_size, metadata=metadata,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"TFRecordDataset", filenames=filenames,
|
|
compression_type=compression_type,
|
|
buffer_size=buffer_size, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("metadata", _op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"TFRecordDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
TFRecordDataset = tf_export("raw_ops.TFRecordDataset")(_ops.to_raw_op(tf_record_dataset))
|
|
|
|
|
|
def tf_record_dataset_eager_fallback(filenames: Annotated[Any, _atypes.String], compression_type: Annotated[Any, _atypes.String], buffer_size: Annotated[Any, _atypes.Int64], metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
filenames = _ops.convert_to_tensor(filenames, _dtypes.string)
|
|
compression_type = _ops.convert_to_tensor(compression_type, _dtypes.string)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
_inputs_flat = [filenames, compression_type, buffer_size]
|
|
_attrs = ("metadata", metadata)
|
|
_result = _execute.execute(b"TFRecordDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"TFRecordDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def tf_record_dataset_v2(filenames: Annotated[Any, _atypes.String], compression_type: Annotated[Any, _atypes.String], buffer_size: Annotated[Any, _atypes.Int64], byte_offsets: Annotated[Any, _atypes.Int64], metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that emits the records from one or more TFRecord files.
|
|
|
|
Args:
|
|
filenames: A `Tensor` of type `string`.
|
|
A scalar or vector containing the name(s) of the file(s) to be
|
|
read.
|
|
compression_type: A `Tensor` of type `string`.
|
|
A scalar containing either (i) the empty string (no
|
|
compression), (ii) "ZLIB", or (iii) "GZIP".
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
A scalar representing the number of bytes to buffer. A value of
|
|
0 means no buffering will be performed.
|
|
byte_offsets: A `Tensor` of type `int64`.
|
|
A scalar or vector containing the number of bytes for each file
|
|
that will be skipped prior to reading.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "TFRecordDatasetV2", name, filenames, compression_type,
|
|
buffer_size, byte_offsets, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return tf_record_dataset_v2_eager_fallback(
|
|
filenames, compression_type, buffer_size, byte_offsets,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"TFRecordDatasetV2", filenames=filenames,
|
|
compression_type=compression_type,
|
|
buffer_size=buffer_size,
|
|
byte_offsets=byte_offsets, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("metadata", _op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"TFRecordDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
TFRecordDatasetV2 = tf_export("raw_ops.TFRecordDatasetV2")(_ops.to_raw_op(tf_record_dataset_v2))
|
|
|
|
|
|
def tf_record_dataset_v2_eager_fallback(filenames: Annotated[Any, _atypes.String], compression_type: Annotated[Any, _atypes.String], buffer_size: Annotated[Any, _atypes.Int64], byte_offsets: Annotated[Any, _atypes.Int64], metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
filenames = _ops.convert_to_tensor(filenames, _dtypes.string)
|
|
compression_type = _ops.convert_to_tensor(compression_type, _dtypes.string)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
byte_offsets = _ops.convert_to_tensor(byte_offsets, _dtypes.int64)
|
|
_inputs_flat = [filenames, compression_type, buffer_size, byte_offsets]
|
|
_attrs = ("metadata", metadata)
|
|
_result = _execute.execute(b"TFRecordDatasetV2", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"TFRecordDatasetV2", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def take_dataset(input_dataset: Annotated[Any, _atypes.Variant], count: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that contains `count` elements from the `input_dataset`.
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
count: A `Tensor` of type `int64`.
|
|
A scalar representing the number of elements from the `input_dataset`
|
|
that should be taken. A value of `-1` indicates that all of `input_dataset`
|
|
is taken.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "TakeDataset", name, input_dataset, count, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return take_dataset_eager_fallback(
|
|
input_dataset, count, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'take_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'take_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"TakeDataset", input_dataset=input_dataset, count=count,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"TakeDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
TakeDataset = tf_export("raw_ops.TakeDataset")(_ops.to_raw_op(take_dataset))
|
|
|
|
|
|
def take_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], count: Annotated[Any, _atypes.Int64], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'take_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'take_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
count = _ops.convert_to_tensor(count, _dtypes.int64)
|
|
_inputs_flat = [input_dataset, count]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"TakeDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"TakeDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def tensor_dataset(components, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that emits `components` as a tuple of tensors once.
|
|
|
|
Args:
|
|
components: A list of `Tensor` objects.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "TensorDataset", name, components, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return tensor_dataset_eager_fallback(
|
|
components, output_shapes=output_shapes, metadata=metadata,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'tensor_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"TensorDataset", components=components, output_shapes=output_shapes,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("Toutput_types", _op.get_attr("Toutput_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"TensorDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
TensorDataset = tf_export("raw_ops.TensorDataset")(_ops.to_raw_op(tensor_dataset))
|
|
|
|
|
|
def tensor_dataset_eager_fallback(components, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'tensor_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_attr_Toutput_types, components = _execute.convert_to_mixed_eager_tensors(components, ctx)
|
|
_inputs_flat = list(components)
|
|
_attrs = ("Toutput_types", _attr_Toutput_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
_result = _execute.execute(b"TensorDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"TensorDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def tensor_slice_dataset(components, output_shapes, is_files:bool=False, metadata:str="", replicate_on_split:bool=False, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that emits each dim-0 slice of `components` once.
|
|
|
|
Args:
|
|
components: A list of `Tensor` objects.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
is_files: An optional `bool`. Defaults to `False`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
replicate_on_split: An optional `bool`. Defaults to `False`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "TensorSliceDataset", name, components, "output_shapes",
|
|
output_shapes, "is_files", is_files, "metadata", metadata,
|
|
"replicate_on_split", replicate_on_split)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return tensor_slice_dataset_eager_fallback(
|
|
components, output_shapes=output_shapes, is_files=is_files,
|
|
metadata=metadata, replicate_on_split=replicate_on_split, name=name,
|
|
ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'tensor_slice_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if is_files is None:
|
|
is_files = False
|
|
is_files = _execute.make_bool(is_files, "is_files")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
if replicate_on_split is None:
|
|
replicate_on_split = False
|
|
replicate_on_split = _execute.make_bool(replicate_on_split, "replicate_on_split")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"TensorSliceDataset", components=components,
|
|
output_shapes=output_shapes, is_files=is_files,
|
|
metadata=metadata,
|
|
replicate_on_split=replicate_on_split,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("Toutput_types", _op.get_attr("Toutput_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "is_files",
|
|
_op._get_attr_bool("is_files"), "metadata",
|
|
_op.get_attr("metadata"), "replicate_on_split",
|
|
_op._get_attr_bool("replicate_on_split"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"TensorSliceDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
TensorSliceDataset = tf_export("raw_ops.TensorSliceDataset")(_ops.to_raw_op(tensor_slice_dataset))
|
|
|
|
|
|
def tensor_slice_dataset_eager_fallback(components, output_shapes, is_files: bool, metadata: str, replicate_on_split: bool, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'tensor_slice_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if is_files is None:
|
|
is_files = False
|
|
is_files = _execute.make_bool(is_files, "is_files")
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
if replicate_on_split is None:
|
|
replicate_on_split = False
|
|
replicate_on_split = _execute.make_bool(replicate_on_split, "replicate_on_split")
|
|
_attr_Toutput_types, components = _execute.convert_to_mixed_eager_tensors(components, ctx)
|
|
_inputs_flat = list(components)
|
|
_attrs = ("Toutput_types", _attr_Toutput_types, "output_shapes",
|
|
output_shapes, "is_files", is_files, "metadata", metadata,
|
|
"replicate_on_split", replicate_on_split)
|
|
_result = _execute.execute(b"TensorSliceDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"TensorSliceDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def text_line_dataset(filenames: Annotated[Any, _atypes.String], compression_type: Annotated[Any, _atypes.String], buffer_size: Annotated[Any, _atypes.Int64], metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that emits the lines of one or more text files.
|
|
|
|
Args:
|
|
filenames: A `Tensor` of type `string`.
|
|
A scalar or a vector containing the name(s) of the file(s) to be
|
|
read.
|
|
compression_type: A `Tensor` of type `string`.
|
|
A scalar containing either (i) the empty string (no
|
|
compression), (ii) "ZLIB", or (iii) "GZIP".
|
|
buffer_size: A `Tensor` of type `int64`.
|
|
A scalar containing the number of bytes to buffer.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "TextLineDataset", name, filenames, compression_type,
|
|
buffer_size, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return text_line_dataset_eager_fallback(
|
|
filenames, compression_type, buffer_size, metadata=metadata,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"TextLineDataset", filenames=filenames,
|
|
compression_type=compression_type,
|
|
buffer_size=buffer_size, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("metadata", _op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"TextLineDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
TextLineDataset = tf_export("raw_ops.TextLineDataset")(_ops.to_raw_op(text_line_dataset))
|
|
|
|
|
|
def text_line_dataset_eager_fallback(filenames: Annotated[Any, _atypes.String], compression_type: Annotated[Any, _atypes.String], buffer_size: Annotated[Any, _atypes.Int64], metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
filenames = _ops.convert_to_tensor(filenames, _dtypes.string)
|
|
compression_type = _ops.convert_to_tensor(compression_type, _dtypes.string)
|
|
buffer_size = _ops.convert_to_tensor(buffer_size, _dtypes.int64)
|
|
_inputs_flat = [filenames, compression_type, buffer_size]
|
|
_attrs = ("metadata", metadata)
|
|
_result = _execute.execute(b"TextLineDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"TextLineDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def unwrap_dataset_variant(input_handle: Annotated[Any, _atypes.Variant], name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
input_handle: A `Tensor` of type `variant`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "UnwrapDatasetVariant", name, input_handle)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return unwrap_dataset_variant_eager_fallback(
|
|
input_handle, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"UnwrapDatasetVariant", input_handle=input_handle, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"UnwrapDatasetVariant", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
UnwrapDatasetVariant = tf_export("raw_ops.UnwrapDatasetVariant")(_ops.to_raw_op(unwrap_dataset_variant))
|
|
|
|
|
|
def unwrap_dataset_variant_eager_fallback(input_handle: Annotated[Any, _atypes.Variant], name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
input_handle = _ops.convert_to_tensor(input_handle, _dtypes.variant)
|
|
_inputs_flat = [input_handle]
|
|
_attrs = None
|
|
_result = _execute.execute(b"UnwrapDatasetVariant", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"UnwrapDatasetVariant", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def window_dataset(input_dataset: Annotated[Any, _atypes.Variant], size: Annotated[Any, _atypes.Int64], shift: Annotated[Any, _atypes.Int64], stride: Annotated[Any, _atypes.Int64], drop_remainder: Annotated[Any, _atypes.Bool], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r""" Combines (nests of) input elements into a dataset of (nests of) windows.
|
|
|
|
A "window" is a finite dataset of flat elements of size `size` (or possibly
|
|
fewer if there are not enough input elements to fill the window and
|
|
`drop_remainder` evaluates to false).
|
|
|
|
The `shift` argument determines the number of input elements by which
|
|
the window moves on each iteration. The first element in the `k`th window
|
|
will be element
|
|
|
|
```
|
|
1 + (k-1) * shift
|
|
```
|
|
|
|
of the input dataset. In particular, the first element of the first window
|
|
will always be the first element of the input dataset.
|
|
|
|
If the `stride` parameter is greater than 1, then each window will skip
|
|
`(stride - 1)` input elements between each element that appears in the
|
|
window. Output windows will still contain `size` elements regardless of
|
|
the value of `stride`.
|
|
|
|
The `stride` argument determines the stride of the input elements, and the
|
|
`shift` argument determines the shift of the window.
|
|
|
|
For example, letting `{...}` to represent a Dataset:
|
|
|
|
- `tf.data.Dataset.range(7).window(2)` produces
|
|
`{{0, 1}, {2, 3}, {4, 5}, {6}}`
|
|
- `tf.data.Dataset.range(7).window(3, 2, 1, True)` produces
|
|
`{{0, 1, 2}, {2, 3, 4}, {4, 5, 6}}`
|
|
- `tf.data.Dataset.range(7).window(3, 1, 2, True)` produces
|
|
`{{0, 2, 4}, {1, 3, 5}, {2, 4, 6}}`
|
|
|
|
Note that when the `window` transformation is applied to a dataset of
|
|
nested elements, it produces a dataset of nested windows.
|
|
|
|
For example:
|
|
|
|
- `tf.data.Dataset.from_tensor_slices((range(4), range(4))).window(2)`
|
|
produces `{({0, 1}, {0, 1}), ({2, 3}, {2, 3})}`
|
|
- `tf.data.Dataset.from_tensor_slices({"a": range(4)}).window(2)`
|
|
produces `{{"a": {0, 1}}, {"a": {2, 3}}}`
|
|
|
|
Args:
|
|
input_dataset: A `Tensor` of type `variant`.
|
|
size: A `Tensor` of type `int64`.
|
|
An integer scalar, representing the number of elements
|
|
of the input dataset to combine into a window. Must be positive.
|
|
shift: A `Tensor` of type `int64`.
|
|
An integer scalar, representing the number of input elements
|
|
by which the window moves in each iteration. Defaults to `size`.
|
|
Must be positive.
|
|
stride: A `Tensor` of type `int64`.
|
|
An integer scalar, representing the stride of the input elements
|
|
in the sliding window. Must be positive. The default value of 1 means
|
|
"retain every input element".
|
|
drop_remainder: A `Tensor` of type `bool`.
|
|
A Boolean scalar, representing whether the last window should be
|
|
dropped if its size is smaller than `window_size`.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "WindowDataset", name, input_dataset, size, shift, stride,
|
|
drop_remainder, "output_types", output_types, "output_shapes",
|
|
output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return window_dataset_eager_fallback(
|
|
input_dataset, size, shift, stride, drop_remainder,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'window_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'window_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"WindowDataset", input_dataset=input_dataset, size=size, shift=shift,
|
|
stride=stride, drop_remainder=drop_remainder,
|
|
output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata,
|
|
name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "metadata",
|
|
_op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"WindowDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
WindowDataset = tf_export("raw_ops.WindowDataset")(_ops.to_raw_op(window_dataset))
|
|
|
|
|
|
def window_dataset_eager_fallback(input_dataset: Annotated[Any, _atypes.Variant], size: Annotated[Any, _atypes.Int64], shift: Annotated[Any, _atypes.Int64], stride: Annotated[Any, _atypes.Int64], drop_remainder: Annotated[Any, _atypes.Bool], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'window_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'window_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_dataset = _ops.convert_to_tensor(input_dataset, _dtypes.variant)
|
|
size = _ops.convert_to_tensor(size, _dtypes.int64)
|
|
shift = _ops.convert_to_tensor(shift, _dtypes.int64)
|
|
stride = _ops.convert_to_tensor(stride, _dtypes.int64)
|
|
drop_remainder = _ops.convert_to_tensor(drop_remainder, _dtypes.bool)
|
|
_inputs_flat = [input_dataset, size, shift, stride, drop_remainder]
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"metadata", metadata)
|
|
_result = _execute.execute(b"WindowDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"WindowDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def window_op(inputs, output_types, output_shapes, name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
inputs: A list of `Tensor` objects.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "WindowOp", name, inputs, "output_types", output_types,
|
|
"output_shapes", output_shapes)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return window_op_eager_fallback(
|
|
inputs, output_types=output_types, output_shapes=output_shapes,
|
|
name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'window_op' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'window_op' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"WindowOp", inputs=inputs, output_types=output_types,
|
|
output_shapes=output_shapes, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "Tinputs",
|
|
_op.get_attr("Tinputs"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"WindowOp", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
WindowOp = tf_export("raw_ops.WindowOp")(_ops.to_raw_op(window_op))
|
|
|
|
|
|
def window_op_eager_fallback(inputs, output_types, output_shapes, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'window_op' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'window_op' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
_attr_Tinputs, inputs = _execute.convert_to_mixed_eager_tensors(inputs, ctx)
|
|
_inputs_flat = list(inputs)
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes,
|
|
"Tinputs", _attr_Tinputs)
|
|
_result = _execute.execute(b"WindowOp", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"WindowOp", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def wrap_dataset_variant(input_handle: Annotated[Any, _atypes.Variant], name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""TODO: add doc.
|
|
|
|
Args:
|
|
input_handle: A `Tensor` of type `variant`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "WrapDatasetVariant", name, input_handle)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return wrap_dataset_variant_eager_fallback(
|
|
input_handle, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"WrapDatasetVariant", input_handle=input_handle, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ()
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"WrapDatasetVariant", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
WrapDatasetVariant = tf_export("raw_ops.WrapDatasetVariant")(_ops.to_raw_op(wrap_dataset_variant))
|
|
|
|
|
|
def wrap_dataset_variant_eager_fallback(input_handle: Annotated[Any, _atypes.Variant], name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
input_handle = _ops.convert_to_tensor(input_handle, _dtypes.variant)
|
|
_inputs_flat = [input_handle]
|
|
_attrs = None
|
|
_result = _execute.execute(b"WrapDatasetVariant", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"WrapDatasetVariant", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
|
|
def zip_dataset(input_datasets: Annotated[List[Any], _atypes.Variant], output_types, output_shapes, metadata:str="", name=None) -> Annotated[Any, _atypes.Variant]:
|
|
r"""Creates a dataset that zips together `input_datasets`.
|
|
|
|
The elements of the resulting dataset are created by zipping corresponding
|
|
elements from each of the input datasets.
|
|
|
|
The size of the resulting dataset will match the size of the smallest input
|
|
dataset, and no error will be raised if input datasets have different sizes.
|
|
|
|
Args:
|
|
input_datasets: A list of at least 1 `Tensor` objects with type `variant`.
|
|
List of `N` variant Tensors representing datasets to be zipped together.
|
|
output_types: A list of `tf.DTypes` that has length `>= 1`.
|
|
output_shapes: A list of shapes (each a `tf.TensorShape` or list of `ints`) that has length `>= 1`.
|
|
metadata: An optional `string`. Defaults to `""`.
|
|
name: A name for the operation (optional).
|
|
|
|
Returns:
|
|
A `Tensor` of type `variant`.
|
|
"""
|
|
_ctx = _context._context or _context.context()
|
|
tld = _ctx._thread_local_data
|
|
if tld.is_eager:
|
|
try:
|
|
_result = pywrap_tfe.TFE_Py_FastPathExecute(
|
|
_ctx, "ZipDataset", name, input_datasets, "output_types",
|
|
output_types, "output_shapes", output_shapes, "metadata", metadata)
|
|
return _result
|
|
except _core._NotOkStatusException as e:
|
|
_ops.raise_from_not_ok_status(e, name)
|
|
except _core._FallbackException:
|
|
pass
|
|
try:
|
|
return zip_dataset_eager_fallback(
|
|
input_datasets, output_types=output_types,
|
|
output_shapes=output_shapes, metadata=metadata, name=name, ctx=_ctx)
|
|
except _core._SymbolicException:
|
|
pass # Add nodes to the TensorFlow graph.
|
|
# Add nodes to the TensorFlow graph.
|
|
if not isinstance(input_datasets, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'input_datasets' argument to "
|
|
"'zip_dataset' Op, not %r." % input_datasets)
|
|
_attr_N = len(input_datasets)
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'zip_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'zip_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
_, _, _op, _outputs = _op_def_library._apply_op_helper(
|
|
"ZipDataset", input_datasets=input_datasets,
|
|
output_types=output_types, output_shapes=output_shapes,
|
|
metadata=metadata, name=name)
|
|
_result = _outputs[:]
|
|
if _execute.must_record_gradient():
|
|
_attrs = ("output_types", _op.get_attr("output_types"), "output_shapes",
|
|
_op.get_attr("output_shapes"), "N", _op._get_attr_int("N"),
|
|
"metadata", _op.get_attr("metadata"))
|
|
_inputs_flat = _op.inputs
|
|
_execute.record_gradient(
|
|
"ZipDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|
|
ZipDataset = tf_export("raw_ops.ZipDataset")(_ops.to_raw_op(zip_dataset))
|
|
|
|
|
|
def zip_dataset_eager_fallback(input_datasets: Annotated[List[Any], _atypes.Variant], output_types, output_shapes, metadata: str, name, ctx) -> Annotated[Any, _atypes.Variant]:
|
|
if not isinstance(input_datasets, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'input_datasets' argument to "
|
|
"'zip_dataset' Op, not %r." % input_datasets)
|
|
_attr_N = len(input_datasets)
|
|
if not isinstance(output_types, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_types' argument to "
|
|
"'zip_dataset' Op, not %r." % output_types)
|
|
output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
|
|
if not isinstance(output_shapes, (list, tuple)):
|
|
raise TypeError(
|
|
"Expected list for 'output_shapes' argument to "
|
|
"'zip_dataset' Op, not %r." % output_shapes)
|
|
output_shapes = [_execute.make_shape(_s, "output_shapes") for _s in output_shapes]
|
|
if metadata is None:
|
|
metadata = ""
|
|
metadata = _execute.make_str(metadata, "metadata")
|
|
input_datasets = _ops.convert_n_to_tensor(input_datasets, _dtypes.variant)
|
|
_inputs_flat = list(input_datasets)
|
|
_attrs = ("output_types", output_types, "output_shapes", output_shapes, "N",
|
|
_attr_N, "metadata", metadata)
|
|
_result = _execute.execute(b"ZipDataset", 1, inputs=_inputs_flat,
|
|
attrs=_attrs, ctx=ctx, name=name)
|
|
if _execute.must_record_gradient():
|
|
_execute.record_gradient(
|
|
"ZipDataset", _inputs_flat, _attrs, _result)
|
|
_result, = _result
|
|
return _result
|
|
|