diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index f729b985456..376b3aff94c 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -71,6 +71,7 @@ 'AudioTensorFlowTensor', 'VideoTensorFlowTensor', ] +__all_test__ = __all__ + _torch_tensors def __getattr__(name: str): @@ -86,7 +87,6 @@ def __getattr__(name: str): import docarray.typing.tensor tensor_cls = getattr(docarray.typing.tensor, name) - if name not in __all__: __all__.append(name) diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index 3dab08230a7..c0ed96a0803 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -46,7 +46,8 @@ def _to_node_protobuf(self: T) -> 'NodeProto': def load(self) -> Tuple[AudioNdArray, int]: """ - Load the Audio from the AudioBytes into an AudioNdArray + Load the Audio from the [`AudioBytes`][docarray.typing.AudioBytes] into an + [`AudioNdArray`][docarray.typing.AudioNdArray]. --- @@ -75,8 +76,8 @@ class MyAudio(BaseDoc): ``` --- - :return: tuple of an AudioNdArray representing the audio bytes content, - and an integer representing the frame rate. + :return: tuple of an [`AudioNdArray`][docarray.typing.AudioNdArray] representing the + audio bytes content, and an integer representing the frame rate. """ pydub = import_library('pydub', raise_error=True) # noqa: F841 from pydub import AudioSegment diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index fe13f99e39c..67a933b2396 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -84,7 +84,8 @@ def load( axis_layout: Tuple[str, str, str] = ('H', 'W', 'C'), ) -> ImageNdArray: """ - Load the image from the ImageBytes into an ImageNdArray + Load the image from the [`ImageBytes`][docarray.typing.ImageBytes] into an + [`ImageNdArray`][docarray.typing.ImageNdArray]. --- @@ -119,7 +120,7 @@ class MyDoc(BaseDoc): :param height: height of the image tensor. :param axis_layout: ordering of the different image axes. 'H' = height, 'W' = width, 'C' = color channel - :return: ImageNdArray representing the image as RGB values + :return: [`ImageNdArray`][docarray.typing.ImageNdArray] representing the image as RGB values """ raw_img = self.load_pil() diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index 1f4e22e4fd1..7a870727c1c 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -51,10 +51,11 @@ def _to_node_protobuf(self: T) -> 'NodeProto': def load(self, **kwargs) -> VideoLoadResult: """ - Load the video from the bytes into a VideoLoadResult object consisting of a - VideoNdArray (`VideoLoadResult.video`), an AudioNdArray - (`VideoLoadResult.audio`) and an NdArray containing the key frame indices - (`VideoLoadResult.key_frame_indices`). + Load the video from the bytes into a VideoLoadResult object consisting of: + + - a [`VideoNdArray`][docarray.typing.VideoNdArray] (`VideoLoadResult.video`) + - an [`AudioNdArray`][docarray.typing.AudioNdArray] (`VideoLoadResult.audio`) + - an [`NdArray`][docarray.typing.NdArray] containing the key frame indices (`VideoLoadResult.key_frame_indices`). --- @@ -82,7 +83,7 @@ class MyDoc(BaseDoc): :param kwargs: supports all keyword arguments that are being supported by av.open() as described [here](https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open) - :return: a VideoLoadResult instance with video, audio and keyframe indices + :return: a `VideoLoadResult` instance with video, audio and keyframe indices """ if TYPE_CHECKING: import av diff --git a/docarray/typing/tensor/abstract_tensor.py b/docarray/typing/tensor/abstract_tensor.py index b74cc06697f..e74f0e6f66e 100644 --- a/docarray/typing/tensor/abstract_tensor.py +++ b/docarray/typing/tensor/abstract_tensor.py @@ -114,7 +114,6 @@ def _to_node_protobuf(self: T) -> 'NodeProto': """Convert itself into a NodeProto protobuf message. This function should be called when the Document is nested into another Document that need to be converted into a protobuf - :param field: field in which to store the content in the node proto :return: the nested item protobuf message """ from docarray.proto import NodeProto @@ -128,12 +127,15 @@ def __docarray_validate_shape__(cls, t: T, shape: Tuple[Union[int, str], ...]) - enable syntax of the form AnyTensor[shape]. It is called when a tensor is assigned to a field of this type. i.e. when a tensor is passed to a Document field of type AnyTensor[shape]. + The intended behaviour is as follows: + - If the shape of `t` is equal to `shape`, return `t`. - If the shape of `t` is not equal to `shape`, but can be reshaped to `shape`, return `t` reshaped to `shape`. - If the shape of `t` is not equal to `shape` and cannot be reshaped to `shape`, raise a ValueError. + :param t: The tensor to validate. :param shape: The shape to validate against. :return: The validated tensor. @@ -197,7 +199,7 @@ def __docarray_validate_shape__(cls, t: T, shape: Tuple[Union[int, str], ...]) - @classmethod def __docarray_validate_getitem__(cls, item: Any) -> Tuple[int]: - """This method validates the input to __class_getitem__. + """This method validates the input to `AbstractTensor.__class_getitem__`. It is called at "class creation time", i.e. when a class is created with syntax of the form AnyTensor[shape]. @@ -206,11 +208,13 @@ def __docarray_validate_getitem__(cls, item: Any) -> Tuple[int]: A subclass can override this method to implement custom validation logic. The output of this is eventually passed to - {ref}`AbstractTensor.__validate_shape__` as its `shape` argument. + [`AbstractTensor.__docarray_validate_shape__`] + [docarray.typing.tensor.abstract_tensor.AbstractTensor.__docarray_validate_shape__] + as its `shape` argument. Raises `ValueError` if the input `item` does not pass validation. - :param item: The item to validate, passed to __class_getitem__ (`Tensor[item]`). + :param item: The item to validate, passed to `__class_getitem__` (`Tensor[item]`). :return: The validated item == the target shape of this tensor. """ if isinstance(item, int): diff --git a/docarray/typing/tensor/audio/audio_ndarray.py b/docarray/typing/tensor/audio/audio_ndarray.py index 6bafc85c54e..3b15c0bc932 100644 --- a/docarray/typing/tensor/audio/audio_ndarray.py +++ b/docarray/typing/tensor/audio/audio_ndarray.py @@ -6,7 +6,7 @@ @_register_proto(proto_type_name='audio_ndarray') class AudioNdArray(AbstractAudioTensor, NdArray): """ - Subclass of NdArray, to represent an audio tensor. + Subclass of [`NdArray`][docarray.typing.NdArray], to represent an audio tensor. Adds audio-specific features to the tensor. @@ -33,7 +33,7 @@ class MyAudioDoc(BaseDoc): audio_tensor=np.random.rand(1000, 2), ) - doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') + # doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') doc_1.bytes_ = doc_1.audio_tensor.to_bytes() # from url @@ -43,7 +43,7 @@ class MyAudioDoc(BaseDoc): ) doc_2.audio_tensor, _ = doc_2.url.load() - doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') + # doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') doc_2.bytes_ = doc_1.audio_tensor.to_bytes() ``` diff --git a/docarray/typing/tensor/audio/audio_tensorflow_tensor.py b/docarray/typing/tensor/audio/audio_tensorflow_tensor.py index 1c7a1e83b40..034cc0faba9 100644 --- a/docarray/typing/tensor/audio/audio_tensorflow_tensor.py +++ b/docarray/typing/tensor/audio/audio_tensorflow_tensor.py @@ -12,18 +12,15 @@ class AudioTensorFlowTensor( AbstractAudioTensor, TensorFlowTensor, metaclass=metaTensorFlow ): """ - Subclass of TensorFlowTensor, to represent an audio tensor. - Adds audio-specific features to the tensor. - - - - ``` + Subclass of [`TensorFlowTensor`][docarray.typing.TensorFlowTensor], + to represent an audio tensor. Adds audio-specific features to the tensor. --- + + ```python from typing import Optional import tensorflow as tf - from pydantic import parse_obj_as from docarray import BaseDoc from docarray.typing import AudioBytes, AudioTensorFlowTensor, AudioUrl @@ -41,19 +38,16 @@ class MyAudioDoc(BaseDoc): audio_tensor=tf.random.normal((1000, 2)), ) - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') + # doc_1.audio_tensor.save(file_path='file_1.wav') doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_2 = MyAudioDoc( title='my_second_audio_doc', url='https://www.kozco.com/tech/piano2.wav', ) - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') + doc_2.audio_tensor, _ = doc_2.url.load() doc_2.bytes_ = doc_1.audio_tensor.to_bytes() - ``` --- diff --git a/docarray/typing/tensor/audio/audio_torch_tensor.py b/docarray/typing/tensor/audio/audio_torch_tensor.py index 66fa0f468c5..974ddff120b 100644 --- a/docarray/typing/tensor/audio/audio_torch_tensor.py +++ b/docarray/typing/tensor/audio/audio_torch_tensor.py @@ -6,10 +6,9 @@ @_register_proto(proto_type_name='audio_torch_tensor') class AudioTorchTensor(AbstractAudioTensor, TorchTensor, metaclass=metaTorchAndNode): """ - Subclass of TorchTensor, to represent an audio tensor. + Subclass of [`TorchTensor`][docarray.typing.TorchTensor], to represent an audio tensor. Adds audio-specific features to the tensor. - --- ```python @@ -33,7 +32,7 @@ class MyAudioDoc(BaseDoc): audio_tensor=torch.zeros(1000, 2), ) - doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') + # doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') doc_1.bytes_ = doc_1.audio_tensor.to_bytes() doc_2 = MyAudioDoc( @@ -42,7 +41,7 @@ class MyAudioDoc(BaseDoc): ) doc_2.audio_tensor, _ = doc_2.url.load() - doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') + # doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') doc_2.bytes_ = doc_1.audio_tensor.to_bytes() ``` diff --git a/docarray/typing/tensor/embedding/torch.py b/docarray/typing/tensor/embedding/torch.py index 555e22c8f14..275d0697f8c 100644 --- a/docarray/typing/tensor/embedding/torch.py +++ b/docarray/typing/tensor/embedding/torch.py @@ -18,7 +18,7 @@ class TorchEmbedding(TorchTensor, EmbeddingMixin, metaclass=metaTorchAndEmbeddin def new_empty(self, *args, **kwargs): """ - This method enables the deepcopy of TorchEmbedding by returning another instance of this subclass. + This method enables the deepcopy of `TorchEmbedding` by returning another instance of this subclass. If this function is not implemented, the deepcopy will throw an RuntimeError from Torch. """ return self.__class__(TorchTensor.new_empty(self, *args, **kwargs)) diff --git a/docarray/typing/tensor/image/image_ndarray.py b/docarray/typing/tensor/image/image_ndarray.py index 3366b3a7c28..1ff3a14eaa0 100644 --- a/docarray/typing/tensor/image/image_ndarray.py +++ b/docarray/typing/tensor/image/image_ndarray.py @@ -8,10 +8,10 @@ @_register_proto(proto_type_name='image_ndarray') class ImageNdArray(AbstractImageTensor, NdArray): """ - Subclass of NdArray, to represent an image tensor. + Subclass of [`NdArray`][docarray.typing.NdArray], to represent an image tensor. Adds image-specific features to the tensor. For instance the ability convert the tensor back to image bytes which are - optimized to send over the wire + optimized to send over the wire. --- diff --git a/docarray/typing/tensor/image/image_tensorflow_tensor.py b/docarray/typing/tensor/image/image_tensorflow_tensor.py index ed89868239a..c95b001e704 100644 --- a/docarray/typing/tensor/image/image_tensorflow_tensor.py +++ b/docarray/typing/tensor/image/image_tensorflow_tensor.py @@ -12,10 +12,10 @@ class ImageTensorFlowTensor( TensorFlowTensor, AbstractImageTensor, metaclass=metaTensorFlow ): """ - Subclass of TensorFlowTensor, to represent an image tensor. - Adds image-specific features to the tensor. + Subclass of [`TensorFlowTensor`][docarray.typing.TensorFlowTensor], + to represent an image tensor. Adds image-specific features to the tensor. For instance the ability convert the tensor back to image bytes which are - optimized to send over the wire + optimized to send over the wire. --- diff --git a/docarray/typing/tensor/image/image_torch_tensor.py b/docarray/typing/tensor/image/image_torch_tensor.py index 55f22cf5310..249030c00f6 100644 --- a/docarray/typing/tensor/image/image_torch_tensor.py +++ b/docarray/typing/tensor/image/image_torch_tensor.py @@ -10,10 +10,10 @@ @_register_proto(proto_type_name='image_torch_tensor') class ImageTorchTensor(AbstractImageTensor, TorchTensor, metaclass=metaTorchAndNode): """ - Subclass of TorchTensor, to represent an image tensor. + Subclass of [`TorchTensor`][docarray.typing.TorchTensor], to represent an image tensor. Adds image-specific features to the tensor. For instance the ability convert the tensor back to image bytes which are - optimized to send over the wire + optimized to send over the wire. --- diff --git a/docarray/typing/tensor/ndarray.py b/docarray/typing/tensor/ndarray.py index 49b23ea8e9c..40901d290fc 100644 --- a/docarray/typing/tensor/ndarray.py +++ b/docarray/typing/tensor/ndarray.py @@ -40,9 +40,9 @@ class metaNumpy(AbstractTensor.__parametrized_meta__, tensor_base): # type: ign @_register_proto(proto_type_name='ndarray') class NdArray(np.ndarray, AbstractTensor, Generic[ShapeT]): """ - Subclass of np.ndarray, intended for use in a Document. + Subclass of `np.ndarray`, intended for use in a Document. This enables (de)serialization from/to protobuf and json, data validation, - and coersion from compatible types like torch.Tensor. + and coersion from compatible types like `torch.Tensor`. This type can also be used in a parametrized way, specifying the shape of the array. @@ -143,7 +143,7 @@ def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: def _docarray_to_json_compatible(self) -> np.ndarray: """ - Convert tensor into a json compatible object + Convert `NdArray` into a json compatible object :return: a representation of the tensor compatible with orjson """ return self.unwrap() @@ -152,30 +152,33 @@ def unwrap(self) -> np.ndarray: """ Return the original ndarray without any memory copy. - The original view rest intact and is still a Document NdArray - but the return object is a pure np.ndarray but both object share + The original view rest intact and is still a Document `NdArray` + but the return object is a pure `np.ndarray` but both object share the same memory layout. - EXAMPLE USAGE - .. code-block:: python - from docarray.typing import NdArray - import numpy as np + --- - t1 = NdArray.validate(np.zeros((3, 224, 224)), None, None) - # here t1 is a docarray NdArray - t2 = t.unwrap() - # here t2 is a pure np.ndarray but t1 is still a Docarray NdArray - # But both share the same underlying memory + ```python + from docarray.typing import NdArray + import numpy as np + t1 = NdArray.validate(np.zeros((3, 224, 224)), None, None) + # here t1 is a docarray NdArray + t2 = t1.unwrap() + # here t2 is a pure np.ndarray but t1 is still a Docarray NdArray + # But both share the same underlying memory + ``` - :return: a numpy ndarray + --- + + :return: a `numpy.ndarray` """ return self.view(np.ndarray) @classmethod def from_protobuf(cls: Type[T], pb_msg: 'NdArrayProto') -> 'T': """ - read ndarray from a proto msg + Read ndarray from a proto msg :param pb_msg: :return: a numpy array """ @@ -190,7 +193,7 @@ def from_protobuf(cls: Type[T], pb_msg: 'NdArrayProto') -> 'T': def to_protobuf(self) -> 'NdArrayProto': """ - transform self into a NdArrayProto protobuf message + Transform self into a NdArrayProto protobuf message """ from docarray.proto import NdArrayProto diff --git a/docarray/typing/tensor/tensorflow_tensor.py b/docarray/typing/tensor/tensorflow_tensor.py index f0d79aa0b30..fc0ce8df331 100644 --- a/docarray/typing/tensor/tensorflow_tensor.py +++ b/docarray/typing/tensor/tensorflow_tensor.py @@ -38,8 +38,8 @@ class metaTensorFlow( @_register_proto(proto_type_name='tensorflow_tensor') class TensorFlowTensor(AbstractTensor, Generic[ShapeT], metaclass=metaTensorFlow): """ - TensorFlowTensor class with a :attr:`~docarray.typing.TensorFlowTensor.tensor` - attribute of type :class:`tf.Tensor`, intended for use in a Document. + TensorFlowTensor class with a `.tensor` attribute of type `tf.Tensor`, + intended for use in a Document. This enables (de)serialization from/to protobuf and json, data validation, and coersion from compatible types like numpy.ndarray. @@ -47,95 +47,114 @@ class TensorFlowTensor(AbstractTensor, Generic[ShapeT], metaclass=metaTensorFlow This type can also be used in a parametrized way, specifying the shape of the tensor. - In comparison to :class:`~docarray.typing.TorchTensor` and - :class:`~docarray.typing.NdArray`, :class:`~docarray.typing.TensorFlowTensor` is not - a subclass of :class:`tf.Tensor` (or :class:`torch.Tensor`, :class:`np.ndarray` - respectively). - Instead, the :class:`tf.Tensor` is stored in - :attr:`~docarray.typing.TensorFlowTensor.tensor`. + In comparison to [`TorchTensor`][docarray.typing.TorchTensor] and + [`NdArray`][docarray.typing.tensor.ndarray.NdArray], + [`TensorFlowTensor`][docarray.typing.tensor.tensorflow_tensor.TensorFlowTensor] + is not a subclass of `tf.Tensor` (or `torch.Tensor`, `np.ndarray` respectively). + Instead, the `tf.Tensor` is stored in + [`TensorFlowTensor.tensor`][docarray.typing.tensor.tensorflow_tensor.TensorFlowTensor]. Therefore, to do operations on the actual tensor data you have to always access the - :attr:`~docarray.typing.TensorFlowTensor.tensor` attribute. + [`TensorFlowTensor.tensor`][docarray.typing.tensor.tensorflow_tensor.TensorFlowTensor] + attribute. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + import tensorflow as tf + from docarray.typing import TensorFlowTensor - import tensorflow as tf - from docarray.typing import TensorFlowTensor + t = TensorFlowTensor(tensor=tf.zeros((224, 224))) - t = TensorFlowTensor(tensor=tf.zeros((224, 224))) + # tensorflow functions + broadcasted = tf.broadcast_to(t.tensor, (3, 224, 224)) + broadcasted = tf.broadcast_to(t.unwrap(), (3, 224, 224)) - # tensorflow functions - broadcasted = tf.broadcast_to(t.tensor, (3, 224, 224)) - broadcasted = tf.broadcast_to(t.unwrap(), (3, 224, 224)) - broadcasted = tf.broadcast_to(t, (3, 224, 224)) # this will fail + # this will fail: + # broadcasted = tf.broadcast_to(t, (3, 224, 224)) - # tensorflow.Tensor methods: - arr = t.tensor.numpy() - arr = t.unwrap().numpy() - arr = t.numpy() # this will fail + # tensorflow.Tensor methods: + arr = t.tensor.numpy() + arr = t.unwrap().numpy() - The :class:`~docarray.computation.tensorflow_backend.TensorFlowBackend` however, - operates on our :class:`~docarray.typing.TensorFlowTensor` instances. - Here, you do not have to access the :attr:`~docarray.typing.TensorFlowTensor.tensor` - but can instead just hand over your :class:`~docarray.typing.TensorFlowTensor` - instance. + # this will fail: + # arr = t.numpy() + ``` - .. code-block:: python + --- - import tensorflow as tf - from docarray.typing import TensorFlowTensor + The [`TensorFlowBackend`] however, operates on our + [`TensorFlowTensor`][docarray.typing.TensorFlowTensor] instances. + Here, you do not have to access the `.tensor` attribute, + but can instead just hand over your + [`TensorFlowTensor`][docarray.typing.TensorFlowTensor] instance. + --- - zeros = TensorFlowTensor(tensor=tf.zeros((3, 224, 224))) + ```python + import tensorflow as tf + from docarray.typing import TensorFlowTensor - comp_be = zeros.get_comp_backend() - reshaped = comp_be.reshape(zeros, (224, 224, 3)) - assert comp_be.shape(reshaped) == (224, 224, 3) - You can use :class:`~docarray.typing.TensorFlowTensor` in a Document as follows: + zeros = TensorFlowTensor(tensor=tf.zeros((3, 224, 224))) - .. code-block:: python + comp_be = zeros.get_comp_backend() + reshaped = comp_be.reshape(zeros, (224, 224, 3)) + assert comp_be.shape(reshaped) == (224, 224, 3) + ``` - from docarray import BaseDoc - from docarray.typing import TensorFlowTensor - import tensorflow as tf + --- + You can use [`TensorFlowTensor`][docarray.typing.TensorFlowTensor] in a Document as follows: - class MyDoc(BaseDoc): - tensor: TensorFlowTensor - image_tensor: TensorFlowTensor[3, 224, 224] - square_crop: TensorFlowTensor[3, 'x', 'x'] - random_image: TensorFlowTensor[ - 3, ... - ] # first dimension is fixed, can have arbitrary shape + --- + ```python + from docarray import BaseDoc + from docarray.typing import TensorFlowTensor + import tensorflow as tf - # create a document with tensors - doc = MyDoc( - tensor=tf.zeros((128,)), - image_tensor=tf.zeros((3, 224, 224)), - square_crop=tf.zeros((3, 64, 64)), - random_image=tf.zeros(3, 128, 256), - ) - # automatic shape conversion - doc = MyDoc( - tensor=tf.zeros((128,)), - image_tensor=tf.zeros((224, 224, 3)), # will reshape to (3, 224, 224) - square_crop=tf.zeros((3, 128, 128)), - random_image=tf.zeros(3, 64, 128), - ) + class MyDoc(BaseDoc): + tensor: TensorFlowTensor + image_tensor: TensorFlowTensor[3, 224, 224] + square_crop: TensorFlowTensor[3, 'x', 'x'] + random_image: TensorFlowTensor[ + 3, ... + ] # first dimension is fixed, can have arbitrary shape + + + # create a document with tensors + doc = MyDoc( + tensor=tf.zeros((128,)), + image_tensor=tf.zeros((3, 224, 224)), + square_crop=tf.zeros((3, 64, 64)), + random_image=tf.zeros((3, 128, 256)), + ) - # !! The following will raise an error due to shape mismatch !! + # automatic shape conversion + doc = MyDoc( + tensor=tf.zeros((128,)), + image_tensor=tf.zeros((224, 224, 3)), # will reshape to (3, 224, 224) + square_crop=tf.zeros((3, 128, 128)), + random_image=tf.zeros((3, 64, 128)), + ) + + # !! The following will raise an error due to shape mismatch !! + from pydantic import ValidationError + + try: doc = MyDoc( tensor=tf.zeros((128,)), image_tensor=tf.zeros((224, 224)), # this will fail validation square_crop=tf.zeros((3, 128, 64)), # this will also fail validation random_image=tf.zeros(4, 64, 128), # this will also fail validation ) + except ValidationError as e: + pass + ``` + --- """ __parametrized_meta__ = metaTensorFlow @@ -153,7 +172,7 @@ def __getitem__(self, item): return TensorFlowCompBackend._cast_output(t=tensor) def __setitem__(self, index, value): - """Set a slice of this tensor's tf.Tensor""" + """Set a slice of this tensor's `tf.Tensor`""" t = self.unwrap() value = tf.cast(value, dtype=t.dtype) var = tf.Variable(t) @@ -161,7 +180,7 @@ def __setitem__(self, index, value): self.tensor = tf.constant(var) def __iter__(self): - """Iterate over the elements of this tensor's tf.Tensor.""" + """Iterate over the elements of this tensor's `tf.Tensor`.""" for i in range(len(self)): yield self[i] @@ -196,11 +215,11 @@ def validate( @classmethod def _docarray_from_native(cls: Type[T], value: Union[tf.Tensor, T]) -> T: """ - Create a TensorFlowTensor from a tensorflow.Tensor or TensorFlowTensor + Create a `TensorFlowTensor` from a `tf.Tensor` or `TensorFlowTensor` instance. - :param value: instance of tf.Tensor or TensorFlowTensor - :return: a TensorFlowTensor + :param value: instance of `tf.Tensor` or `TensorFlowTensor` + :return: a `TensorFlowTensor` """ if isinstance(value, TensorFlowTensor): if cls.__unparametrizedcls__: # None if the tensor is parametrized @@ -231,7 +250,7 @@ def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: def _docarray_to_json_compatible(self) -> np.ndarray: """ - Convert TensorFlowTensor into a json compatible object + Convert `TensorFlowTensor` into a json compatible object :return: a representation of the tensor compatible with orjson """ return self.unwrap().numpy() @@ -257,7 +276,7 @@ def from_protobuf(cls: Type[T], pb_msg: 'NdArrayProto') -> 'T': """ Read ndarray from a proto msg. :param pb_msg: - :return: a TensorFlowTensor + :return: a `TensorFlowTensor` """ source = pb_msg.dense if source.buffer: @@ -272,33 +291,35 @@ def from_protobuf(cls: Type[T], pb_msg: 'NdArrayProto') -> 'T': @classmethod def from_ndarray(cls: Type[T], value: np.ndarray) -> T: - """Create a TensorFlowTensor from a numpy array. + """Create a `TensorFlowTensor` from a numpy array. :param value: the numpy array - :return: a TensorFlowTensor + :return: a `TensorFlowTensor` """ return cls._docarray_from_native(tf.convert_to_tensor(value)) def unwrap(self) -> tf.Tensor: """ - Return the original tensorflow.Tensor without any memory copy. + Return the original `tf.Tensor` without any memory copy. - The original view rest intact and is still a Document TensorFlowTensor - but the return object is a pure tf.Tensor but both object share + The original view rest intact and is still a Document `TensorFlowTensor` + but the return object is a pure `tf.Tensor` but both object share the same memory layout. - EXAMPLE USAGE - .. code-block:: python - from docarray.typing import TensorFlowTensor - import tensorflow as tf + --- - t1 = TensorFlowTensor.validate(tf.zeros((3, 224, 224)), None, None) - # here t1 is a docarray TensorFlowTensor - t2 = t.unwrap() - # here t2 is a pure tf.Tensor but t1 is still a Docarray TensorFlowTensor + ```python + from docarray.typing import TensorFlowTensor + import tensorflow as tf + t1 = TensorFlowTensor.validate(tf.zeros((3, 224, 224)), None, None) + # here t1 is a docarray TensorFlowTensor + t2 = t1.unwrap() + # here t2 is a pure tf.Tensor but t1 is still a Docarray TensorFlowTensor + ``` - :return: a tf.Tensor + --- + :return: a `tf.Tensor` """ return self.tensor diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index 1ec05df0086..27a33186f3f 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -46,7 +46,7 @@ class TorchTensor( # Subclassing torch.Tensor following the advice from here: # https://pytorch.org/docs/stable/notes/extending.html#subclassing-torch-tensor """ - Subclass of torch.Tensor, intended for use in a Document. + Subclass of `torch.Tensor`, intended for use in a Document. This enables (de)serialization from/to protobuf and json, data validation, and coersion from compatible types like numpy.ndarray. @@ -139,32 +139,35 @@ def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: def _docarray_to_json_compatible(self) -> np.ndarray: """ - Convert torchTensor into a json compatible object + Convert `TorchTensor` into a json compatible object :return: a representation of the tensor compatible with orjson """ return self.numpy() ## might need to check device later def unwrap(self) -> torch.Tensor: """ - Return the original torch.Tensor without any memory copy. + Return the original `torch.Tensor` without any memory copy. - The original view rest intact and is still a Document TorchTensor - but the return object is a pure torch Tensor but both object share + The original view rest intact and is still a Document `TorchTensor` + but the return object is a pure `torch.Tensor` but both object share the same memory layout. - EXAMPLE USAGE - .. code-block:: python - from docarray.typing import TorchTensor - import torch + --- - t = TorchTensor.validate(torch.zeros(3, 224, 224), None, None) - # here t is a docarray TorchTensor - t2 = t.unwrap() - # here t2 is a pure torch.Tensor but t1 is still a Docarray TorchTensor - # But both share the same underlying memory + ```python + from docarray.typing import TorchTensor + import torch + t = TorchTensor.validate(torch.zeros(3, 224, 224), None, None) + # here t is a docarray TorchTensor + t2 = t.unwrap() + # here t2 is a pure torch.Tensor but t1 is still a Docarray TorchTensor + # But both share the same underlying memory + ``` - :return: a torch Tensor + --- + + :return: a `torch.Tensor` """ value = copy(self) # as unintuitive as it sounds, this # does not do any relevant memory copying, just shallow @@ -174,10 +177,10 @@ def unwrap(self) -> torch.Tensor: @classmethod def _docarray_from_native(cls: Type[T], value: torch.Tensor) -> T: - """Create a TorchTensor from a native torch.Tensor + """Create a `TorchTensor` from a native `torch.Tensor` - :param value: the native torch.Tensor - :return: a TorchTensor + :param value: the native `torch.Tensor` + :return: a `TorchTensor` """ if cls.__unparametrizedcls__: # This is not None if the tensor is parametrized value.__class__ = cls.__unparametrizedcls__ # type: ignore @@ -187,19 +190,19 @@ def _docarray_from_native(cls: Type[T], value: torch.Tensor) -> T: @classmethod def from_ndarray(cls: Type[T], value: np.ndarray) -> T: - """Create a TorchTensor from a numpy array + """Create a `TorchTensor` from a numpy array :param value: the numpy array - :return: a TorchTensor + :return: a `TorchTensor` """ return cls._docarray_from_native(torch.from_numpy(value)) @classmethod def from_protobuf(cls: Type[T], pb_msg: 'NdArrayProto') -> 'T': """ - read ndarray from a proto msg + Read ndarray from a proto msg :param pb_msg: - :return: a torch tensor + :return: a `TorchTensor` """ source = pb_msg.dense if source.buffer: @@ -212,7 +215,7 @@ def from_protobuf(cls: Type[T], pb_msg: 'NdArrayProto') -> 'T': def to_protobuf(self) -> 'NdArrayProto': """ - transform self into a NdArrayProto protobuf message + Transform self into a `NdArrayProto` protobuf message """ from docarray.proto import NdArrayProto diff --git a/docarray/typing/tensor/video/video_ndarray.py b/docarray/typing/tensor/video/video_ndarray.py index c237de5760e..4077cb6b161 100644 --- a/docarray/typing/tensor/video/video_ndarray.py +++ b/docarray/typing/tensor/video/video_ndarray.py @@ -16,7 +16,7 @@ @_register_proto(proto_type_name='video_ndarray') class VideoNdArray(NdArray, VideoTensorMixin): """ - Subclass of NdArray, to represent a video tensor. + Subclass of [`NdArray`][docarray.typing.NdArray], to represent a video tensor. Adds video-specific features to the tensor. --- @@ -42,15 +42,13 @@ class MyVideoDoc(BaseDoc): video_tensor=np.random.random((100, 224, 224, 3)), ) - doc_1.video_tensor.save(file_path='/tmp/file_1.mp4') - doc_2 = MyVideoDoc( title='my_second_video_doc', - url='/tmp/file_1.mp4', + url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', ) doc_2.video_tensor = parse_obj_as(VideoNdArray, doc_2.url.load().video) - doc_2.video_tensor.save(file_path='/tmp/file_2.mp4') + # doc_2.video_tensor.save(file_path='/tmp/file_2.mp4') ``` --- diff --git a/docarray/typing/tensor/video/video_tensorflow_tensor.py b/docarray/typing/tensor/video/video_tensorflow_tensor.py index 39a7fce87ec..9a652c2e4ca 100644 --- a/docarray/typing/tensor/video/video_tensorflow_tensor.py +++ b/docarray/typing/tensor/video/video_tensorflow_tensor.py @@ -18,44 +18,42 @@ class VideoTensorFlowTensor( TensorFlowTensor, VideoTensorMixin, metaclass=metaTensorFlow ): """ - Subclass of TensorFlowTensor, to represent a video tensor. - Adds video-specific features to the tensor. + Subclass of [`TensorFlowTensor`][docarray.typing.TensorFlowTensor], + to represent a video tensor. Adds video-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + import tensorflow as tf - import tensorflow as tf - from pydantic import parse_obj_as + from docarray import BaseDoc + from docarray.typing import VideoTensorFlowTensor, VideoUrl - from docarray import BaseDoc - from docarray.typing import VideoTensorFlowTensor, VideoUrl + class MyVideoDoc(BaseDoc): + title: str + url: Optional[VideoUrl] + video_tensor: Optional[VideoTensorFlowTensor] - class MyVideoDoc(BaseDoc): - title: str - url: Optional[VideoUrl] - video_tensor: Optional[VideoTensorFlowTensor] + doc_1 = MyVideoDoc( + title='my_first_video_doc', + video_tensor=tf.random.normal((100, 224, 224, 3)), + ) + # doc_1.video_tensor.save(file_path='file_1.mp4') - doc_1 = MyVideoDoc( - title='my_first_video_doc', - video_tensor=tf.random.normal((100, 224, 224, 3)), - ) + doc_2 = MyVideoDoc( + title='my_second_video_doc', + url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', + ) - doc_1.video_tensor.save(file_path='file_1.wav') - - - doc_2 = MyVideoDoc( - title='my_second_video_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) - - doc_2.video_tensor = parse_obj_as(VideoTensorFlowTensor, doc_2.url.load()) - doc_2.video_tensor.save(file_path='file_2.wav') + doc_2.video_tensor = doc_2.url.load().video + # doc_2.video_tensor.save(file_path='file_2.wav') + ``` + --- """ @classmethod diff --git a/docarray/typing/tensor/video/video_torch_tensor.py b/docarray/typing/tensor/video/video_torch_tensor.py index 9c20159e7e4..bdef42d2fc2 100644 --- a/docarray/typing/tensor/video/video_torch_tensor.py +++ b/docarray/typing/tensor/video/video_torch_tensor.py @@ -16,43 +16,42 @@ @_register_proto(proto_type_name='video_torch_tensor') class VideoTorchTensor(TorchTensor, VideoTensorMixin, metaclass=metaTorchAndNode): """ - Subclass of TorchTensor, to represent a video tensor. + Subclass of [`TorchTensor`][docarray.typing.TorchTensor], to represent a video tensor. Adds video-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + import torch - import torch - from pydantic import parse_obj_as + from docarray import BaseDoc + from docarray.typing import VideoTorchTensor, VideoUrl - from docarray import BaseDoc - from docarray.typing import VideoTorchTensor, VideoUrl + class MyVideoDoc(BaseDoc): + title: str + url: Optional[VideoUrl] + video_tensor: Optional[VideoTorchTensor] - class MyVideoDoc(BaseDoc): - title: str - url: Optional[VideoUrl] - video_tensor: Optional[VideoTorchTensor] + doc_1 = MyVideoDoc( + title='my_first_video_doc', + video_tensor=torch.randn(size=(100, 224, 224, 3)), + ) + # doc_1.video_tensor.save(file_path='file_1.mp4') - doc_1 = MyVideoDoc( - title='my_first_video_doc', - video_tensor=torch.randn(size=(100, 224, 224, 3)), - ) + doc_2 = MyVideoDoc( + title='my_second_video_doc', + url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', + ) - doc_1.video_tensor.save(file_path='file_1.wav') + doc_2.video_tensor = doc_2.url.load().video + # doc_2.video_tensor.save(file_path='file_2.wav') + ``` - - doc_2 = MyVideoDoc( - title='my_second_video_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) - - doc_2.video_tensor = parse_obj_as(VideoTorchTensor, doc_2.url.load()) - doc_2.video_tensor.save(file_path='file_2.wav') + --- """ diff --git a/docarray/typing/url/any_url.py b/docarray/typing/url/any_url.py index e414a5572e2..6d930aa53f3 100644 --- a/docarray/typing/url/any_url.py +++ b/docarray/typing/url/any_url.py @@ -133,7 +133,7 @@ def build( @classmethod def from_protobuf(cls: Type[T], pb_msg: 'str') -> T: """ - read url from a proto msg + Read url from a proto msg. :param pb_msg: :return: url """ diff --git a/docarray/typing/url/audio_url.py b/docarray/typing/url/audio_url.py index 84cafecf752..6851b8a98c8 100644 --- a/docarray/typing/url/audio_url.py +++ b/docarray/typing/url/audio_url.py @@ -19,7 +19,8 @@ class AudioUrl(AnyUrl): def load(self: T) -> Tuple[AudioNdArray, int]: """ - Load the data from the url into an AudioNdArray and the frame rate. + Load the data from the url into an [`AudioNdArray`][docarray.typing.AudioNdArray] + and the frame rate. --- @@ -42,8 +43,8 @@ class MyDoc(BaseDoc): --- - :return: tuple of an AudioNdArray representing the Audio file content, - and an integer representing the frame rate. + :return: tuple of an [`AudioNdArray`][docarray.typing.AudioNdArray] representing + the audio file content, and an integer representing the frame rate. """ bytes_ = self.load_bytes() @@ -51,11 +52,11 @@ class MyDoc(BaseDoc): def load_bytes(self, timeout: Optional[float] = None) -> AudioBytes: """ - Convert url to AudioBytes. This will either load or download the file and save - it into an AudioBytes object. + Convert url to [`AudioBytes`][docarray.typing.AudioBytes]. This will either load or + download the file and save it into an [`AudioBytes`][docarray.typing.AudioBytes] object. :param timeout: timeout for urlopen. Only relevant if url is not local - :return: AudioBytes object + :return: [`AudioBytes`][docarray.typing.AudioBytes] object """ bytes_ = super().load_bytes(timeout=timeout) return AudioBytes(bytes_) diff --git a/docarray/typing/url/image_url.py b/docarray/typing/url/image_url.py index c025346fd44..43758cf7436 100644 --- a/docarray/typing/url/image_url.py +++ b/docarray/typing/url/image_url.py @@ -57,7 +57,7 @@ def load( timeout: Optional[float] = None, ) -> ImageNdArray: """ - Load the data from the url into an ImageNdArray + Load the data from the url into an [`ImageNdArray`][docarray.typing.ImageNdArray] --- @@ -94,7 +94,7 @@ class MyDoc(BaseDoc): 'H' = height, 'W' = width, 'C' = color channel :param timeout: timeout (sec) for urlopen network request. Only relevant if URL is not local - :return: ImageNdArray representing the image as RGB values + :return: [`ImageNdArray`][docarray.typing.ImageNdArray] representing the image as RGB values """ from docarray.typing.bytes.image_bytes import ImageBytes @@ -103,11 +103,11 @@ class MyDoc(BaseDoc): def load_bytes(self, timeout: Optional[float] = None) -> ImageBytes: """ - Convert url to ImageBytes. This will either load or download the file and save - it into an ImageBytes object. + Convert url to [`ImageBytes`][docarray.typing.ImageBytes]. This will either load or + download the file and save it into an [`ImageBytes`][docarray.typing.ImageBytes] object. :param timeout: timeout for urlopen. Only relevant if url is not local - :return: ImageBytes object + :return: [`ImageBytes`][docarray.typing.ImageBytes] object """ bytes_ = super().load_bytes(timeout=timeout) return ImageBytes(bytes_) diff --git a/docarray/typing/url/url_3d/point_cloud_url.py b/docarray/typing/url/url_3d/point_cloud_url.py index 614bd7d89f1..dd3f17be0df 100644 --- a/docarray/typing/url/url_3d/point_cloud_url.py +++ b/docarray/typing/url/url_3d/point_cloud_url.py @@ -93,7 +93,7 @@ def display( """ Plot point cloud from url. - First, it loads the point cloud into a :class:`PointsAndColors` object, and then + First, it loads the point cloud into a `PointsAndColors` object, and then calls display on it. The following is therefore equivalent: .. code-block:: python diff --git a/docarray/typing/url/video_url.py b/docarray/typing/url/video_url.py index feaed7308a6..db9dd4b5080 100644 --- a/docarray/typing/url/video_url.py +++ b/docarray/typing/url/video_url.py @@ -18,8 +18,10 @@ class VideoUrl(AnyUrl): def load(self: T, **kwargs) -> VideoLoadResult: """ - Load the data from the url into a named Tuple of VideoNdArray, AudioNdArray and - NdArray. + Load the data from the url into a `NamedTuple` of + [`VideoNdArray`][docarray.typing.VideoNdArray], + [`AudioNdArray`][docarray.typing.AudioNdArray] + and [`NdArray`][docarray.typing.NdArray]. --- @@ -74,19 +76,20 @@ class MyDoc(BaseDoc): av.open() as described in: https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open - :return: AudioNdArray representing the audio content, VideoNdArray representing - the images of the video, NdArray of the key frame indices. + :return: [`AudioNdArray`][docarray.typing.AudioNdArray] representing the audio content, + [`VideoNdArray`][docarray.typing.VideoNdArray] representing the images of the video, + [`NdArray`][docarray.typing.NdArray] of the key frame indices. """ buffer = self.load_bytes(**kwargs) return buffer.load() def load_bytes(self, timeout: Optional[float] = None) -> VideoBytes: """ - Convert url to VideoBytes. This will either load or download the file and save - it into an VideoBytes object. + Convert url to [`VideoBytes`][docarray.typing.VideoBytes]. This will either load or download + the file and save it into an [`VideoBytes`][docarray.typing.VideoBytes] object. :param timeout: timeout for urlopen. Only relevant if url is not local - :return: VideoBytes object + :return: [`VideoBytes`][docarray.typing.VideoBytes] object """ bytes_ = super().load_bytes(timeout=timeout) return VideoBytes(bytes_) diff --git a/docarray/utils/find.py b/docarray/utils/find.py index 405f3e75f15..b1ea28ecb13 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -39,7 +39,7 @@ def find( !!! note This is a simple implementation of exact search. If you need to do advance search using approximate nearest neighbours search or hybrid search or - multi vector search please take a look at the [BaseDoc][docarray.base_doc.doc.BaseDoc] + multi vector search please take a look at the [`BaseDoc`][docarray.base_doc.doc.BaseDoc]. --- @@ -122,7 +122,7 @@ def find_batched( !!! note This is a simple implementation of exact search. If you need to do advance search using approximate nearest neighbours search or hybrid search or - multi vector search please take a look at the [BaseDoc][docarray.base_doc.doc.BaseDoc] + multi vector search please take a look at the [`BaseDoc`][docarray.base_doc.doc.BaseDoc] --- diff --git a/docarray/utils/map.py b/docarray/utils/map.py index 09c7b1ae2ef..682f8fa5aae 100644 --- a/docarray/utils/map.py +++ b/docarray/utils/map.py @@ -57,8 +57,8 @@ def load_url_to_tensor(img: ImageDoc) -> ImageDoc: --- :param docs: DocList to apply function to - :param func: a function that takes a :class:`BaseDoc` as input and outputs - a :class:`BaseDoc`. + :param func: a function that takes a [`BaseDoc`][docarray.base_doc.doc.BaseDoc] + as input and outputs a [`BaseDoc`][docarray.base_doc.doc.BaseDoc]. :param backend: `thread` for multithreading and `process` for multiprocessing. Defaults to `thread`. In general, if `func` is IO-bound then `thread` is a good choice. @@ -69,7 +69,7 @@ def load_url_to_tensor(img: ImageDoc) -> ImageDoc: Note that computation that is offloaded to non-python code (e.g. through np/torch/tf) falls under the "IO-bound" category. - .. warning:: + !!! warning When using `process` backend, your `func` should not modify elements in-place. This is because the multiprocessing backend passes the variable via pickle and works in another process. @@ -116,7 +116,7 @@ def map_docs_batched( """ Return an iterator that applies `func` to every **minibatch** of iterable in parallel, yielding the results. - Each element in the returned iterator is an :class:`AnyDocArray`. + Each element in the returned iterator is an `AnyDocArray`. --- @@ -168,7 +168,7 @@ def upper_case_name(docs: DocList[MyDoc]) -> DocList[MyDoc]: Note that computation that is offloaded to non-python code (e.g. through np/torch/tf) falls under the "IO-bound" category. - .. warning:: + !!! warning When using `process` backend, your `func` should not modify elements in-place. This is because the multiprocessing backend passes the variable via pickle and works in another process. diff --git a/docarray/utils/reduce.py b/docarray/utils/reduce.py index f60ad0a1671..f615b9aeaeb 100644 --- a/docarray/utils/reduce.py +++ b/docarray/utils/reduce.py @@ -36,7 +36,7 @@ def reduce( return left -def reduce_all(docarrays: List[DocList]) -> DocList: +def reduce_all(docs: List[DocList]) -> DocList: """ Reduces a list of DocLists into one DocList. Changes are applied to the first DocList in-place. @@ -49,22 +49,24 @@ def reduce_all(docarrays: List[DocList]) -> DocList: DocList is kept). Nested DocLists belonging to many DocLists are also reduced in the same way. - .. note:: + + !!! note + - Nested DocLists order does not follow any specific rule. You might want to re-sort them in a later step. - The final result depends on the order of DocLists when applying reduction. - :param docarrays: List of DocLists to be reduced + :param docs: List of DocLists to be reduced :return: the resulting DocList """ - if len(docarrays) <= 1: + if len(docs) <= 1: raise Exception( 'In order to reduce DocLists' ' we should have more than one DocList' ) - left = docarrays[0] - others = docarrays[1:] + left = docs[0] + others = docs[1:] left_id_map = {doc.id: i for i, doc in enumerate(left)} - for docs in others: - reduce(left, docs, left_id_map) + for other_docs in others: + reduce(left, other_docs, left_id_map) return left diff --git a/docs/api_references/typing/tensor.md b/docs/api_references/typing/tensor.md deleted file mode 100644 index f8040705f32..00000000000 --- a/docs/api_references/typing/tensor.md +++ /dev/null @@ -1,3 +0,0 @@ -# Tensor - -::: docarray.typing.tensor diff --git a/docs/api_references/typing/tensor/audio.md b/docs/api_references/typing/tensor/audio.md new file mode 100644 index 00000000000..07ec2652520 --- /dev/null +++ b/docs/api_references/typing/tensor/audio.md @@ -0,0 +1,6 @@ +# AudioTensor + +::: docarray.typing.tensor.audio.audio_ndarray +::: docarray.typing.tensor.audio.abstract_audio_tensor +::: docarray.typing.tensor.audio.audio_tensorflow_tensor +::: docarray.typing.tensor.audio.audio_torch_tensor diff --git a/docs/api_references/typing/tensor/embedding.md b/docs/api_references/typing/tensor/embedding.md new file mode 100644 index 00000000000..47da823f7b4 --- /dev/null +++ b/docs/api_references/typing/tensor/embedding.md @@ -0,0 +1,7 @@ +# Embedding + +::: docarray.typing.tensor.embedding.embedding +::: docarray.typing.tensor.embedding.embedding_mixin +::: docarray.typing.tensor.embedding.ndarray +::: docarray.typing.tensor.embedding.tensorflow +::: docarray.typing.tensor.embedding.torch diff --git a/docs/api_references/typing/tensor/image.md b/docs/api_references/typing/tensor/image.md new file mode 100644 index 00000000000..ee5184fa92a --- /dev/null +++ b/docs/api_references/typing/tensor/image.md @@ -0,0 +1,6 @@ +# ImageTensor + +::: docarray.typing.tensor.image.image_ndarray +::: docarray.typing.tensor.image.abstract_image_tensor +::: docarray.typing.tensor.image.image_tensorflow_tensor +::: docarray.typing.tensor.image.image_torch_tensor diff --git a/docs/api_references/typing/tensor/tensor.md b/docs/api_references/typing/tensor/tensor.md new file mode 100644 index 00000000000..7273dea9476 --- /dev/null +++ b/docs/api_references/typing/tensor/tensor.md @@ -0,0 +1,6 @@ +# Tensor + +::: docarray.typing.tensor.abstract_tensor +::: docarray.typing.tensor.ndarray +::: docarray.typing.tensor.tensorflow_tensor +::: docarray.typing.tensor.torch_tensor diff --git a/docs/api_references/typing/tensor/video.md b/docs/api_references/typing/tensor/video.md new file mode 100644 index 00000000000..73fd671105f --- /dev/null +++ b/docs/api_references/typing/tensor/video.md @@ -0,0 +1,6 @@ +# VideoTensor + +::: docarray.typing.tensor.video.video_ndarray +::: docarray.typing.tensor.video.video_tensor_mixin +::: docarray.typing.tensor.video.video_tensorflow_tensor +::: docarray.typing.tensor.video.video_torch_tensor diff --git a/docs/api_references/utils/filter.md b/docs/api_references/utils/filter.md index b40c59c510a..56cf5e8c776 100644 --- a/docs/api_references/utils/filter.md +++ b/docs/api_references/utils/filter.md @@ -1,6 +1,6 @@ # filter -::: docarray.utils.filter.filter_docs +::: docarray.utils.filter diff --git a/docs/api_references/utils/find.md b/docs/api_references/utils/find.md index e94a9401149..e73ac17a354 100644 --- a/docs/api_references/utils/find.md +++ b/docs/api_references/utils/find.md @@ -1,7 +1,6 @@ # find -::: docarray.utils.find.find -::: docarray.utils.find.find_batched +::: docarray.utils.find diff --git a/docs/api_references/utils/maps_docs.md b/docs/api_references/utils/maps_docs.md index da71bc867e9..d715f08474e 100644 --- a/docs/api_references/utils/maps_docs.md +++ b/docs/api_references/utils/maps_docs.md @@ -1,7 +1,6 @@ # map -::: docarray.utils.map.map_docs -::: docarray.utils.map.map_docs_batched +::: docarray.utils.map diff --git a/docs/api_references/utils/reduce.md b/docs/api_references/utils/reduce.md index 9b6db3eea02..23939b4fe48 100644 --- a/docs/api_references/utils/reduce.md +++ b/docs/api_references/utils/reduce.md @@ -1,7 +1,6 @@ # reduce -::: docarray.utils.reduce.reduce -::: docarray.utils.reduce.reduce_all +::: docarray.utils.reduce diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py index 91bc43468c1..9bb6e01aeb2 100644 --- a/tests/documentation/test_docstring.py +++ b/tests/documentation/test_docstring.py @@ -32,7 +32,12 @@ def get_obj_to_check(lib): obj_to_check = [] - for obj in lib.__all__: + all_test = getattr(lib, '__all__') + try: + all_test = getattr(lib, '__all_test__') + except (AttributeError, ImportError): + pass + for obj in all_test: obj_to_check.append(getattr(lib, obj)) return obj_to_check