From d3fcbac54e8d5a65299aba087af6a5b301d045ae Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 11:41:23 +0200 Subject: [PATCH 01/19] wip Signed-off-by: samsja --- docarray/store/file.py | 4 ++-- docarray/store/jac.py | 6 +++--- docarray/store/s3.py | 4 ++-- docarray/utils/__init__.py | 5 +++++ docarray/utils/cache.py | 2 +- docarray/utils/progress_bar.py | 4 ++-- tests/documentation/test_docstring.py | 0 tests/integrations/store/test_file.py | 4 ++-- 8 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 tests/documentation/test_docstring.py diff --git a/docarray/store/file.py b/docarray/store/file.py index aeb355e2e17..e88e87b23f8 100644 --- a/docarray/store/file.py +++ b/docarray/store/file.py @@ -7,7 +7,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.exceptions import ConcurrentPushException from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import get_cache_path +from docarray.utils.cache import _get_cache_path if TYPE_CHECKING: from docarray import BaseDoc, DocArray @@ -23,7 +23,7 @@ def _abs_filepath(name: str) -> Path: If it is a path, it is resolved to an absolute path. """ if not (name.startswith('/') or name.startswith('~') or name.startswith('.')): - name = str(get_cache_path() / name) + name = str(_get_cache_path() / name) if name.startswith('~'): name = str(Path.home() / name[2:]) return Path(name).resolve() diff --git a/docarray/store/jac.py b/docarray/store/jac.py index 75685e61cb4..969df0a6233 100644 --- a/docarray/store/jac.py +++ b/docarray/store/jac.py @@ -24,7 +24,7 @@ get_version_info, raise_req_error, ) -from docarray.utils.cache import get_cache_path +from docarray.utils.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover import io @@ -331,7 +331,7 @@ def pull_stream( _BufferedCachingRequestReader, io.BufferedReader ] = _BufferedCachingRequestReader(r, tmp_cache_file) - cache_file = get_cache_path() / f'{save_name}.da' + cache_file = _get_cache_path() / f'{save_name}.da' if local_cache and cache_file.exists(): _cache_len = cache_file.stat().st_size if _cache_len == int(r.headers['Content-length']): @@ -354,7 +354,7 @@ def pull_stream( if local_cache: if isinstance(_source, _BufferedCachingRequestReader): - Path(get_cache_path()).mkdir(parents=True, exist_ok=True) + Path(_get_cache_path()).mkdir(parents=True, exist_ok=True) tmp_cache_file.rename(cache_file) else: _source.close() diff --git a/docarray/store/s3.py b/docarray/store/s3.py index b038e7c4b41..8c5604b6d35 100644 --- a/docarray/store/s3.py +++ b/docarray/store/s3.py @@ -10,7 +10,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import get_cache_path +from docarray.utils.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover from docarray import BaseDoc, DocArray @@ -212,7 +212,7 @@ def pull_stream( bucket, name = name.split('/', 1) save_name = name.replace('/', '_') - cache_path = get_cache_path() / f'{save_name}.da' + cache_path = _get_cache_path() / f'{save_name}.da' source = _BufferedCachingReader( open(f"s3://{bucket}/{name}.da", 'rb', compression='.gz'), diff --git a/docarray/utils/__init__.py b/docarray/utils/__init__.py index e69de29bb2d..a1a68cf806e 100644 --- a/docarray/utils/__init__.py +++ b/docarray/utils/__init__.py @@ -0,0 +1,5 @@ +from docarray.utils.filter import filter_docs +from docarray.utils.find import find, find_batched +from docarray.utils.map import map_docs, map_docs_batch + +__all__ = ['filter_docs', 'find', 'find_batched', 'map_docs', 'map_docs_batch'] diff --git a/docarray/utils/cache.py b/docarray/utils/cache.py index 4df305414b3..249c4f9d179 100644 --- a/docarray/utils/cache.py +++ b/docarray/utils/cache.py @@ -4,7 +4,7 @@ @lru_cache(maxsize=None) -def get_cache_path() -> Path: +def _get_cache_path() -> Path: """ Get the path to the cache directory. diff --git a/docarray/utils/progress_bar.py b/docarray/utils/progress_bar.py index ad7bb3dcd5d..4750c509a1a 100644 --- a/docarray/utils/progress_bar.py +++ b/docarray/utils/progress_bar.py @@ -12,7 +12,7 @@ ) -class QPSColumn(TextColumn): +class _QPSColumn(TextColumn): def render(self, task) -> Text: if task.speed: _text = f'{task.speed:.0f} QPS' @@ -34,7 +34,7 @@ def _get_pbar(disable: bool, total: Optional[int] = None): BarColumn(), MofNCompleteColumn(), '•', - QPSColumn('{task.speed} QPS', justify='right', style='progress.data.speed'), + _QPSColumn('{task.speed} QPS', justify='right', style='progress.data.speed'), '•', TimeRemainingColumn() if total else TimeElapsedColumn(), '•', diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integrations/store/test_file.py b/tests/integrations/store/test_file.py index 43205260abd..0019f6970c0 100644 --- a/tests/integrations/store/test_file.py +++ b/tests/integrations/store/test_file.py @@ -6,7 +6,7 @@ from docarray import DocArray from docarray.documents import TextDoc from docarray.store.file import ConcurrentPushException, FileDocStore -from docarray.utils.cache import get_cache_path +from docarray.utils.cache import _get_cache_path from tests.integrations.store import gen_text_docs, get_test_da, profile_memory DA_LEN: int = 2**10 @@ -14,7 +14,7 @@ def test_path_resolution(): - assert FileDocStore._abs_filepath('meow') == get_cache_path() / 'meow' + assert FileDocStore._abs_filepath('meow') == _get_cache_path() / 'meow' assert FileDocStore._abs_filepath('/meow') == Path('/meow') assert FileDocStore._abs_filepath('~/meow') == Path.home() / 'meow' assert FileDocStore._abs_filepath('./meow') == Path.cwd() / 'meow' From df3ac0223d089d4c93249c4e4b6b2ff869e0ac89 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 12:26:05 +0200 Subject: [PATCH 02/19] fix: cleanup namespace utils Signed-off-by: samsja --- docarray/array/abstract_array.py | 2 +- docarray/array/array/io.py | 8 ++++---- docarray/array/array/sequence_indexing_mixin.py | 2 +- docarray/array/stacked/array_stacked.py | 4 ++-- docarray/base_doc/mixins/io.py | 4 ++-- docarray/data/torch_dataset.py | 2 +- docarray/documents/audio.py | 2 +- docarray/documents/image.py | 2 +- docarray/documents/point_cloud/point_cloud_3d.py | 2 +- docarray/documents/point_cloud/points_and_colors.py | 2 +- docarray/documents/video.py | 2 +- docarray/index/abstract.py | 4 ++-- docarray/index/backends/hnswlib.py | 2 +- docarray/store/file.py | 2 +- docarray/store/helpers.py | 2 +- docarray/store/jac.py | 2 +- docarray/store/s3.py | 2 +- docarray/typing/__init__.py | 2 +- docarray/typing/tensor/__init__.py | 2 +- docarray/typing/tensor/audio/__init__.py | 2 +- docarray/typing/tensor/audio/abstract_audio_tensor.py | 2 +- docarray/typing/tensor/audio/audio_tensor.py | 2 +- docarray/typing/tensor/embedding/__init__.py | 2 +- docarray/typing/tensor/embedding/embedding.py | 2 +- docarray/typing/tensor/image/__init__.py | 2 +- docarray/typing/tensor/image/abstract_image_tensor.py | 2 +- docarray/typing/tensor/image/image_tensor.py | 2 +- docarray/typing/tensor/tensor.py | 2 +- docarray/typing/tensor/video/__init__.py | 2 +- docarray/typing/tensor/video/video_tensor.py | 2 +- docarray/typing/tensor/video/video_tensor_mixin.py | 2 +- docarray/typing/url/audio_url.py | 2 +- docarray/typing/url/image_url.py | 2 +- docarray/typing/url/video_url.py | 2 +- docarray/utils/__init__.py | 5 ----- docarray/utils/{query_language => _internal}/__init__.py | 0 docarray/utils/{ => _internal}/_typing.py | 0 docarray/utils/{ => _internal}/cache.py | 0 docarray/utils/{ => _internal}/compress.py | 2 +- docarray/utils/{ => _internal}/misc.py | 0 docarray/utils/{ => _internal}/progress_bar.py | 0 docarray/utils/_internal/query_language/__init__.py | 0 docarray/utils/{ => _internal}/query_language/lookup.py | 0 .../utils/{ => _internal}/query_language/query_parser.py | 2 +- docarray/utils/filter.py | 2 +- tests/integrations/document/test_proto.py | 2 +- tests/integrations/predefined_document/test_audio.py | 2 +- tests/integrations/predefined_document/test_image.py | 2 +- .../integrations/predefined_document/test_point_cloud.py | 2 +- tests/integrations/predefined_document/test_video.py | 2 +- tests/integrations/store/test_file.py | 2 +- tests/integrations/typing/test_tensor.py | 2 +- tests/integrations/typing/test_tensorflow_tensor.py | 2 +- tests/integrations/typing/test_tensors_interop.py | 2 +- tests/units/array/stack/test_array_stacked_tf.py | 2 +- tests/units/array/test_array.py | 2 +- .../tensorflow_backend/test_basics.py | 2 +- .../tensorflow_backend/test_metrics.py | 2 +- .../tensorflow_backend/test_retrieval.py | 2 +- tests/units/document/proto/test_document_proto.py | 2 +- tests/units/typing/tensor/test_audio_tensor.py | 2 +- tests/units/typing/tensor/test_tensor_flow_tensor.py | 2 +- tests/units/typing/tensor/test_video_tensor.py | 2 +- tests/units/typing/url/test_audio_url.py | 2 +- tests/units/typing/url/test_video_url.py | 2 +- tests/units/util/query_language/test_lookup.py | 2 +- tests/units/util/test_typing.py | 4 ++-- 67 files changed, 66 insertions(+), 71 deletions(-) rename docarray/utils/{query_language => _internal}/__init__.py (100%) rename docarray/utils/{ => _internal}/_typing.py (100%) rename docarray/utils/{ => _internal}/cache.py (100%) rename docarray/utils/{ => _internal}/compress.py (97%) rename docarray/utils/{ => _internal}/misc.py (100%) rename docarray/utils/{ => _internal}/progress_bar.py (100%) create mode 100644 docarray/utils/_internal/query_language/__init__.py rename docarray/utils/{ => _internal}/query_language/lookup.py (100%) rename docarray/utils/{ => _internal}/query_language/query_parser.py (98%) diff --git a/docarray/array/abstract_array.py b/docarray/array/abstract_array.py index 762f37cfb42..0669a927451 100644 --- a/docarray/array/abstract_array.py +++ b/docarray/array/abstract_array.py @@ -22,7 +22,7 @@ from docarray.base_doc import BaseDoc from docarray.display.document_array_summary import DocArraySummary from docarray.typing.abstract_type import AbstractType -from docarray.utils._typing import change_cls_name +from docarray.utils._internal._typing import change_cls_name if TYPE_CHECKING: from docarray.proto import DocArrayProto, NodeProto diff --git a/docarray/array/array/io.py b/docarray/array/array/io.py index 8659af08588..1d8de498638 100644 --- a/docarray/array/array/io.py +++ b/docarray/array/array/io.py @@ -31,7 +31,7 @@ _all_access_paths_valid, _dict_to_access_paths, ) -from docarray.utils.compress import _decompress_bytes, _get_compress_ctx +from docarray.utils._internal.compress import _decompress_bytes, _get_compress_ctx if TYPE_CHECKING: import pandas as pd @@ -201,7 +201,7 @@ def to_binary_stream( from rich import filesize if show_progress: - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar pbar, t = _get_progressbar( 'Serializing', disable=not show_progress, total=len(self) @@ -564,7 +564,7 @@ def _load_binary_all( else: from rich import filesize - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar # 1 byte (uint8) # 8 bytes (uint64) @@ -629,7 +629,7 @@ def _load_binary_stream( num_docs = int.from_bytes(version_numdocs_lendoc0[1:9], 'big', signed=False) if show_progress: - from docarray.utils.progress_bar import _get_progressbar + from docarray.utils._internal.progress_bar import _get_progressbar pbar, t = _get_progressbar( 'Deserializing', disable=not show_progress, total=num_docs diff --git a/docarray/array/array/sequence_indexing_mixin.py b/docarray/array/array/sequence_indexing_mixin.py index cc12b6a9a9f..01db74c0218 100644 --- a/docarray/array/array/sequence_indexing_mixin.py +++ b/docarray/array/array/sequence_indexing_mixin.py @@ -14,7 +14,7 @@ import numpy as np -from docarray.utils.misc import is_torch_available +from docarray.utils._internal.misc import is_torch_available T_item = TypeVar('T_item') T = TypeVar('T', bound='IndexingSequenceMixin') diff --git a/docarray/array/stacked/array_stacked.py b/docarray/array/stacked/array_stacked.py index d2630a4b655..1445b97379e 100644 --- a/docarray/array/stacked/array_stacked.py +++ b/docarray/array/stacked/array_stacked.py @@ -26,8 +26,8 @@ from docarray.base_doc.mixins.io import _type_to_protobuf from docarray.typing import NdArray from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._typing import is_tensor_union -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal._typing import is_tensor_union +from docarray.utils._internal.misc import is_tf_available, is_torch_available if TYPE_CHECKING: from pydantic.fields import ModelField diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py index 13f723df3a3..e0fde4b5d4f 100644 --- a/docarray/base_doc/mixins/io.py +++ b/docarray/base_doc/mixins/io.py @@ -20,8 +20,8 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing import NdArray from docarray.typing.proto_register import _PROTO_TYPE_NAME_TO_CLASS -from docarray.utils.compress import _compress_bytes, _decompress_bytes -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.compress import _compress_bytes, _decompress_bytes +from docarray.utils._internal.misc import is_tf_available, is_torch_available tf_available = is_tf_available() if tf_available: diff --git a/docarray/data/torch_dataset.py b/docarray/data/torch_dataset.py index 3b5cb0f87a9..dd58035cd33 100644 --- a/docarray/data/torch_dataset.py +++ b/docarray/data/torch_dataset.py @@ -4,7 +4,7 @@ from docarray import BaseDoc, DocArray, DocArrayStacked from docarray.typing import TorchTensor -from docarray.utils._typing import change_cls_name +from docarray.utils._internal._typing import change_cls_name T_doc = TypeVar('T_doc', bound=BaseDoc) diff --git a/docarray/documents/audio.py b/docarray/documents/audio.py index ab0bbe00b34..e27abcb5408 100644 --- a/docarray/documents/audio.py +++ b/docarray/documents/audio.py @@ -7,7 +7,7 @@ from docarray.typing.bytes.audio_bytes import AudioBytes from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/image.py b/docarray/documents/image.py index f2375637de6..bf281f83d8f 100644 --- a/docarray/documents/image.py +++ b/docarray/documents/image.py @@ -6,7 +6,7 @@ from docarray.typing import AnyEmbedding, ImageBytes, ImageUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.image.image_tensor import ImageTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available T = TypeVar('T', bound='ImageDoc') diff --git a/docarray/documents/point_cloud/point_cloud_3d.py b/docarray/documents/point_cloud/point_cloud_3d.py index 3cee613f596..25e0e6437d1 100644 --- a/docarray/documents/point_cloud/point_cloud_3d.py +++ b/docarray/documents/point_cloud/point_cloud_3d.py @@ -6,7 +6,7 @@ from docarray.documents.point_cloud.points_and_colors import PointsAndColors from docarray.typing import AnyEmbedding, PointCloud3DUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index af4917bf5a7..82ce82f9ba1 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -5,7 +5,7 @@ from docarray.base_doc import BaseDoc from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/documents/video.py b/docarray/documents/video.py index 3770ee245fc..679f10ea8ba 100644 --- a/docarray/documents/video.py +++ b/docarray/documents/video.py @@ -8,7 +8,7 @@ from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.video.video_tensor import VideoTensor from docarray.typing.url.video_url import VideoUrl -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 3eac3d7f869..c587846c8c4 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -26,9 +26,9 @@ from docarray import BaseDoc, DocArray from docarray.array.abstract_array import AnyDocArray from docarray.typing import AnyTensor -from docarray.utils._typing import unwrap_optional_type +from docarray.utils._internal._typing import unwrap_optional_type +from docarray.utils._internal.misc import is_tf_available, torch_imported from docarray.utils.find import FindResult, _FindResult -from docarray.utils.misc import is_tf_available, torch_imported if TYPE_CHECKING: from pydantic.fields import ModelField diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 38b827539ab..ab8aa6e56c1 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -29,9 +29,9 @@ _raise_not_supported, ) from docarray.proto import DocumentProto +from docarray.utils._internal.misc import is_np_int, is_tf_available, is_torch_available from docarray.utils.filter import filter_docs from docarray.utils.find import _FindResult -from docarray.utils.misc import is_np_int, is_tf_available, is_torch_available TSchema = TypeVar('TSchema', bound=BaseDoc) T = TypeVar('T', bound='HnswDocumentIndex') diff --git a/docarray/store/file.py b/docarray/store/file.py index e88e87b23f8..bb79162109b 100644 --- a/docarray/store/file.py +++ b/docarray/store/file.py @@ -7,7 +7,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.exceptions import ConcurrentPushException from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import _get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: from docarray import BaseDoc, DocArray diff --git a/docarray/store/helpers.py b/docarray/store/helpers.py index 25e40991e68..457d160d41c 100644 --- a/docarray/store/helpers.py +++ b/docarray/store/helpers.py @@ -6,7 +6,7 @@ from rich import filesize from typing_extensions import TYPE_CHECKING, Protocol -from docarray.utils.progress_bar import _get_progressbar +from docarray.utils._internal.progress_bar import _get_progressbar if TYPE_CHECKING: from pathlib import Path diff --git a/docarray/store/jac.py b/docarray/store/jac.py index 969df0a6233..c9350fae18a 100644 --- a/docarray/store/jac.py +++ b/docarray/store/jac.py @@ -24,7 +24,7 @@ get_version_info, raise_req_error, ) -from docarray.utils.cache import _get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover import io diff --git a/docarray/store/s3.py b/docarray/store/s3.py index 8c5604b6d35..7711432900c 100644 --- a/docarray/store/s3.py +++ b/docarray/store/s3.py @@ -10,7 +10,7 @@ from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.helpers import _from_binary_stream, _to_binary_stream -from docarray.utils.cache import _get_cache_path +from docarray.utils._internal.cache import _get_cache_path if TYPE_CHECKING: # pragma: no cover from docarray import BaseDoc, DocArray diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index cd48ff7c26b..daf89af9fe8 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -38,7 +38,7 @@ 'ImageNdArray', ] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 07a27292aee..f2be5ecc0ac 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -13,7 +13,7 @@ 'TensorFlowTensor', ] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index 4b1b7edc34d..4f730451603 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -2,7 +2,7 @@ __all__ = ['AudioNdArray'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/audio/abstract_audio_tensor.py b/docarray/typing/tensor/audio/abstract_audio_tensor.py index 7e677bcf952..e84b4ebc13e 100644 --- a/docarray/typing/tensor/audio/abstract_audio_tensor.py +++ b/docarray/typing/tensor/audio/abstract_audio_tensor.py @@ -3,7 +3,7 @@ from typing import Any, BinaryIO, Dict, TypeVar, Union from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook T = TypeVar('T', bound='AbstractAudioTensor') diff --git a/docarray/typing/tensor/audio/audio_tensor.py b/docarray/typing/tensor/audio/audio_tensor.py index 188ccba26a5..7d1f4a9e315 100644 --- a/docarray/typing/tensor/audio/audio_tensor.py +++ b/docarray/typing/tensor/audio/audio_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index 6cc8b1d6008..f844b7abaa9 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -3,7 +3,7 @@ __all__ = ['NdArrayEmbedding', 'AnyEmbedding'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/embedding/embedding.py b/docarray/typing/tensor/embedding/embedding.py index a8bb78a33d2..2eb1fa7e842 100644 --- a/docarray/typing/tensor/embedding/embedding.py +++ b/docarray/typing/tensor/embedding/embedding.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.embedding.ndarray import NdArrayEmbedding -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 03bd92a5342..2fb32d452ab 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -3,7 +3,7 @@ __all__ = ['ImageNdArray', 'ImageTensor'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/image/abstract_image_tensor.py b/docarray/typing/tensor/image/abstract_image_tensor.py index 97b8fd8e23a..b02d1452f63 100644 --- a/docarray/typing/tensor/image/abstract_image_tensor.py +++ b/docarray/typing/tensor/image/abstract_image_tensor.py @@ -3,7 +3,7 @@ from abc import ABC from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook class AbstractImageTensor(AbstractTensor, ABC): diff --git a/docarray/typing/tensor/image/image_tensor.py b/docarray/typing/tensor/image/image_tensor.py index 747b080b455..af439ce607e 100644 --- a/docarray/typing/tensor/image/image_tensor.py +++ b/docarray/typing/tensor/image/image_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.image.image_ndarray import ImageNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/tensor.py b/docarray/typing/tensor/tensor.py index 6cff0b99e26..19a89cbed3f 100644 --- a/docarray/typing/tensor/tensor.py +++ b/docarray/typing/tensor/tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.ndarray import NdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index add0c958d86..424758cfc91 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -2,7 +2,7 @@ __all__ = ['VideoNdArray'] -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/video/video_tensor.py b/docarray/typing/tensor/video/video_tensor.py index 974f409fcb3..fa4fbb47d6b 100644 --- a/docarray/typing/tensor/video/video_tensor.py +++ b/docarray/typing/tensor/video/video_tensor.py @@ -1,7 +1,7 @@ from typing import Union from docarray.typing.tensor.video.video_ndarray import VideoNdArray -from docarray.utils.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: diff --git a/docarray/typing/tensor/video/video_tensor_mixin.py b/docarray/typing/tensor/video/video_tensor_mixin.py index cfa5a46ea70..5dd11fb2da1 100644 --- a/docarray/typing/tensor/video/video_tensor_mixin.py +++ b/docarray/typing/tensor/video/video_tensor_mixin.py @@ -7,7 +7,7 @@ from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook T = TypeVar('T', bound='VideoTensorMixin') diff --git a/docarray/typing/url/audio_url.py b/docarray/typing/url/audio_url.py index c971f6d364f..dbfdd84029b 100644 --- a/docarray/typing/url/audio_url.py +++ b/docarray/typing/url/audio_url.py @@ -7,7 +7,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl from docarray.typing.url.filetypes import AUDIO_FILE_FORMATS -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig diff --git a/docarray/typing/url/image_url.py b/docarray/typing/url/image_url.py index b5c40b71e2d..25f07ed4707 100644 --- a/docarray/typing/url/image_url.py +++ b/docarray/typing/url/image_url.py @@ -5,7 +5,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig diff --git a/docarray/typing/url/video_url.py b/docarray/typing/url/video_url.py index 44c2d33f9b2..1ff5e96c9c4 100644 --- a/docarray/typing/url/video_url.py +++ b/docarray/typing/url/video_url.py @@ -4,7 +4,7 @@ from docarray.typing.bytes.video_bytes import VideoLoadResult from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils.misc import is_notebook +from docarray.utils._internal.misc import is_notebook if TYPE_CHECKING: from pydantic import BaseConfig diff --git a/docarray/utils/__init__.py b/docarray/utils/__init__.py index a1a68cf806e..e69de29bb2d 100644 --- a/docarray/utils/__init__.py +++ b/docarray/utils/__init__.py @@ -1,5 +0,0 @@ -from docarray.utils.filter import filter_docs -from docarray.utils.find import find, find_batched -from docarray.utils.map import map_docs, map_docs_batch - -__all__ = ['filter_docs', 'find', 'find_batched', 'map_docs', 'map_docs_batch'] diff --git a/docarray/utils/query_language/__init__.py b/docarray/utils/_internal/__init__.py similarity index 100% rename from docarray/utils/query_language/__init__.py rename to docarray/utils/_internal/__init__.py diff --git a/docarray/utils/_typing.py b/docarray/utils/_internal/_typing.py similarity index 100% rename from docarray/utils/_typing.py rename to docarray/utils/_internal/_typing.py diff --git a/docarray/utils/cache.py b/docarray/utils/_internal/cache.py similarity index 100% rename from docarray/utils/cache.py rename to docarray/utils/_internal/cache.py diff --git a/docarray/utils/compress.py b/docarray/utils/_internal/compress.py similarity index 97% rename from docarray/utils/compress.py rename to docarray/utils/_internal/compress.py index d427e998c89..0ff00f8eecd 100644 --- a/docarray/utils/compress.py +++ b/docarray/utils/_internal/compress.py @@ -1,4 +1,4 @@ -from typing import Optional, Callable, IO +from typing import IO, Callable, Optional def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: diff --git a/docarray/utils/misc.py b/docarray/utils/_internal/misc.py similarity index 100% rename from docarray/utils/misc.py rename to docarray/utils/_internal/misc.py diff --git a/docarray/utils/progress_bar.py b/docarray/utils/_internal/progress_bar.py similarity index 100% rename from docarray/utils/progress_bar.py rename to docarray/utils/_internal/progress_bar.py diff --git a/docarray/utils/_internal/query_language/__init__.py b/docarray/utils/_internal/query_language/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docarray/utils/query_language/lookup.py b/docarray/utils/_internal/query_language/lookup.py similarity index 100% rename from docarray/utils/query_language/lookup.py rename to docarray/utils/_internal/query_language/lookup.py diff --git a/docarray/utils/query_language/query_parser.py b/docarray/utils/_internal/query_language/query_parser.py similarity index 98% rename from docarray/utils/query_language/query_parser.py rename to docarray/utils/_internal/query_language/query_parser.py index 8bce97f6e0d..b8381584a94 100644 --- a/docarray/utils/query_language/query_parser.py +++ b/docarray/utils/_internal/query_language/query_parser.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional, Union -from docarray.utils.query_language.lookup import ( +from docarray.utils._internal.query_language.lookup import ( LookupLeaf, LookupNode, LookupTreeElem, diff --git a/docarray/utils/filter.py b/docarray/utils/filter.py index 7a9887364fc..a68641a7b13 100644 --- a/docarray/utils/filter.py +++ b/docarray/utils/filter.py @@ -63,7 +63,7 @@ class MyDocument(BaseDoc): :return: A DocArray containing the Documents in `docs` that fulfill the filter conditions in the `query` """ - from docarray.utils.query_language.query_parser import QueryParser + from docarray.utils._internal.query_language.query_parser import QueryParser if query: query = query if not isinstance(query, str) else json.loads(query) diff --git a/tests/integrations/document/test_proto.py b/tests/integrations/document/test_proto.py index c1f38d05b41..02b9aae3f00 100644 --- a/tests/integrations/document/test_proto.py +++ b/tests/integrations/document/test_proto.py @@ -18,7 +18,7 @@ TorchTensor, ) from docarray.typing.tensor import NdArrayEmbedding -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/predefined_document/test_audio.py b/tests/integrations/predefined_document/test_audio.py index c25aab9b1ab..2ba207245f7 100644 --- a/tests/integrations/predefined_document/test_audio.py +++ b/tests/integrations/predefined_document/test_audio.py @@ -10,7 +10,7 @@ from docarray.documents import AudioDoc from docarray.typing import AudioUrl from docarray.typing.tensor.audio import AudioNdArray, AudioTorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/predefined_document/test_image.py b/tests/integrations/predefined_document/test_image.py index 92e19d09a14..f77e945d714 100644 --- a/tests/integrations/predefined_document/test_image.py +++ b/tests/integrations/predefined_document/test_image.py @@ -5,7 +5,7 @@ from docarray import BaseDoc from docarray.documents import ImageDoc -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/predefined_document/test_point_cloud.py b/tests/integrations/predefined_document/test_point_cloud.py index 76b7e0236d8..b8a75914f26 100644 --- a/tests/integrations/predefined_document/test_point_cloud.py +++ b/tests/integrations/predefined_document/test_point_cloud.py @@ -5,7 +5,7 @@ from docarray import BaseDoc from docarray.documents import PointCloud3D -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/predefined_document/test_video.py b/tests/integrations/predefined_document/test_video.py index e208e3890c9..606dad6a6c1 100644 --- a/tests/integrations/predefined_document/test_video.py +++ b/tests/integrations/predefined_document/test_video.py @@ -6,7 +6,7 @@ from docarray import BaseDoc from docarray.documents import VideoDoc from docarray.typing import AudioNdArray, NdArray, VideoNdArray -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/integrations/store/test_file.py b/tests/integrations/store/test_file.py index 0019f6970c0..4b6a72c5b62 100644 --- a/tests/integrations/store/test_file.py +++ b/tests/integrations/store/test_file.py @@ -6,7 +6,7 @@ from docarray import DocArray from docarray.documents import TextDoc from docarray.store.file import ConcurrentPushException, FileDocStore -from docarray.utils.cache import _get_cache_path +from docarray.utils._internal.cache import _get_cache_path from tests.integrations.store import gen_text_docs, get_test_da, profile_memory DA_LEN: int = 2**10 diff --git a/tests/integrations/typing/test_tensor.py b/tests/integrations/typing/test_tensor.py index ba15e2d5c94..6b32432d971 100644 --- a/tests/integrations/typing/test_tensor.py +++ b/tests/integrations/typing/test_tensor.py @@ -4,7 +4,7 @@ from docarray import BaseDoc from docarray.typing import AnyTensor, NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/typing/test_tensorflow_tensor.py b/tests/integrations/typing/test_tensorflow_tensor.py index 84505968090..edd64e3dcbb 100644 --- a/tests/integrations/typing/test_tensorflow_tensor.py +++ b/tests/integrations/typing/test_tensorflow_tensor.py @@ -1,7 +1,7 @@ import pytest from docarray import BaseDoc -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/integrations/typing/test_tensors_interop.py b/tests/integrations/typing/test_tensors_interop.py index d59751f21af..47023dca96a 100644 --- a/tests/integrations/typing/test_tensors_interop.py +++ b/tests/integrations/typing/test_tensors_interop.py @@ -8,7 +8,7 @@ NdArrayEmbedding, TorchEmbedding, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/array/stack/test_array_stacked_tf.py b/tests/units/array/stack/test_array_stacked_tf.py index e82bfc7716a..0ec91268575 100644 --- a/tests/units/array/stack/test_array_stacked_tf.py +++ b/tests/units/array/stack/test_array_stacked_tf.py @@ -5,7 +5,7 @@ from docarray import BaseDoc, DocArray from docarray.array import DocArrayStacked from docarray.typing import AnyTensor, NdArray -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/array/test_array.py b/tests/units/array/test_array.py index bab9ccd1313..be4fa6fa505 100644 --- a/tests/units/array/test_array.py +++ b/tests/units/array/test_array.py @@ -6,7 +6,7 @@ from docarray import BaseDoc, DocArray from docarray.typing import ImageUrl, NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_basics.py b/tests/units/computation_backends/tensorflow_backend/test_basics.py index 6eed1a58cf0..ae5f9b44264 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_basics.py +++ b/tests/units/computation_backends/tensorflow_backend/test_basics.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_metrics.py b/tests/units/computation_backends/tensorflow_backend/test_metrics.py index 863dd18fe4c..196297adf0b 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_metrics.py +++ b/tests/units/computation_backends/tensorflow_backend/test_metrics.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/computation_backends/tensorflow_backend/test_retrieval.py b/tests/units/computation_backends/tensorflow_backend/test_retrieval.py index feac992d760..283d7f8b44a 100644 --- a/tests/units/computation_backends/tensorflow_backend/test_retrieval.py +++ b/tests/units/computation_backends/tensorflow_backend/test_retrieval.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/document/proto/test_document_proto.py b/tests/units/document/proto/test_document_proto.py index e6e8a58fa99..1642c17631d 100644 --- a/tests/units/document/proto/test_document_proto.py +++ b/tests/units/document/proto/test_document_proto.py @@ -7,7 +7,7 @@ from docarray import DocArray from docarray.base_doc import BaseDoc from docarray.typing import NdArray, TorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available if is_tf_available(): import tensorflow as tf diff --git a/tests/units/typing/tensor/test_audio_tensor.py b/tests/units/typing/tensor/test_audio_tensor.py index 3bc20a1c7f6..4a145f3c4b7 100644 --- a/tests/units/typing/tensor/test_audio_tensor.py +++ b/tests/units/typing/tensor/test_audio_tensor.py @@ -8,7 +8,7 @@ from docarray import BaseDoc from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/tensor/test_tensor_flow_tensor.py b/tests/units/typing/tensor/test_tensor_flow_tensor.py index 5c7d942a02d..3a51f5a95aa 100644 --- a/tests/units/typing/tensor/test_tensor_flow_tensor.py +++ b/tests/units/typing/tensor/test_tensor_flow_tensor.py @@ -4,7 +4,7 @@ from pydantic.tools import parse_obj_as from docarray.base_doc.io.json import orjson_dumps -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/tensor/test_video_tensor.py b/tests/units/typing/tensor/test_video_tensor.py index f9981dedbc9..23ae8115e64 100644 --- a/tests/units/typing/tensor/test_video_tensor.py +++ b/tests/units/typing/tensor/test_video_tensor.py @@ -12,7 +12,7 @@ VideoNdArray, VideoTorchTensor, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: diff --git a/tests/units/typing/url/test_audio_url.py b/tests/units/typing/url/test_audio_url.py index 6b876d81de7..9168302aada 100644 --- a/tests/units/typing/url/test_audio_url.py +++ b/tests/units/typing/url/test_audio_url.py @@ -8,7 +8,7 @@ from docarray import BaseDoc from docarray.base_doc.io.json import orjson_dumps from docarray.typing import AudioTorchTensor, AudioUrl -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/units/typing/url/test_video_url.py b/tests/units/typing/url/test_video_url.py index a1401bde896..e0dba53b6fe 100644 --- a/tests/units/typing/url/test_video_url.py +++ b/tests/units/typing/url/test_video_url.py @@ -14,7 +14,7 @@ VideoTorchTensor, VideoUrl, ) -from docarray.utils.misc import is_tf_available +from docarray.utils._internal.misc import is_tf_available from tests import TOYDATA_DIR tf_available = is_tf_available() diff --git a/tests/units/util/query_language/test_lookup.py b/tests/units/util/query_language/test_lookup.py index 37b3c2a1a4c..dd30f51c8f0 100644 --- a/tests/units/util/query_language/test_lookup.py +++ b/tests/units/util/query_language/test_lookup.py @@ -1,6 +1,6 @@ import pytest -from docarray.utils.query_language.lookup import dunder_get, lookup +from docarray.utils._internal.query_language.lookup import dunder_get, lookup class A: diff --git a/tests/units/util/test_typing.py b/tests/units/util/test_typing.py index 2db742522f5..5446cf3ce04 100644 --- a/tests/units/util/test_typing.py +++ b/tests/units/util/test_typing.py @@ -4,8 +4,8 @@ from docarray.typing import NdArray, TorchTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._typing import is_tensor_union, is_type_tensor -from docarray.utils.misc import is_tf_available +from docarray.utils._internal._typing import is_tensor_union, is_type_tensor +from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() if tf_available: From 090091f978ca7e7da3e6ee1ad242ab4adcbc3895 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 13:08:47 +0200 Subject: [PATCH 03/19] feat: add docstring test Signed-off-by: samsja --- docarray/typing/__init__.py | 3 +- tests/documentation/test_docstring.py | 48 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index daf89af9fe8..3deefe96f01 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -31,7 +31,6 @@ 'AnyUrl', 'ID', 'AnyTensor', - 'TensorFlowTensor', 'NdArrayEmbedding', 'ImageBytes', 'ImageTensor', @@ -59,7 +58,7 @@ tf_available = is_tf_available() if tf_available: - from docarray.typing.tensor import TensorFlowTensor + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py index e69de29bb2d..c6a8e3d9a40 100644 --- a/tests/documentation/test_docstring.py +++ b/tests/documentation/test_docstring.py @@ -0,0 +1,48 @@ +""" +this test check the docstring of all of our public API. It does it +by checking the `__all__` of each of our namespace. + +to add a new namespace you need to +* import it +* add it to the `SUB_MODULE_TO_CHECK` list +""" + +import pytest +from mktestdocs import check_docstring, get_codeblock_members + +import docarray.data +import docarray.documents +import docarray.index +import docarray.store +import docarray.typing + +SUB_MODULE_TO_CHECK = [ + docarray, + docarray.index, + docarray.data, + docarray.documents, + docarray.store, + docarray.typing, +] + + +def get_obj_to_check(lib): + obj_to_check = [] + for obj in lib.__all__: + obj_to_check.append(getattr(lib, obj)) + return obj_to_check + + +obj_to_check = [] + +for lib in SUB_MODULE_TO_CHECK: + obj_to_check.extend(get_obj_to_check(lib)) + +members = [] +for obj in obj_to_check: + members.extend(get_codeblock_members(obj)) + + +@pytest.mark.parametrize("obj", members, ids=lambda d: d.__qualname__) +def test_member(obj): + check_docstring(obj) From 9dd34496012a770c19d55c95d5a9777ff44b952c Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 13:49:34 +0200 Subject: [PATCH 04/19] fix: fix video url docstring Signed-off-by: samsja --- docarray/typing/url/video_url.py | 70 +++++++++++++++++--------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/docarray/typing/url/video_url.py b/docarray/typing/url/video_url.py index 1ff5e96c9c4..e6b35642f5e 100644 --- a/docarray/typing/url/video_url.py +++ b/docarray/typing/url/video_url.py @@ -43,57 +43,61 @@ def load(self: T, **kwargs) -> VideoLoadResult: Load the data from the url into a named Tuple of VideoNdArray, AudioNdArray and NdArray. - :param kwargs: supports all keyword arguments that are being supported by - av.open() as described in: - https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open + --- - :return: AudioNdArray representing the audio content, VideoNdArray representing - the images of the video, NdArray of the key frame indices. + ```python + from typing import Optional + from docarray import BaseDoc - EXAMPLE USAGE + from docarray.typing import VideoUrl, VideoNdArray, AudioNdArray, NdArray - .. code-block:: python - from typing import Optional + class MyDoc(BaseDoc): + video_url: VideoUrl + video: Optional[VideoNdArray] + audio: Optional[AudioNdArray] + key_frame_indices: Optional[NdArray] - from docarray import BaseDoc - from docarray.typing import VideoUrl, VideoNdArray, AudioNdArray, NdArray + doc = MyDoc( + video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' + ) + doc.video, doc.audio, doc.key_frame_indices = doc.video_url.load() + assert isinstance(doc.video, VideoNdArray) + assert isinstance(doc.audio, AudioNdArray) + assert isinstance(doc.key_frame_indices, NdArray) + ``` - class MyDoc(BaseDoc): - video_url: VideoUrl - video: Optional[VideoNdArray] - audio: Optional[AudioNdArray] - key_frame_indices: Optional[NdArray] + --- + You can load only the key frames (or video, audio respectively): - doc = MyDoc( - video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' - ) - doc.video, doc.audio, doc.key_frame_indices = doc.video_url.load() + --- - assert isinstance(doc.video, VideoNdArray) - assert isinstance(doc.audio, AudioNdArray) - assert isinstance(doc.key_frame_indices, NdArray) + ```python + from pydantic import parse_obj_as - You can load only the key frames (or video, audio respectively): + from docarray.typing import NdArray, VideoUrl - .. code-block:: python - from pydantic import parse_obj_as + url = parse_obj_as( + VideoUrl, + 'https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', + ) + key_frame_indices = url.load().key_frame_indices + assert isinstance(key_frame_indices, NdArray) + ``` - from docarray.typing import NdArray, VideoUrl + --- + :param kwargs: supports all keyword arguments that are being supported by + av.open() as described in: + https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open - url = parse_obj_as( - VideoUrl, - 'https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', - ) - key_frame_indices = url.load().key_frame_indices - assert isinstance(key_frame_indices, NdArray) - + :return: AudioNdArray representing the audio content, VideoNdArray representing + the images of the video, NdArray of the key frame indices. """ from docarray.typing.bytes.video_bytes import VideoBytes From f18ffd18b377be4c459eb339f94e727d733c6cd8 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 14:38:47 +0200 Subject: [PATCH 05/19] fix: fix text url Signed-off-by: samsja --- docarray/typing/url/text_url.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/docarray/typing/url/text_url.py b/docarray/typing/url/text_url.py index 049de511d1f..1ebfad1c511 100644 --- a/docarray/typing/url/text_url.py +++ b/docarray/typing/url/text_url.py @@ -43,31 +43,26 @@ def load(self, charset: str = 'utf-8', timeout: Optional[float] = None) -> str: """ Load the text file into a string. - EXAMPLE USAGE - .. code-block:: python + --- - from docarray import BaseDoc - from docarray.typing import TextUrl + ```python + from docarray import BaseDoc + from docarray.typing import TextUrl - class MyDoc(BaseDoc): - remote_url: TextUrl - local_url: TextUrl + class MyDoc(BaseDoc): + remote_url: TextUrl - doc = MyDoc( - remote_url='https://de.wikipedia.org/wiki/Brixen', - local_url='home/username/my_file.txt', - ) + doc = MyDoc( + remote_url='https://de.wikipedia.org/wiki/Brixen', + ) - remote_txt = doc.remote_url.load() - print(remote_txt) - # prints: ```\n ...``` + remote_txt = doc.remote_url.load() + ``` - local_txt = doc.local_url.load() - print(local_txt) - # prints content of my_file.txt + --- :param timeout: timeout (sec) for urlopen network request. From 0f26a96b7f16df1c207ff68f963ceeed66e8f558 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 14:56:43 +0200 Subject: [PATCH 06/19] fix: fix image url Signed-off-by: samsja --- docarray/typing/url/image_url.py | 39 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/docarray/typing/url/image_url.py b/docarray/typing/url/image_url.py index 25f07ed4707..4c25e68094e 100644 --- a/docarray/typing/url/image_url.py +++ b/docarray/typing/url/image_url.py @@ -49,34 +49,35 @@ def load( """ Load the data from the url into a numpy.ndarray image tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import ImageUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import ImageUrl - import numpy as np + class MyDoc(BaseDoc): + img_url: ImageUrl - class MyDoc(BaseDoc): - img_url: ImageUrl + doc = MyDoc( + img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" + ) - doc = MyDoc( - img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" - ) - - img_tensor = doc.img_url.load() - assert isinstance(img_tensor, np.ndarray) + img_tensor = doc.img_url.load() + assert isinstance(img_tensor, np.ndarray) - img_tensor = doc.img_url.load(height=224, width=224) - assert img_tensor.shape == (224, 224, 3) + img_tensor = doc.img_url.load(height=224, width=224) + assert img_tensor.shape == (224, 224, 3) - layout = ('C', 'W', 'H') - img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) - assert img_tensor.shape == (3, 200, 100) + layout = ('C', 'W', 'H') + img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) + assert img_tensor.shape == (3, 200, 100) + ``` + --- :param width: width of the image tensor. :param height: height of the image tensor. From 01197e6ff57c183715ffb3effe0f0365ba40cc36 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 15:11:33 +0200 Subject: [PATCH 07/19] fix: fic audio url Signed-off-by: samsja --- docarray/typing/url/audio_url.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docarray/typing/url/audio_url.py b/docarray/typing/url/audio_url.py index dbfdd84029b..ea2a344f9dd 100644 --- a/docarray/typing/url/audio_url.py +++ b/docarray/typing/url/audio_url.py @@ -48,27 +48,30 @@ def load(self: T) -> Tuple[np.ndarray, int]: """ Load the data from the url into an AudioNdArray. - :return: AudioNdArray representing the audio file content. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional + from docarray import BaseDoc + import numpy as np - from docarray import BaseDoc - import numpy as np + from docarray.typing import AudioUrl, AudioNdArray - from docarray.typing import AudioUrl + class MyDoc(BaseDoc): + audio_url: AudioUrl + audio_tensor: Optional[AudioNdArray] - class MyDoc(Document): - audio_url: AudioUrl - audio_tensor: AudioNdArray + doc = MyDoc(audio_url="toydata/hello.wav") + # doc.audio_tensor, doc.frame_rate = doc.audio_url.load() + # assert isinstance(doc.audio_tensor, np.ndarray) + ``` - doc = MyDoc(audio_url="toydata/hello.wav") - doc.audio_tensor, doc.frame_rate = doc.audio_url.load() - assert isinstance(doc.audio_tensor, np.ndarray) + --- + :return: AudioNdArray representing the audio file content. """ bytes_ = AudioBytes(self.load_bytes()) return bytes_.load() From c81cf0dd0f5e5d0180d79c9c6cd0d391db3a08f8 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 15:28:41 +0200 Subject: [PATCH 08/19] fix: mesh 3d url Signed-off-by: samsja --- docarray/typing/url/url_3d/mesh_url.py | 37 +++++++++++++------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/docarray/typing/url/url_3d/mesh_url.py b/docarray/typing/url/url_3d/mesh_url.py index e820ba8a4d5..073f4a006a1 100644 --- a/docarray/typing/url/url_3d/mesh_url.py +++ b/docarray/typing/url/url_3d/mesh_url.py @@ -26,33 +26,34 @@ def load( trimesh_args: Optional[Dict[str, Any]] = None, ) -> 'VerticesAndFaces': """ - Load the data from the url into a VerticesAndFaces object containing - vertices and faces information. + Load the data from the url into a VerticesAndFaces object containing + vertices and faces information. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + import numpy as np - from docarray import BaseDoc - import numpy as np + from docarray.typing import Mesh3DUrl, NdArray - from docarray.typing import Mesh3DUrl, NdArray + class MyDoc(BaseDoc): + mesh_url: Mesh3DUrl - class MyDoc(BaseDoc): - mesh_url: Mesh3DUrl + doc = MyDoc(mesh_url="toydata/tetrahedron.obj") - doc = MyDoc(mesh_url="toydata/tetrahedron.obj") + tensors = doc.mesh_url.load() + assert isinstance(tensors.vertices, NdArray) + assert isinstance(tensors.faces, NdArray) + ``` - tensors = doc.mesh_url.load() - assert isinstance(tensors.vertices, NdArray) - assert isinstance(tensors.faces, NdArray) - - :param skip_materials: Skip materials if True, else skip. - :param trimesh_args: dictionary of additional arguments for `trimesh.load()` - or `trimesh.load_remote()`. - :return: VerticesAndFaces object containing vertices and faces information. + --- + :param skip_materials: Skip materials if True, else skip. + :param trimesh_args: dictionary of additional arguments for `trimesh.load()` + or `trimesh.load_remote()`. + :return: VerticesAndFaces object containing vertices and faces information. """ from docarray.documents.mesh.vertices_and_faces import VerticesAndFaces From 0db5f05f67bac894551323378cc39a06192fb3af Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 15:32:02 +0200 Subject: [PATCH 09/19] fix: mesh 3d url Signed-off-by: samsja --- docarray/typing/url/url_3d/point_cloud_url.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docarray/typing/url/url_3d/point_cloud_url.py b/docarray/typing/url/url_3d/point_cloud_url.py index 502a29c740d..eba73f5da6f 100644 --- a/docarray/typing/url/url_3d/point_cloud_url.py +++ b/docarray/typing/url/url_3d/point_cloud_url.py @@ -31,26 +31,29 @@ def load( """ Load the data from the url into an NdArray containing point cloud information. - EXAMPLE USAGE - .. code-block:: python + --- - import numpy as np - from docarray import BaseDoc + ```python + import numpy as np + from docarray import BaseDoc + + from docarray.typing import PointCloud3DUrl - from docarray.typing import PointCloud3DUrl + class MyDoc(BaseDoc): + point_cloud_url: PointCloud3DUrl - class MyDoc(BaseDoc): - point_cloud_url: PointCloud3DUrl + doc = MyDoc(point_cloud_url="toydata/tetrahedron.obj") - doc = MyDoc(point_cloud_url="toydata/tetrahedron.obj") + # point_cloud = doc.point_cloud_url.load(samples=100) - point_cloud = doc.point_cloud_url.load(samples=100) + # assert isinstance(point_cloud, np.ndarray) + # assert point_cloud.shape == (100, 3) + ``` - assert isinstance(point_cloud, np.ndarray) - assert point_cloud.shape == (100, 3) + --- :param samples: number of points to sample from the mesh :param multiple_geometries: if False, store point cloud in 2D np.ndarray. From 98e4804a38840f16cf4b27b33986ba5fa8b3db45 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 15:38:13 +0200 Subject: [PATCH 10/19] fix: remove useless data Signed-off-by: samsja --- docarray/typing/url/data/05978.jpg | Bin 22715 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docarray/typing/url/data/05978.jpg diff --git a/docarray/typing/url/data/05978.jpg b/docarray/typing/url/data/05978.jpg deleted file mode 100644 index 3f0bf32e01d4f2737c6b5898fe0203499c6a04f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22715 zcmbTdd03M9|2|67R8v;AnWZVyVoteDxzJ+rohdVO9H&LxQl`w6kSTLVp3f|0X2zJQ zDU(c@nhTN(kfJ=LsVS1FsR;t2VWlFA$rAAJd;0v&b-=%fp^N8|fQR>bzwXz4 zzuu;|rU}HllgE9JBbF>#g189(Lzo1JqlmBImrIs>xqkKK@%7R#*JWRSz4Ytl%a$+y z?{meTEArJ`X#0p2qXfrWZ8e4w*>KjJeGX5^y_8ISHR6MgFnDn2RDD| zQn=B};1@! z*>cNPE4$tH-|gA!=I((yaPZL4W8TMoPWYZYb2bnY6dV#7c`+(FCN?fU@oG|XN@`mA zpMPDuo^>Pp=AHbz1%*ZTic21rJu0uLd|XxC(AdOavYs}#w0FGd?CO5m(>plyW_V=u z?HFG&IrUCD{rd#P0L5 z=Pz^pcJ1FL#PCVB2{Dv;EpxF;gg zLS1S?{Lu~#fLkV?4qO%fh+h2>W!5~mdH2&hQ&5WoNk&qc5NqYem7FxDkzqo_W;z5O zB478~8_)OTy4=hEkncos0Mdxpuq9OY)6ExvA4}J2txbrpZz(f+5+%s7el;*`g)W?Y zHS{dq!Qa}2ry-KrD&x$1(&1EV=o`8Ikl_GF%TrdW&ui_=Kd8TV%f>}17JP6uT>{nf z)n3OxUL?$$5auF0=uX-)4m>{!#$qkQ1w19T^KWgtAJz$<E-&yW#VNe7u`o@1z9AfRKq>KoSW&|~VX|7& zlQ)ZIQVpBHiIh(Ax`-m%;*8n#;==@Nt zyK^ph$I+BN6C$SZwz~=O9~o9%QaIV~pLwQ-DYR>iW2Goh2!JeTS>R>->YIP-D1fwP zCosi$bYWW;P2F616t}AKFURd&vOBWzL7l&ApTvpRxD#Z-Bx}Rr(Ve=Kkkm5C zG$j)W9a%0nq(9Bb_-6hzj`FeOz*IIxs$&7GXs3dIYDf#vgYrfbV$FAtf7e*HleG?& z9l)yY3#A9(S&k?14!zHJ$YjaD<)0W~*9f}lfn&54T%)uU`~iqE*X{uf)}zY7nf{~N z4*fF24lSYjryOe!y5diteE&ETqJ@|;W_i-+P(i1Q|dc5lE7 z6JiY2xo%wRcP8JzvBXGl!DBGEkn%viDAX|Enkn`=(rDo6`9_@aE-*0>Fh-vGxu{Zm zbTWnPEF=F^3+7t8M;V-ccxS!W?m_GQ6lCu}6JjvFuf>>+o}h#k zJEA8_fF+|)i@|O{U_#()nie>Or(EVgLz(aC|8%Anb-txve-sIAxqtUc>o!u)bPDxJ zkzwQf`{aJDedV*%6aa2F*@SpGEWt8HuE9U_r!n`}Ap*%A>NFu%;>E#2BL!DsupOL^ z+e*?1ne;Nqok-I9={-}u2)2euC9m|)@(z_wbJ{o!&Cr!W@5F}9N7aMIQZ(+ZVWSDL z%AvC{o>$h!#m+yugLFv|>W4r4);L*e$J=APz`#bz6a7j~k=rq@Gr^u{_Lc=0Mijpl zJqs8^KLZp$_b)YsYgg*swHu$Ol)QhD>3-(4;{BhOn)H=T`F?LMs&?J^W|%o2EFfqK z0Je9439;fUZN6_aJUz#W76TP+SY~)C_e9H0^U2t$fklgG6N2;T_Dy5i@9`IqYoEV* zB$zI3NC7Mp-@86OI*TjPMe1x*;6eD>gvdXpbTQs#gGSWbATd9}gec@Jg>SpWgy{J< zH6QBr7ejx?4QtdCh<2KDoBrZAYb)Gcfhv>F4?o%4KH{g>ce1W;TCgje#5x}s)eAs2 zvlXj;rfr)9{$oNk+TY^r(;niF;Lmh?b`1~lCQ%*;o4>&Bvl4J8Iu_JA0y2UUl!(d| zlt9kF)f}66yR1`1$DXL7afHd=FXF|HRA;%Ggq_3>Z~1)@YuI!KG7E??n?bIEF)56j zk&`*Y3@sylwwBS@GS?pm3?pJk(mAYZIIwjw@1m;A(5M7*h^Hn4HiV~kbe7t;ykX>Y z3oln;g=s@4#GMN$=vs&-sEC#C(N=rd|MG8Qd&+_vRTg+~dovjOx`R>Rl?Jqh9{2ng z(_9i)5oZ^jSIlxfoB)q-dhxasGXn9n36WfzNl6=9{?WsP*Z@O_VR@6qoKsUYvltbT z2F?_^EaIRcV2<%*Yd!sHfL?}H$KQDsb6t`@jdhhK>8r(aL*CL}3G+BXM*w6fEc?6v>19 z2@}SX{E@@OwYxBJm}8>cXrULtiV#|9wEjqKE$G@%4X=YRXj#a>BUj&si?PF`(D6ZnlVoI<{$%C@Iyl1Jn-Ht< zWy8qWBzECJ!$FFFtO)9V4i%Pr2o9&E_-*blW*P**1a>ccqv~gsy9-7~ElQ8u4Xiwz|jK#e`^JypFfJ(@_=gA4M9e7DN_o2nSmUtu;_D zq>Qg9n;33>S}^A_T!P+Lj;kU?37R?Zd?%OjeI$P)j~n1={)0}uqS#gf+;k#!<0jeG z{rj;S$9$OnCtX|!d6{NJ*7q>=GG9b~;->Jhm1u>ZSnHfX*8ilf>xKicd`(BW_U6ar zF<}V!`N$~OIHCtTvDeeRu04!r&XKE)g$qzf8oO|_3Gq8%ZbC$bm0Du0h)0wd%A|Nr zYNW`SVJJOa@}tvpZn*dECUL$(b(SfE$6rvC$W|-zz;~p zDZ+KC;uJEMy8ddogW6Te>&lUc=dOn*U|BS3k(X4!mM);%G`M%bH4te+Bp}}|Rlj{r zT4}hVWn7+g$c%>uZBF;jhCr{c{o71%xzb4N?HzuPUW=Z12q=CD(C(q%B)aMk8|;;r zNLz+!b6)+`UiUN9Xx4t?$7zb(SyrQ>B?4Q)7;Fx*K`whRLH*irNqIG0-Ii4Qf&W*_ ztsoUo#TW-1+!(56XCKeaYeN#w&qUU3v!weZjouB5dVes)BKM*OQeL1}$ydF3S+ zlHq~p5nw#|PZNUK)i#0r3v47YMZ4eZY>i&HL<=bkvDabT33B2acprp!Ozs!gEclph z;qtHO>!+o9@3ZPNGw2!e2^@LNE#rgsj@nwaG8Mef@R|y7tfav4kAG9|WllfNJi2m#Bztc{ls6V3 zSLJMVP?MoMw+PyAJq75C*bVl^vpGx{$jw&ir~F&r)_}=!$yvxnyN0+72LBCj(H99D zVxXJeVu&*5Ah&T^FPKlr;}A+Y*H3Bscc1p`hgN+{PSme8{F2dA5Q$xR?j?23$Txib z`cnO3_@d^$K^D3 zsPDV`VH%uiAPDa3Z5gz?+RQazCze9V$*>?emB;8F5heo(3#vC)`sl>W(w5K2 z;^lYqx~-UF1d9~6T!;&NKx>9ifgI-dX~`#;N)uw+N%o_Bwo@vIXL+;ff(j?2gTiGb#*;Z z8whFoKuv$&@MozaMKW6S?^}@_;~yZ(d(0xg#&LgB29ZHvs|=uGuda30#?v$HLZ-dB zw5eW879l;xNgY-@ABnbS9DGE189evU8x#93N;M1!!?`8|k?|V@W9T0w-9QI${?~!p z5jIiW`g)$A=7fv)kA6suEx|X;6{}ye!+MX8gQD=^HN$^7*OxPz(0c){O93Fls%zfC zTcds*^+o2pr-T&;4EFEjT`prgJ3~Vof@AfyG7Q!oFO4h9n%4arf|l0KiD!4gLsjU` zXS^dFHc}1eC2#cp5M>uBfS2UiP(6>Y76fU57SNbW8jT=2D1NTo`|(U1Mn&XYsOS`0 znGj}pI#T;lzr%!Re>ntmjq|HUoHq0jHp&FnnL-g(2xE2}3$No~i);ALm7AiRLy(7y zfBWZwZG6nsD{2J~zm2>vw<0&>f{&hE(ztjr14nNR_WCd?#eR;%#-s|=2xX;V*PV#n z+02oH$(@_9d+Ex|5dk}eXYe!5ESL}*N!`>9@I|+KWmqsTD9Z7F9?DxmlU%Ydo#3LqLoMDt3&5R|B z-2O3_sh_iBc9)y~IgJVCXbPH(j8YR~vZgO7U^$MoffJilv+_YPFnmn8zXoVyYe|bK zKI>05_n$Gx6m@QNN}8$=LsE+}JXggMV%aPc!Y1_1falNZ5mg)=RYbVT)sJEe!i?qB z=~;g=DEi%_wM}CsJtXXr$#FKAGabs)`_~8zjlS<)6kJOoOh~3_&F2s3D-X)RdK1Fn z7-mB713F0GXb(HxuxY|?~>=4mU!e(auGDw)LI-_Ra=VrYhsT12+DNA}V_7Q9CX{YXSdt*Wz9d?we61Ky! z-McP8o%Uq|K%Llz%}+$G3RITNgkXR$HeJ$Yoz|ayO*gtm>lwPGlVkx^-K8yhd`W38 zTRIsXmK}6;GYk3&(Da4j)H~J7jQOn_Y0yir6-p|cax}T;hJ`lXQK_d+x5jz$cyL3d zBz`e_x&CJnpKi`Tvqwju+RwTx}AS_L-QJX93vL*c+`QmUN z<%O=}w@1PSmU`Dftg+$vMC2dOGDUlmC4n5a+ZBul%-O5#F1>{O^I@NNG`86Iw zvBJvXQ)|BcgO0RI-(QV6*ojMFH&4Ckn~A=nUOXS&{|eOF2N^bzpNZ?cpREa#jIb0M z%m~9lzIlA+ROZde1Crnp{-8@SRm(bRaG!5M8!Ri^!Pv7S=$P=+2Y_5!(g(gbyhg5! zPNT9=#%up^L}wV>ANH$p9>vlqI8$2^FPH95BX44adb3=+uU~;cJ1Md2y>)VL`vXoOBL&7MEaOI2~OQGtN5|Xtnu$YfH(2hbg*W z{p*27LtsWg%bhu=HtCe(_s5hvMkAaXBdC)AYy=^blA%UndtH1A&@~goy z(u@HV=~93-q)Tw%_quAc++|&i{}ASQZdo|B^$xBIf{9J_IM0Omm#%atl@gEWJ|$;j zqzNuoE}~kj+hv0wxf_!63}H2r#+j@S=NtmYeD2j(eE4YQ&h6fH$49gNOtM_=0@JDV z3KQb5bjy8$9chUtpo@(x_@+y+hU6Br@a5BEg)uPU#qRx%vxtlP?CI)??aR}pn6WN_ z`n{vwZ=f^ssu5Un(H?V7NI{PkQ=-Kvk`RPRNOsLBEhgrv>hxwpzNSjk+7C-xbHg94 zfUF0tjkv=OQFjig8}T}(433ual6rz94DT`Zy}+IAyID+4fuRNmeU5)$wddG}+}NMJ zlv1=J`$!Ydy)=6p&vry8b=W@-&0}a6^TO{D2ix`#eQaI#bWCoSA2al5>-8IFTF}?$ zH?&)4@4UX7&>(1Nbi-=;_2Le!t@cmJUG?Y2SoNW!jzbB@SWgRra0!q`Q=_K5DBYM9 z9G3eLj&f$k2OiyaFwH&>y~3npnnVYm3}Ft3Ah~UC!6fh+b3Rn>Sad+m$oz-==j7_^ zTZm|88S*cKq2=~WyOCCrTXtu&N{DV5o(Go`oGJz*8+}UxoG>R?#$h28+A5f2=BUvd z{V%WSOIg=AC*D*oPGTD6ZQ3Lf}L+(7smn_ThGA+uU_ zK)pX9dR*Lzebe0muDS2bR%PAni4z~No#F8ul2-v@9${9&{sFfpI$7y|DL z_ty0&4d*}{oT1*$jKBF7M#1(=q)wR6b}9}+nr=Lt-u=`vY_T@wP@GTUX7qjPsRPX~x%gi519VP*g{O%oxh7K&53@ zz-q&0@=v&MRLAE_E!TCx$hmXE9!7DIHt3QA%Q#)@^|nm@lh*3yceFKGKB9`S=An1T z1qQTxP!;+wVy{^5gosZ~l0bt&`ZHnOR`Y46IPvpMC`w@c9sYqw*Lih2$?ua@?{uKF zLc9oO%p2vJnmdY|kG~23eC*@)yvTU&rmeKh{K=PD15M#Kt?eB!R^Tt7gS?3sldu8o)qquPtyB=KhXS8I~wg({iJm&p5W0Z z{`KbaYXxBmGEg;|EG~jYr)Hq=z8V8MMN85sXhbi3 zKzb#*IMGZ^MlytDc_ z1C;^k-;earuW9({ z7TYRZz=`G$ze8JiS%#(N`%-*H+(wL_exhyW^@?RcTMOsm8@CfzEkJn8XQew-`zkymIJg(?-O<{_(3-zV9hto?ag)PcbEUVDkX%zI)b$$}U)+PY+i z+vN~pTA>)lyL_paW((^{zvK0c>@qFGa0I|d+BD&9IvzH$hr?2!YlgtZ6oUm$a-3qU zdT2P+$c!qE!h5Mw9}3w7`A#@s#cT7i%^7(7#VoByOCr>-sab5b38+0?xl>6DMAnQJ zxXJ;o{p~*>Ph*W2@`Hyc(!Kk}T-j9!xk!bEfzu^6)O@RsMq}vS(FGVdzFyxLfP}`y z=4ZGen@3EDS3n+%Sk1!@RTS(x9Goc}p5aQe-w#2KVI{*x&uTj2ny zW$43VSePyNiK>tRSqk!`R$2)&=I37Z-qBZ)_mGPJS^0NrlyT`gPM~!H3W1l1&H~5w_vDSm9Fqh>n3N=~v@aZK&;`CbWm_tKG1KJ!?XA z2Cam@+D(RBF`|nBLf!;>H&$;#Tmeyrl{hr>7-Hl&5=>J{YZR<1)Dn>O-nXi)WPg7`M>J-MXs&ZXESt)WLHKLqnC) z_POLu+FYPl-~w_wm4nrP1Yz+j(Pt>SB+9q$VidO63ox#LzNTq#*@MKNDVVg~xdqhs z*-^S;4wen^l5Nf3)S{E?EMb^9YeFmx8O4TVP68~OJk&_!Hg@3k^>FquP&>b~C65z; znifEE!}1oq2VcdHl_i{!tpeS!u}|+D$7;<`+@BRPMrnEFT!wpGK?-RR4X@&A#_Ow< zjSDm?SMN0}-b(}(cCh3F7tR@n1f14+quDUQ?uH(TZ15f*@5xKoMu?Z8-; zHv{Cug1UaV|3+mHRdnb(uK9s0F9u64Hs0~yyXnr>UBEt)swP2KeY>hWZJ(|`O;v3W zctr_<(}%<#E5w0A2_@dx!eFW#cm}XT`3dD|z=g-xg5nK-qx-nf0w8w(2W*Yw9=P&y zD#?GtnA5PWlv#^KD_vayhQYmEt2x;Wm@^C98(!8l1G6RbkVem#vAWjSv7vPj<3UkM z4}lFMMfy|fjZpJ{SXOnqvN$lcXS!|}J1GeKIRf_Y0#7VZsc07APe)H*MaqkH;^4hT z^2^rjeEIQ#{smwP)Vq@cQ{_MB`zUvn{M6q}8;yhQw%ag8W$0b>=d}Cr0nbR^4M?s< zhsid4-cpouPlWTq-WH)syTg^iXiD^Q#R?|#~ujs@3tx8ADHmV}N zt(>bjbbq89tXPsT9zop)y=^|O`jlxkEeYqy6?CX?*k~Jf=nh^do{l#hgjvfu$hMr=!f7yHQc`ds0)m9%k=9Z1OJFoLf!1ggPB9p%u2fuB#bSJ)K76qFiMRo z)+h)i=-XDr3_vunNQUfUYhgluSOxj@Yj*>`cfgi%_Yo9Kggy^j6pyY!3x3n4i92;f@Oa&IN<<` zYg0||&3CNGt@Yfe$N2v%>b2A-c8q#FiQ;^!9(V~8F=9Z28% z2~Sdg9KzhKF%05|rn347>Mydun#7T+xw5^~-m4S8HFiZ#9Y@FN-b)B`!-~u!;ADFrz<4=rX=s)=h9 znI^=IR@8*Xce}vyo`xA@9Zi<%xhR8~w&r!~<%yaamzCqBS4it7>+mpNHimLy*y}}& zdplFUcm?1(JcW67NFX@jwQL*zR1I%WB5+kd7i2CW(dW|A;(a8r0I`dNDR}P!$;{hU z+`;ap8QkT_ruTE+lj8rhB++x5a<2$N6{0E!pD4g`{v8I}w&`ycuQ{1zT!=9L?eWEn z$D#zrna_LOtAeuJE)v&VI&$tv$eAjoWvh$C7}VLa$jWL8OTII$I<@UYn#VeQ_ z{>)bBk2Mbw4}sClVEzHin7^qhM!v4@+b)2c|KTaYydM*|D}DFv&vlInER@^|MybLR zloIqjoU3s#tfrdoO!sh?DV9J}{G+mVj@m5-C=qxE9f1fg+87kc?x2sJA7(Z{3r_@ih% zI5s4?#jBV_T6_xQo!knL3RC=cbQ@)T>;gcjhpn(Cs&+4R4*##9ClOHkXnJ$4oUZ33 zqU1nU;{Li}J?J5pKzkLy4QxI28U32aWwF68>lcg;<8Y%qRt@tMx6?j{69RQaW;D!z zIsRJ7Wd9i9M;1r1ct7hV!=*8zw51#7lJ)V;W;u|ul zdQh<$ULA$t4_-?l5y@WJfBpP`mn*iz<_$`=ru(fO)~O&F)?bYuEdc#yy8YIoj>2;d z(?>3?JulA~ka{&KH@>xT3U^NKVi(C_^9}Tn>t`Abu5=*It#I9ZE=CyQY=xU0a3Z*CYFb+5MfpaYA z2=?%q6<(1uRg-%%oa1b!;+Yw(RQp61p32n$1#27sj+#$Z-BE4LztC5kJKf}~GRg~E z2~$O|@-;zi7p4)9Bu)x|gV==rJ(vJkhppDVpF|lp`%TOjsA~?xEx@gD8%X=j6YGG| z`$7G@@V7*|g-b)Kwr|SpB(b%-o|rszo`P$^tHgb&ETdQsP}MjUS?y;mAg+O{e_9*0 z+o$jWr&+A5pwf}ausuOi)fgvG2+fSPrm~?`LW$h^XqxMXh0Gc#WEVIy2IRnWzVjS; z&T@*kF&HL+`Kdtn=RDBr}pI?inE(J-Is2G05@p zOCu~G{b6H3KGAwhus282<2*zxb=l2St zweosnS^FrGPhFQFAo{VdbYuVL2MM@Nhlg$Lv;t6YQjh&jyVY;KR|xVRRgr?qbzfo3 zhBl`jTFD#}5syQ!3ti;CHrM9j#hSjsQ|GV-n~vHJydqzM{8QRYmi;y}DD?e`6Y3tj z8(R$U1>Tp>YRQB(hMh|41f6U+rJ#4VU|$fH#RHMUP2t`NweQE+)8U|2E)rq`>9SLB z_=wcQ*Y;OT#ntqm8JUWKCS&wy@`w3dS3+CB0-3@QgDv`k2&{ z-pMS`pGw_pt)daC;;H9bj_F7tV2TEu@H@Aqg*0mRT`9*eY)02+MHP9;wFkefv`X1} zw~=KO@5aG7+N4n~#4v9k=Zh6~^a?ayzEvC4jb4@H;DiiRP_N@Z>CNKGhuzbrLUzZ# z3-Q84MGL1L0Y|_UFEC(HTr@(`tU$GINdoIx0%rz}5|%v)CwP6Lh>}M%4tY2B7=)WfQXJssDoIo0gRTm`@HCW`kBKAoiPPQTF@R(of%m+iuE$9*`;`20syntTa8=ujD zcI|(xx`1xYgva{pac%gK(HpyzPR6EN%=*lS!aTTYNf$;in8AWQeV~5{TNdV)oLB7G zFEUP(4fAvgZ~?groyA!bDrW}IJ6B?;@AzESJRLd`VVuY~nBS3{EOKhTZQ~oYz%+mv zDO?ROCd9h@bKwYQ?`f}&EXCyqf08xiRBd>9$V2|a|7ma~3U>H|>1|*9A52WZMZd#= zPj9_rQ6K7~kYR`pfFnK-D2(bgiOAjW-?9w%pkz0Ru2U7ONO!v>3 z>34`oU1)|Gkzo9{iMly{KuZd{XE>NtE-<#t{*ph(TV;Guf)zUc#uU~}JIhY!$sn&zWl2V}9}FbaZN{O6KiIr+Vt@dHGOdI0(^3d;uGEJn#1Fpc%aYmh zybDps3xy#RNC&gg)|BoDb714F^>#&jcZ*YxLj~-H%#mctn6ALi)6%?E6pT&+)%1FEx~cVWNY$sDQ_3D;}A>lrzz5lHN)we4)NQ3Nz!yZQeZe@ zJ}*e&=WDK@^kR<@{TCWZ!;$*aMfGq2sX0$_cCg@qs<`~AkWr=!Nv?*+PKqsQ+vxwM z^<%B*l{8Y|_3`vB{jg+&cIa&RRzK0hKZ5^OPcfW@j6` z?P8q&>1#N;Kx8MI5Edi~G-~_J$S0S#$j*|lLKC2l!HHlGSC0=~0yxk#=%^YVai!i~ z8#!5Y%j-yL%rV90qKbfOkE)HKb&tf&(HLlV-JHk%Qyph%E{%rn{2R}@XHQjboJWtz zyU~i%_!Z9;g4ErDNjDD^k`qh2z`lKO7ok;v8{DomH&`4!ormY}l-r94$|J92@S4ob zoH%T*-OFQU#YAT-N{1tJ2%_dYEVRTLUMh4^f{K`?s2EL50JH_;J*+R-cB^Px^gp7x z$H2sCxNbAxs4?sr;0P+VP|1Me3^|L+pk=k&xJ+8gwn_rlcM$Avep}B1N5d&ssWI%R z+>m1M8eB%niKSPgCnk`qAc2+rT!@4hpeAdxCX8X;`IGot$xM;Wz#~=xEKfEvY-r7b z0zQn;e%_g4^Jw=($L0%rdr^3UrBsJK)7|y!hw~0Cm*2BW-@nhAo6?&TNyZ0WYlOrQ zt@ne_nGy9N*(++&uSI4ygH6dJgY$tvK|vub6>xA%yG)3sBs6i^sGE3Os$Al8!!7t~ z=E;>-D)sZSTx_y@k*csVeglugDuM}d{g`rRrNVH6tq0ZSPv}ALe^(ifD3P7!Z~yEn zA!NxyeFkN}j@Sk40fz9_#Acax=5h5$J&D>xVBro$bHQaec$t zp&jy5Xl{o;dKx#D;C}C+Dq)w7gIVA#p!$ZD2l_m#F4%2FGJjS6()sJ2H zet?SjRKqo2^^s{=w68W4`u zQuNyB0cf0#Ds1>s6q*LM!_|7w;fM$ScBuQEr>)I}HFX+&tymuP=S{Lajaz+M2kQn5sn@b${GG1R zi(Zkg4tN1O-Y70%Xb zRvB&A4W9ZhT``y=C1@@75bdLp=KVpUgOqHB{dIqLmt9f%dRG^ShiL*<366B%2flVv zxlo$baMc1uKm3@VK#7TQ=XO}N*ON5qjR_EPp{28ptVbh7dfUzBn zetXNtT)KN`&Z$;bcD>^#KARJ#Nr=|LKJaASolPl?x7(+AtzDTf^(47|xMzejheEb z23wmcvcZrey>)W2bGK@^2xc;DtTV(Pu{^Z9xm?@dF$&u&;pns3v9805w&FRdxGID|t!tI4a>HXms29g4%i#bH(_PpcHq> z$#<}3E3k)g$%XO!gv79M>OjJ!^Zr36!!UU$Hzr-r#C);4g*2!5zS#|KdlJ>;-NPj= zl5sISore|jb{3Ol^Y-t?qcNAIM+Ge%!Sm{U`wO!OLQph>ENVOHJ=2a+3&) zX5pJi)&oNo*$JS!2Zxr-7Z}Nsj3e$4=DJ^FVnv~w01K#`UeH`O7vM6Q?BjD8t(zI^!2GgyH)v%{aV5c%j9I% zYVqv9AzLj|GzA4XIXJddU8l8IrabrXNbh>mg1u3D0qZf$3GwWNz7yyf3vk7IqQsx| zAMlcyc;iP5_<^)*XtEVgO@faVU=3+0fH+VUQS?|G7o>!r8-BRF`sRS&Qdtx{Z`7^7 zmy^roM4#6eo2Pf~p}J*$;7(@i-Sz(QDWvVjk|g|kui@W#8-)KOZWn|lqKpgEK|R1S zVwg>>c#P#tgOTi^aYWt%>ODr_^@{*-0et%om?sk4kXurd1GZ>WFoRN7OUqD2VC!`c zG7e^xV!6{5a7EJ7^TV5Ge8ykT2kJN)9;}rF^>?grbd1_+t3K@3KBYW6b4KqoT)O0S zQqEy6?ZPW#NJ8!A15Gl{#mle0R5jSrk15u&YJ4)&DrUdu*JRcc$@eMAusT^tl@S>o z29$Egxep3Kh#)Pm+j*9j+f0$_#ah*9VA*r`(#MTQ7mg1$-R3Urm(iW;Kb+5eN&O+! z;}z1`y5|A*`xj5Rdr`ghaG0Njy-RseS!zmHrftBJIRa(=5gYupS0(-{eepi+R`RHv zpf2~m9TpO09M6F;Z2U>p*4CGJVdR2y>x)z{RggSNKJEHm>P1=M9H+Y9MS%-Nc2OiD zmi>ZyJNy8w%Q0bmD`eAF=sgZ&554DNU3+s+pB3(4$|aDCe!3Ogk*>aP{FnJM1TWC^ zNQrFtp|$j_&l6boK|a#eWYRpu3O@GxP! z{xIsZCl&=;IG=ccwi@<4|0(BgJaq5napd}1(2O#O9r?al4uMQasIU z@4UWu_#Lp`gqV0j+N%x1C9Prox#)lLL>TvGhkp6=RuqtaJeElp!AH7o2VfG7<7v0C zvfkielIp=H&GR;lphLW&jH9ktJPd3vwOkD-N9u^Z%Jx!x#@RLg5WM%XqROAS9HHh8pJY#QWpwKa5LTW)+0m;d#aB*i$k(QR4yZfs zBU@~_bj~iK+G9AK(=YbgWhnT6!dtS|^T`*T46EJr9>R&>=ur`V<>vnoH^K;PaRq%5 zSPzVyV8f@7;Ogmiw}pZ{6p^ce7XORTusodat+Y4Z#ev3epU~Ufgw$tV8vWs#QtVc$ zzza5K{}?CrZ`9^Xy7HBgKV|h%{0knm0F(SL7JYY(>b8m}dn8`Ki?OdqWxp#1yqrj{ zI6D;SVY}}5|Jp|tjWe_>rUJ|(Vh=4&XQy4_+R=s_=!R7l`|n1t^LlLjXFkB-pa<-B z>GK*5VweZR@mzaE?4z_$Q#7HkAb66IlR9`{>T1;CC2&YFLSAEg08^S2M61obPCaIQsDN+&A1$=vz8S?=WK)E z1U(y+7s7*F(nfhUnjf55I<}q`lHl>E%JQDr0ndle&A=Vbm>;E{{>S{^+=LT}wmtuT zWcH-8LeJLgE(>(jHXS8JI1Y9g0-G-b03yu-w7#mG)-Bm%_c11x`QNEj>745GuAk7d)nKw$~CqDuF4GR*N zTaU6Pede0!VZsP}iP_jZjq)m-$0)*OiYa;q)Ap2ppXT-}0^(?DL0O5nfR$oc7G|Wj zMG)X5aoI9OC!=*g(7rLULYGk}*2hEy6APhF@ioSo&K7uEP;r)pXJSPbgH2%{ALR5D zI1%`>a+h8%g9C@>1=zqEc09fp*^&L>OwY`#h)*{X5=c&|aCl69!b}hB2>s}F&O_;T z3V#P^v*$5V@Zni)_w?sS;QmjR}NFbWn@wX#&Oo4_yXKw1&c@-;WEj@1HLyl4#B z81uOcE73D5EZ5(>kCl`rL=|)_I~g%pX>7>|nzL4d5reH!gtAB6ZI}6-HgB*Qu#{)! zqG8JDY|kGitq;Sm81HuW-64qZ3nVN>{)LrO(Mt^*@ee};T7l1X@5Y4-*GI4pgkn|Q zG)z*Iz?#h8ACAvpfVCLB@APfohpLGBKr&g~$k1248+4+vD z{Vs-#me^|1Q|9%p@eOf_V_fZ+B?Ap~lM$;C-e|6dh z`Vx3#st-3NDg?cWCeTo-5##Wfsm-hnyMS-+IAh_>VP*E9WSaPGlD3>@G&DCFil} zaYyIbG@>Pp%E(M=1NKIv3E^!8+ejxI9pp}403}8->Iu_aKat0->(<<@PD}@%9kQ!% zjDpwhb6@bh6QLcu$=%#GHOPlBC!md*OYONfBxV>dHUG!Lm$ftvI7as{HZ6+(5UtYM zSL4K6iBV-Az8A$c3O<7Tl$_N?`TOAmMc2Zn3t(lvO51VIaL-tPjk?x)cq=7+P6rRQ zf)P)8En1$&2qwrlaTXrT7clw6Am_T>T*Lj{zN@Zrq%9C@;d6dpV(`4kQ6Var+``|~ z824zbB_tgEC8s6w;xZh4_NjwCJ@*B#J@X#@mA0C&Li@)7&dj)MG^_`PylM*MagtT~V?awo`O)0JEj<)c&rhR`DwIZVuLaYx7{k$`x3Hl4Oy$ z4=lPo!6|xnI*)mvc#OgpyPg~5eeVaVf|dmp`n(mJeWIm5za(otx?dO(`h0pLCaY1v9$;@h5adJ>kvNa*lqGvUhsHJJ*HOf7xwt9*io&{U~4oHL#GZ z=SJWIMEydy+=-p$Tafs!L*JaYHCZNPPh<5n)o_0GziPM^N2d4w z@2OLro=!PVlx{pHCyHvxts;BQDRh(MZn0A$wlgA^30s{Gl2|3gLMbW34w_-(o90sH zlIvo&F;R`#ja}Ns^YJ{t-=FaPemO`s_h{gW zkYsFHj⋙k*L*cwm3N*9qhTRCHzQbJDKv05L>8w&jkkxRrpLaUY|?er)*T)_=o33 z&u-1XHD49?y&tB1CJX=RT^F>_j|&BUxt1p<39`JV$REw<1IrT{2|EwAtE*@&QD6|) zf$t#P#wV{Qf7|gm=QkcKSDuE(4i~h91<`;vR+cU=uhBO8nzj*aJsu>xi%s8LUtGv{ zwjWvS@qCui(rKh=xs1`Vm@)@RieM^4X)~F56xG+};B1gBJ?adQ(INJJH!z6edHaFe zks|nZp2vqWoWfZ7g^D!WZHPmGFnjDi9LO!q3ppB)7Le-L{)L@yo4iee7I+T1bu)wn zMZOY$r}_J?D%fqwwKL3r+)khWss@JYO7YtZUypeFRkAd(?SG`;=0b*^L1|C7uJ> zU4@H1VQfFfrmg#*XH+^AghV&q_1o40DmGd7Cp6E>chf(jeA!{q%eVIaTk>#u)Gv~d zu@Ot1GQ=6wV*^X$@S2tH=WaIy$p(BefP@kv$>T7kUDSj|&eaF>gG9tYkq{y&5vc}8 zsCGvNr4rP~FD0j(yvBH>eVk6#Y#>_|QCx@Rk$n=McSX*h73gub+a49{|G_jFh>M}3 zWiU0TxS0MIbJx59ZddRE09~}V&p*Wanjf>|xc9n$0ns5nVM*`#CL=|Bfg({oEJkX6 zEE|Pi=7E;?IPHHeFXLp4aC>-(4QyOz`8mHM6n6WWWN<1!6}qDTr^zqMsj15mmB|ie zl)=C@cAx-BTVG2Rs~*^*Y1#_SY9&ch&QBLwNKi69yxqfyBN%%@O8KQvPJ9tcb;WRx zTk-^@J}Mu-X&*8bKGDddERydy{aUJf$NgUNYs26D96Pu%g+y}OCFj(SEGHtvr^4<- zVyI_S)^zQozAOFjF%anC82L4qxz6bftbv~x|Hf#>5yMCCHMX#sD; zg7IE%UogS>>{9RNp`O)!BR|ru7-Q&X{Oq1t?*@{2D%7_r##Qcxud^!|JKt|jmZShq zyj3}Oxmgz?tS!VhOgky~Yi`XH8hPxrCrn>C>2(diS%GHY_9xqp*q$aRPc9(B#%hIm z{-vsrDBvsGEym5{Ub7=QeW-lknzg;=4;4aN1p>V@$-hoK3U(xgM~0viOF1i5r?`kn zHRQmgPA`K-RtO00nVbzT_5*uE{?K&|J*{t~qYpb&!X20vYA}MDQX{7Fg+<+YHt5aw zI#}zr_g^^0iv#vpLCY)*2<21VbnB5pRG`n}yn3}F(pw8sD&Vlgs?`&n9X*k6 zx8e2A5TB=O&ugDehw0{ppin)!>x5!Fl3Z10baL*0+x0}_Qg_-hNedK){o7(=xUH}v z>I|lfNsUFuk!tHCN1HWWU@UOxLwl2eub)4o+X28vjuWujaHh)6)kG{yajv_*Ho)OM ziWmr@?N&NgH)P{imb_70O1GYDcK$Y_Gc^~SfMR<#x?6x|J~KfAtfA_$$!`#>d1J+) z+j#56hxXTkFSr)I@hvoz744)3eVE743h<-8Yy}$DHs*z@(Pp!J!iA&LX#(GTX1E+U z#@Zq_2|g)m`$Tj}2guA$G`kX3qCgZ}Qay`O;G{x8MygSHn{&bygddnqj#(qfsAk_B zWk^tuq99(VLWu0H4KZZh^1qcY8Qgm!;{bd_z#==z z=Il0lU&C{}i%2DOwFf<%W#zCup zlC|Aah6oQL-qo9;D6+^cm!(Ge+7JizK9EBkiOvdiYxXcmC7o8J-+Mm^o?*TFDt=;& z8yF4t4NurF%Ed(~P#rm9LEot`Nsq>AJNk1cdeg{wj})A+_LbG&k}$#s;`7cw6!u`1 zO^)K8dAn}`VkU?LM)H>QeyjH2ci!Dcx1PXWtZRJWn3>XoRzWtfM-T@bAo>~603Iui z9zLMNZii(%Te?paosvpNC+ix}K?C&jLZ8s6NuB6hJ-LrJwsOjzBq4MR2DdyPh>*+b z0s-i_f)U;lrGVwGKBxO2!?OKb#PSSml{_;|jTyiv=n{d?-$aieUJ1Ggo=nQ>_hfYH z>x)d+-EQ|VFhjVx&po@Mh70``7D1zDpxg>zZAtHU=m%{no7ph@FLmsap7{u{R8%At zQ%k_D6+k9BXDo6uz9eh_k3xA?s=eYrQj>?Z7+fDtuTB{uE_V@QQfoUmlw3624F>yJ zVXK{FGlx;h+smIIG(^t zqjeL=yUzzTjxXNJkp`$jcZQpJ)iuqz#5?Sko^8oAd`51IR7ifP`G@+1l08}ewhA$s zkoH}?L_p1QYP#dN_91wtHJ3~ znkl{Vu#f$a7W}?Zuems@qWuD%FD0cG0j;PE4yfu1+Es}0iF)m#>vjh^&psc@@w`m| z5C{iIQ%^{HEk;ahTWYTGnU5Az?3{hZYTK9eoMwfD^sCPz+->{n-7CnlNN|yc$vyfX zJNZ88H*{9)FD>SD|Jde0pR@qvJi|d#hgX2*0i!4_T$u!Z7Kz)<)F8ls)UpC7X{2nq zIJ=}5tKOSePGQ`y&yC4PerXLTa7fY8{Iq1Xpr}G=EfP*OVnckWnF?ex+k6^KV`(|V zW2jMm85w1Zdo2K|A-haRcM z8CHri%V_7IfyJ+{*#Ub+=1mxA7L&|_&8YHY$V{5DBEkv5N3P-6mzC}67;XqC%{4_Q zLuv=L4+qPX#}5B7iN5l8Xej<5?~*E0hYOUJ7(_I2bFYp?QeX5)+ojKGdG=96(OyX7 zbKQG-t*Jv`Q6SRL<9DTpIt&qj@(MtYRg7^?sLLq1f%Z|{NRYlvx&~zB(0VZ2+VVh& zSNr*+;}38xvYtk~C_W71Q#dk*uKxM)Zju5tQ=xz^#S3`iEWxRfu@2%o%jz$M*cM-) z8k6i(R`>Q4%36zUhF>@StfG^GPIqS^5)`2v4U$ux;aTS0TWncnw&>N)GjW=YYaE~i znI??l@Mckus%GlH4Clv-nk_)r=CzDy$`d%V3(2R6=jjbYTXcOtUD)}KxrVDEf+7Gy ze#jN?B{2D|{weQ)7OPS2F`Dkuxr6g+URx!Piz<&(U(xjk|1`*Qb5HtJ%9?heauh^0 zyG6l$m5tQYI>ISsht>V?bM$Bl{B4VI_SAXFdM$Zg-m}-wRN8SAN&@^yC&b}(TvMuV z5kKqfY-_OI$hMiVYykqf7)=J~gCQ5_cIm*nFsH=nuiRfG6vl6vic8BCbc*uX+zk&k zlBnx)-#%=9o_)bJ)R%tvY-wN6U}U6Oz0HXpGmxBOxjY+vZ_8X^J`*;U`{qjst_DSP zz$wab1bnM8exhiMCybEfu{un4fA|!&{)<$V%~~rRKMgG3#b8T9vvJ$x-DYlxkf3xZ z(XDNVr$*+>vy=r-ge~+8eC>b_`%`BVgsQxF;-bH&5h$2bI0KFC2b{6emGx`?oLQP9 zwqBSKPi6*{wY2$EciCzm9zHWI+Vn)^w^Mhdw7RHb2qyErfddH8I*pfRidNu9uMaHg zEikJ+-83eCOM1Io#TMH9EYMV*X__{Q{-{5b=DZ;IBjxMRo3EO~(RQ^6<=qHo-0G5p zj6@(EM|K_xV4EeKWUdSan9EoMB;4vDs&y6uCtMT z{{@H?gUuin`P06Degi7uonW%9Ch3*KTj(H0N)^iB%NjH26-c!WAA((F4?=yT3PQP% zQlXVy=*uXUx#!HWM_dEqH6;!`Iv}JN`FB1ZDBp#QZEX^-zIAPT-qleXH*lNlDEJAu zg2k!!IH!G>Y(0hkn&aR9v#N+ruIs9##cLp#n93!erw$cO1ypxD(?6R0g0c>@2l8eO2jY@OV#WA7v^i^z&keqFp^jpGPmNh+RBt zl-f?Vq}SH>u}^1^nOJ>sIPe5(I%s(XKn59dc1i{-*wPkMPu)-LsGvcyB`^+<<1?p3 zrXR8O5ykA@hBP-74w*}$=_7u`Dl4IND?f426Q6HKEr#_kusedF>;8cC9<53?PgC&s z6U46Y&4HuZXERMh@@hTj*}qbpn9am2RE0oI=-tMS$=%i4T}hq=G;z~dC|@`gVL zg@=u#Je>mHfPJ9&ZTRAYZ6wFEwE4rhg%}bB+^HM2!pcp1(~D!!Zx-+KZG9=mt?tGnnG}&8jP)-jzL|`-;vs-Ru}1Z!T=EaMpNiHrVP->atei z`jlrG@+-=ViIS2j)?dzs5npui5Pu!oj{RkM4u+z|fcFnZ&q4ro#dU#3%NAPu<%suz z_v+7+OutQ!3in_(`W)|9rrmW@CDOGTvn9L6Z*0bP^wZ zR%WfkPx7c^Rlk9aalJ1)mh|lWvoCv0sUVuh$7S=R8c_9sEFDS;-QiJ0bDiQH0XS_WdW)>|dR5Tn_1Mi2j1#FySV^QCm|*wh+IokX9#FuRMIHp8{brMMo-#C}&wz3{`m zqpR1T!pcJo#{z~*GWJzF(MxY!xL$(4zPo&-lbeKaS^L=80$a;yEllmmdU>?CYT^K> zE~#5VEwXc`> z9kDcP_%^02Ek1u$eetb-p6n3pCKG$VdKK;Plp9-846pWnYv|I&v*(Yxq3uiL1@(td zK=EfnVsv&n6@89-V&hcKUvpPdXTnQcTz7x__)tj-QNlw*^+p#X;k5M~2J;xR z&70W$%DyjL`$DtwIo8cf(wCm>gzJ0p)|O`w5Rm$=KJoZ>Z_1r4wT(9-8`2J3yCdGS z-_PT1LalBdh{2K+>^Nph~!g>gVEd~Y)ITPoBbw5&w0T!Xiff!8cae9R@jE!-}u7@_yN5h z9H2t25zqt)WRQ6bQC3su+v#wZ-+}Uk=Ulpw- zzf$G|VnY0e_s^4Gbm#Db#|vIOED_OSctxk4o;zu>_Iqt#;O-|m;ANsyWE$JSs#~i9 Y>?n_J9O}Do!Mr(kXZI6I&C=+90Is@cWdHyG From 5bcc8c6a29f60c4b09dd299ecef02bc7123a629c Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 15:45:49 +0200 Subject: [PATCH 11/19] fix: fix docstring ndarray and torch tensor Signed-off-by: samsja --- docarray/typing/tensor/ndarray.py | 79 ++++++++++++++------------ docarray/typing/tensor/torch_tensor.py | 75 +++++++++++++----------- 2 files changed, 85 insertions(+), 69 deletions(-) diff --git a/docarray/typing/tensor/ndarray.py b/docarray/typing/tensor/ndarray.py index 0fc41a0c184..49b23ea8e9c 100644 --- a/docarray/typing/tensor/ndarray.py +++ b/docarray/typing/tensor/ndarray.py @@ -46,47 +46,54 @@ class NdArray(np.ndarray, AbstractTensor, Generic[ShapeT]): This type can also be used in a parametrized way, specifying the shape of the array. - EXAMPLE USAGE - - .. code-block:: python - - from docarray import BaseDoc - from docarray.typing import NdArray - import numpy as np - - - class MyDoc(BaseDoc): - arr: NdArray - image_arr: NdArray[3, 224, 224] - square_crop: NdArray[3, 'x', 'x'] - random_image: NdArray[3, ...] # first dimension is fixed, can have arbitrary shape - - - # create a document with tensors - doc = MyDoc( - arr=np.zeros((128,)), - image_arr=np.zeros((3, 224, 224)), - square_crop=np.zeros((3, 64, 64)), - random_image=np.zeros(3, 128, 256), - ) - assert doc.image_arr.shape == (3, 224, 224) - - # automatic shape conversion - doc = MyDoc( - arr=np.zeros((128,)), - image_arr=np.zeros((224, 224, 3)), # will reshape to (3, 224, 224) - square_crop=np.zeros((3, 128, 128)), - random_image=np.zeros(3, 64, 128), - ) - assert doc.image_arr.shape == (3, 224, 224) - - # !! The following will raise an error due to shape mismatch !! + --- + + ```python + from docarray import BaseDoc + from docarray.typing import NdArray + import numpy as np + + + class MyDoc(BaseDoc): + arr: NdArray + image_arr: NdArray[3, 224, 224] + square_crop: NdArray[3, 'x', 'x'] + random_image: NdArray[3, ...] # first dimension is fixed, can have arbitrary shape + + + # create a document with tensors + doc = MyDoc( + arr=np.zeros((128,)), + image_arr=np.zeros((3, 224, 224)), + square_crop=np.zeros((3, 64, 64)), + random_image=np.zeros((3, 128, 256)), + ) + assert doc.image_arr.shape == (3, 224, 224) + + # automatic shape conversion + doc = MyDoc( + arr=np.zeros((128,)), + image_arr=np.zeros((224, 224, 3)), # will reshape to (3, 224, 224) + square_crop=np.zeros((3, 128, 128)), + random_image=np.zeros((3, 64, 128)), + ) + assert doc.image_arr.shape == (3, 224, 224) + + # !! The following will raise an error due to shape mismatch !! + from pydantic import ValidationError + + try: doc = MyDoc( arr=np.zeros((128,)), image_arr=np.zeros((224, 224)), # this will fail validation square_crop=np.zeros((3, 128, 64)), # this will also fail validation - random_image=np.zeros(4, 64, 128), # this will also fail validation + random_image=np.zeros((4, 64, 128)), # this will also fail validation ) + except ValidationError as e: + pass + ``` + + --- """ __parametrized_meta__ = metaNumpy diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index 7ca9cf70a89..6972cd321d4 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -48,45 +48,54 @@ class TorchTensor( EXAMPLE USAGE - .. code-block:: python - - from docarray import BaseDoc - from docarray.typing import TorchTensor - import torch - - - class MyDoc(BaseDoc): - tensor: TorchTensor - image_tensor: TorchTensor[3, 224, 224] - square_crop: TorchTensor[3, 'x', 'x'] - random_image: TorchTensor[ - 3, ... - ] # first dimension is fixed, can have arbitrary shape - - - # create a document with tensors - doc = MyDoc( - tensor=torch.zeros(128), - image_tensor=torch.zeros(3, 224, 224), - square_crop=torch.zeros(3, 64, 64), - random_image=torch.zeros(3, 128, 256), - ) - - # automatic shape conversion - doc = MyDoc( - tensor=torch.zeros(128), - image_tensor=torch.zeros(224, 224, 3), # will reshape to (3, 224, 224) - square_crop=torch.zeros(3, 128, 128), - random_image=torch.zeros(3, 64, 128), - ) - - # !! The following will raise an error due to shape mismatch !! + --- + + ```python + from docarray import BaseDoc + from docarray.typing import TorchTensor + import torch + + + class MyDoc(BaseDoc): + tensor: TorchTensor + image_tensor: TorchTensor[3, 224, 224] + square_crop: TorchTensor[3, 'x', 'x'] + random_image: TorchTensor[ + 3, ... + ] # first dimension is fixed, can have arbitrary shape + + + # create a document with tensors + doc = MyDoc( + tensor=torch.zeros(128), + image_tensor=torch.zeros(3, 224, 224), + square_crop=torch.zeros(3, 64, 64), + random_image=torch.zeros(3, 128, 256), + ) + + # automatic shape conversion + doc = MyDoc( + tensor=torch.zeros(128), + image_tensor=torch.zeros(224, 224, 3), # will reshape to (3, 224, 224) + square_crop=torch.zeros(3, 128, 128), + random_image=torch.zeros(3, 64, 128), + ) + + # !! The following will raise an error due to shape mismatch !! + from pydantic import ValidationError + + try: doc = MyDoc( tensor=torch.zeros(128), image_tensor=torch.zeros(224, 224), # this will fail validation square_crop=torch.zeros(3, 128, 64), # this will also fail validation random_image=torch.zeros(4, 64, 128), # this will also fail validation ) + except ValidationError as e: + pass + ``` + + --- """ __parametrized_meta__ = metaTorchAndNode From d15cdaaaf21bf676e7db3f804fa2ee9827154abb Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 15:46:39 +0200 Subject: [PATCH 12/19] fix: fix docstring ndarray and torch tensor Signed-off-by: samsja --- docarray/typing/tensor/torch_tensor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index 6972cd321d4..011208b02bc 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -46,8 +46,6 @@ class TorchTensor( This type can also be used in a parametrized way, specifying the shape of the tensor. - EXAMPLE USAGE - --- ```python From 4747ed07e05f63187ed46d1630efefb0eb50c7c4 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 15:49:36 +0200 Subject: [PATCH 13/19] fix: fix fix audio url and audio ndarray Signed-off-by: samsja --- docarray/typing/tensor/audio/audio_ndarray.py | 55 ++++++++++--------- docarray/typing/url/audio_url.py | 6 +- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/docarray/typing/tensor/audio/audio_ndarray.py b/docarray/typing/tensor/audio/audio_ndarray.py index bae64ed36c2..e6f5388ae45 100644 --- a/docarray/typing/tensor/audio/audio_ndarray.py +++ b/docarray/typing/tensor/audio/audio_ndarray.py @@ -10,43 +10,44 @@ class AudioNdArray(AbstractAudioTensor, NdArray): Adds audio-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import AudioNdArray, AudioUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import AudioNdArray, AudioUrl - import numpy as np + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioNdArray] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioNdArray] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + # from tensor + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=np.random.rand(1000, 2), + ) - # from tensor - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=np.random.rand(1000, 2), - ) + doc_1.audio_tensor.save(file_path='path/to/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + # from url + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - # from url - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) - - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2.audio_tensor = doc_2.url.load() + doc_2.audio_tensor.save(file_path='path/to/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` + --- """ ... diff --git a/docarray/typing/url/audio_url.py b/docarray/typing/url/audio_url.py index ea2a344f9dd..bab24b518cf 100644 --- a/docarray/typing/url/audio_url.py +++ b/docarray/typing/url/audio_url.py @@ -64,9 +64,9 @@ class MyDoc(BaseDoc): audio_tensor: Optional[AudioNdArray] - doc = MyDoc(audio_url="toydata/hello.wav") - # doc.audio_tensor, doc.frame_rate = doc.audio_url.load() - # assert isinstance(doc.audio_tensor, np.ndarray) + doc = MyDoc(audio_url='https://www.kozco.com/tech/piano2.wav') + doc.audio_tensor, _ = doc.audio_url.load() + assert isinstance(doc.audio_tensor, np.ndarray) ``` --- From 24dbcc19579de319bc98527861de02eea00527ed Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 16:01:15 +0200 Subject: [PATCH 14/19] fix: fix fix audio url and audio ndarray Signed-off-by: samsja --- docarray/typing/tensor/audio/audio_ndarray.py | 54 +++++++++---------- .../tensor/audio/audio_tensorflow_tensor.py | 53 +++++++++--------- .../typing/tensor/audio/audio_torch_tensor.py | 50 ++++++++--------- docarray/typing/tensor/image/image_ndarray.py | 39 +++++++------- .../tensor/image/image_tensorflow_tensor.py | 37 ++++++------- .../typing/tensor/image/image_torch_tensor.py | 36 +++++++------ 6 files changed, 139 insertions(+), 130 deletions(-) diff --git a/docarray/typing/tensor/audio/audio_ndarray.py b/docarray/typing/tensor/audio/audio_ndarray.py index e6f5388ae45..f47f40b9d8a 100644 --- a/docarray/typing/tensor/audio/audio_ndarray.py +++ b/docarray/typing/tensor/audio/audio_ndarray.py @@ -12,42 +12,42 @@ class AudioNdArray(AbstractAudioTensor, NdArray): --- - ```python - from typing import Optional + ```python + from typing import Optional - from docarray import BaseDoc - from docarray.typing import AudioNdArray, AudioUrl - import numpy as np + from docarray import BaseDoc + from docarray.typing import AudioNdArray, AudioUrl + import numpy as np - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioNdArray] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioNdArray] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - # from tensor - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=np.random.rand(1000, 2), - ) + # from tensor + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=np.random.rand(1000, 2), + ) - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - # from url - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) + # from url + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() - ``` + doc_2.audio_tensor, _ = doc_2.url.load() + doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` - --- + --- """ ... diff --git a/docarray/typing/tensor/audio/audio_tensorflow_tensor.py b/docarray/typing/tensor/audio/audio_tensorflow_tensor.py index a91b5b3183a..497a57f9464 100644 --- a/docarray/typing/tensor/audio/audio_tensorflow_tensor.py +++ b/docarray/typing/tensor/audio/audio_tensorflow_tensor.py @@ -16,44 +16,47 @@ class AudioTensorFlowTensor( Adds audio-specific features to the tensor. - EXAMPLE USAGE - .. code-block:: python + ``` - from typing import Optional + --- + from typing import Optional - import tensorflow as tf - from pydantic import parse_obj_as + import tensorflow as tf + from pydantic import parse_obj_as - from docarray import BaseDoc - from docarray.typing import AudioTensorFlowTensor, AudioUrl + from docarray import BaseDoc + from docarray.typing import AudioTensorFlowTensor, AudioUrl - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioTensorFlowTensor] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioTensorFlowTensor] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=tf.random.normal((1000, 2)), - ) + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=tf.random.normal((1000, 2)), + ) - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + doc_1.audio_tensor.save(file_path='path/to/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2.audio_tensor = doc_2.url.load() + doc_2.audio_tensor.save(file_path='path/to/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` + + --- """ ... diff --git a/docarray/typing/tensor/audio/audio_torch_tensor.py b/docarray/typing/tensor/audio/audio_torch_tensor.py index 86d09e2212f..532d97a7f8e 100644 --- a/docarray/typing/tensor/audio/audio_torch_tensor.py +++ b/docarray/typing/tensor/audio/audio_torch_tensor.py @@ -10,41 +10,43 @@ class AudioTorchTensor(AbstractAudioTensor, TorchTensor, metaclass=metaTorchAndN Adds audio-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + import torch - import torch + from docarray import BaseDoc + from docarray.typing import AudioTorchTensor, AudioUrl - from docarray import BaseDoc - from docarray.typing import AudioTorchTensor, AudioUrl + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioTorchTensor] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioTorchTensor] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=torch.zeros(1000, 2), + ) - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=torch.randn(size=(1000, 2)), - ) + doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) + doc_2.audio_tensor, _ = doc_2.url.load() + doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + --- """ ... diff --git a/docarray/typing/tensor/image/image_ndarray.py b/docarray/typing/tensor/image/image_ndarray.py index df304cd1d21..4ac022b300b 100644 --- a/docarray/typing/tensor/image/image_ndarray.py +++ b/docarray/typing/tensor/image/image_ndarray.py @@ -14,34 +14,35 @@ class ImageNdArray(AbstractImageTensor, NdArray): optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageNdArray, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageNdArray, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageNdArray] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageNdArray] - url: Optional[ImageUrl] - bytes: Optional[bytes] + # from url + doc = MyImageDoc( + title='my_second_audio_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) - # from url - doc = MyImageDoc( - title='my_second_audio_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) + doc.tensor = doc.url.load() - doc.tensor = doc.url.load() - - doc.bytes = doc.tensor.to_bytes() + doc.bytes = doc.tensor.to_bytes() + ``` + --- """ ... diff --git a/docarray/typing/tensor/image/image_tensorflow_tensor.py b/docarray/typing/tensor/image/image_tensorflow_tensor.py index 7afcbb38086..95aba9d5337 100644 --- a/docarray/typing/tensor/image/image_tensorflow_tensor.py +++ b/docarray/typing/tensor/image/image_tensorflow_tensor.py @@ -18,31 +18,32 @@ class ImageTensorFlowTensor( optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageTensorFlowTensor, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageTensorFlowTensor, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageTensorFlowTensor] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageTensorFlowTensor] - url: Optional[ImageUrl] - bytes: Optional[bytes] + doc = MyImageDoc( + title='my_second_image_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) + doc.tensor = doc.url.load() + doc.bytes = doc.tensor.to_bytes() + ``` - doc = MyImageDoc( - title='my_second_image_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) - doc.tensor = doc.url.load() - doc.bytes = doc.tensor.to_bytes() - + --- """ ... diff --git a/docarray/typing/tensor/image/image_torch_tensor.py b/docarray/typing/tensor/image/image_torch_tensor.py index 90ecac13750..bc2a8dbc7db 100644 --- a/docarray/typing/tensor/image/image_torch_tensor.py +++ b/docarray/typing/tensor/image/image_torch_tensor.py @@ -16,31 +16,33 @@ class ImageTorchTensor(AbstractImageTensor, TorchTensor, metaclass=metaTorchAndN optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageTorchTensor, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageTorchTensor, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageTorchTensor] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageTorchTensor] - url: Optional[ImageUrl] - bytes: Optional[bytes] + doc = MyImageDoc( + title='my_second_image_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) - doc = MyImageDoc( - title='my_second_image_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) + doc.tensor = doc.url.load() + doc.bytes = doc.tensor.to_bytes() + ``` - doc.tensor = doc.url.load() - doc.bytes = doc.tensor.to_bytes() + --- """ ... From 9976aa144c0868af0417d854c1f9e92c968a26e6 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 16:12:11 +0200 Subject: [PATCH 15/19] fix: fix video tensor Signed-off-by: samsja --- docarray/typing/tensor/video/video_ndarray.py | 48 +++++++-------- .../typing/tensor/video/video_tensor_mixin.py | 58 ++++++++++--------- 2 files changed, 55 insertions(+), 51 deletions(-) diff --git a/docarray/typing/tensor/video/video_ndarray.py b/docarray/typing/tensor/video/video_ndarray.py index d52e9a1ae28..f6b3cd8d3b1 100644 --- a/docarray/typing/tensor/video/video_ndarray.py +++ b/docarray/typing/tensor/video/video_ndarray.py @@ -19,41 +19,41 @@ class VideoNdArray(NdArray, VideoTensorMixin): Subclass of NdArray, to represent a video tensor. Adds video-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + import numpy as np + from pydantic import parse_obj_as - import numpy as np - from pydantic import parse_obj_as + from docarray import BaseDoc + from docarray.typing import VideoNdArray, VideoUrl - from docarray import BaseDoc - from docarray.typing import VideoNdArray, VideoUrl + class MyVideoDoc(BaseDoc): + title: str + url: Optional[VideoUrl] + video_tensor: Optional[VideoNdArray] - class MyVideoDoc(BaseDoc): - title: str - url: Optional[VideoUrl] - video_tensor: Optional[VideoNdArray] + doc_1 = MyVideoDoc( + title='my_first_video_doc', + video_tensor=np.random.random((100, 224, 224, 3)), + ) - doc_1 = MyVideoDoc( - title='my_first_video_doc', - video_tensor=np.random.random((100, 224, 224, 3)), - ) + doc_1.video_tensor.save(file_path='file_1.mp4') - doc_1.video_tensor.save(file_path='file_1.mp4') + doc_2 = MyVideoDoc( + title='my_second_video_doc', + url='file_1.mp4', + ) + doc_2.video_tensor = parse_obj_as(VideoNdArray, doc_2.url.load().video) + doc_2.video_tensor.save(file_path='file_2.mp4') + ``` - doc_2 = MyVideoDoc( - title='my_second_video_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) - - doc_2.video_tensor = parse_obj_as(VideoNdArray, doc_2.url.load().video) - doc_2.video_tensor.save(file_path='file_2.mp4') - + --- """ @classmethod diff --git a/docarray/typing/tensor/video/video_tensor_mixin.py b/docarray/typing/tensor/video/video_tensor_mixin.py index 5dd11fb2da1..d43344ae6ab 100644 --- a/docarray/typing/tensor/video/video_tensor_mixin.py +++ b/docarray/typing/tensor/video/video_tensor_mixin.py @@ -38,42 +38,46 @@ def save( """ Save video tensor to a .mp4 file. - :param file_path: path to a .mp4 file. If file is a string, open the file by - that name, otherwise treat it as a file-like object. - :param audio_tensor: AudioTensor containing the video's soundtrack. - :param video_frame_rate: video frames per second. - :param video_codec: the name of a video decoder/encoder. - :param audio_frame_rate: audio frames per second. - :param audio_codec: the name of an audio decoder/encoder. - :param audio_format: the name of one of the audio formats supported by PyAV, - such as 'flt', 'fltp', 's16' or 's16p'. - EXAMPLE USAGE - .. code-block:: python - import numpy as np - from docarray import BaseDoc - from docarray.typing.tensor.audio.audio_tensor import AudioTensor - from docarray.typing.tensor.video.video_tensor import VideoTensor + --- + ```python + import numpy as np - class MyDoc(BaseDoc): - video_tensor: VideoTensor - audio_tensor: AudioTensor + from docarray import BaseDoc + from docarray.typing.tensor.audio.audio_tensor import AudioTensor + from docarray.typing.tensor.video.video_tensor import VideoTensor - doc = MyDoc( - video_tensor=np.random.randint(low=0, high=256, size=(10, 200, 300, 3)), - audio_tensor=np.random.randn(100, 1, 1024).astype("float32"), - ) + class MyDoc(BaseDoc): + video_tensor: VideoTensor + audio_tensor: AudioTensor - doc.video_tensor.save( - file_path="toydata/mp_.mp4", - audio_tensor=doc.audio_tensor, - audio_format="flt", - ) + doc = MyDoc( + video_tensor=np.random.randint(low=0, high=256, size=(10, 200, 300, 3)), + audio_tensor=np.random.randn(100, 1, 1024).astype("float32"), + ) + + doc.video_tensor.save( + file_path="/tmp/mp_.mp4", + audio_tensor=doc.audio_tensor, + audio_format="flt", + ) + ``` + + --- + :param file_path: path to a .mp4 file. If file is a string, open the file by + that name, otherwise treat it as a file-like object. + :param audio_tensor: AudioTensor containing the video's soundtrack. + :param video_frame_rate: video frames per second. + :param video_codec: the name of a video decoder/encoder. + :param audio_frame_rate: audio frames per second. + :param audio_codec: the name of an audio decoder/encoder. + :param audio_format: the name of one of the audio formats supported by PyAV, + such as 'flt', 'fltp', 's16' or 's16p'. """ import av From 40aab666b4604f338691ec6e196d8cb89da7d571 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 16:12:38 +0200 Subject: [PATCH 16/19] fix: fix video tensor Signed-off-by: samsja --- docarray/typing/tensor/video/video_tensor_mixin.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docarray/typing/tensor/video/video_tensor_mixin.py b/docarray/typing/tensor/video/video_tensor_mixin.py index d43344ae6ab..3d19e499bf4 100644 --- a/docarray/typing/tensor/video/video_tensor_mixin.py +++ b/docarray/typing/tensor/video/video_tensor_mixin.py @@ -38,9 +38,6 @@ def save( """ Save video tensor to a .mp4 file. - - - --- ```python From 71c6ffc10c76a612042fd79a83c928a754e6fdcd Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 16:24:02 +0200 Subject: [PATCH 17/19] fix: fix audio bytes Signed-off-by: samsja --- docarray/typing/__init__.py | 5 +++- docarray/typing/bytes/__init__.py | 4 ++- docarray/typing/bytes/audio_bytes.py | 35 +++++++++++++++------------ docarray/typing/bytes/video_bytes.py | 2 +- docarray/typing/tensor/__init__.py | 33 +++++++++++++++++++++---- file_1.mp4 | Bin 0 -> 8232 bytes file_2.mp4 | Bin 0 -> 8232 bytes 7 files changed, 55 insertions(+), 24 deletions(-) create mode 100644 file_1.mp4 create mode 100644 file_2.mp4 diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index 3deefe96f01..c9dceb600c9 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -1,4 +1,4 @@ -from docarray.typing.bytes import ImageBytes +from docarray.typing.bytes import AudioBytes, ImageBytes, VideoBytes from docarray.typing.id import ID from docarray.typing.tensor import ImageNdArray, ImageTensor from docarray.typing.tensor.audio import AudioNdArray @@ -35,6 +35,9 @@ 'ImageBytes', 'ImageTensor', 'ImageNdArray', + 'ImageBytes', + 'VideoBytes', + 'AudioBytes', ] from docarray.utils._internal.misc import is_tf_available, is_torch_available diff --git a/docarray/typing/bytes/__init__.py b/docarray/typing/bytes/__init__.py index 9b622669919..2cf8524bcc0 100644 --- a/docarray/typing/bytes/__init__.py +++ b/docarray/typing/bytes/__init__.py @@ -1,3 +1,5 @@ +from docarray.typing.bytes.audio_bytes import AudioBytes from docarray.typing.bytes.image_bytes import ImageBytes +from docarray.typing.bytes.video_bytes import VideoBytes -__all__ = ['ImageBytes'] +__all__ = ['ImageBytes', 'VideoBytes', 'AudioBytes'] diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index 6a6c4e14bfd..649d06bac03 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -46,32 +46,35 @@ def load(self) -> Tuple[np.ndarray, int]: """ Load the Audio from the bytes into a numpy.ndarray Audio tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional + from docarray import BaseDoc + from docarray.typing import AudioUrl, NdArray, AudioBytes - from docarray import BaseDoc - import numpy as np + import numpy as np - from docarray.typing import AudioUrl + class MyAudio(BaseDoc): + url: AudioUrl + tensor: Optional[NdArray] + bytes: Optional[AudioBytes] + frame_rate: Optional[float] - class MyAudio(Document): - url: AudioUrl - tensor: Optional[NdArray] - bytes: Optional[bytes] + doc = MyAudio(url='https://www.kozco.com/tech/piano2.wav') + doc.bytes = doc.url.load_bytes() + doc.tensor, doc.frame_rate = doc.bytes.load() - doc = MyAudio(url="toydata/hello.wav") - doc.bytes = doc.url.load_bytes() - doc.tensor, doc.frame_rate = doc.bytes.load() + # Note this is equivalent to do - # Note this is equivalent to do + doc.tensor, doc.frame_rate = doc.url.load() - doc.tensor, doc.frame_rate = doc.url.load() - - assert isinstance(doc.audio_tensor, np.ndarray) + assert isinstance(doc.tensor, np.ndarray) + ``` + --- :return: np.ndarray representing the Audio as RGB values """ diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index fb45a36063f..a7660e0a898 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -5,9 +5,9 @@ from pydantic import parse_obj_as from pydantic.validators import bytes_validator -from docarray.typing import AudioNdArray, NdArray, VideoNdArray from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto +from docarray.typing.tensor import AudioNdArray, NdArray, VideoNdArray if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index f2be5ecc0ac..4185f23c22d 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -1,7 +1,9 @@ +from docarray.typing.tensor.audio import AudioNdArray from docarray.typing.tensor.embedding import AnyEmbedding, NdArrayEmbedding from docarray.typing.tensor.image import ImageNdArray, ImageTensor from docarray.typing.tensor.ndarray import NdArray from docarray.typing.tensor.tensor import AnyTensor +from docarray.typing.tensor.video import VideoNdArray __all__ = [ 'NdArray', @@ -10,26 +12,47 @@ 'NdArrayEmbedding', 'ImageNdArray', 'ImageTensor', - 'TensorFlowTensor', + 'AudioNdArray', + 'VideoNdArray', ] from docarray.utils._internal.misc import is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 - - __all__.extend(['TorchEmbedding', 'TorchTensor', 'ImageTorchTensor']) + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 + + __all__.extend( + [ + 'TorchEmbedding', + 'TorchTensor', + 'ImageTorchTensor', + 'AudioTorchTensor', + 'VideoTorchTensor', + ] + ) torch_available = is_torch_available() tf_available = is_tf_available() if tf_available: + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 - - __all__.extend(['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor']) + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 + + __all__.extend( + [ + 'TensorFlowEmbedding', + 'TensorFlowTensor', + 'ImageTensorFlowTensor', + 'AudioTensorFlowTensor', + 'VideoTensorFlowTensor', + ] + ) diff --git a/file_1.mp4 b/file_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15b58e871afd89ea39501de2fd6f2618a77bd9fe GIT binary patch literal 8232 zcmeHMZ)hCV7QdTmW372rOSDz$W2EuVhh#Im*`#e?uxnadLBR@Ikq67&nY-CBGdq)+ zNxJciQHxgm@;*rArO#Ar_4%}-RQk$8eTh)vJrNO9w4&0|So~9JDQLPDUC+65?`CJ( z!t;ZEkR38}&%b-``Q0<;p4?eNh*}CKJS%V=LfQznaA)bbk#W3Sh7hu(Ax)o^{LW;nSy4Wi!L@7TDhC#UA` z9s-@o46qq;y$PF@LN%*teLYz%n+0Jx486hL-fi2qrN=Fkxi+n&U4OV2$4!?*#|9hM z3oW-245|h#QbW(EJ}c>cs>zDBYmDfbL2Xb|X@%Mo0n@c=-=J2_WcpNx>19?`gRvq| zYg8|ofUfW?_@U56H+Yeo%yX4_T()Y|G= z(+GhuMjc%P7pQr^TVZ-O)1AqvB^rbUFBq{rw1@^6^$KpO6tGb5$*N)5haLeqZ@caY zEdx_nkOj77fG<*zQMmxED?XphupAnqLslhZzD>azzZAw0M7!Fk|coRTdrk(|wn$$x$#X`}d0ZwB1j04=ptl@H41fA=#N@3XbKpP1? z=sLnCU~E0xuL-i?SQR~+Qv-ulm@yXWISt+PDXy*0f--dXjY0!A>U~3}MPGG_u)qkb zXn{Qf)LhpX;mLs2ZA7Eo}N=J526Qe3iN=n=_tGy7-ljF zFLRdd*iNpR`FZ|>o8H}jVyf%Fu@~m|OkZCZ8oMsFYv-X;N544B-aD}As?Sfa_DJgQ zM5I#Hel{Uy+2?uH$`50J&Tb5^NEEY~hgP17(xQYY;n zqB7OC?)ZugSI&@&KjExWi&(A&TjM@%`;oLA*o(G@r7dUMQM)`pRlfCweEDy;E$M%N zd;c0pJHeOdw@+=q^06moAKuDQE&7tnm0CjHhV~u#=|?tR-hK59A>(fWkLbTLw=93_ zr8*(6KFB3Xts&ty;N^2+{_#3Vy$yOq^v%`{D`!aClbls*5i7LQ|F{p97lxk&Z3h~U z@5Sj}o#Z@NM?BX9sC@v+UI2w5K~OIq0#N^30EL0+%qz8ooQDTMy|eL(IH(hQx7-;A z)sS#dm!zh@4&#oVd~!p~QNeRZ!Tj3&2a|YW{!*elDGsf zabfXl+UI4wc#%3!ekJ5GUXLVkL8(gY^9_l_1;s1oD3mLXOFAleY2K<|{GW|h{o?;` zwCb14FWlai!GB<_`X%=HR{e^~T~yEHKHsWeeBWr*FS&1EnZ(U@v;eR)1`bx$Yq;Iis0YE6=Oi0;MXr$o{wrH zr2R+uZaf0Yzl0-vPNm6Ej0W>2uf#m(o%aZI!kdsTW`+T@tC$_Y?;-qx5pdi^!smA1 zF)a!j)iIOUH4y)o+;xce>nWSI&nbA|?;euI_HO(gWHYUpHba~CYv6--35d>dhY#B9 zc9$9HR+;dX9XrE?`i?Q*RwH`x=RsJsfqpp%0~rvsc?y5*ffldkF(b(lfV~+=yzx5M zB9bpWSJvU%D(Qn!hf*f#tRE#ShN0bec<9mcdO{Qej}GPSrED@q7MH(0vA_GLGiR`S zCw%{pR`k-RflX6*tPDcF+CcKD!Ep#Q*a8TQ0$%76+{%r{f~^tBAR=71RJ4*u7W?qA zpl#`+^=Pe6v<~=s{E^;|C5Q)JcK*Jvg51IUz)R!~fxJRGwHI{vb4~OBxk7>Lhr7G! zz6I!<;CrE!Jkl-xw_Oj|`wSP>883_6!lg?<`FJvAx)o^{LW;nSy4Wi!L@7TDhC#UA` z9s-@o46qq;y$PF@LN%*teLYz%n+0Jx486hL-fi2qrN=Fkxi+n&U4OV2$4!?*#|9hM z3oW-245|h#QbW(EJ}c>cs>zDBYmDfbL2Xb|X@%Mo0n@c=-=J2_WcpNx>19?`gRvq| zYg8|ofUfW?_@U56H+Yeo%yX4_T()Y|G= z(+GhuMjc%P7pQr^TVZ-O)1AqvB^rbUFBq{rw1@^6^$KpO6tGb5$*N)5haLeqZ@caY zEdx_nkOj77fG<*zQMmxED?XphupAnqLslhZzD>azzZAw0M7!Fk|coRTdrk(|wn$$x$#X`}d0ZwB1j04=ptl@H41fA=#N@3XbKpP1? z=sLnCU~E0xuL-i?SQR~+Qv-ulm@yXWISt+PDXy*0f--dXjY0!A>U~3}MPGG_u)qkb zXn{Qf)LhpX;mLs2ZA7Eo}N=J526Qe3iN=n=_tGy7-ljF zFLRdd*iNpR`FZ|>o8H}jVyf%Fu@~m|OkZCZ8oMsFYv-X;N544B-aD}As?Sfa_DJgQ zM5I#Hel{Uy+2?uH$`50J&Tb5^NEEY~hgP17(xQYY;n zqB7OC?)ZugSI&@&KjExWi&(A&TjM@%`;oLA*o(G@r7dUMQM)`pRlfCweEDy;E$M%N zd;c0pJHeOdw@+=q^06moAKuDQE&7tnm0CjHhV~u#=|?tR-hK59A>(fWkLbTLw=93_ zr8*(6KFB3Xts&ty;N^2+{_#3Vy$yOq^v%`{D`!aClbls*5i7LQ|F{p97lxk&Z3h~U z@5Sj}o#Z@NM?BX9sC@v+UI2w5K~OIq0#N^30EL0+%qz8ooQDTMy|eL(IH(hQx7-;A z)sS#dm!zh@4&#oVd~!p~QNeRZ!Tj3&2a|YW{!*elDGsf zabfXl+UI4wc#%3!ekJ5GUXLVkL8(gY^9_l_1;s1oD3mLXOFAleY2K<|{GW|h{o?;` zwCb14FWlai!GB<_`X%=HR{e^~T~yEHKHsWeeBWr*FS&1EnZ(U@v;eR)1`bx$Yq;Iis0YE6=Oi0;MXr$o{wrH zr2R+uZaf0Yzl0-vPNm6Ej0W>2uf#m(o%aZI!kdsTW`+T@tC$_Y?;-qx5pdi^!smA1 zF)a!j)iIOUH4y)o+;xce>nWSI&nbA|?;euI_HO(gWHYUpHba~CYv6--35d>dhY#B9 zc9$9HR+;dX9XrE?`i?Q*RwH`x=RsJsfqpp%0~rvsc?y5*ffldkF(b(lfV~+=yzx5M zB9bpWSJvU%D(Qn!hf*f#tRE#ShN0bec<9mcdO{Qej}GPSrED@q7MH(0vA_GLGiR`S zCw%{pR`k-RflX6*tPDcF+CcKD!Ep#Q*a8TQ0$%76+{%r{f~^tBAR=71RJ4*u7W?qA zpl#`+^=Pe6v<~=s{E^;|C5Q)JcK*Jvg51IUz)R!~fxJRGwHI{vb4~OBxk7>Lhr7G! zz6I!<;CrE!Jkl-xw_Oj|`wSP>883_6!lg?<`FJv Date: Tue, 28 Mar 2023 16:27:59 +0200 Subject: [PATCH 18/19] fix: video and image bytes Signed-off-by: samsja --- docarray/typing/bytes/image_bytes.py | 39 ++++++++++++++-------------- docarray/typing/bytes/video_bytes.py | 32 +++++++++++++---------- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index 2888f60fadd..f8e68ba03f0 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -51,34 +51,35 @@ def load( """ Load the image from the bytes into a numpy.ndarray image tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import ImageUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import ImageUrl - import numpy as np + class MyDoc(BaseDoc): + img_url: ImageUrl - class MyDoc(BaseDoc): - img_url: ImageUrl + doc = MyDoc( + img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" + ) - doc = MyDoc( - img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" - ) + img_tensor = doc.img_url.load() + assert isinstance(img_tensor, np.ndarray) - img_tensor = doc.img_url.load() - assert isinstance(img_tensor, np.ndarray) + img_tensor = doc.img_url.load(height=224, width=224) + assert img_tensor.shape == (224, 224, 3) - img_tensor = doc.img_url.load(height=224, width=224) - assert img_tensor.shape == (224, 224, 3) - - layout = ('C', 'W', 'H') - img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) - assert img_tensor.shape == (3, 200, 100) + layout = ('C', 'W', 'H') + img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) + assert img_tensor.shape == (3, 200, 100) + ``` + --- :param width: width of the image tensor. :param height: height of the image tensor. diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index a7660e0a898..a48face2b6b 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -55,29 +55,33 @@ def load(self, **kwargs) -> VideoLoadResult: (`VideoLoadResult.audio`) and an NdArray containing the key frame indices (`VideoLoadResult.key_frame_indices`). - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import VideoUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import VideoUrl - import numpy as np + class MyDoc(BaseDoc): + video_url: VideoUrl - class MyDoc(BaseDoc): - video_url: VideoUrl + doc = MyDoc( + video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' + ) - doc = MyDoc(video_url="toydata/mp_.mp4") + video, audio, key_frame_indices = doc.video_url.load() + assert isinstance(video, np.ndarray) + assert isinstance(audio, np.ndarray) + assert isinstance(key_frame_indices, np.ndarray) + ``` + + --- - video, audio, key_frame_indices = doc.video_url.load() - assert isinstance(video, np.ndarray) - assert isinstance(audio, np.ndarray) - assert isinstance(key_frame_indices, np.ndarray) :param kwargs: supports all keyword arguments that are being supported by - av.open() as described in: - https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open + av.open() as described [here](https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open) :return: a VideoLoadResult instance with video, audio and keyframe indices """ import av From 0ab736ff9535b08e081b565ca1b7fdb9f42eb43f Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 28 Mar 2023 16:39:47 +0200 Subject: [PATCH 19/19] docs: move typing section Signed-off-by: samsja --- docs/api_references/typing/bytes.md | 6 ++++++ docs/api_references/typing/id.md | 6 ++++++ docs/api_references/typing/tensor.md | 3 +++ docs/api_references/typing/typing.md | 4 ---- docs/api_references/typing/url.md | 6 ++++++ 5 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 docs/api_references/typing/bytes.md create mode 100644 docs/api_references/typing/id.md create mode 100644 docs/api_references/typing/tensor.md delete mode 100644 docs/api_references/typing/typing.md create mode 100644 docs/api_references/typing/url.md diff --git a/docs/api_references/typing/bytes.md b/docs/api_references/typing/bytes.md new file mode 100644 index 00000000000..a263581af0d --- /dev/null +++ b/docs/api_references/typing/bytes.md @@ -0,0 +1,6 @@ +# Bytes + +::: docarray.typing.bytes + + + diff --git a/docs/api_references/typing/id.md b/docs/api_references/typing/id.md new file mode 100644 index 00000000000..6dbfbd39dea --- /dev/null +++ b/docs/api_references/typing/id.md @@ -0,0 +1,6 @@ +# Id + +::: docarray.typing.id + + + diff --git a/docs/api_references/typing/tensor.md b/docs/api_references/typing/tensor.md new file mode 100644 index 00000000000..f8040705f32 --- /dev/null +++ b/docs/api_references/typing/tensor.md @@ -0,0 +1,3 @@ +# Tensor + +::: docarray.typing.tensor diff --git a/docs/api_references/typing/typing.md b/docs/api_references/typing/typing.md deleted file mode 100644 index fa29d35750c..00000000000 --- a/docs/api_references/typing/typing.md +++ /dev/null @@ -1,4 +0,0 @@ -# Typing - -::: docarray.typing - diff --git a/docs/api_references/typing/url.md b/docs/api_references/typing/url.md new file mode 100644 index 00000000000..f58c3b6c722 --- /dev/null +++ b/docs/api_references/typing/url.md @@ -0,0 +1,6 @@ +# Url + +::: docarray.typing.url + + +